// // Copyright © 2024 Hardcore Engineering Inc. // // Licensed under the Eclipse Public License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. You may // obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // // See the License for the specific language governing permissions and // limitations under the License. // import core, { type Account, AccountRole, type Class, type Doc, type DocumentUpdate, type Domain, type FieldIndexConfig, type MeasureContext, type MixinUpdate, platformNow, platformNowDiff, type Projection, type Ref, systemAccountUuid, type WorkspaceUuid } from '@hcengineering/core' import { type DomainHelperOperations } from '@hcengineering/server-core' import type postgres from 'postgres' import { type ParameterOrJSON } from 'postgres' import { addSchema, type DataType, getDocFieldsByDomains, getIndex, getSchema, getSchemaAndFields, type Schema, type SchemaAndFields, translateDomain } from './schemas' import { retryTxn, type DBClient } from '@hcengineering/postgres-base' const loadedDomains = new Set() let loadedTables = new Set() export const NumericTypes = [ core.class.TypeNumber, core.class.TypeTimestamp, core.class.TypeDate, core.class.Collection ] export async function createTables ( ctx: MeasureContext, client: postgres.Sql, url: string, domains: string[] ): Promise { const filtered = domains.filter((d) => !loadedDomains.has(url + translateDomain(d))) if (filtered.length === 0) { return } const mapped = filtered.map((p) => translateDomain(p)) const t = platformNow() loadedTables = loadedTables.size === 0 ? new Set( ( await ctx.with('load-table', {}, () => client.unsafe(` SELECT table_name FROM information_schema.tables WHERE table_schema NOT IN ('pg_catalog', 'information_schema') AND table_name NOT LIKE 'pg_%' AND table_name NOT LIKE 'cluster_%' AND table_name NOT LIKE 'kv_%' AND table_name NOT LIKE 'node_%'`) ) ).map((it) => it.table_name) ) : loadedTables console.log('load-table', platformNowDiff(t)) const domainsToLoad = mapped.filter((it) => loadedTables.has(it)) if (domainsToLoad.length > 0) { await ctx.with('load-schemas', {}, () => getTableSchema(client, domainsToLoad)) } const domainsToCreate: string[] = [] for (const domain of mapped) { if (!loadedTables.has(domain)) { domainsToCreate.push(domain) } else { loadedDomains.add(url + domain) } } if (domainsToCreate.length > 0) { await retryTxn(client, async (client) => { for (const domain of domainsToCreate) { await ctx.with('create-table', {}, () => createTable(client, domain)) loadedDomains.add(url + domain) } }) } } async function getTableSchema (client: postgres.Sql, domains: string[]): Promise { const res = await client.unsafe(`SELECT column_name::name, data_type::text, is_nullable::text, table_name::name FROM information_schema.columns WHERE table_name IN (${domains.map((it) => `'${it}'`).join(', ')}) and table_schema = 'public'::name ORDER BY table_name::name, ordinal_position::int ASC;`) const schemas: Record = {} for (const column of res) { if (column.column_name === 'workspaceId' || column.column_name === 'data') { continue } const schema = schemas[column.table_name] ?? {} schemas[column.table_name] = schema schema[column.column_name] = { type: parseDataType(column.data_type), notNull: column.is_nullable === 'NO', index: false } } for (const [domain, schema] of Object.entries(schemas)) { addSchema(domain, schema) } } function parseDataType (type: string): DataType { switch (type) { case 'text': return 'text' case 'bigint': return 'bigint' case 'boolean': return 'bool' case 'ARRAY': return 'text[]' } return 'text' } async function createTable (client: postgres.Sql, domain: string): Promise { const schema = getSchema(domain) const fields: string[] = [] for (const key in schema) { const val = schema[key] fields.push(`"${key}" ${val.type} ${val.notNull ? 'NOT NULL' : ''}`) } const colums = fields.join(', ') const res = await client.unsafe(`CREATE TABLE IF NOT EXISTS ${domain} ( "workspaceId" uuid NOT NULL, ${colums}, data JSONB NOT NULL, PRIMARY KEY("workspaceId", _id) )`) if (res.count > 0) { for (const key in schema) { const val = schema[key] if (val.index) { await client.unsafe(` CREATE INDEX ${domain}_${key} ON ${domain} ${getIndex(val)} ("${key}") `) } fields.push(`"${key}" ${val.type} ${val.notNull ? 'NOT NULL' : ''}`) } } } export function convertDoc ( domain: string, doc: T, workspaceId: WorkspaceUuid, schemaAndFields?: SchemaAndFields ): DBDoc { const extractedFields: Doc & Record = { _id: doc._id, space: doc.space, createdBy: doc.createdBy, modifiedBy: doc.modifiedBy, modifiedOn: doc.modifiedOn, createdOn: doc.createdOn ?? doc.modifiedOn, _class: doc._class, '%hash%': (doc as any)['%hash%'] ?? Date.now().toString(16) } const remainingData: Partial = {} const extractedFieldsKeys = new Set(Object.keys(extractedFields)) schemaAndFields = schemaAndFields ?? getSchemaAndFields(domain) for (const key in doc) { if (extractedFieldsKeys.has(key)) { continue } if (schemaAndFields.domainFields.has(key)) { extractedFields[key] = doc[key] } else { remainingData[key] = doc[key] } } // Check if some fields are missing for (const [key, _type] of Object.entries(schemaAndFields.schema)) { if (_type.notNull) { if (!(key in doc) || (doc as any)[key] == null) { // We missing required field, and we need to add a dummy value for it. // Null value is not allowed switch (_type.type) { case 'bigint': extractedFields[key] = 0 break case 'bool': extractedFields[key] = false break case 'text': extractedFields[key] = '' break case 'text[]': extractedFields[key] = [] break } } } } const res: any = { ...extractedFields, workspaceId, data: remainingData } return res } export function inferType (val: any): string { if (typeof val === 'string') { return '::text' } if (typeof val === 'number') { return '::numeric' } if (typeof val === 'boolean') { return '::boolean' } if (Array.isArray(val)) { const type = inferType(val[0] ?? val[1]) if (type !== '') { return type + '[]' } } if (typeof val === 'object') { if (val instanceof Date) { return '::text' } return '::jsonb' } return '' } export function parseUpdate ( ops: DocumentUpdate | MixinUpdate, schemaFields: SchemaAndFields ): { extractedFields: Partial remainingData: Partial } { const extractedFields: Partial = {} const remainingData: Partial = {} for (const key in ops) { const val = (ops as any)[key] if (key.startsWith('$')) { for (const k in val) { if (schemaFields.domainFields.has(k)) { ;(extractedFields as any)[k] = val[key] } else { ;(remainingData as any)[k] = val[key] } } } else { if (schemaFields.domainFields.has(key)) { ;(extractedFields as any)[key] = val } else { ;(remainingData as any)[key] = val } } } return { extractedFields, remainingData } } export function escapeBackticks (str: string): string { if (typeof str !== 'string') return str return str.replaceAll("'", "''") } export function isOwner (account: Account): boolean { return account.role === AccountRole.Owner || account.uuid === systemAccountUuid } export class DBCollectionHelper implements DomainHelperOperations { constructor ( protected readonly client: DBClient, protected readonly workspaceId: WorkspaceUuid ) {} async dropIndex (domain: Domain, name: string): Promise {} domains = new Set() async create (domain: Domain): Promise {} async exists (domain: Domain): Promise { // Always exists. We don't need to check for index existence return true } async listDomains (): Promise> { return this.domains } async createIndex (domain: Domain, value: string | FieldIndexConfig, options?: { name: string }): Promise {} async listIndexes (domain: Domain): Promise<{ name: string }[]> { return [] } async estimatedCount (domain: Domain): Promise { // We should always return 0, since no controlled index stuff is required for postgres driver return 0 } } export function decodeArray (value: string): string[] { if (value === 'NULL') return [] // Remove first and last character (the array brackets) const inner = value.substring(1, value.length - 1) const items = inner.split(',') return items.map((item) => { // Remove quotes at start/end if they exist let result = item if (result.startsWith('"')) { result = result.substring(1) } if (result.endsWith('"')) { result = result.substring(0, result.length - 1) } // Replace escaped quotes with regular quotes let final = '' for (let i = 0; i < result.length; i++) { if (result[i] === '\\' && result[i + 1] === '"') { final += '"' i++ // Skip next char } else { final += result[i] } } return final }) } export function convertArrayParams (parameters?: ParameterOrJSON[]): any[] | undefined { if (parameters === undefined) return undefined return parameters.map((param) => { if (Array.isArray(param)) { if (param.length === 0) return '{}' const sanitized = param.map((item) => { if (item === null) return 'NULL' if (typeof item === 'string') return `"${item.replace(/"/g, '\\"')}"` return String(item) }) return `{${sanitized.join(',')}}` } return param }) } export function filterProjection (data: any, projection: Projection | undefined): any { if (projection === undefined) { return data } for (const key in data) { if (!Object.prototype.hasOwnProperty.call(projection, key) || (projection as any)[key] === 0) { // check nested projections in case of object let value = data[key] if (typeof value === 'object' && !Array.isArray(value) && value != null) { // We need to filter projection for nested objects const innerP = Object.entries(projection as any) .filter((it) => it[0].startsWith(key)) .map((it) => [it[0].substring(key.length + 1), it[1]]) if (innerP.length > 0) { value = filterProjection(value, Object.fromEntries(innerP)) data[key] = value continue } } // eslint-disable-next-line @typescript-eslint/no-dynamic-delete delete data[key] } } return data } export function parseDocWithProjection ( doc: DBDoc, domain: string, projection?: Projection | undefined ): T { const { workspaceId, data, '%hash%': hash, ...rest } = doc const schema = getSchema(domain) for (const key in rest) { if ((rest as any)[key] === 'NULL' || (rest as any)[key] === null) { if (key === 'attachedTo') { // eslint-disable-next-line @typescript-eslint/no-dynamic-delete delete rest[key] } else { ;(rest as any)[key] = null } } else if (schema[key] !== undefined && schema[key].type === 'bigint') { ;(rest as any)[key] = Number.parseInt((rest as any)[key]) } else if (schema[key] !== undefined && schema[key].type === 'text[]' && typeof (rest as any)[key] === 'string') { ;(rest as any)[key] = decodeArray((rest as any)[key]) } } let resultData = data if (projection !== undefined) { resultData = filterProjection(data, projection) } const res = { ...resultData, ...rest } as any as T return res } export function parseDoc (doc: DBDoc, schema: Schema): T { const { workspaceId, data, ...rest } = doc for (const key in rest) { if ((rest as any)[key] === 'NULL' || (rest as any)[key] === null) { if (key === 'attachedTo') { // eslint-disable-next-line @typescript-eslint/no-dynamic-delete delete rest[key] } else { ;(rest as any)[key] = null } } else if (schema[key] !== undefined && schema[key].type === 'bigint') { ;(rest as any)[key] = Number.parseInt((rest as any)[key]) } else if (schema[key] !== undefined && schema[key].type === 'text[]' && typeof (rest as any)[key] === 'string') { ;(rest as any)[key] = decodeArray((rest as any)[key]) } } const res = { ...data, ...rest } as any as T return res } export interface DBDoc extends Doc { workspaceId: WorkspaceUuid data: Record [key: string]: any } export function isDataField (domain: string, field: string): boolean { return !getDocFieldsByDomains(domain).includes(field) } export interface JoinProps { table: string // table to join path: string // _id.roles, attachedTo.attachedTo, space... fromAlias: string fromField: string toAlias: string // alias for the table toField: string // field to join on isReverse: boolean toClass?: Ref> classes?: Ref>[] // filter by classes } export function escape (str: T): T { if (typeof str === 'string') { // Remove all characters except a-z, A-Z, 0-9 and _ . // Add cyrillic for support old custom attributes // eslint-disable-next-line no-useless-escape return str.replace(/[^a-zA-ZА-Яа-яЁё0-9_.:\- ]/g, '') as T } return str }