diff --git a/dev/tool/src/clean.ts b/dev/tool/src/clean.ts index d049f81491..da7d4f6c4a 100644 --- a/dev/tool/src/clean.ts +++ b/dev/tool/src/clean.ts @@ -1152,7 +1152,9 @@ function isPersonAccount (tx: TxCUD): boolean { } async function update (h: Hierarchy, db: Db, doc: T, update: DocumentUpdate): Promise { - await db.collection(h.getDomain(doc._class)).updateOne({ _id: doc._id }, { $set: { ...update, '%hash%': null } }) + await db + .collection(h.getDomain(doc._class)) + .updateOne({ _id: doc._id }, { $set: { ...update, '%hash%': Date.now().toString(16) } }) } async function updateId ( @@ -1173,12 +1175,14 @@ async function updateId ( const newId = generateId() // update txes - await db.collection(DOMAIN_TX).updateMany({ objectId: doc._id }, { $set: { objectId: newId, '%hash%': null } }) + await db + .collection(DOMAIN_TX) + .updateMany({ objectId: doc._id }, { $set: { objectId: newId, '%hash%': Date.now().toString(16) } }) // update nested txes await db .collection(DOMAIN_TX) - .updateMany({ 'tx.objectId': doc._id }, { $set: { 'tx.objectId': newId, '%hash%': null } }) + .updateMany({ 'tx.objectId': doc._id }, { $set: { 'tx.objectId': newId, '%hash%': Date.now().toString(16) } }) // we have generated ids for calendar, let's update in if (h.isDerived(doc._class, core.class.Account)) { @@ -1232,7 +1236,7 @@ async function updateId ( await db.collection(domain).insertOne({ ...raw, _id: newId as any, - '%hash%': null + '%hash%': Date.now().toString(16) }) await db.collection(domain).deleteOne({ _id: doc._id }) } diff --git a/models/activity/src/migration.ts b/models/activity/src/migration.ts index 898984596f..34c1ff8b34 100644 --- a/models/activity/src/migration.ts +++ b/models/activity/src/migration.ts @@ -217,12 +217,6 @@ export const activityOperation: MigrateOperation = { state: 'migrate-activity-markup', func: migrateActivityMarkup }, - { - state: 'fix-rename-backups', - func: async (client: MigrationClient): Promise => { - await client.update(DOMAIN_ACTIVITY, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } }) - } - }, { state: 'move-reactions', func: async (client: MigrationClient): Promise => { diff --git a/models/attachment/src/migration.ts b/models/attachment/src/migration.ts index 73b55dd12b..df8c59ae94 100644 --- a/models/attachment/src/migration.ts +++ b/models/attachment/src/migration.ts @@ -24,12 +24,6 @@ import attachment, { attachmentId, DOMAIN_ATTACHMENT } from '.' export const attachmentOperation: MigrateOperation = { async migrate (client: MigrationClient): Promise { await tryMigrate(client, attachmentId, [ - { - state: 'fix-rename-backups', - func: async (client: MigrationClient): Promise => { - await client.update(DOMAIN_ATTACHMENT, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } }) - } - }, { state: 'fix-attachedTo', func: async (client: MigrationClient): Promise => { diff --git a/models/chunter/src/migration.ts b/models/chunter/src/migration.ts index 65438a982b..70d2c2151c 100644 --- a/models/chunter/src/migration.ts +++ b/models/chunter/src/migration.ts @@ -362,12 +362,6 @@ export const chunterOperation: MigrateOperation = { 'attributeUpdates.attrKey': 'members' }) } - }, - { - state: 'fix-rename-backups', - func: async (client: MigrationClient): Promise => { - await client.update(DOMAIN_CHUNTER, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } }) - } } ]) }, diff --git a/models/contact/src/migration.ts b/models/contact/src/migration.ts index a109668aa8..d78df3c00e 100644 --- a/models/contact/src/migration.ts +++ b/models/contact/src/migration.ts @@ -26,7 +26,7 @@ import activity, { DOMAIN_ACTIVITY } from '@hcengineering/model-activity' import core, { DOMAIN_SPACE } from '@hcengineering/model-core' import { DOMAIN_VIEW } from '@hcengineering/model-view' -import contact, { contactId, DOMAIN_CHANNEL, DOMAIN_CONTACT } from './index' +import contact, { contactId, DOMAIN_CONTACT } from './index' async function createEmployeeEmail (client: TxOperations): Promise { const employees = await client.findAll(contact.mixin.Employee, {}) @@ -300,13 +300,6 @@ export const contactOperation: MigrateOperation = { { state: 'create-person-spaces-v1', func: createPersonSpaces - }, - { - state: 'fix-rename-backups', - func: async (client: MigrationClient): Promise => { - await client.update(DOMAIN_CONTACT, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } }) - await client.update(DOMAIN_CHANNEL, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } }) - } } ]) }, diff --git a/models/core/src/migration.ts b/models/core/src/migration.ts index ccebe20b86..f922129f28 100644 --- a/models/core/src/migration.ts +++ b/models/core/src/migration.ts @@ -424,10 +424,12 @@ export const coreOperation: MigrateOperation = { func: migrateCollaborativeContentToStorage }, { - state: 'fix-rename-backups', + state: 'fix-backups-hash-timestamp', func: async (client: MigrationClient): Promise => { - await client.update(DOMAIN_TX, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } }) - await client.update(DOMAIN_SPACE, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } }) + const now = Date.now().toString(16) + for (const d of client.hierarchy.domains()) { + await client.update(d, { '%hash%': { $in: [null, ''] } }, { $set: { '%hash%': now } }) + } } }, { diff --git a/models/document/src/migration.ts b/models/document/src/migration.ts index 6bc4efb4a4..79dcd9fd4a 100644 --- a/models/document/src/migration.ts +++ b/models/document/src/migration.ts @@ -335,12 +335,6 @@ export const documentOperation: MigrateOperation = { state: 'renameFields', func: renameFields }, - { - state: 'fix-rename-backups', - func: async (client: MigrationClient): Promise => { - await client.update(DOMAIN_DOCUMENT, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } }) - } - }, { state: 'renameFieldsRevert', func: renameFieldsRevert diff --git a/models/drive/src/migration.ts b/models/drive/src/migration.ts index 6ea69d90d2..e2762c8b13 100644 --- a/models/drive/src/migration.ts +++ b/models/drive/src/migration.ts @@ -132,12 +132,6 @@ export const driveOperation: MigrateOperation = { { state: 'renameFields', func: renameFields - }, - { - state: 'fix-rename-backups', - func: async (client: MigrationClient): Promise => { - await client.update(DOMAIN_DRIVE, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } }) - } } ]) }, diff --git a/models/notification/src/migration.ts b/models/notification/src/migration.ts index 56fe3affe2..972d3f77a2 100644 --- a/models/notification/src/migration.ts +++ b/models/notification/src/migration.ts @@ -393,12 +393,6 @@ export const notificationOperation: MigrateOperation = { ) } }, - { - state: 'fix-rename-backups', - func: async (client: MigrationClient): Promise => { - await client.update(DOMAIN_DOC_NOTIFY, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } }) - } - }, { state: 'remove-update-txes-docnotify-ctx-v2', func: async (client) => { diff --git a/models/task/src/migration.ts b/models/task/src/migration.ts index ffb516d8de..03c06f7c69 100644 --- a/models/task/src/migration.ts +++ b/models/task/src/migration.ts @@ -569,12 +569,6 @@ export const taskOperation: MigrateOperation = { await migrateSpace(client, task.space.Sequence, core.space.Workspace, [DOMAIN_KANBAN]) } }, - { - state: 'fix-rename-backups', - func: async (client: MigrationClient): Promise => { - await client.update(DOMAIN_TASK, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } }) - } - }, { state: 'migrateRanks', func: migrateRanks diff --git a/models/time/src/migration.ts b/models/time/src/migration.ts index 481e28dfe0..91e282515c 100644 --- a/models/time/src/migration.ts +++ b/models/time/src/migration.ts @@ -172,12 +172,6 @@ export const timeOperation: MigrateOperation = { func: async (client) => { await fillProps(client) } - }, - { - state: 'fix-rename-backups', - func: async (client: MigrationClient): Promise => { - await client.update(DOMAIN_TIME, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } }) - } } ]) }, diff --git a/models/view/src/migration.ts b/models/view/src/migration.ts index 572a60d8d7..f5587c0547 100644 --- a/models/view/src/migration.ts +++ b/models/view/src/migration.ts @@ -86,12 +86,6 @@ export const viewOperation: MigrateOperation = { { state: 'remove-done-state-filter', func: removeDoneStateFilter - }, - { - state: 'fix-rename-backups', - func: async (client: MigrationClient): Promise => { - await client.update(DOMAIN_VIEW, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } }) - } } ]) }, diff --git a/packages/core/src/client.ts b/packages/core/src/client.ts index fc7e458ee3..8d56516f27 100644 --- a/packages/core/src/client.ts +++ b/packages/core/src/client.ts @@ -236,7 +236,7 @@ export async function createClient ( let hierarchy = new Hierarchy() let model = new ModelDb(hierarchy) - let lastTx: number + let lastTx: number = 0 function txHandler (...tx: Tx[]): void { if (tx == null || tx.length === 0) { @@ -295,6 +295,10 @@ export async function createClient ( } // We need to look for last {transactionThreshold} transactions and if it is more since lastTx one we receive, we need to perform full refresh. + if (lastTx === 0) { + await oldOnConnect?.(ClientConnectEvent.Refresh, data) + return + } const atxes = await ctx.with('find-atx', {}, () => conn.findAll( core.class.Tx, diff --git a/pods/backup/package.json b/pods/backup/package.json index 6b1e15c3a2..06d542cb3b 100644 --- a/pods/backup/package.json +++ b/pods/backup/package.json @@ -18,6 +18,7 @@ "bundle": "rushx get-model && node ../../common/scripts/esbuild.js --entry=src/index.ts --keep-names=true --bundle=true --sourcemap=external --external=*.node", "docker:build": "../../common/scripts/docker_build.sh hardcoreeng/backup", "docker:tbuild": "docker build -t hardcoreeng/backup . --platform=linux/amd64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/backup", + "docker:abuild": "docker build -t hardcoreeng/backup . --platform=linux/arm64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/backup", "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/backup staging", "docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/backup", "run-local": "cross-env ACCOUNTS_URL=http://localhost:3000/ SECRET=secret MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost BUCKET_NAME=backups INTERVAL=30 ts-node src/index.ts", diff --git a/server/middleware/src/spaceSecurity.ts b/server/middleware/src/spaceSecurity.ts index 6c792db300..5cbe879627 100644 --- a/server/middleware/src/spaceSecurity.ts +++ b/server/middleware/src/spaceSecurity.ts @@ -40,6 +40,7 @@ import core, { TxUpdateDoc, TxWorkspaceEvent, WorkspaceEvent, + clone, generateId, systemAccountEmail, toFindResult, @@ -69,14 +70,14 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar wasInit: Promise | boolean = false - private readonly mainSpaces = [ + private readonly mainSpaces = new Set([ core.space.Configuration, core.space.DerivedTx, core.space.Model, core.space.Space, core.space.Workspace, core.space.Tx - ] + ]) private constructor ( private readonly skipFindCheck: boolean, @@ -424,7 +425,7 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar ctx.contextData.broadcast.targets.spaceSec = (tx) => { const space = this.spacesMap.get(tx.objectSpace) if (space === undefined) return undefined - if (this.systemSpaces.has(space._id) || this.mainSpaces.includes(space._id)) return undefined + if (this.systemSpaces.has(space._id) || this.mainSpaces.has(space._id)) return undefined return space.members.length === 0 ? undefined : this.getTargets(space?.members) } @@ -455,12 +456,12 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar ctx: MeasureContext, domain: Domain, spaces: Ref[] - ): Promise<{ result: Ref[], allDomainSpaces: boolean, domainSpaces: Set> }> { + ): Promise<{ result: Set>, allDomainSpaces: boolean, domainSpaces: Set> }> { const domainSpaces = await this.getDomainSpaces(ctx, domain) - const result = spaces.filter((p) => domainSpaces.has(p)) + const result = new Set(spaces.filter((p) => domainSpaces.has(p))) return { - result: spaces.filter((p) => domainSpaces.has(p)), - allDomainSpaces: result.length === domainSpaces.size, + result, + allDomainSpaces: result.size === domainSpaces.size, domainSpaces } } @@ -477,14 +478,14 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar if (spaces.allDomainSpaces) { return undefined } - return { $in: spaces.result } + return { $in: Array.from(spaces.result) } } if (typeof query === 'string') { - if (!spaces.result.includes(query)) { + if (!spaces.result.has(query)) { return { $in: [] } } } else if (query.$in != null) { - query.$in = query.$in.filter((p) => spaces.result.includes(p)) + query.$in = query.$in.filter((p) => spaces.result.has(p)) if (query.$in.length === spaces.domainSpaces.size) { // all domain spaces delete query.$in @@ -493,7 +494,7 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar if (spaces.allDomainSpaces) { delete query.$in } else { - query.$in = spaces.result + query.$in = Array.from(spaces.result) } } if (Object.keys(query).length === 0) { @@ -515,7 +516,7 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar await this.init(ctx) const domain = this.context.hierarchy.getDomain(_class) - const newQuery = { ...query } + const newQuery = clone(query) const account = ctx.contextData.account const isSpace = this.context.hierarchy.isDerived(_class, core.class.Space) const field = this.getKey(domain) @@ -528,12 +529,12 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar const res = await this.mergeQuery(ctx, account, query[field], domain, isSpace) if (res === undefined) { // eslint-disable-next-line @typescript-eslint/no-dynamic-delete - delete (newQuery as any)[field] + delete newQuery[field] } else { - ;(newQuery as any)[field] = res + newQuery[field] = res if (typeof res === 'object') { if (Array.isArray(res.$in) && res.$in.length === 1 && Object.keys(res).length === 1) { - ;(newQuery as any)[field] = res.$in[0] + newQuery[field] = res.$in[0] } } } @@ -541,25 +542,25 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar const spaces = await this.filterByDomain(ctx, domain, this.getAllAllowedSpaces(account, !isSpace)) if (spaces.allDomainSpaces) { // eslint-disable-next-line @typescript-eslint/no-dynamic-delete - delete (newQuery as any)[field] - } else if (spaces.result.length === 1) { - ;(newQuery as any)[field] = spaces.result[0] + delete newQuery[field] + } else if (spaces.result.size === 1) { + newQuery[field] = Array.from(spaces.result)[0] if (options !== undefined) { - options.allowedSpaces = spaces.result + options.allowedSpaces = Array.from(spaces.result) } else { - options = { allowedSpaces: spaces.result } + options = { allowedSpaces: Array.from(spaces.result) } } } else { // Check if spaces > 85% of all domain spaces, in this case return all and filter on client. - if (spaces.result.length / spaces.domainSpaces.size > 0.85 && options?.limit === undefined) { - clientFilterSpaces = new Set(spaces.result) + if (spaces.result.size / spaces.domainSpaces.size > 0.85 && options?.limit === undefined) { + clientFilterSpaces = spaces.result delete newQuery.space } else { - ;(newQuery as any)[field] = { $in: spaces.result } + newQuery[field] = { $in: Array.from(spaces.result) } if (options !== undefined) { - options.allowedSpaces = spaces.result + options.allowedSpaces = Array.from(spaces.result) } else { - options = { allowedSpaces: spaces.result } + options = { allowedSpaces: Array.from(spaces.result) } } } } @@ -625,19 +626,19 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar if (Object.keys(lookup).length === 0) return const account = ctx.contextData.account if (isSystem(account, ctx)) return - const allowedSpaces = this.getAllAllowedSpaces(account, true) + const allowedSpaces = new Set(this.getAllAllowedSpaces(account, true)) for (const key in lookup) { const val = lookup[key] if (Array.isArray(val)) { const arr: AttachedDoc[] = [] for (const value of val) { - if (allowedSpaces.includes(value.space)) { + if (allowedSpaces.has(value.space)) { arr.push(value) } } lookup[key] = arr as any } else if (val !== undefined) { - if (!allowedSpaces.includes(val.space)) { + if (!allowedSpaces.has(val.space)) { lookup[key] = undefined } } diff --git a/server/mongo/src/storage.ts b/server/mongo/src/storage.ts index fa8cb11239..de49e832fc 100644 --- a/server/mongo/src/storage.ts +++ b/server/mongo/src/storage.ts @@ -1025,7 +1025,10 @@ abstract class MongoAdapterBase implements DbAdapter { return Date.now().toString(16) // Current hash value } - strimSize (str: string): string { + strimSize (str?: string): string { + if (str == null) { + return '' + } const pos = str.indexOf('|') if (pos > 0) { return str.substring(0, pos) @@ -1041,8 +1044,6 @@ abstract class MongoAdapterBase implements DbAdapter { return { next: async () => { if (iterator === undefined) { - await coll.updateMany({ '%hash%': { $in: [null, ''] } }, { $set: { '%hash%': this.curHash() } }) - iterator = coll.find( {}, { diff --git a/server/postgres/src/schemas.ts b/server/postgres/src/schemas.ts index f41d54a8a3..d676413005 100644 --- a/server/postgres/src/schemas.ts +++ b/server/postgres/src/schemas.ts @@ -152,6 +152,11 @@ const docIndexStateSchema: Schema = { type: 'bool', notNull: true, index: true + }, + objectClass: { + type: 'text', + notNull: true, + index: false } } @@ -212,6 +217,40 @@ const eventSchema: Schema = { } } +const docSyncInfo: Schema = { + ...baseSchema, + needSync: { + type: 'text', + notNull: false, + index: false + }, + externalVersion: { + type: 'text', + notNull: false, + index: false + }, + repository: { + type: 'text', + notNull: false, + index: false + }, + url: { + type: 'text', + notNull: false, + index: false + }, + objectClass: { + type: 'text', + notNull: false, + index: false + }, + deleted: { + type: 'bool', + notNull: false, + index: false + } +} + export function addSchema (domain: string, schema: Schema): void { domainSchemas[translateDomain(domain)] = schema domainSchemaFields.set(domain, createSchemaFields(schema)) @@ -231,7 +270,8 @@ export const domainSchemas: Record = { [translateDomain(DOMAIN_DOC_INDEX_STATE)]: docIndexStateSchema, notification: notificationSchema, [translateDomain('notification-dnc')]: dncSchema, - [translateDomain('notification-user')]: userNotificationSchema + [translateDomain('notification-user')]: userNotificationSchema, + github: docSyncInfo } export function getSchema (domain: string): Schema { diff --git a/server/postgres/src/storage.ts b/server/postgres/src/storage.ts index 8bc5951e15..2b4ecb8272 100644 --- a/server/postgres/src/storage.ts +++ b/server/postgres/src/storage.ts @@ -1276,7 +1276,10 @@ abstract class PostgresAdapterBase implements DbAdapter { return [] } - strimSize (str: string): string { + strimSize (str?: string): string { + if (str == null) { + return '' + } const pos = str.indexOf('|') if (pos > 0) { return str.substring(0, pos) @@ -1308,12 +1311,6 @@ abstract class PostgresAdapterBase implements DbAdapter { if (client === undefined) { client = await this.client.reserve() } - - // We need update hash to be set properly - await client.unsafe( - `UPDATE ${tdomain} SET "%hash%" = '${this.curHash()}' WHERE "workspaceId" = '${this.workspaceId.name}' AND "%hash%" IS NULL OR "%hash%" = ''` - ) - initialized = true bulk = createBulk('_id, "%hash%"') } diff --git a/server/postgres/src/utils.ts b/server/postgres/src/utils.ts index 58666f8b2e..040f569ec6 100644 --- a/server/postgres/src/utils.ts +++ b/server/postgres/src/utils.ts @@ -283,7 +283,7 @@ export function convertDoc ( modifiedOn: doc.modifiedOn, createdOn: doc.createdOn ?? doc.modifiedOn, _class: doc._class, - '%hash%': (doc as any)['%hash%'] ?? null + '%hash%': (doc as any)['%hash%'] ?? Date.now().toString(16) } const remainingData: Partial = {} diff --git a/server/rpc/src/rpc.ts b/server/rpc/src/rpc.ts index 56809a9519..5ae60ae81f 100644 --- a/server/rpc/src/rpc.ts +++ b/server/rpc/src/rpc.ts @@ -106,7 +106,13 @@ export class RPCHandler { const decoder = new TextDecoder() _data = decoder.decode(_data) } - return JSON.parse(_data.toString(), receiver) + try { + return JSON.parse(_data.toString(), receiver) + } catch (err: any) { + if (((err.message as string) ?? '').includes('Unexpected token')) { + return this.packr.unpack(new Uint8Array(data)) + } + } } return this.packr.unpack(new Uint8Array(data)) }