UBERF-8856: Fix space security query and schema update (#7413)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

dev/tool/src/clean.ts

@@ -1152,7 +1152,9 @@ function isPersonAccount (tx: TxCUD<Doc>): boolean {
 }
 
 async function update<T extends Doc> (h: Hierarchy, db: Db, doc: T, update: DocumentUpdate<T>): Promise<void> {
-  await db.collection(h.getDomain(doc._class)).updateOne({ _id: doc._id }, { $set: { ...update, '%hash%': null } })
+  await db
+    .collection(h.getDomain(doc._class))
+    .updateOne({ _id: doc._id }, { $set: { ...update, '%hash%': Date.now().toString(16) } })
 }
 
 async function updateId (
@@ -1173,12 +1175,14 @@ async function updateId (
     const newId = generateId()
 
     // update txes
-    await db.collection(DOMAIN_TX).updateMany({ objectId: doc._id }, { $set: { objectId: newId, '%hash%': null } })
+    await db
+      .collection(DOMAIN_TX)
+      .updateMany({ objectId: doc._id }, { $set: { objectId: newId, '%hash%': Date.now().toString(16) } })
 
     // update nested txes
     await db
       .collection(DOMAIN_TX)
-      .updateMany({ 'tx.objectId': doc._id }, { $set: { 'tx.objectId': newId, '%hash%': null } })
+      .updateMany({ 'tx.objectId': doc._id }, { $set: { 'tx.objectId': newId, '%hash%': Date.now().toString(16) } })
 
     // we have generated ids for calendar, let's update in
     if (h.isDerived(doc._class, core.class.Account)) {
@@ -1232,7 +1236,7 @@ async function updateId (
       await db.collection(domain).insertOne({
         ...raw,
         _id: newId as any,
-        '%hash%': null
+        '%hash%': Date.now().toString(16)
       })
       await db.collection(domain).deleteOne({ _id: doc._id })
     }

models/activity/src/migration.ts

@@ -217,12 +217,6 @@ export const activityOperation: MigrateOperation = {
         state: 'migrate-activity-markup',
         func: migrateActivityMarkup
       },
-      {
-        state: 'fix-rename-backups',
-        func: async (client: MigrationClient): Promise<void> => {
-          await client.update(DOMAIN_ACTIVITY, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
-        }
-      },
       {
         state: 'move-reactions',
         func: async (client: MigrationClient): Promise<void> => {

models/attachment/src/migration.ts

@@ -24,12 +24,6 @@ import attachment, { attachmentId, DOMAIN_ATTACHMENT } from '.'
 export const attachmentOperation: MigrateOperation = {
   async migrate (client: MigrationClient): Promise<void> {
     await tryMigrate(client, attachmentId, [
-      {
-        state: 'fix-rename-backups',
-        func: async (client: MigrationClient): Promise<void> => {
-          await client.update(DOMAIN_ATTACHMENT, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
-        }
-      },
       {
         state: 'fix-attachedTo',
         func: async (client: MigrationClient): Promise<void> => {

models/chunter/src/migration.ts

@@ -362,12 +362,6 @@ export const chunterOperation: MigrateOperation = {
             'attributeUpdates.attrKey': 'members'
           })
         }
-      },
-      {
-        state: 'fix-rename-backups',
-        func: async (client: MigrationClient): Promise<void> => {
-          await client.update(DOMAIN_CHUNTER, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
-        }
       }
     ])
   },

models/contact/src/migration.ts

@@ -26,7 +26,7 @@ import activity, { DOMAIN_ACTIVITY } from '@hcengineering/model-activity'
 import core, { DOMAIN_SPACE } from '@hcengineering/model-core'
 import { DOMAIN_VIEW } from '@hcengineering/model-view'
 
-import contact, { contactId, DOMAIN_CHANNEL, DOMAIN_CONTACT } from './index'
+import contact, { contactId, DOMAIN_CONTACT } from './index'
 
 async function createEmployeeEmail (client: TxOperations): Promise<void> {
   const employees = await client.findAll(contact.mixin.Employee, {})
@@ -300,13 +300,6 @@ export const contactOperation: MigrateOperation = {
       {
         state: 'create-person-spaces-v1',
         func: createPersonSpaces
-      },
-      {
-        state: 'fix-rename-backups',
-        func: async (client: MigrationClient): Promise<void> => {
-          await client.update(DOMAIN_CONTACT, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
-          await client.update(DOMAIN_CHANNEL, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
-        }
       }
     ])
   },

models/core/src/migration.ts

@@ -424,10 +424,12 @@ export const coreOperation: MigrateOperation = {
         func: migrateCollaborativeContentToStorage
       },
       {
-        state: 'fix-rename-backups',
+        state: 'fix-backups-hash-timestamp',
         func: async (client: MigrationClient): Promise<void> => {
-          await client.update(DOMAIN_TX, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
-          await client.update(DOMAIN_SPACE, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
+          const now = Date.now().toString(16)
+          for (const d of client.hierarchy.domains()) {
+            await client.update(d, { '%hash%': { $in: [null, ''] } }, { $set: { '%hash%': now } })
+          }
         }
       },
       {

models/document/src/migration.ts

@@ -335,12 +335,6 @@ export const documentOperation: MigrateOperation = {
         state: 'renameFields',
         func: renameFields
       },
-      {
-        state: 'fix-rename-backups',
-        func: async (client: MigrationClient): Promise<void> => {
-          await client.update(DOMAIN_DOCUMENT, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
-        }
-      },
       {
         state: 'renameFieldsRevert',
         func: renameFieldsRevert

models/drive/src/migration.ts

@@ -132,12 +132,6 @@ export const driveOperation: MigrateOperation = {
       {
         state: 'renameFields',
         func: renameFields
-      },
-      {
-        state: 'fix-rename-backups',
-        func: async (client: MigrationClient): Promise<void> => {
-          await client.update(DOMAIN_DRIVE, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
-        }
       }
     ])
   },

models/notification/src/migration.ts

@@ -393,12 +393,6 @@ export const notificationOperation: MigrateOperation = {
           )
         }
       },
-      {
-        state: 'fix-rename-backups',
-        func: async (client: MigrationClient): Promise<void> => {
-          await client.update(DOMAIN_DOC_NOTIFY, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
-        }
-      },
       {
         state: 'remove-update-txes-docnotify-ctx-v2',
         func: async (client) => {

models/task/src/migration.ts

@@ -569,12 +569,6 @@ export const taskOperation: MigrateOperation = {
           await migrateSpace(client, task.space.Sequence, core.space.Workspace, [DOMAIN_KANBAN])
         }
       },
-      {
-        state: 'fix-rename-backups',
-        func: async (client: MigrationClient): Promise<void> => {
-          await client.update(DOMAIN_TASK, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
-        }
-      },
       {
         state: 'migrateRanks',
         func: migrateRanks

models/time/src/migration.ts

@@ -172,12 +172,6 @@ export const timeOperation: MigrateOperation = {
         func: async (client) => {
           await fillProps(client)
         }
-      },
-      {
-        state: 'fix-rename-backups',
-        func: async (client: MigrationClient): Promise<void> => {
-          await client.update(DOMAIN_TIME, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
-        }
       }
     ])
   },

models/view/src/migration.ts

@@ -86,12 +86,6 @@ export const viewOperation: MigrateOperation = {
       {
         state: 'remove-done-state-filter',
         func: removeDoneStateFilter
-      },
-      {
-        state: 'fix-rename-backups',
-        func: async (client: MigrationClient): Promise<void> => {
-          await client.update(DOMAIN_VIEW, { '%hash%': { $exists: true } }, { $set: { '%hash%': null } })
-        }
       }
     ])
   },

packages/core/src/client.ts

@@ -236,7 +236,7 @@ export async function createClient (
   let hierarchy = new Hierarchy()
   let model = new ModelDb(hierarchy)
 
-  let lastTx: number
+  let lastTx: number = 0
 
   function txHandler (...tx: Tx[]): void {
     if (tx == null || tx.length === 0) {
@@ -295,6 +295,10 @@ export async function createClient (
     }
 
     // We need to look for last {transactionThreshold} transactions and if it is more since lastTx one we receive, we need to perform full refresh.
+    if (lastTx === 0) {
+      await oldOnConnect?.(ClientConnectEvent.Refresh, data)
+      return
+    }
     const atxes = await ctx.with('find-atx', {}, () =>
       conn.findAll(
         core.class.Tx,

pods/backup/package.json

@@ -18,6 +18,7 @@
     "bundle": "rushx get-model && node ../../common/scripts/esbuild.js --entry=src/index.ts --keep-names=true --bundle=true --sourcemap=external --external=*.node",
     "docker:build": "../../common/scripts/docker_build.sh hardcoreeng/backup",
     "docker:tbuild": "docker build -t hardcoreeng/backup . --platform=linux/amd64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/backup",
+    "docker:abuild": "docker build -t hardcoreeng/backup . --platform=linux/arm64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/backup",
     "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/backup staging",
     "docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/backup",
     "run-local": "cross-env ACCOUNTS_URL=http://localhost:3000/ SECRET=secret MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost BUCKET_NAME=backups INTERVAL=30 ts-node src/index.ts",

server/middleware/src/spaceSecurity.ts

@@ -40,6 +40,7 @@ import core, {
   TxUpdateDoc,
   TxWorkspaceEvent,
   WorkspaceEvent,
+  clone,
   generateId,
   systemAccountEmail,
   toFindResult,
@@ -69,14 +70,14 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar
 
   wasInit: Promise<void> | boolean = false
 
-  private readonly mainSpaces = [
+  private readonly mainSpaces = new Set([
     core.space.Configuration,
     core.space.DerivedTx,
     core.space.Model,
     core.space.Space,
     core.space.Workspace,
     core.space.Tx
-  ]
+  ])
 
   private constructor (
     private readonly skipFindCheck: boolean,
@@ -424,7 +425,7 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar
     ctx.contextData.broadcast.targets.spaceSec = (tx) => {
       const space = this.spacesMap.get(tx.objectSpace)
       if (space === undefined) return undefined
-      if (this.systemSpaces.has(space._id) || this.mainSpaces.includes(space._id)) return undefined
+      if (this.systemSpaces.has(space._id) || this.mainSpaces.has(space._id)) return undefined
 
       return space.members.length === 0 ? undefined : this.getTargets(space?.members)
     }
@@ -455,12 +456,12 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar
     ctx: MeasureContext,
     domain: Domain,
     spaces: Ref<Space>[]
-  ): Promise<{ result: Ref<Space>[], allDomainSpaces: boolean, domainSpaces: Set<Ref<Space>> }> {
+  ): Promise<{ result: Set<Ref<Space>>, allDomainSpaces: boolean, domainSpaces: Set<Ref<Space>> }> {
     const domainSpaces = await this.getDomainSpaces(ctx, domain)
-    const result = spaces.filter((p) => domainSpaces.has(p))
+    const result = new Set(spaces.filter((p) => domainSpaces.has(p)))
     return {
-      result: spaces.filter((p) => domainSpaces.has(p)),
-      allDomainSpaces: result.length === domainSpaces.size,
+      result,
+      allDomainSpaces: result.size === domainSpaces.size,
       domainSpaces
     }
   }
@@ -477,14 +478,14 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar
       if (spaces.allDomainSpaces) {
         return undefined
       }
-      return { $in: spaces.result }
+      return { $in: Array.from(spaces.result) }
     }
     if (typeof query === 'string') {
-      if (!spaces.result.includes(query)) {
+      if (!spaces.result.has(query)) {
         return { $in: [] }
       }
     } else if (query.$in != null) {
-      query.$in = query.$in.filter((p) => spaces.result.includes(p))
+      query.$in = query.$in.filter((p) => spaces.result.has(p))
       if (query.$in.length === spaces.domainSpaces.size) {
         // all domain spaces
         delete query.$in
@@ -493,7 +494,7 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar
       if (spaces.allDomainSpaces) {
         delete query.$in
       } else {
-        query.$in = spaces.result
+        query.$in = Array.from(spaces.result)
       }
     }
     if (Object.keys(query).length === 0) {
@@ -515,7 +516,7 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar
     await this.init(ctx)
 
     const domain = this.context.hierarchy.getDomain(_class)
-    const newQuery = { ...query }
+    const newQuery = clone(query)
     const account = ctx.contextData.account
     const isSpace = this.context.hierarchy.isDerived(_class, core.class.Space)
     const field = this.getKey(domain)
@@ -528,12 +529,12 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar
           const res = await this.mergeQuery(ctx, account, query[field], domain, isSpace)
           if (res === undefined) {
             // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
-            delete (newQuery as any)[field]
+            delete newQuery[field]
           } else {
-            ;(newQuery as any)[field] = res
+            newQuery[field] = res
             if (typeof res === 'object') {
               if (Array.isArray(res.$in) && res.$in.length === 1 && Object.keys(res).length === 1) {
-                ;(newQuery as any)[field] = res.$in[0]
+                newQuery[field] = res.$in[0]
               }
             }
           }
@@ -541,25 +542,25 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar
           const spaces = await this.filterByDomain(ctx, domain, this.getAllAllowedSpaces(account, !isSpace))
           if (spaces.allDomainSpaces) {
             // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
-            delete (newQuery as any)[field]
-          } else if (spaces.result.length === 1) {
-            ;(newQuery as any)[field] = spaces.result[0]
+            delete newQuery[field]
+          } else if (spaces.result.size === 1) {
+            newQuery[field] = Array.from(spaces.result)[0]
             if (options !== undefined) {
-              options.allowedSpaces = spaces.result
+              options.allowedSpaces = Array.from(spaces.result)
             } else {
-              options = { allowedSpaces: spaces.result }
+              options = { allowedSpaces: Array.from(spaces.result) }
             }
           } else {
             // Check if spaces > 85% of all domain spaces, in this case return all and filter on client.
-            if (spaces.result.length / spaces.domainSpaces.size > 0.85 && options?.limit === undefined) {
-              clientFilterSpaces = new Set(spaces.result)
+            if (spaces.result.size / spaces.domainSpaces.size > 0.85 && options?.limit === undefined) {
+              clientFilterSpaces = spaces.result
               delete newQuery.space
             } else {
-              ;(newQuery as any)[field] = { $in: spaces.result }
+              newQuery[field] = { $in: Array.from(spaces.result) }
               if (options !== undefined) {
-                options.allowedSpaces = spaces.result
+                options.allowedSpaces = Array.from(spaces.result)
               } else {
-                options = { allowedSpaces: spaces.result }
+                options = { allowedSpaces: Array.from(spaces.result) }
               }
             }
           }
@@ -625,19 +626,19 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar
     if (Object.keys(lookup).length === 0) return
     const account = ctx.contextData.account
     if (isSystem(account, ctx)) return
-    const allowedSpaces = this.getAllAllowedSpaces(account, true)
+    const allowedSpaces = new Set(this.getAllAllowedSpaces(account, true))
     for (const key in lookup) {
       const val = lookup[key]
       if (Array.isArray(val)) {
         const arr: AttachedDoc[] = []
         for (const value of val) {
-          if (allowedSpaces.includes(value.space)) {
+          if (allowedSpaces.has(value.space)) {
             arr.push(value)
           }
         }
         lookup[key] = arr as any
       } else if (val !== undefined) {
-        if (!allowedSpaces.includes(val.space)) {
+        if (!allowedSpaces.has(val.space)) {
           lookup[key] = undefined
         }
       }

server/mongo/src/storage.ts

@@ -1025,7 +1025,10 @@ abstract class MongoAdapterBase implements DbAdapter {
     return Date.now().toString(16) // Current hash value
   }
 
-  strimSize (str: string): string {
+  strimSize (str?: string): string {
+    if (str == null) {
+      return ''
+    }
     const pos = str.indexOf('|')
     if (pos > 0) {
       return str.substring(0, pos)
@@ -1041,8 +1044,6 @@ abstract class MongoAdapterBase implements DbAdapter {
     return {
       next: async () => {
         if (iterator === undefined) {
-          await coll.updateMany({ '%hash%': { $in: [null, ''] } }, { $set: { '%hash%': this.curHash() } })
-
           iterator = coll.find(
             {},
             {

server/postgres/src/schemas.ts

@@ -152,6 +152,11 @@ const docIndexStateSchema: Schema = {
     type: 'bool',
     notNull: true,
     index: true
+  },
+  objectClass: {
+    type: 'text',
+    notNull: true,
+    index: false
   }
 }
 
@@ -212,6 +217,40 @@ const eventSchema: Schema = {
   }
 }
 
+const docSyncInfo: Schema = {
+  ...baseSchema,
+  needSync: {
+    type: 'text',
+    notNull: false,
+    index: false
+  },
+  externalVersion: {
+    type: 'text',
+    notNull: false,
+    index: false
+  },
+  repository: {
+    type: 'text',
+    notNull: false,
+    index: false
+  },
+  url: {
+    type: 'text',
+    notNull: false,
+    index: false
+  },
+  objectClass: {
+    type: 'text',
+    notNull: false,
+    index: false
+  },
+  deleted: {
+    type: 'bool',
+    notNull: false,
+    index: false
+  }
+}
+
 export function addSchema (domain: string, schema: Schema): void {
   domainSchemas[translateDomain(domain)] = schema
   domainSchemaFields.set(domain, createSchemaFields(schema))
@@ -231,7 +270,8 @@ export const domainSchemas: Record<string, Schema> = {
   [translateDomain(DOMAIN_DOC_INDEX_STATE)]: docIndexStateSchema,
   notification: notificationSchema,
   [translateDomain('notification-dnc')]: dncSchema,
-  [translateDomain('notification-user')]: userNotificationSchema
+  [translateDomain('notification-user')]: userNotificationSchema,
+  github: docSyncInfo
 }
 
 export function getSchema (domain: string): Schema {

server/postgres/src/storage.ts

@@ -1276,7 +1276,10 @@ abstract class PostgresAdapterBase implements DbAdapter {
     return []
   }
 
-  strimSize (str: string): string {
+  strimSize (str?: string): string {
+    if (str == null) {
+      return ''
+    }
     const pos = str.indexOf('|')
     if (pos > 0) {
       return str.substring(0, pos)
@@ -1308,12 +1311,6 @@ abstract class PostgresAdapterBase implements DbAdapter {
           if (client === undefined) {
             client = await this.client.reserve()
           }
-
-          // We need update hash to be set properly
-          await client.unsafe(
-            `UPDATE ${tdomain} SET "%hash%" = '${this.curHash()}' WHERE "workspaceId" = '${this.workspaceId.name}' AND "%hash%" IS NULL OR "%hash%" = ''`
-          )
-
           initialized = true
           bulk = createBulk('_id, "%hash%"')
         }

server/postgres/src/utils.ts

@@ -283,7 +283,7 @@ export function convertDoc<T extends Doc> (
     modifiedOn: doc.modifiedOn,
     createdOn: doc.createdOn ?? doc.modifiedOn,
     _class: doc._class,
-    '%hash%': (doc as any)['%hash%'] ?? null
+    '%hash%': (doc as any)['%hash%'] ?? Date.now().toString(16)
   }
   const remainingData: Partial<T> = {}
 

server/rpc/src/rpc.ts

@@ -106,7 +106,13 @@ export class RPCHandler {
         const decoder = new TextDecoder()
         _data = decoder.decode(_data)
       }
-      return JSON.parse(_data.toString(), receiver)
+      try {
+        return JSON.parse(_data.toString(), receiver)
+      } catch (err: any) {
+        if (((err.message as string) ?? '').includes('Unexpected token')) {
+          return this.packr.unpack(new Uint8Array(data))
+        }
+      }
     }
     return this.packr.unpack(new Uint8Array(data))
   }