CF Transactor(initial) (#7037)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
Signed-off-by: Nikolay Chunosov <Chunosov.N@gmail.com>

dev/docker-compose.yaml

@@ -22,6 +22,8 @@ services:
   postgres:
     image: postgres
     container_name: postgres
+    extra_hosts:
+      - "host.docker.internal:host-gateway"    
     environment:
       - POSTGRES_PASSWORD=example
     volumes:

dev/prod/package.json

@@ -10,6 +10,7 @@
     "analyze": "rm -rf ./dist && cross-env NODE_ENV=production webpack --profile --json > stats.json",
     "show": "webpack-bundle-analyzer stats.json dist",
     "dev-server": "cross-env USE_CACHE=false CLIENT_TYPE=dev-server webpack serve",
+    "dev-worker": "cross-env USE_CACHE=false CLIENT_TYPE=dev-worker webpack serve",
     "dev-server-cache": "cross-env USE_CACHE=true CLIENT_TYPE=dev-server webpack serve",
     "tunnel-qms": "socat tcp-listen:8081,reuseaddr,fork,bind=localhost tcp:localhost:8080",
     "dev-production": "cross-env CLIENT_TYPE=dev-production webpack serve",

dev/prod/public/config-worker.json

@@ -0,0 +1,53 @@
+{
+  "ACCOUNTS_URL": "https://account.hc.engineering/",
+
+  "AI_URL": "https://ai.hc.engineering",
+  
+  "ANALYTICS_COLLECTOR_URL": "https://collector.hc.engineering",
+  
+  "BRANDING_URL": "https://front.hc.engineering/stage-branding_v2.json",
+  
+  "CALENDAR_URL": "https://calendar.hc.engineering/",
+  
+  "COLLABORATOR_URL": "wss://collaborator.hc.engineering",
+  
+  "DESKTOP_UPDATES_CHANNEL": "front",
+  
+  "FILES_URL": "https://dl.hc.engineering/blob/:workspace/:blobId",
+  
+  "GITHUB_APP": "huly-github-staging",
+  
+  "GITHUB_CLIENTID": "Iv1.cd9d3f7987474b5e" ,
+  
+  "GITHUB_URL": "https://github.hc.engineering",
+  
+  "GMAIL_URL": "https://gmail.hc.engineering/" ,
+  
+  "INTERCOM_API_URL": "",
+  
+  "INTERCOM_APP_ID": "",
+    
+  "LOVE_ENDPOINT": "https://love.hc.engineering/",
+  
+  "MODEL_VERSION": "",
+  
+  "PREVIEW_CONFIG": "https://dl.hc.engineering/image/fit=scale-down,width=:size/:workspace/:blobId",
+  
+  "PRINT_URL": "https://print.hc.engineering",
+    
+  "REKONI_URL": "https://rekoni.hc.engineering",
+  
+  "SIGN_URL": "https://sign.hc.engineering",
+  
+  "STATS_URL": "https://stats.hc.engineering",
+  
+  "TELEGRAM_BOT_URL": "https://telegram-bot.hc.engineering",
+  
+  "TELEGRAM_URL": "https://telegram.hc.engineering",
+  
+  "UPLOAD_CONFIG": "",
+  
+  "UPLOAD_URL": "https://dl.hc.engineering/upload/form-data/:workspace",  
+  "USE_BINARY_PROTOCOL": false,
+  "TRANSACTOR_OVERRIDE": "wss://cloud-transactor.andrey-47f.workers.dev"
+}

dev/prod/src/platform.ts

@@ -16,12 +16,17 @@
 import platform, { type Plugin, addLocation, addStringsLoader, platformId } from '@hcengineering/platform'
 
 import { activityId } from '@hcengineering/activity'
+import aiBot, { aiBotId } from '@hcengineering/ai-bot'
+import analyticsCollector, { analyticsCollectorId } from '@hcengineering/analytics-collector'
 import { attachmentId } from '@hcengineering/attachment'
 import { boardId } from '@hcengineering/board'
 import calendar, { calendarId } from '@hcengineering/calendar'
 import { chunterId } from '@hcengineering/chunter'
 import client, { clientId } from '@hcengineering/client'
 import contactPlugin, { contactId } from '@hcengineering/contact'
+import { documentsId } from '@hcengineering/controlled-documents'
+import { desktopPreferencesId } from '@hcengineering/desktop-preferences'
+import { diffviewId } from '@hcengineering/diffview'
 import { documentId } from '@hcengineering/document'
 import { driveId } from '@hcengineering/drive'
 import gmail, { gmailId } from '@hcengineering/gmail'
@@ -31,47 +36,46 @@ import { imageCropperId } from '@hcengineering/image-cropper'
 import { inventoryId } from '@hcengineering/inventory'
 import { leadId } from '@hcengineering/lead'
 import login, { loginId } from '@hcengineering/login'
+import love, { loveId } from '@hcengineering/love'
 import notification, { notificationId } from '@hcengineering/notification'
 import onboard, { onboardId } from '@hcengineering/onboard'
+import print, { printId } from '@hcengineering/print'
+import { productsId } from '@hcengineering/products'
+import { questionsId } from '@hcengineering/questions'
 import { recruitId } from '@hcengineering/recruit'
 import rekoni from '@hcengineering/rekoni'
 import { requestId } from '@hcengineering/request'
 import { settingId } from '@hcengineering/setting'
+import sign from '@hcengineering/sign'
 import { supportId } from '@hcengineering/support'
 import { tagsId } from '@hcengineering/tags'
 import { taskId } from '@hcengineering/task'
 import telegram, { telegramId } from '@hcengineering/telegram'
 import { templatesId } from '@hcengineering/templates'
+import textEditor, { textEditorId } from '@hcengineering/text-editor'
 import { timeId } from '@hcengineering/time'
 import tracker, { trackerId } from '@hcengineering/tracker'
+import { trainingId } from '@hcengineering/training'
 import uiPlugin from '@hcengineering/ui'
+import { uploaderId } from '@hcengineering/uploader'
 import { viewId } from '@hcengineering/view'
 import workbench, { workbenchId } from '@hcengineering/workbench'
-import love, { loveId } from '@hcengineering/love'
-import print, { printId } from '@hcengineering/print'
-import sign from '@hcengineering/sign'
-import { desktopPreferencesId } from '@hcengineering/desktop-preferences'
-import { diffviewId } from '@hcengineering/diffview'
-import { productsId } from '@hcengineering/products'
-import { questionsId } from '@hcengineering/questions'
-import { trainingId } from '@hcengineering/training'
-import { documentsId } from '@hcengineering/controlled-documents'
-import textEditor, { textEditorId } from '@hcengineering/text-editor'
-import analyticsCollector, {analyticsCollectorId} from '@hcengineering/analytics-collector'
-import { uploaderId } from '@hcengineering/uploader'
-import aiBot, { aiBotId } from '@hcengineering/ai-bot'
 import { testManagementId } from '@hcengineering/test-management'
 import { surveyId } from '@hcengineering/survey'
 
 import { bitrixId } from '@hcengineering/bitrix'
 
 import '@hcengineering/activity-assets'
+import '@hcengineering/analytics-collector-assets'
 import '@hcengineering/attachment-assets'
 import '@hcengineering/bitrix-assets'
 import '@hcengineering/board-assets'
 import '@hcengineering/calendar-assets'
 import '@hcengineering/chunter-assets'
 import '@hcengineering/contact-assets'
+import '@hcengineering/controlled-documents-assets'
+import '@hcengineering/desktop-preferences-assets'
+import '@hcengineering/diffview-assets'
 import '@hcengineering/document-assets'
 import '@hcengineering/drive-assets'
 import '@hcengineering/gmail-assets'
@@ -80,8 +84,12 @@ import '@hcengineering/hr-assets'
 import '@hcengineering/inventory-assets'
 import '@hcengineering/lead-assets'
 import '@hcengineering/login-assets'
+import '@hcengineering/love-assets'
 import '@hcengineering/notification-assets'
 import '@hcengineering/preference-assets'
+import '@hcengineering/print-assets'
+import '@hcengineering/products-assets'
+import '@hcengineering/questions-assets'
 import '@hcengineering/recruit-assets'
 import '@hcengineering/request-assets'
 import '@hcengineering/setting-assets'
@@ -90,21 +98,13 @@ import '@hcengineering/tags-assets'
 import '@hcengineering/task-assets'
 import '@hcengineering/telegram-assets'
 import '@hcengineering/templates-assets'
+import '@hcengineering/text-editor-assets'
 import '@hcengineering/time-assets'
 import '@hcengineering/tracker-assets'
+import '@hcengineering/training-assets'
+import '@hcengineering/uploader-assets'
 import '@hcengineering/view-assets'
 import '@hcengineering/workbench-assets'
-import '@hcengineering/love-assets'
-import '@hcengineering/print-assets'
-import '@hcengineering/desktop-preferences-assets'
-import '@hcengineering/diffview-assets'
-import '@hcengineering/questions-assets'
-import '@hcengineering/training-assets'
-import '@hcengineering/products-assets'
-import '@hcengineering/controlled-documents-assets'
-import '@hcengineering/analytics-collector-assets'
-import '@hcengineering/text-editor-assets'
-import '@hcengineering/uploader-assets'
 import '@hcengineering/test-management-assets'
 import '@hcengineering/survey-assets'
 
@@ -120,12 +120,12 @@ import presentation, {
 } from '@hcengineering/presentation'
 
 import { setMetadata } from '@hcengineering/platform'
-import { setDefaultLanguage, initThemeStore } from '@hcengineering/theme'
+import { initThemeStore, setDefaultLanguage } from '@hcengineering/theme'
 
+import { Analytics } from '@hcengineering/analytics'
 import { preferenceId } from '@hcengineering/preference'
 import { uiId } from '@hcengineering/ui/src/plugin'
 import { configureAnalytics } from './analytics'
-import { Analytics } from '@hcengineering/analytics'
 
 export interface Config {
   ACCOUNTS_URL: string
@@ -161,6 +161,8 @@ export interface Config {
   PREVIEW_CONFIG?: string
   UPLOAD_CONFIG?: string
   STATS_URL?: string
+  USE_BINARY_PROTOCOL?: boolean,
+  TRANSACTOR_OVERRIDE?: string
 }
 
 export interface Branding {
@@ -182,9 +184,14 @@ export interface Branding {
 
 export type BrandingMap = Record<string, Branding>
 
-const devConfig = process.env.CLIENT_TYPE === 'dev-production'
-const devConfigHuly = process.env.CLIENT_TYPE === 'dev-huly'
-const devConfigBold = process.env.CLIENT_TYPE === 'dev-bold'
+const clientType = process.env.CLIENT_TYPE
+const configs: Record<string, string> = { 
+  'dev-production': '/config-dev.json',
+  'dev-huly': '/config-huly.json',
+  'dev-bold': '/config.json',
+  'dev-server': '/config.json',
+  'dev-worker': '/config-worker.json',
+}
 
 function configureI18n(): void {
    //Add localization
@@ -255,12 +262,7 @@ export async function configurePlatform() {
   })
   configureI18n()
 
-  const config: Config = await loadServerConfig(
-    devConfigHuly
-      ? '/config-huly.json' : (
-        devConfigBold ? '/config-bold.json' : ( 
-          devConfig ? '/config-dev.json' : '/config.json'))
-  )
+  const config: Config = await loadServerConfig( configs[clientType] ?? '/config.json')
   const branding: BrandingMap = config.BRANDING_URL !== undefined ? await (await fetch(config.BRANDING_URL)).json() : {}
   const myBranding = branding[window.location.host] ?? {}
 
@@ -300,6 +302,7 @@ export async function configurePlatform() {
 
   setMetadata(login.metadata.AccountsUrl, config.ACCOUNTS_URL)
   setMetadata(login.metadata.DisableSignUp, config.DISABLE_SIGNUP === 'true')
+  
   setMetadata(presentation.metadata.FilesURL, config.FILES_URL)
   setMetadata(presentation.metadata.UploadURL, config.UPLOAD_URL)
   setMetadata(presentation.metadata.CollaboratorUrl, config.COLLABORATOR_URL)
@@ -410,9 +413,10 @@ export async function configurePlatform() {
 
   setMetadata(client.metadata.FilterModel, 'ui')
   setMetadata(client.metadata.ExtraPlugins, ['preference' as Plugin])
+  setMetadata(login.metadata.TransactorOverride, config.TRANSACTOR_OVERRIDE)
 
   // Use binary response transfer for faster performance and small transfer sizes.
-  setMetadata(client.metadata.UseBinaryProtocol, true)
+  setMetadata(client.metadata.UseBinaryProtocol, config.USE_BINARY_PROTOCOL ?? true)
   // Disable for now, since it causes performance issues on linux/docker/kubernetes boxes for now.
   setMetadata(client.metadata.UseProtocolCompression, true)
 

dev/prod/webpack.config.js

@@ -22,10 +22,12 @@ const { Configuration } = require('webpack')
 
 const mode = process.env.NODE_ENV || 'development'
 const prod = mode === 'production'
-const devServer = (process.env.CLIENT_TYPE ?? '') === 'dev-server'
-const devProduction = (process.env.CLIENT_TYPE ?? '') === 'dev-production'
-const devProductionHuly = (process.env.CLIENT_TYPE ?? '') === 'dev-huly'
-const devProductionBold = (process.env.CLIENT_TYPE ?? '') === 'dev-bold'
+const clientType = process.env.CLIENT_TYPE ?? ''
+const devServer = clientType === 'dev-server'
+const devServerWorker = clientType === 'dev-worker'
+const devProduction = clientType === 'dev-production'
+const devProductionHuly = clientType === 'dev-huly'
+const devProductionBold = clientType === 'dev-bold'
 const dev =
   (process.env.CLIENT_TYPE ?? '') === 'dev' || devServer || devProduction || devProductionHuly || devProductionBold
 const ForkTsCheckerWebpackPlugin = require('fork-ts-checker-webpack-plugin')
@@ -38,6 +40,117 @@ const useCache = process.env.USE_CACHE === 'true'
 
 const BundleAnalyzerPlugin = require('webpack-bundle-analyzer').BundleAnalyzerPlugin
 
+const devProxy = {
+  '/account': {
+    target: 'http://localhost:3000',
+    changeOrigin: true,
+    pathRewrite: { '^/account': '' },
+    logLevel: 'debug'
+  },
+  '/files': {
+    target: 'http://localhost:8087',
+    changeOrigin: true,
+    logLevel: 'debug'
+  },
+  '/api/v1': {
+    target: 'http://localhost:8087',
+    changeOrigin: true,
+    logLevel: 'debug'
+  },
+  '/import': {
+    target: 'http://localhost:8087',
+    changeOrigin: true,
+    logLevel: 'debug'
+  },
+  '/rekoni/recognize': {
+    target: 'http://localhost:4004',
+    changeOrigin: true,
+    pathRewrite: { '^/rekoni/recognize': '/recognize' },
+    logLevel: 'debug'
+  }
+}
+
+const devHulyProxy = {
+  '/account': {
+    target: 'https://account.huly.app/',
+    changeOrigin: true,
+    pathRewrite: { '^/account': '' },
+    logLevel: 'debug'
+  },
+  '/api/v1': {
+    target: 'http://huly.app',
+    changeOrigin: true,
+    logLevel: 'debug'
+  },
+  '/files': {
+    target: 'https://huly.app/files',
+    changeOrigin: true,
+    pathRewrite: { '^/files': '' },
+    logLevel: 'debug'
+  },
+  '/rekoni/recognize': {
+    target: 'https://rekoni.huly.app',
+    changeOrigin: true,
+    pathRewrite: { '^/rekoni/recognize': '/recognize' },
+    logLevel: 'debug'
+  }
+}
+
+const devBoldProxy = {
+  '/account': {
+    target: 'https://account.bold.ru/',
+    changeOrigin: true,
+    pathRewrite: { '^/account': '' },
+    logLevel: 'debug'
+  },
+  '/files': {
+    target: 'https://app.bold.ru/files',
+    changeOrigin: true,
+    pathRewrite: { '^/files': '' },
+    logLevel: 'debug'
+  },
+  '/api/v1': {
+    target: 'http://app.bold.ru',
+    changeOrigin: true,
+    logLevel: 'debug'
+  },
+  '/rekoni/recognize': {
+    target: 'https://rekoni.bold.ru',
+    changeOrigin: true,
+    pathRewrite: { '^/rekoni/recognize': '/recognize' },
+    logLevel: 'debug'
+  }
+}
+
+const devFrontProxy = {
+  '/account': {
+    target: 'https://account.hc.engineering/',
+    changeOrigin: true,
+    pathRewrite: { '^/account': '' },
+    logLevel: 'debug'
+  },
+  '/files': {
+    target: 'https://front.hc.engineering/files',
+    changeOrigin: true,
+    pathRewrite: { '^/files': '' },
+    logLevel: 'debug'
+  },
+  '/rekoni/recognize': {
+    target: 'https://rekoni.hc.enigneering',
+    changeOrigin: true,
+    pathRewrite: { '^/rekoni/recognize': '/recognize' },
+    logLevel: 'debug'
+  }
+}
+
+const proxy = {
+  'dev-worker': devProxy,
+  'dev-server': devProxy,
+  'dev-production': devFrontProxy,
+  'dev-bold': devBoldProxy,
+  'dev-huly': devHulyProxy
+}
+
 /**
  * @type {Configuration}
  */
@@ -343,109 +456,7 @@ module.exports = [
         },
         progress: false
       },
-      proxy:
-        devServer && !devProduction && !devProductionHuly && !devProductionBold
-          ? {
-              '/account': {
-                target: 'http://localhost:3000',
-                changeOrigin: true,
-                pathRewrite: { '^/account': '' },
-                logLevel: 'debug'
-              },
-              '/files': {
-                target: 'http://localhost:8087',
-                changeOrigin: true,
-                logLevel: 'debug'
-              },
-              '/api/v1': {
-                target: 'http://localhost:8087',
-                changeOrigin: true,
-                logLevel: 'debug'
-              },
-              '/import': {
-                target: 'http://localhost:8087',
-                changeOrigin: true,
-                logLevel: 'debug'
-              },
-              '/rekoni/recognize': {
-                target: 'http://localhost:4004',
-                changeOrigin: true,
-                pathRewrite: { '^/rekoni/recognize': '/recognize' },
-                logLevel: 'debug'
-              }
-            }
-          : !devProductionHuly && !devProductionBold
-          ? {
-              '/account': {
-                target: 'https://account.hc.engineering/',
-                changeOrigin: true,
-                pathRewrite: { '^/account': '' },
-                logLevel: 'debug'
-              },
-              '/files': {
-                target: 'https://front.hc.engineering/files',
-                changeOrigin: true,
-                pathRewrite: { '^/files': '' },
-                logLevel: 'debug'
-              },
-              '/rekoni/recognize': {
-                target: 'https://rekoni.hc.enigneering',
-                changeOrigin: true,
-                pathRewrite: { '^/rekoni/recognize': '/recognize' },
-                logLevel: 'debug'
-              }
-            }
-          : !devProductionBold
-          ? {
-              '/account': {
-                target: 'https://account.huly.app/',
-                changeOrigin: true,
-                pathRewrite: { '^/account': '' },
-                logLevel: 'debug'
-              },
-              '/api/v1': {
-                target: 'http://huly.app',
-                changeOrigin: true,
-                logLevel: 'debug'
-              },
-              '/files': {
-                target: 'https://huly.app/files',
-                changeOrigin: true,
-                pathRewrite: { '^/files': '' },
-                logLevel: 'debug'
-              },
-              '/rekoni/recognize': {
-                target: 'https://rekoni.huly.app',
-                changeOrigin: true,
-                pathRewrite: { '^/rekoni/recognize': '/recognize' },
-                logLevel: 'debug'
-              }
-            }
-          : {
-              '/account': {
-                target: 'https://account.bold.ru/',
-                changeOrigin: true,
-                pathRewrite: { '^/account': '' },
-                logLevel: 'debug'
-              },
-              '/files': {
-                target: 'https://app.bold.ru/files',
-                changeOrigin: true,
-                pathRewrite: { '^/files': '' },
-                logLevel: 'debug'
-              },
-              '/api/v1': {
-                target: 'http://app.bold.ru',
-                changeOrigin: true,
-                logLevel: 'debug'
-              },
-              '/rekoni/recognize': {
-                target: 'https://rekoni.bold.ru',
-                changeOrigin: true,
-                pathRewrite: { '^/rekoni/recognize': '/recognize' },
-                logLevel: 'debug'
-              }
-            }
+      proxy: proxy[clientType]
     }
   }
 ]

dev/tool/src/index.ts

@@ -58,7 +58,14 @@ import serverClientPlugin, {
   listAccountWorkspaces,
   updateBackupInfo
 } from '@hcengineering/server-client'
-import { createBackupPipeline, getConfig, getWorkspaceDestroyAdapter } from '@hcengineering/server-pipeline'
+import {
+  createBackupPipeline,
+  getConfig,
+  getWorkspaceDestroyAdapter,
+  registerAdapterFactry,
+  registerDestroyFactry,
+  registerTxAdapterFactry
+} from '@hcengineering/server-pipeline'
 import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token'
 import { FileModelLogger } from '@hcengineering/server-tool'
 import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service'
@@ -89,12 +96,19 @@ import core, {
 } from '@hcengineering/core'
 import { consoleModelLogger, type MigrateOperation } from '@hcengineering/model'
 import contact from '@hcengineering/model-contact'
-import { getMongoClient, getWorkspaceMongoDB, shutdown } from '@hcengineering/mongo'
+import {
+  createMongoAdapter,
+  createMongoDestroyAdapter,
+  createMongoTxAdapter,
+  getMongoClient,
+  getWorkspaceMongoDB,
+  shutdown
+} from '@hcengineering/mongo'
 import { backupDownload } from '@hcengineering/server-backup/src/backup'
 
-import { createDatalakeClient, DatalakeService, type DatalakeConfig } from '@hcengineering/datalake'
+import { createDatalakeClient, CONFIG_KIND as DATALAKE_CONFIG_KIND, type DatalakeConfig } from '@hcengineering/datalake'
 import { getModelVersion } from '@hcengineering/model-all'
-import { S3Service, type S3Config } from '@hcengineering/s3'
+import { CONFIG_KIND as S3_CONFIG_KIND, S3Service, type S3Config } from '@hcengineering/s3'
 import type { PipelineFactory, StorageAdapter, StorageAdapterEx } from '@hcengineering/server-core'
 import { deepEqual } from 'fast-equals'
 import { createWriteStream, readFileSync } from 'fs'
@@ -125,6 +139,7 @@ import { restoreControlledDocContentMongo, restoreWikiContentMongo } from './mar
 import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin'
 import { fixAccountEmails, renameAccount } from './renameAccount'
 import { copyToDatalake, moveFiles, showLostFiles } from './storage'
+import { createPostgresTxAdapter, createPostgresAdapter, createPostgreeDestroyAdapter } from '@hcengineering/postgres'
 
 const colorConstants = {
   colorRed: '\u001b[31m',
@@ -152,6 +167,14 @@ export function devTool (
 ): void {
   const toolCtx = new MeasureMetricsContext('tool', {})
 
+  registerTxAdapterFactry('mongodb', createMongoTxAdapter)
+  registerAdapterFactry('mongodb', createMongoAdapter)
+  registerDestroyFactry('mongodb', createMongoDestroyAdapter)
+
+  registerTxAdapterFactry('postgresql', createPostgresTxAdapter, true)
+  registerAdapterFactry('postgresql', createPostgresAdapter, true)
+  registerDestroyFactry('postgresql', createPostgreeDestroyAdapter, true)
+
   const serverSecret = process.env.SERVER_SECRET
   if (serverSecret === undefined) {
     console.error('please provide server secret')
@@ -1170,12 +1193,12 @@ export function devTool (
 
       const storageConfig = storageConfigFromEnv(process.env.STORAGE)
 
-      const storages = storageConfig.storages.filter((p) => p.kind === S3Service.config) as S3Config[]
+      const storages = storageConfig.storages.filter((p) => p.kind === S3_CONFIG_KIND) as S3Config[]
       if (storages.length === 0) {
         throw new Error('S3 storage config is required')
       }
 
-      const datalakeConfig = storageConfig.storages.find((p) => p.kind === DatalakeService.config)
+      const datalakeConfig = storageConfig.storages.find((p) => p.kind === DATALAKE_CONFIG_KIND)
       if (datalakeConfig === undefined) {
         throw new Error('Datalake storage config is required')
       }