CF Transactor(initial) (#7037)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
Signed-off-by: Nikolay Chunosov <Chunosov.N@gmail.com>
This commit is contained in:
Andrey Sobolev
2024-12-19 12:06:38 +07:00
committed by GitHub
parent 6e53e76fbb
commit cdcec3efca
43 changed files with 1606 additions and 451 deletions
+419 -210
View File
File diff suppressed because it is too large Load Diff
+2
View File
@@ -22,6 +22,8 @@ services:
postgres:
image: postgres
container_name: postgres
extra_hosts:
- "host.docker.internal:host-gateway"
environment:
- POSTGRES_PASSWORD=example
volumes:
+1
View File
@@ -10,6 +10,7 @@
"analyze": "rm -rf ./dist && cross-env NODE_ENV=production webpack --profile --json > stats.json",
"show": "webpack-bundle-analyzer stats.json dist",
"dev-server": "cross-env USE_CACHE=false CLIENT_TYPE=dev-server webpack serve",
"dev-worker": "cross-env USE_CACHE=false CLIENT_TYPE=dev-worker webpack serve",
"dev-server-cache": "cross-env USE_CACHE=true CLIENT_TYPE=dev-server webpack serve",
"tunnel-qms": "socat tcp-listen:8081,reuseaddr,fork,bind=localhost tcp:localhost:8080",
"dev-production": "cross-env CLIENT_TYPE=dev-production webpack serve",
+53
View File
@@ -0,0 +1,53 @@
{
"ACCOUNTS_URL": "https://account.hc.engineering/",
"AI_URL": "https://ai.hc.engineering",
"ANALYTICS_COLLECTOR_URL": "https://collector.hc.engineering",
"BRANDING_URL": "https://front.hc.engineering/stage-branding_v2.json",
"CALENDAR_URL": "https://calendar.hc.engineering/",
"COLLABORATOR_URL": "wss://collaborator.hc.engineering",
"DESKTOP_UPDATES_CHANNEL": "front",
"FILES_URL": "https://dl.hc.engineering/blob/:workspace/:blobId",
"GITHUB_APP": "huly-github-staging",
"GITHUB_CLIENTID": "Iv1.cd9d3f7987474b5e" ,
"GITHUB_URL": "https://github.hc.engineering",
"GMAIL_URL": "https://gmail.hc.engineering/" ,
"INTERCOM_API_URL": "",
"INTERCOM_APP_ID": "",
"LOVE_ENDPOINT": "https://love.hc.engineering/",
"MODEL_VERSION": "",
"PREVIEW_CONFIG": "https://dl.hc.engineering/image/fit=scale-down,width=:size/:workspace/:blobId",
"PRINT_URL": "https://print.hc.engineering",
"REKONI_URL": "https://rekoni.hc.engineering",
"SIGN_URL": "https://sign.hc.engineering",
"STATS_URL": "https://stats.hc.engineering",
"TELEGRAM_BOT_URL": "https://telegram-bot.hc.engineering",
"TELEGRAM_URL": "https://telegram.hc.engineering",
"UPLOAD_CONFIG": "",
"UPLOAD_URL": "https://dl.hc.engineering/upload/form-data/:workspace",
"USE_BINARY_PROTOCOL": false,
"TRANSACTOR_OVERRIDE": "wss://cloud-transactor.andrey-47f.workers.dev"
}
+40 -36
View File
@@ -16,12 +16,17 @@
import platform, { type Plugin, addLocation, addStringsLoader, platformId } from '@hcengineering/platform'
import { activityId } from '@hcengineering/activity'
import aiBot, { aiBotId } from '@hcengineering/ai-bot'
import analyticsCollector, { analyticsCollectorId } from '@hcengineering/analytics-collector'
import { attachmentId } from '@hcengineering/attachment'
import { boardId } from '@hcengineering/board'
import calendar, { calendarId } from '@hcengineering/calendar'
import { chunterId } from '@hcengineering/chunter'
import client, { clientId } from '@hcengineering/client'
import contactPlugin, { contactId } from '@hcengineering/contact'
import { documentsId } from '@hcengineering/controlled-documents'
import { desktopPreferencesId } from '@hcengineering/desktop-preferences'
import { diffviewId } from '@hcengineering/diffview'
import { documentId } from '@hcengineering/document'
import { driveId } from '@hcengineering/drive'
import gmail, { gmailId } from '@hcengineering/gmail'
@@ -31,47 +36,46 @@ import { imageCropperId } from '@hcengineering/image-cropper'
import { inventoryId } from '@hcengineering/inventory'
import { leadId } from '@hcengineering/lead'
import login, { loginId } from '@hcengineering/login'
import love, { loveId } from '@hcengineering/love'
import notification, { notificationId } from '@hcengineering/notification'
import onboard, { onboardId } from '@hcengineering/onboard'
import print, { printId } from '@hcengineering/print'
import { productsId } from '@hcengineering/products'
import { questionsId } from '@hcengineering/questions'
import { recruitId } from '@hcengineering/recruit'
import rekoni from '@hcengineering/rekoni'
import { requestId } from '@hcengineering/request'
import { settingId } from '@hcengineering/setting'
import sign from '@hcengineering/sign'
import { supportId } from '@hcengineering/support'
import { tagsId } from '@hcengineering/tags'
import { taskId } from '@hcengineering/task'
import telegram, { telegramId } from '@hcengineering/telegram'
import { templatesId } from '@hcengineering/templates'
import textEditor, { textEditorId } from '@hcengineering/text-editor'
import { timeId } from '@hcengineering/time'
import tracker, { trackerId } from '@hcengineering/tracker'
import { trainingId } from '@hcengineering/training'
import uiPlugin from '@hcengineering/ui'
import { uploaderId } from '@hcengineering/uploader'
import { viewId } from '@hcengineering/view'
import workbench, { workbenchId } from '@hcengineering/workbench'
import love, { loveId } from '@hcengineering/love'
import print, { printId } from '@hcengineering/print'
import sign from '@hcengineering/sign'
import { desktopPreferencesId } from '@hcengineering/desktop-preferences'
import { diffviewId } from '@hcengineering/diffview'
import { productsId } from '@hcengineering/products'
import { questionsId } from '@hcengineering/questions'
import { trainingId } from '@hcengineering/training'
import { documentsId } from '@hcengineering/controlled-documents'
import textEditor, { textEditorId } from '@hcengineering/text-editor'
import analyticsCollector, {analyticsCollectorId} from '@hcengineering/analytics-collector'
import { uploaderId } from '@hcengineering/uploader'
import aiBot, { aiBotId } from '@hcengineering/ai-bot'
import { testManagementId } from '@hcengineering/test-management'
import { surveyId } from '@hcengineering/survey'
import { bitrixId } from '@hcengineering/bitrix'
import '@hcengineering/activity-assets'
import '@hcengineering/analytics-collector-assets'
import '@hcengineering/attachment-assets'
import '@hcengineering/bitrix-assets'
import '@hcengineering/board-assets'
import '@hcengineering/calendar-assets'
import '@hcengineering/chunter-assets'
import '@hcengineering/contact-assets'
import '@hcengineering/controlled-documents-assets'
import '@hcengineering/desktop-preferences-assets'
import '@hcengineering/diffview-assets'
import '@hcengineering/document-assets'
import '@hcengineering/drive-assets'
import '@hcengineering/gmail-assets'
@@ -80,8 +84,12 @@ import '@hcengineering/hr-assets'
import '@hcengineering/inventory-assets'
import '@hcengineering/lead-assets'
import '@hcengineering/login-assets'
import '@hcengineering/love-assets'
import '@hcengineering/notification-assets'
import '@hcengineering/preference-assets'
import '@hcengineering/print-assets'
import '@hcengineering/products-assets'
import '@hcengineering/questions-assets'
import '@hcengineering/recruit-assets'
import '@hcengineering/request-assets'
import '@hcengineering/setting-assets'
@@ -90,21 +98,13 @@ import '@hcengineering/tags-assets'
import '@hcengineering/task-assets'
import '@hcengineering/telegram-assets'
import '@hcengineering/templates-assets'
import '@hcengineering/text-editor-assets'
import '@hcengineering/time-assets'
import '@hcengineering/tracker-assets'
import '@hcengineering/training-assets'
import '@hcengineering/uploader-assets'
import '@hcengineering/view-assets'
import '@hcengineering/workbench-assets'
import '@hcengineering/love-assets'
import '@hcengineering/print-assets'
import '@hcengineering/desktop-preferences-assets'
import '@hcengineering/diffview-assets'
import '@hcengineering/questions-assets'
import '@hcengineering/training-assets'
import '@hcengineering/products-assets'
import '@hcengineering/controlled-documents-assets'
import '@hcengineering/analytics-collector-assets'
import '@hcengineering/text-editor-assets'
import '@hcengineering/uploader-assets'
import '@hcengineering/test-management-assets'
import '@hcengineering/survey-assets'
@@ -120,12 +120,12 @@ import presentation, {
} from '@hcengineering/presentation'
import { setMetadata } from '@hcengineering/platform'
import { setDefaultLanguage, initThemeStore } from '@hcengineering/theme'
import { initThemeStore, setDefaultLanguage } from '@hcengineering/theme'
import { Analytics } from '@hcengineering/analytics'
import { preferenceId } from '@hcengineering/preference'
import { uiId } from '@hcengineering/ui/src/plugin'
import { configureAnalytics } from './analytics'
import { Analytics } from '@hcengineering/analytics'
export interface Config {
ACCOUNTS_URL: string
@@ -161,6 +161,8 @@ export interface Config {
PREVIEW_CONFIG?: string
UPLOAD_CONFIG?: string
STATS_URL?: string
USE_BINARY_PROTOCOL?: boolean,
TRANSACTOR_OVERRIDE?: string
}
export interface Branding {
@@ -182,9 +184,14 @@ export interface Branding {
export type BrandingMap = Record<string, Branding>
const devConfig = process.env.CLIENT_TYPE === 'dev-production'
const devConfigHuly = process.env.CLIENT_TYPE === 'dev-huly'
const devConfigBold = process.env.CLIENT_TYPE === 'dev-bold'
const clientType = process.env.CLIENT_TYPE
const configs: Record<string, string> = {
'dev-production': '/config-dev.json',
'dev-huly': '/config-huly.json',
'dev-bold': '/config.json',
'dev-server': '/config.json',
'dev-worker': '/config-worker.json',
}
function configureI18n(): void {
//Add localization
@@ -255,12 +262,7 @@ export async function configurePlatform() {
})
configureI18n()
const config: Config = await loadServerConfig(
devConfigHuly
? '/config-huly.json' : (
devConfigBold ? '/config-bold.json' : (
devConfig ? '/config-dev.json' : '/config.json'))
)
const config: Config = await loadServerConfig( configs[clientType] ?? '/config.json')
const branding: BrandingMap = config.BRANDING_URL !== undefined ? await (await fetch(config.BRANDING_URL)).json() : {}
const myBranding = branding[window.location.host] ?? {}
@@ -300,6 +302,7 @@ export async function configurePlatform() {
setMetadata(login.metadata.AccountsUrl, config.ACCOUNTS_URL)
setMetadata(login.metadata.DisableSignUp, config.DISABLE_SIGNUP === 'true')
setMetadata(presentation.metadata.FilesURL, config.FILES_URL)
setMetadata(presentation.metadata.UploadURL, config.UPLOAD_URL)
setMetadata(presentation.metadata.CollaboratorUrl, config.COLLABORATOR_URL)
@@ -410,9 +413,10 @@ export async function configurePlatform() {
setMetadata(client.metadata.FilterModel, 'ui')
setMetadata(client.metadata.ExtraPlugins, ['preference' as Plugin])
setMetadata(login.metadata.TransactorOverride, config.TRANSACTOR_OVERRIDE)
// Use binary response transfer for faster performance and small transfer sizes.
setMetadata(client.metadata.UseBinaryProtocol, true)
setMetadata(client.metadata.UseBinaryProtocol, config.USE_BINARY_PROTOCOL ?? true)
// Disable for now, since it causes performance issues on linux/docker/kubernetes boxes for now.
setMetadata(client.metadata.UseProtocolCompression, true)
+118 -107
View File
@@ -22,10 +22,12 @@ const { Configuration } = require('webpack')
const mode = process.env.NODE_ENV || 'development'
const prod = mode === 'production'
const devServer = (process.env.CLIENT_TYPE ?? '') === 'dev-server'
const devProduction = (process.env.CLIENT_TYPE ?? '') === 'dev-production'
const devProductionHuly = (process.env.CLIENT_TYPE ?? '') === 'dev-huly'
const devProductionBold = (process.env.CLIENT_TYPE ?? '') === 'dev-bold'
const clientType = process.env.CLIENT_TYPE ?? ''
const devServer = clientType === 'dev-server'
const devServerWorker = clientType === 'dev-worker'
const devProduction = clientType === 'dev-production'
const devProductionHuly = clientType === 'dev-huly'
const devProductionBold = clientType === 'dev-bold'
const dev =
(process.env.CLIENT_TYPE ?? '') === 'dev' || devServer || devProduction || devProductionHuly || devProductionBold
const ForkTsCheckerWebpackPlugin = require('fork-ts-checker-webpack-plugin')
@@ -38,6 +40,117 @@ const useCache = process.env.USE_CACHE === 'true'
const BundleAnalyzerPlugin = require('webpack-bundle-analyzer').BundleAnalyzerPlugin
const devProxy = {
'/account': {
target: 'http://localhost:3000',
changeOrigin: true,
pathRewrite: { '^/account': '' },
logLevel: 'debug'
},
'/files': {
target: 'http://localhost:8087',
changeOrigin: true,
logLevel: 'debug'
},
'/api/v1': {
target: 'http://localhost:8087',
changeOrigin: true,
logLevel: 'debug'
},
'/import': {
target: 'http://localhost:8087',
changeOrigin: true,
logLevel: 'debug'
},
'/rekoni/recognize': {
target: 'http://localhost:4004',
changeOrigin: true,
pathRewrite: { '^/rekoni/recognize': '/recognize' },
logLevel: 'debug'
}
}
const devHulyProxy = {
'/account': {
target: 'https://account.huly.app/',
changeOrigin: true,
pathRewrite: { '^/account': '' },
logLevel: 'debug'
},
'/api/v1': {
target: 'http://huly.app',
changeOrigin: true,
logLevel: 'debug'
},
'/files': {
target: 'https://huly.app/files',
changeOrigin: true,
pathRewrite: { '^/files': '' },
logLevel: 'debug'
},
'/rekoni/recognize': {
target: 'https://rekoni.huly.app',
changeOrigin: true,
pathRewrite: { '^/rekoni/recognize': '/recognize' },
logLevel: 'debug'
}
}
const devBoldProxy = {
'/account': {
target: 'https://account.bold.ru/',
changeOrigin: true,
pathRewrite: { '^/account': '' },
logLevel: 'debug'
},
'/files': {
target: 'https://app.bold.ru/files',
changeOrigin: true,
pathRewrite: { '^/files': '' },
logLevel: 'debug'
},
'/api/v1': {
target: 'http://app.bold.ru',
changeOrigin: true,
logLevel: 'debug'
},
'/rekoni/recognize': {
target: 'https://rekoni.bold.ru',
changeOrigin: true,
pathRewrite: { '^/rekoni/recognize': '/recognize' },
logLevel: 'debug'
}
}
const devFrontProxy = {
'/account': {
target: 'https://account.hc.engineering/',
changeOrigin: true,
pathRewrite: { '^/account': '' },
logLevel: 'debug'
},
'/files': {
target: 'https://front.hc.engineering/files',
changeOrigin: true,
pathRewrite: { '^/files': '' },
logLevel: 'debug'
},
'/rekoni/recognize': {
target: 'https://rekoni.hc.enigneering',
changeOrigin: true,
pathRewrite: { '^/rekoni/recognize': '/recognize' },
logLevel: 'debug'
}
}
const proxy = {
'dev-worker': devProxy,
'dev-server': devProxy,
'dev-production': devFrontProxy,
'dev-bold': devBoldProxy,
'dev-huly': devHulyProxy
}
/**
* @type {Configuration}
*/
@@ -343,109 +456,7 @@ module.exports = [
},
progress: false
},
proxy:
devServer && !devProduction && !devProductionHuly && !devProductionBold
? {
'/account': {
target: 'http://localhost:3000',
changeOrigin: true,
pathRewrite: { '^/account': '' },
logLevel: 'debug'
},
'/files': {
target: 'http://localhost:8087',
changeOrigin: true,
logLevel: 'debug'
},
'/api/v1': {
target: 'http://localhost:8087',
changeOrigin: true,
logLevel: 'debug'
},
'/import': {
target: 'http://localhost:8087',
changeOrigin: true,
logLevel: 'debug'
},
'/rekoni/recognize': {
target: 'http://localhost:4004',
changeOrigin: true,
pathRewrite: { '^/rekoni/recognize': '/recognize' },
logLevel: 'debug'
}
}
: !devProductionHuly && !devProductionBold
? {
'/account': {
target: 'https://account.hc.engineering/',
changeOrigin: true,
pathRewrite: { '^/account': '' },
logLevel: 'debug'
},
'/files': {
target: 'https://front.hc.engineering/files',
changeOrigin: true,
pathRewrite: { '^/files': '' },
logLevel: 'debug'
},
'/rekoni/recognize': {
target: 'https://rekoni.hc.enigneering',
changeOrigin: true,
pathRewrite: { '^/rekoni/recognize': '/recognize' },
logLevel: 'debug'
}
}
: !devProductionBold
? {
'/account': {
target: 'https://account.huly.app/',
changeOrigin: true,
pathRewrite: { '^/account': '' },
logLevel: 'debug'
},
'/api/v1': {
target: 'http://huly.app',
changeOrigin: true,
logLevel: 'debug'
},
'/files': {
target: 'https://huly.app/files',
changeOrigin: true,
pathRewrite: { '^/files': '' },
logLevel: 'debug'
},
'/rekoni/recognize': {
target: 'https://rekoni.huly.app',
changeOrigin: true,
pathRewrite: { '^/rekoni/recognize': '/recognize' },
logLevel: 'debug'
}
}
: {
'/account': {
target: 'https://account.bold.ru/',
changeOrigin: true,
pathRewrite: { '^/account': '' },
logLevel: 'debug'
},
'/files': {
target: 'https://app.bold.ru/files',
changeOrigin: true,
pathRewrite: { '^/files': '' },
logLevel: 'debug'
},
'/api/v1': {
target: 'http://app.bold.ru',
changeOrigin: true,
logLevel: 'debug'
},
'/rekoni/recognize': {
target: 'https://rekoni.bold.ru',
changeOrigin: true,
pathRewrite: { '^/rekoni/recognize': '/recognize' },
logLevel: 'debug'
}
}
proxy: proxy[clientType]
}
}
]
+29 -6
View File
@@ -58,7 +58,14 @@ import serverClientPlugin, {
listAccountWorkspaces,
updateBackupInfo
} from '@hcengineering/server-client'
import { createBackupPipeline, getConfig, getWorkspaceDestroyAdapter } from '@hcengineering/server-pipeline'
import {
createBackupPipeline,
getConfig,
getWorkspaceDestroyAdapter,
registerAdapterFactry,
registerDestroyFactry,
registerTxAdapterFactry
} from '@hcengineering/server-pipeline'
import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token'
import { FileModelLogger } from '@hcengineering/server-tool'
import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service'
@@ -89,12 +96,19 @@ import core, {
} from '@hcengineering/core'
import { consoleModelLogger, type MigrateOperation } from '@hcengineering/model'
import contact from '@hcengineering/model-contact'
import { getMongoClient, getWorkspaceMongoDB, shutdown } from '@hcengineering/mongo'
import {
createMongoAdapter,
createMongoDestroyAdapter,
createMongoTxAdapter,
getMongoClient,
getWorkspaceMongoDB,
shutdown
} from '@hcengineering/mongo'
import { backupDownload } from '@hcengineering/server-backup/src/backup'
import { createDatalakeClient, DatalakeService, type DatalakeConfig } from '@hcengineering/datalake'
import { createDatalakeClient, CONFIG_KIND as DATALAKE_CONFIG_KIND, type DatalakeConfig } from '@hcengineering/datalake'
import { getModelVersion } from '@hcengineering/model-all'
import { S3Service, type S3Config } from '@hcengineering/s3'
import { CONFIG_KIND as S3_CONFIG_KIND, S3Service, type S3Config } from '@hcengineering/s3'
import type { PipelineFactory, StorageAdapter, StorageAdapterEx } from '@hcengineering/server-core'
import { deepEqual } from 'fast-equals'
import { createWriteStream, readFileSync } from 'fs'
@@ -125,6 +139,7 @@ import { restoreControlledDocContentMongo, restoreWikiContentMongo } from './mar
import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin'
import { fixAccountEmails, renameAccount } from './renameAccount'
import { copyToDatalake, moveFiles, showLostFiles } from './storage'
import { createPostgresTxAdapter, createPostgresAdapter, createPostgreeDestroyAdapter } from '@hcengineering/postgres'
const colorConstants = {
colorRed: '\u001b[31m',
@@ -152,6 +167,14 @@ export function devTool (
): void {
const toolCtx = new MeasureMetricsContext('tool', {})
registerTxAdapterFactry('mongodb', createMongoTxAdapter)
registerAdapterFactry('mongodb', createMongoAdapter)
registerDestroyFactry('mongodb', createMongoDestroyAdapter)
registerTxAdapterFactry('postgresql', createPostgresTxAdapter, true)
registerAdapterFactry('postgresql', createPostgresAdapter, true)
registerDestroyFactry('postgresql', createPostgreeDestroyAdapter, true)
const serverSecret = process.env.SERVER_SECRET
if (serverSecret === undefined) {
console.error('please provide server secret')
@@ -1170,12 +1193,12 @@ export function devTool (
const storageConfig = storageConfigFromEnv(process.env.STORAGE)
const storages = storageConfig.storages.filter((p) => p.kind === S3Service.config) as S3Config[]
const storages = storageConfig.storages.filter((p) => p.kind === S3_CONFIG_KIND) as S3Config[]
if (storages.length === 0) {
throw new Error('S3 storage config is required')
}
const datalakeConfig = storageConfig.storages.find((p) => p.kind === DatalakeService.config)
const datalakeConfig = storageConfig.storages.find((p) => p.kind === DATALAKE_CONFIG_KIND)
if (datalakeConfig === undefined) {
throw new Error('Datalake storage config is required')
}
+2 -1
View File
@@ -71,7 +71,8 @@ export default plugin(loginId, {
LastToken: '' as Metadata<string>,
LoginEndpoint: '' as Metadata<string>,
LoginEmail: '' as Metadata<string>,
DisableSignUp: '' as Metadata<boolean>
DisableSignUp: '' as Metadata<boolean>,
TransactorOverride: '' as Metadata<string>
},
component: {
LoginApp: '' as AnyComponent,
+1 -1
View File
@@ -121,7 +121,7 @@ export async function connect (title: string): Promise<Client | undefined> {
setMetadataLocalStorage(login.metadata.LoginEndpoint, workspaceLoginInfo?.endpoint)
const endpoint = workspaceLoginInfo?.endpoint // fetchMetadataLocalStorage(login.metadata.LoginEndpoint)
const endpoint = getMetadata(login.metadata.TransactorOverride) ?? workspaceLoginInfo?.endpoint // fetchMetadataLocalStorage(login.metadata.LoginEndpoint)
const email = workspaceLoginInfo?.email // fetchMetadataLocalStorage(login.metadata.LoginEmail)
if (token == null || endpoint == null || email == null) {
const navigateUrl = encodeURIComponent(JSON.stringify(loc))
+3 -1
View File
@@ -73,6 +73,8 @@
"@hcengineering/server-collaboration": "^0.6.0",
"@hcengineering/middleware": "^0.6.0",
"@hcengineering/server-client": "^0.6.0",
"@hcengineering/server-storage": "^0.6.0"
"@hcengineering/server-storage": "^0.6.0",
"@hcengineering/postgres": "^0.6.0",
"@hcengineering/mongo": "^0.6.1"
}
}
+18 -1
View File
@@ -30,7 +30,9 @@ import {
LowLevelMiddleware,
ModelMiddleware
} from '@hcengineering/middleware'
import { createMongoAdapter, createMongoDestroyAdapter, createMongoTxAdapter } from '@hcengineering/mongo'
import { PlatformError, setMetadata, unknownError } from '@hcengineering/platform'
import { createPostgreeDestroyAdapter, createPostgresAdapter, createPostgresTxAdapter } from '@hcengineering/postgres'
import serverClientPlugin, { getTransactorEndpoint, getWorkspaceInfo } from '@hcengineering/server-client'
import serverCore, {
createContentAdapter,
@@ -43,7 +45,14 @@ import serverCore, {
type StorageAdapter
} from '@hcengineering/server-core'
import { FullTextIndexPipeline, searchFulltext, type FulltextDBConfiguration } from '@hcengineering/server-indexer'
import { getConfig, registerServerPlugins, registerStringLoaders } from '@hcengineering/server-pipeline'
import {
getConfig,
registerAdapterFactry,
registerDestroyFactry,
registerServerPlugins,
registerStringLoaders,
registerTxAdapterFactry
} from '@hcengineering/server-pipeline'
import serverToken, { decodeToken, generateToken, type Token } from '@hcengineering/server-token'
import cors from '@koa/cors'
import Koa from 'koa'
@@ -198,6 +207,14 @@ export async function startIndexer (
setMetadata(serverCore.metadata.ElasticIndexName, opt.elasticIndexName)
setMetadata(serverClientPlugin.metadata.Endpoint, opt.accountsUrl)
registerTxAdapterFactry('mongodb', createMongoTxAdapter)
registerAdapterFactry('mongodb', createMongoAdapter)
registerDestroyFactry('mongodb', createMongoDestroyAdapter)
registerTxAdapterFactry('postgresql', createPostgresTxAdapter, true)
registerAdapterFactry('postgresql', createPostgresAdapter, true)
registerDestroyFactry('postgresql', createPostgreeDestroyAdapter, true)
registerServerPlugins()
registerStringLoaders()
+2 -1
View File
@@ -75,6 +75,7 @@
"ws": "^8.18.0",
"bufferutil": "^4.0.8",
"msgpackr": "^1.11.0",
"msgpackr-extract": "^3.0.3"
"msgpackr-extract": "^3.0.3",
"@hcengineering/postgres": "^0.6.0"
}
}
+8 -4
View File
@@ -10,6 +10,7 @@ import core, {
import builder from '@hcengineering/model-all'
import { wrapPipeline } from '@hcengineering/server-core'
import { getServerPipeline } from '@hcengineering/server-pipeline'
import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage'
const model = builder().getTxes()
// const dbURL = 'postgresql://root@localhost:26257/defaultdb?sslmode=disable'
@@ -22,8 +23,10 @@ describe.skip('test-backup-find', () => {
const toolCtx = new MeasureMetricsContext('-', {})
// We should setup a DB with docuemnts and try to backup them.
const wsUrl = { name: 'testdb-backup-test', workspaceName: 'test', workspaceUrl: 'test' }
const { pipeline, storageAdapter } = await getServerPipeline(toolCtx, model, dbURL, wsUrl, {
storageConfig: STORAGE_CONFIG,
const storageConfig = storageConfigFromEnv(STORAGE_CONFIG)
const storageAdapter = buildStorageFromConfig(storageConfig)
const pipeline = await getServerPipeline(toolCtx, model, dbURL, wsUrl, storageAdapter, {
disableTriggers: true
})
try {
@@ -65,8 +68,9 @@ describe.skip('test-backup-find', () => {
const toolCtx = new MeasureMetricsContext('-', {})
// We should setup a DB with docuemnts and try to backup them.
const wsUrl = { name: 'testdb-backup-test', workspaceName: 'test', workspaceUrl: 'test' }
const { pipeline, storageAdapter } = await getServerPipeline(toolCtx, model, dbURL, wsUrl, {
storageConfig: STORAGE_CONFIG,
const storageConfig = storageConfigFromEnv(STORAGE_CONFIG)
const storageAdapter = buildStorageFromConfig(storageConfig)
const pipeline = await getServerPipeline(toolCtx, model, dbURL, wsUrl, storageAdapter, {
disableTriggers: true
})
try {
+18 -1
View File
@@ -33,9 +33,18 @@ import {
} from '@hcengineering/server-core'
import { type Token } from '@hcengineering/server-token'
import { createServerPipeline, registerServerPlugins, registerStringLoaders } from '@hcengineering/server-pipeline'
import {
createServerPipeline,
registerAdapterFactry,
registerDestroyFactry,
registerServerPlugins,
registerStringLoaders,
registerTxAdapterFactry
} from '@hcengineering/server-pipeline'
import { readFileSync } from 'node:fs'
import { createMongoAdapter, createMongoDestroyAdapter, createMongoTxAdapter } from '@hcengineering/mongo'
import { createPostgreeDestroyAdapter, createPostgresAdapter, createPostgresTxAdapter } from '@hcengineering/postgres'
const model = JSON.parse(readFileSync(process.env.MODEL_JSON ?? 'model.json').toString()) as Tx[]
registerStringLoaders()
@@ -65,6 +74,14 @@ export function start (
mongoUrl?: string
}
): { shutdown: () => Promise<void>, sessionManager: SessionManager } {
registerTxAdapterFactry('mongodb', createMongoTxAdapter)
registerAdapterFactry('mongodb', createMongoAdapter)
registerDestroyFactry('mongodb', createMongoDestroyAdapter)
registerTxAdapterFactry('postgresql', createPostgresTxAdapter, true)
registerAdapterFactry('postgresql', createPostgresAdapter, true)
registerDestroyFactry('postgresql', createPostgreeDestroyAdapter, true)
registerServerPlugins()
const externalStorage = buildStorageFromConfig(opt.storageConfig)
+5
View File
@@ -2205,6 +2205,11 @@
"packageName": "@hcengineering/survey-resources",
"projectFolder": "plugins/survey-resources",
"shouldPublish": false
},
{
"packageName": "@hcengineering/cloud-transactor",
"projectFolder": "workers/transactor",
"shouldPublish": false
}
]
}
+1 -1
View File
@@ -63,7 +63,6 @@ import { connect } from '@hcengineering/server-tool'
import { randomBytes } from 'crypto'
import otpGenerator from 'otp-generator'
import { getWorkspaceDestroyAdapter } from '@hcengineering/server-pipeline'
import { accountPlugin } from './plugin'
import type {
Account,
@@ -93,6 +92,7 @@ import {
toAccountInfo,
verifyPassword
} from './utils'
import { getWorkspaceDestroyAdapter } from '@hcengineering/server-pipeline'
/**
* @public
+1 -1
View File
@@ -115,5 +115,5 @@ export type DbAdapterFactory = (
url: string,
workspaceId: WorkspaceId,
modelDb: ModelDb,
storage: StorageAdapter
storage?: StorageAdapter
) => Promise<DbAdapter>
+2 -1
View File
@@ -44,11 +44,12 @@ export function createDatalakeClient (opt: DatalakeConfig): DatalakeClient {
return new DatalakeClient(endpoint)
}
export const CONFIG_KIND = 'datalake'
/**
* @public
*/
export class DatalakeService implements StorageAdapter {
static config = 'datalake'
private readonly client: DatalakeClient
constructor (readonly opt: DatalakeConfig) {
-4
View File
@@ -14,7 +14,6 @@
//
import { DOMAIN_TX, type MeasureContext } from '@hcengineering/core'
import { PlatformError, unknownStatus } from '@hcengineering/platform'
import type {
DbAdapter,
DbConfiguration,
@@ -49,9 +48,6 @@ export class DBAdapterMiddleware extends BaseMiddleware implements Middleware {
const adapters = new Map<string, DbAdapter>()
await ctx.with('create-adapters', {}, async (ctx) => {
if (this.context.storageAdapter == null) {
throw new PlatformError(unknownStatus('StorageSdapter is not specified'))
}
for (const key in this.conf.adapters) {
const adapterConf = this.conf.adapters[key]
adapters.set(
+2 -1
View File
@@ -50,11 +50,12 @@ export interface MinioConfig extends StorageConfig {
bucketPrefix?: string
}
export const CONFIG_KIND = 'minio'
/**
* @public
*/
export class MinioService implements StorageAdapter {
static config = 'minio'
client: Client
constructor (readonly opt: MinioConfig) {
this.client = new Client({
+2 -1
View File
@@ -60,11 +60,12 @@ export interface S3Config extends StorageConfig {
expireTime?: string
}
export const CONFIG_KIND = 's3'
/**
* @public
*/
export class S3Service implements StorageAdapter {
static config = 's3'
expireTime: number
client: S3
constructor (readonly opt: S3Config) {
-4
View File
@@ -45,9 +45,6 @@
"@hcengineering/server-collaboration": "^0.6.0",
"@hcengineering/server-collaboration-resources": "^0.6.0",
"@hcengineering/server": "^0.6.4",
"@hcengineering/server-storage": "^0.6.0",
"@hcengineering/postgres": "^0.6.0",
"@hcengineering/mongo": "^0.6.1",
"@hcengineering/server-contact": "^0.6.1",
"@hcengineering/server-contact-resources": "^0.6.0",
"@hcengineering/server-notification": "^0.6.1",
@@ -97,7 +94,6 @@
"@hcengineering/server-training": "^0.1.0",
"@hcengineering/server-training-resources": "^0.1.0",
"@hcengineering/middleware": "^0.6.0",
"@hcengineering/minio": "^0.6.0",
"@hcengineering/login-assets": "^0.6.0",
"@hcengineering/onboard-assets": "^0.6.0",
"@hcengineering/view-assets": "^0.6.11",
@@ -14,28 +14,33 @@
//
import {
Class,
Doc,
DocumentQuery,
DocumentUpdate,
Domain,
FindOptions,
FindResult,
Hierarchy,
IndexingConfiguration,
Iterator,
MeasureContext,
ModelDb,
Ref,
StorageIterator,
type Class,
type Doc,
type DocumentQuery,
type DocumentUpdate,
type Domain,
type FindOptions,
type FindResult,
type Hierarchy,
type IndexingConfiguration,
type Iterator,
type MeasureContext,
type ModelDb,
type Ref,
type StorageIterator,
toFindResult,
Tx,
TxResult,
WorkspaceId,
type Tx,
type TxResult,
type WorkspaceId,
type Blob
} from '@hcengineering/core'
import { PlatformError, unknownError } from '@hcengineering/platform'
import { DbAdapter, DbAdapterHandler, StorageAdapter, type StorageAdapterEx } from '@hcengineering/server-core'
import {
type DbAdapter,
type DbAdapterHandler,
type StorageAdapter,
type StorageAdapterEx
} from '@hcengineering/server-core'
class StorageBlobAdapter implements DbAdapter {
constructor (
@@ -129,10 +134,10 @@ export async function createStorageDataAdapter (
url: string,
workspaceId: WorkspaceId,
modelDb: ModelDb,
storage: StorageAdapter
storage?: StorageAdapter
): Promise<DbAdapter> {
if (storage === undefined) {
throw new Error('minio storage adapter require minio')
throw new Error('Storage adapter required')
}
// We need to create bucket if it doesn't exist
if (!(await storage.exists(ctx, workspaceId))) {
+59 -25
View File
@@ -37,8 +37,6 @@ import {
TriggersMiddleware,
TxMiddleware
} from '@hcengineering/middleware'
import { createMongoAdapter, createMongoDestroyAdapter, createMongoTxAdapter } from '@hcengineering/mongo'
import { createPostgreeDestroyAdapter, createPostgresAdapter, createPostgresTxAdapter } from '@hcengineering/postgres'
import {
createBenchmarkAdapter,
createInMemoryAdapter,
@@ -52,12 +50,10 @@ import {
type PipelineContext,
type PipelineFactory,
type StorageAdapter,
type StorageConfiguration,
type WorkspaceDestroyAdapter
} from '@hcengineering/server-core'
import { buildStorageFromConfig, createStorageDataAdapter, storageConfigFromEnv } from '@hcengineering/server-storage'
import { generateToken } from '@hcengineering/server-token'
import { createStorageDataAdapter } from './blobStorage'
/**
* @public
*/
@@ -171,6 +167,7 @@ export function createBackupPipeline (
const middlewares: MiddlewareCreator[] = [
LowLevelMiddleware.create,
ContextNameMiddleware.create,
// ConnectionMgrMiddleware.create,
DomainFindMiddleware.create,
DBAdapterInitMiddleware.create,
ModelMiddleware.create(systemTx),
@@ -195,37 +192,74 @@ export async function getServerPipeline (
model: Tx[],
dbUrl: string,
wsUrl: WorkspaceIdWithUrl,
storageAdapter: StorageAdapter,
opt?: {
storageConfig: string
disableTriggers?: boolean
}
): Promise<{
pipeline: Pipeline
storageAdapter: StorageAdapter
}> {
const storageConfig: StorageConfiguration = storageConfigFromEnv(opt?.storageConfig)
const storageAdapter = buildStorageFromConfig(storageConfig)
): Promise<Pipeline> {
const pipelineFactory = createServerPipeline(ctx, dbUrl, model, {
externalStorage: storageAdapter,
usePassedCtx: true,
disableTriggers: opt?.disableTriggers ?? false
})
try {
return {
pipeline: await pipelineFactory(ctx, wsUrl, true, () => {}, null),
storageAdapter
}
} catch (err: any) {
await storageAdapter.close()
throw err
return await pipelineFactory(ctx, wsUrl, true, () => {}, null)
}
const txAdapterFactories: Record<string, DbAdapterFactory> = {}
const adapterFactories: Record<string, DbAdapterFactory> = {}
const destroyFactories: Record<string, (url: string) => WorkspaceDestroyAdapter> = {}
export function registerTxAdapterFactry (name: string, factory: DbAdapterFactory, useAsDefault: boolean = true): void {
txAdapterFactories[name] = factory
if (useAsDefault) {
txAdapterFactories[''] = factory
}
}
export function registerAdapterFactry (name: string, factory: DbAdapterFactory, useAsDefault: boolean = true): void {
adapterFactories[name] = factory
if (useAsDefault) {
adapterFactories[''] = factory
}
}
export function registerDestroyFactry (
name: string,
factory: (url: string) => WorkspaceDestroyAdapter,
useAsDefault: boolean = true
): void {
destroyFactories[name] = factory
if (useAsDefault) {
destroyFactories[''] = factory
}
}
function matchTxAdapterFactory (dbUrl: string): DbAdapterFactory {
for (const [k, v] of Object.entries(txAdapterFactories)) {
if (dbUrl.startsWith(k)) {
return v
}
}
return txAdapterFactories['']
}
function matchAdapterFactory (dbUrl: string): DbAdapterFactory {
for (const [k, v] of Object.entries(adapterFactories)) {
if (dbUrl.startsWith(k)) {
return v
}
}
return adapterFactories['']
}
export function getWorkspaceDestroyAdapter (dbUrl: string): WorkspaceDestroyAdapter {
return dbUrl.startsWith('mongodb') ? createMongoDestroyAdapter(dbUrl) : createPostgreeDestroyAdapter(dbUrl)
for (const [k, v] of Object.entries(destroyFactories)) {
if (dbUrl.startsWith(k)) {
return v(dbUrl)
}
}
return destroyFactories[''](dbUrl)
}
export function getConfig (
@@ -255,11 +289,11 @@ export function getConfig (
defaultAdapter: extensions?.defaultAdapter ?? 'Main',
adapters: {
Tx: {
factory: dbUrl.startsWith('mongodb') ? createMongoTxAdapter : createPostgresTxAdapter,
factory: matchTxAdapterFactory(dbUrl),
url: dbUrl
},
Main: {
factory: dbUrl.startsWith('mongodb') ? createMongoAdapter : createPostgresAdapter,
factory: matchAdapterFactory(dbUrl),
url: dbUrl
},
Null: {
-1
View File
@@ -15,5 +15,4 @@
//
export * from './fallback'
export * from './blobStorage'
export * from './starter'
+6 -6
View File
@@ -1,6 +1,6 @@
import { DatalakeService, type DatalakeConfig } from '@hcengineering/datalake'
import { MinioConfig, MinioService, addMinioFallback } from '@hcengineering/minio'
import { S3Service, type S3Config } from '@hcengineering/s3'
import { CONFIG_KIND as DATALAKE_CONFIG_KIND, DatalakeService, type DatalakeConfig } from '@hcengineering/datalake'
import { CONFIG_KIND as MINIO_CONFIG_KIND, MinioConfig, MinioService, addMinioFallback } from '@hcengineering/minio'
import { CONFIG_KIND as S3_CONFIG_KIND, S3Service, type S3Config } from '@hcengineering/s3'
import { StorageAdapter, StorageConfiguration, type StorageConfig } from '@hcengineering/server-core'
import { FallbackStorageAdapter, buildStorage } from './fallback'
@@ -77,19 +77,19 @@ export function parseStorageEnv (storageEnv: string, storageConfig: StorageConfi
export function createStorageFromConfig (config: StorageConfig): StorageAdapter {
const kind = config.kind
if (kind === MinioService.config) {
if (kind === MINIO_CONFIG_KIND) {
const c = config as MinioConfig
if (c.endpoint == null || c.accessKey == null || c.secretKey == null) {
throw new Error('One of endpoint/accessKey/secretKey values are not specified')
}
return new MinioService(c)
} else if (kind === S3Service.config) {
} else if (kind === S3_CONFIG_KIND) {
const c = config as S3Config
if (c.endpoint == null || c.accessKey == null || c.secretKey == null) {
throw new Error('One of endpoint/accessKey/secretKey values are not specified')
}
return new S3Service(c)
} else if (kind === DatalakeService.config) {
} else if (kind === DATALAKE_CONFIG_KIND) {
const c = config as DatalakeConfig
if (c.endpoint == null) {
throw new Error('Endpoint value is not specified')
+10 -3
View File
@@ -415,7 +415,7 @@ class TSessionManager implements SessionManager {
: null) ?? null
if (workspace === undefined) {
ctx.warn('open workspace', {
ctx.info('open workspace', {
email: token.email,
workspace: workspaceInfo.workspaceId,
wsUrl: workspaceInfo.workspaceUrl,
@@ -469,7 +469,14 @@ class TSessionManager implements SessionManager {
})
return { upgrade: true }
}
pipeline = await ctx.with('💤 wait', { workspaceName }, () => (workspace as Workspace).pipeline)
try {
pipeline = await ctx.with('💤 wait', { workspaceName }, () => (workspace as Workspace).pipeline)
} catch (err: any) {
// Failed to create pipeline, etc
Analytics.handleError(err)
this.workspaces.delete(wsString)
throw err
}
}
const session = this.createSession(
@@ -794,7 +801,7 @@ class TSessionManager implements SessionManager {
ignoreSocket?: ConnectionSocket
): Promise<void> {
if (LOGGING_ENABLED) {
this.ctx.warn('closing workspace', {
this.ctx.info('closing workspace', {
workspace: workspace.id,
wsName: workspace.workspaceName,
code,
+3 -1
View File
@@ -56,6 +56,8 @@
"@hcengineering/server-token": "^0.6.11",
"@hcengineering/server-notification": "^0.6.1",
"@hcengineering/analytics": "^0.6.0",
"@hcengineering/server-backup": "^0.6.0"
"@hcengineering/server-backup": "^0.6.0",
"@hcengineering/postgres": "^0.6.0",
"@hcengineering/mongo": "^0.6.1"
}
}
+40 -8
View File
@@ -15,14 +15,20 @@ import core, {
type WorkspaceIdWithUrl
} from '@hcengineering/core'
import { consoleModelLogger, type MigrateOperation, type ModelLogger } from '@hcengineering/model'
import { createMongoTxAdapter, createMongoAdapter, createMongoDestroyAdapter } from '@hcengineering/mongo'
import { createPostgresTxAdapter, createPostgresAdapter, createPostgreeDestroyAdapter } from '@hcengineering/postgres'
import { getTransactorEndpoint } from '@hcengineering/server-client'
import { SessionDataImpl, wrapPipeline, type Pipeline, type StorageAdapter } from '@hcengineering/server-core'
import {
getServerPipeline,
getTxAdapterFactory,
registerAdapterFactry,
registerDestroyFactry,
registerServerPlugins,
registerStringLoaders
registerStringLoaders,
registerTxAdapterFactry
} from '@hcengineering/server-pipeline'
import { buildStorageFromConfig, storageConfigFromEnv } from '@hcengineering/server-storage'
import { generateToken } from '@hcengineering/server-token'
import { initializeWorkspace, initModel, prepareTools, updateModel, upgradeModel } from '@hcengineering/server-tool'
@@ -72,10 +78,21 @@ export async function createWorkspace (
const { dbUrl } = prepareTools([])
const hierarchy = new Hierarchy()
const modelDb = new ModelDb(hierarchy)
registerTxAdapterFactry('mongodb', createMongoTxAdapter)
registerAdapterFactry('mongodb', createMongoAdapter)
registerDestroyFactry('mongodb', createMongoDestroyAdapter)
registerTxAdapterFactry('postgresql', createPostgresTxAdapter, true)
registerAdapterFactry('postgresql', createPostgresAdapter, true)
registerDestroyFactry('postgresql', createPostgreeDestroyAdapter, true)
registerServerPlugins()
registerStringLoaders()
const { pipeline, storageAdapter } = await getServerPipeline(ctx, txes, dbUrl, wsUrl)
const storageConfig = storageConfigFromEnv()
const storageAdapter = buildStorageFromConfig(storageConfig)
const pipeline = await getServerPipeline(ctx, txes, dbUrl, wsUrl, storageAdapter)
try {
const txFactory = getTxAdapterFactory(ctx, dbUrl, wsUrl, null, {
@@ -153,16 +170,31 @@ export async function upgradeWorkspace (
): Promise<void> {
const { dbUrl } = prepareTools([])
let pipeline: Pipeline | undefined
let storageAdapter: StorageAdapter | undefined
registerTxAdapterFactry('mongodb', createMongoTxAdapter)
registerAdapterFactry('mongodb', createMongoAdapter)
registerDestroyFactry('mongodb', createMongoDestroyAdapter)
registerTxAdapterFactry('postgresql', createPostgresTxAdapter, true)
registerAdapterFactry('postgresql', createPostgresAdapter, true)
registerDestroyFactry('postgresql', createPostgreeDestroyAdapter, true)
registerServerPlugins()
registerStringLoaders()
const storageConfig = storageConfigFromEnv()
const storageAdapter = buildStorageFromConfig(storageConfig)
try {
;({ pipeline, storageAdapter } = await getServerPipeline(ctx, txes, dbUrl, {
name: ws.workspace,
workspaceName: ws.workspaceName ?? '',
workspaceUrl: ws.workspaceUrl ?? ''
}))
pipeline = await getServerPipeline(
ctx,
txes,
dbUrl,
{
name: ws.workspace,
workspaceName: ws.workspaceName ?? '',
workspaceUrl: ws.workspaceUrl ?? ''
},
storageAdapter
)
if (pipeline === undefined || storageAdapter === undefined) {
return
}
+1 -1
View File
@@ -22,7 +22,7 @@
"@hcengineering/platform-rig": "*"
},
"devDependencies": {
"wrangler": "^3.80.1",
"wrangler": "^3.97.0",
"@typescript-eslint/eslint-plugin": "^6.11.0",
"@typescript-eslint/parser": "^6.11.0",
"eslint-config-standard-with-typescript": "^40.0.0",
+2 -2
View File
@@ -20,9 +20,9 @@
},
"devDependencies": {
"@hcengineering/platform-rig": "^0.6.0",
"@cloudflare/workers-types": "^4.20240729.0",
"@cloudflare/workers-types": "^4.20241022.0",
"typescript": "^5.3.3",
"wrangler": "^3.80.1",
"wrangler": "^3.97.0",
"jest": "^29.7.0",
"prettier": "^3.1.0",
"ts-jest": "^29.1.1",
+2 -2
View File
@@ -20,9 +20,9 @@
},
"devDependencies": {
"@hcengineering/platform-rig": "^0.6.0",
"@cloudflare/workers-types": "^4.20240729.0",
"@cloudflare/workers-types": "^4.20241022.0",
"typescript": "^5.3.3",
"wrangler": "^3.80.1",
"wrangler": "^3.97.0",
"jest": "^29.7.0",
"prettier": "^3.1.0",
"ts-jest": "^29.1.1",
+12
View File
@@ -0,0 +1,12 @@
# http://editorconfig.org
root = true
[*]
indent_style = tab
end_of_line = lf
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
[*.yml]
indent_style = space
+7
View File
@@ -0,0 +1,7 @@
module.exports = {
extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/eslint.config.json'],
parserOptions: {
tsconfigRootDir: __dirname,
project: './tsconfig.json'
}
}
+173
View File
@@ -0,0 +1,173 @@
# Logs
logs
_.log
npm-debug.log_
yarn-debug.log*
yarn-error.log*
lerna-debug.log*
.pnpm-debug.log*
# Diagnostic reports (https://nodejs.org/api/report.html)
report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
# Runtime data
pids
_.pid
_.seed
\*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
\*.lcov
# nyc test coverage
.nyc_output
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
jspm_packages/
# Snowpack dependency directory (https://snowpack.dev/)
web_modules/
# TypeScript cache
\*.tsbuildinfo
# Optional npm cache directory
.npm
# Optional eslint cache
.eslintcache
# Optional stylelint cache
.stylelintcache
# Microbundle cache
.rpt2_cache/
.rts2_cache_cjs/
.rts2_cache_es/
.rts2_cache_umd/
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
\*.tgz
# Yarn Integrity file
.yarn-integrity
# dotenv environment variable files
.env
.env.development.local
.env.test.local
.env.production.local
.env.local
# parcel-bundler cache (https://parceljs.org/)
.cache
.parcel-cache
# Next.js build output
.next
out
# Nuxt.js build / generate output
.nuxt
dist
# Gatsby files
.cache/
# Comment in the public line in if your project uses Gatsby and not Next.js
# https://nextjs.org/blog/next-9-1#public-directory-support
# public
# vuepress build output
.vuepress/dist
# vuepress v2.x temp and cache directory
.temp
.cache
# Docusaurus cache and generated files
.docusaurus
# Serverless directories
.serverless/
# FuseBox cache
.fusebox/
# DynamoDB Local files
.dynamodb/
# TernJS port file
.tern-port
# Stores VSCode versions used for testing VSCode extensions
.vscode-test
# yarn v2
.yarn/cache
.yarn/unplugged
.yarn/build-state.yml
.yarn/install-state.gz
.pnp.\*
# wrangler project
.dev.vars
.wrangler/
src/model.json
+4
View File
@@ -0,0 +1,4 @@
{
"$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
"rigPackageName": "@hcengineering/platform-rig"
}
+59
View File
@@ -0,0 +1,59 @@
{
"name": "@hcengineering/cloud-transactor",
"version": "0.0.0",
"private": true,
"template": "cloud",
"scripts": {
"deploy": "wrangler deploy",
"dev": "wrangler dev --port 3335",
"dev-local": "wrangler dev --port 3335 --local-upstream --host localhost",
"start": "wrangler dev --port 3335",
"logs": "npx wrangler tail --format pretty",
"cf-typegen": "wrangler types",
"get-model": "mkdir -p bundle && esbuild src/get-model.ts --bundle --keep-names --platform=node --define:process.env.MODEL_VERSION=$(node ../../common/scripts/show_version.js) --define:process.env.VERSION=$(node ../../common/scripts/show_tag.js) --define:process.env.GIT_REVISION=$(../../common/scripts/git_version.sh) --outfile=bundle/bundle.js --log-level=error && node ./bundle/bundle.js > ./src/model.json",
"bundle": "rushx get-model && wrangler deploy --dry-run --outdir dist",
"build": "compile",
"build:watch": "compile",
"test": "jest --passWithNoTests --silent --forceExit",
"format": "format src",
"_phase:bundle": "rushx get-model",
"_phase:build": "compile transpile src",
"_phase:test": "jest --passWithNoTests --silent --forceExit",
"_phase:format": "format src",
"_phase:validate": "compile validate"
},
"devDependencies": {
"@cloudflare/workers-types": "^4.20241022.0",
"@hcengineering/model-all": "^0.6.0",
"@hcengineering/platform-rig": "^0.6.0",
"@types/jest": "^29.5.5",
"@typescript-eslint/eslint-plugin": "^6.11.0",
"@typescript-eslint/parser": "^6.11.0",
"eslint-config-standard-with-typescript": "^40.0.0",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-promise": "^6.1.1",
"eslint-plugin-n": "^15.4.0",
"eslint": "^8.54.0",
"jest": "^29.7.0",
"prettier": "^3.1.0",
"ts-jest": "^29.1.1",
"typescript": "^5.3.3",
"wrangler": "^3.97.0",
"esbuild": "^0.20.0"
},
"dependencies": {
"@hcengineering/core": "^0.6.32",
"@hcengineering/middleware": "^0.6.0",
"@hcengineering/platform-rig": "^0.6.0",
"@hcengineering/rpc": "^0.6.5",
"@hcengineering/platform": "^0.6.11",
"@hcengineering/postgres": "^0.6.0",
"@hcengineering/server": "^0.6.4",
"@hcengineering/server-core": "^0.6.1",
"@hcengineering/server-client": "^0.6.0",
"@hcengineering/server-pipeline": "^0.6.0",
"@hcengineering/server-token": "^0.6.11",
"@hcengineering/storage": "^0.6.0",
"itty-router": "^5.0.18"
}
}
+1
View File
@@ -0,0 +1 @@
import '@hcengineering/model-all/src/show'
+50
View File
@@ -0,0 +1,50 @@
// Copyright © 2024 Huly Labs.
import { Router, error, html } from 'itty-router'
import { decodeToken } from '@hcengineering/server-token'
import type { Transactor } from './transactor'
export { Transactor } from './transactor'
export interface Env {
TRANSACTOR: DurableObjectNamespace<Transactor>
HYPERDRIVE: Hyperdrive
SERVER_SECRET: string
ACCOUNTS_URL: string
}
export default {
async fetch (request: Request, env: Env, ctx: ExecutionContext): Promise<Response> {
const router = Router()
router
.get('/:token', ({ params, headers }) => {
if (headers.get('Upgrade') !== 'websocket') {
return new Response('Expected header Upgrade: websocket', { status: 426 })
}
try {
const decodedToken = decodeToken(params.token, true, env.SERVER_SECRET)
console.log('connecting', decodedToken.email)
const id = env.TRANSACTOR.idFromName(decodedToken.workspace.name)
const stub = env.TRANSACTOR.get(id)
return stub.fetch(request)
} catch (err: any) {
return new Response('Expected header Upgrade: websocket', { status: 426 })
}
})
// TODO: Add statistics using storage
.all('/', () =>
html(
`Huly&reg; Transactor&trade; <a href="https://huly.io">https://huly.io</a>
&copy; 2024 <a href="https://hulylabs.com">Huly Labs</a>`
)
)
return await router.fetch(request).catch(error)
}
} satisfies ExportedHandler<Env>
+287
View File
@@ -0,0 +1,287 @@
// Copyright © 2024 Huly Labs.
import {
Branding,
generateId,
MeasureMetricsContext,
type MeasureContext,
type WorkspaceIdWithUrl
} from '@hcengineering/core'
import { setMetadata } from '@hcengineering/platform'
import { RPCHandler } from '@hcengineering/rpc'
import { ClientSession, createSessionManager, doSessionOp, type WebsocketData } from '@hcengineering/server'
import serverClient from '@hcengineering/server-client'
import {
createDummyStorageAdapter,
loadBrandingMap,
Pipeline,
type ConnectionSocket,
type PipelineFactory,
type SessionManager
} from '@hcengineering/server-core'
// import { registerStringLoaders } from '@hcengineering/server-pipeline'
import serverPlugin, { decodeToken, type Token } from '@hcengineering/server-token'
import { DurableObject } from 'cloudflare:workers'
// Approach usefull only for separate build, after model-all bundle phase is executed.
import { createPostgreeDestroyAdapter, createPostgresAdapter, createPostgresTxAdapter } from '@hcengineering/postgres'
import {
createServerPipeline,
registerAdapterFactry,
registerDestroyFactry,
registerServerPlugins,
registerTxAdapterFactry
} from '@hcengineering/server-pipeline'
import model from './model.json'
export const PREFERRED_SAVE_SIZE = 500
export const PREFERRED_SAVE_INTERVAL = 30 * 1000
export class Transactor extends DurableObject<Env> {
rpcHandler = new RPCHandler()
private workspace: string = ''
private sessionManager!: SessionManager
private readonly measureCtx: MeasureContext
private readonly pipelineFactory: PipelineFactory
private readonly accountsUrl: string
private readonly sessions = new Map<WebSocket, WebsocketData>()
constructor (ctx: DurableObjectState, env: Env) {
super(ctx, env)
registerTxAdapterFactry('postgresql', createPostgresTxAdapter, true)
registerAdapterFactry('postgresql', createPostgresAdapter, true)
registerDestroyFactry('postgresql', createPostgreeDestroyAdapter, true)
registerServerPlugins()
this.accountsUrl = env.ACCOUNTS_URL ?? 'http://127.0.0.1:3000'
this.measureCtx = new MeasureMetricsContext('transactor-' + this.workspace, {})
setMetadata(serverPlugin.metadata.Secret, env.SERVER_SECRET ?? 'secret')
console.log('Connecting DB to', env.DB_URL !== '' ? 'Direct ' : 'Hyperdrive')
// TODO:
const storage = createDummyStorageAdapter()
this.pipelineFactory = async (ctx, ws, upgrade, broadcast, branding) => {
const pipeline = createServerPipeline(
this.measureCtx,
env.DB_URL !== '' ? env.DB_URL : env.HYPERDRIVE.connectionString,
model,
{
externalStorage: storage,
adapterSecurity: false,
disableTriggers: false,
fulltextUrl: undefined // TODO: Pass fulltext service URI.
}
)
return await pipeline(ctx, ws, upgrade, broadcast, branding)
}
void this.ctx.blockConcurrencyWhile(async () => {
setMetadata(serverClient.metadata.Endpoint, env.ACCOUNTS_URL)
this.sessionManager = createSessionManager(
this.measureCtx,
(token: Token, pipeline: Pipeline, workspaceId: WorkspaceIdWithUrl, branding: Branding | null) =>
new ClientSession(token, pipeline, workspaceId, branding, false),
loadBrandingMap(), // TODO: Support branding map
{
pingTimeout: 10000,
reconnectTimeout: 3000
},
undefined,
this.accountsUrl
)
})
}
async fetch (request: Request): Promise<Response> {
const { 0: client, 1: server } = new WebSocketPair()
const url = new URL(request.url ?? '')
const token = url.pathname.substring(1)
try {
const payload = decodeToken(token ?? '')
const sessionId = url.searchParams.get('sessionId')
// By design, all fetches to this durable object will be for the same workspace
if (this.workspace === '') {
this.workspace = payload.workspace.name
}
if (!(await this.handleSession(server, request, payload, token, sessionId))) {
return new Response(null, { status: 404 })
}
return new Response(null, { status: 101, webSocket: client })
} catch (err: any) {
console.error(err)
return new Response(null, { status: 404 })
}
}
async webSocketMessage (ws: WebSocket, message: ArrayBuffer | string): Promise<void> {
const session = this.sessions.get(ws)
if (session === undefined) {
return
}
const cs = session.connectionSocket
if (cs === undefined) {
return
}
doSessionOp(
session,
(s, buff) => {
s.context.measure('receive-data', buff?.length ?? 0)
// processRequest(s.session, cs, s.context, s.workspaceId, buff, handleRequest)
const request = cs.readRequest(buff, s.session.binaryMode)
this.sessionManager.handleRequest(this.measureCtx, s.session, cs, request, this.workspace)
},
typeof message === 'string' ? Buffer.from(message) : Buffer.from(message)
)
}
async webSocketError (ws: WebSocket, error: unknown): Promise<void> {
console.error('WebSocket error:', error)
await this.handleClose(ws, 1011, 'error')
}
async alarm (): Promise<void> {}
async handleSession (
ws: WebSocket,
request: Request,
token: Token,
rawToken: string,
sessionId: string | null
): Promise<boolean> {
const data = {
remoteAddress: request.headers.get('CF-Connecting-IP') ?? '',
userAgent: request.headers.get('user-agent') ?? '',
language: request.headers.get('accept-language') ?? '',
email: token.email,
mode: token.extra?.mode,
model: token.extra?.model
}
const cs = this.createWebsocketClientSocket(ws, data)
const session = await this.sessionManager.addSession(
this.measureCtx,
cs,
token,
rawToken,
this.pipelineFactory,
sessionId ?? undefined
)
const webSocketData: WebsocketData = {
connectionSocket: cs,
payload: token,
token: rawToken,
session,
url: ''
}
if ('error' in session) {
if (session.terminate === true) {
ws.close()
}
throw session.error
}
if ('upgrade' in session) {
cs.send(
this.measureCtx,
{ id: -1, result: { state: 'upgrading', stats: (session as any).upgradeInfo } },
false,
false
)
cs.close()
return true
}
this.sessions.set(ws, webSocketData)
this.ctx.acceptWebSocket(ws)
return true
}
createWebsocketClientSocket (
ws: WebSocket,
data: {
remoteAddress: string
userAgent: string
language: string
email: string
mode: any
model: any
}
): ConnectionSocket {
const cs: ConnectionSocket = {
id: generateId(),
isClosed: false,
close: () => {
cs.isClosed = true
ws.close()
},
checkState: () => {
if (ws.readyState === WebSocket.CLOSED || ws.readyState === WebSocket.CLOSING) {
ws.close()
return false
}
return true
},
readRequest: (buffer: Buffer, binary: boolean) => {
return this.rpcHandler.readRequest(buffer, binary)
},
data: () => data,
send: (ctx: MeasureContext, msg, binary, compression) => {
const smsg = this.rpcHandler.serialize(msg, binary)
ctx.measure('send-data', smsg.length)
if (ws.readyState !== WebSocket.OPEN || cs.isClosed) {
return
}
ws.send(smsg)
}
}
return cs
}
async broadcastMessage (message: Uint8Array, origin?: any): Promise<void> {
const wss = this.ctx.getWebSockets().filter((ws) => ws.readyState === WebSocket.OPEN)
await Promise.all(
wss.map(async (ws) => {
await this.sendMessage(ws, message)
})
)
}
async sendMessage (ws: WebSocket, message: Uint8Array): Promise<void> {
try {
ws.send(message)
} catch (error) {
console.error('Failed to send message:', error)
await this.handleClose(ws, 1011, 'error')
}
}
async handleClose (ws: WebSocket, code: number, reason?: string): Promise<void> {
try {
ws.close(code, reason)
} catch (err) {
console.error('Failed to close WebSocket:', err)
}
const session = this.sessions.get(ws)
if (session !== undefined) {
await this.sessionManager.close(this.measureCtx, session.connectionSocket as ConnectionSocket, this.workspace)
}
}
}
+11
View File
@@ -0,0 +1,11 @@
{
"extends": "./node_modules/@hcengineering/platform-rig/profiles/default/tsconfig.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./lib",
"declarationDir": "./types",
"tsBuildInfoFile": ".build/build.tsbuildinfo",
"types": ["@cloudflare/workers-types", "jest"]
}
}
+13
View File
@@ -0,0 +1,13 @@
// Generated by Wrangler
// After adding bindings to `wrangler.toml`, regenerate this interface via `npm run cf-typegen`
interface Env {
TRANSACTOR: DurableObjectNamespace
SERVER_SECRET: string
HYPERDRIVE: Hyperdrive
ACCOUNTS_URL: string
DB_URL: string
}
+114
View File
@@ -0,0 +1,114 @@
#:schema node_modules/wrangler/config-schema.json
name = "cloud-transactor"
main = "src/index.ts"
compatibility_date = "2024-09-23"
compatibility_flags = ["nodejs_compat_v2"]
keep_vars = true
[observability.logs]
enabled = true
# Automatically place your workloads in an optimal location to minimize latency.
# If you are running back-end logic in a Worker, running it closer to your back-end infrastructure
# rather than the end user may result in better performance.
# Docs: https://developers.cloudflare.com/workers/configuration/smart-placement/#smart-placement
# [placement]
# mode = "smart"
# Variable bindings. These are arbitrary, plaintext strings (similar to environment variables)
# Docs:
# - https://developers.cloudflare.com/workers/wrangler/configuration/#environment-variables
# Note: Use secrets to store sensitive data.
# - https://developers.cloudflare.com/workers/configuration/secrets/
[vars]
# ACCOUNTS_URL = "http://127.0.0.1:3000"
# SERVER_SECRET = "secret"
# Bind the Workers AI model catalog. Run machine learning models, powered by serverless GPUs, on Cloudflares global network
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#workers-ai
# [ai]
# binding = "AI"
# Bind an Analytics Engine dataset. Use Analytics Engine to write analytics within your Pages Function.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#analytics-engine-datasets
# [[analytics_engine_datasets]]
# binding = "MY_DATASET"
# Bind a headless browser instance running on Cloudflare's global network.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#browser-rendering
# [browser]
# binding = "MY_BROWSER"
# Bind a D1 database. D1 is Cloudflares native serverless SQL database.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#d1-databases
# [[d1_databases]]
# binding = "MY_DB"
# database_name = "my-database"
# database_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
# Bind a dispatch namespace. Use Workers for Platforms to deploy serverless functions programmatically on behalf of your customers.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#dispatch-namespace-bindings-workers-for-platforms
# [[dispatch_namespaces]]
# binding = "MY_DISPATCHER"
# namespace = "my-namespace"
# Bind a Durable Object. Durable objects are a scale-to-zero compute primitive based on the actor model.
# Durable Objects can live for as long as needed. Use these when you need a long-running "server", such as in realtime apps.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#durable-objects
[[durable_objects.bindings]]
name = "TRANSACTOR"
class_name = "Transactor"
# Durable Object migrations.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#migrations
[[migrations]]
tag = "v1"
new_classes = ["Transactor"]
# Bind a Hyperdrive configuration. Use to accelerate access to your existing databases from Cloudflare Workers.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#hyperdrive
[[hyperdrive]]
binding = "HYPERDRIVE"
id = "1f713bb2ec1b464cb663983feca4c89e"
localConnectionString = "postgresql://postgree:example@host.docker.internal:26257/defaultdb?sslmode=disable"
# Bind a KV Namespace. Use KV as persistent storage for small key-value pairs.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#kv-namespaces
#[[kv_namespaces]]
#binding = "transactor_model"
#id = ""
# Bind an mTLS certificate. Use to present a client certificate when communicating with another service.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#mtls-certificates
# [[mtls_certificates]]
# binding = "MY_CERTIFICATE"
# certificate_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
# Bind a Queue producer. Use this binding to schedule an arbitrary task that may be processed later by a Queue consumer.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#queues
# [[queues.producers]]
# binding = "MY_QUEUE"
# queue = "my-queue"
# Bind a Queue consumer. Queue Consumers can retrieve tasks scheduled by Producers to act on them.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#queues
# [[queues.consumers]]
# queue = "my-queue"
# Bind an R2 Bucket. Use R2 to store arbitrarily large blobs of data, such as files.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#r2-buckets
# [[r2_buckets]]
# binding = "MY_BUCKET"
# bucket_name = "my-bucket"
# Bind another Worker service. Use this binding to call another Worker without network overhead.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#service-bindings
# [[services]]
# binding = "MY_SERVICE"
# service = "my-service"
# Bind a Vectorize index. Use to store and query vector embeddings for semantic search, classification and other vector search use-cases.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#vectorize-indexes
# [[vectorize]]
# binding = "MY_INDEX"
# index_name = "my-index"