diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index d506b71450..29a2726a83 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -686,6 +686,9 @@ dependencies: '@rush-temp/server-notification-resources': specifier: file:./projects/server-notification-resources.tgz version: file:projects/server-notification-resources.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2) + '@rush-temp/server-pipeline': + specifier: file:./projects/server-pipeline.tgz + version: file:projects/server-pipeline.tgz '@rush-temp/server-preference': specifier: file:./projects/server-preference.tgz version: file:projects/server-preference.tgz(esbuild@0.20.1)(ts-node@10.9.2) @@ -21355,7 +21358,7 @@ packages: dev: false file:projects/model-chunter.tgz: - resolution: {integrity: sha512-s99PJII281vT3N2c/Z82EDGmjU1xLi1CXolixGNHtP2OaOKDFIL3z7TGABOz/RrNtT8yTIoEy1LBSy/qCiuNYQ==, tarball: file:projects/model-chunter.tgz} + resolution: {integrity: sha512-es0oJ+MzCyOR75Kdx2An+tBhga3cZCaPV24k7V4LtlxXyxmboZjiDhZgh9/56psQwWgT7qqL6XBPzCR4nZBV1g==, tarball: file:projects/model-chunter.tgz} name: '@rush-temp/model-chunter' version: 0.0.0 dependencies: @@ -23059,7 +23062,7 @@ packages: dev: false file:projects/pod-server.tgz: - resolution: {integrity: sha512-wPRlkGQX23QexM6xt1m5VtJVyxJ4TlT87nVR6Ibf/ty1nrpKTqCYTS/7xxPB6cSC/S8hCb1VheKNkR4+vXXM0Q==, tarball: file:projects/pod-server.tgz} + resolution: {integrity: sha512-GjJqgPc/ux0jjGydLB1tmiwo9f4tTZL4KECyHmiLemLahnZojN8AIIFQTsqUkiFh0dwULxPvJkRWnYdGLA0iZg==, tarball: file:projects/pod-server.tgz} name: '@rush-temp/pod-server' version: 0.0.0 dependencies: @@ -25029,6 +25032,40 @@ packages: - ts-node dev: false + file:projects/server-pipeline.tgz: + resolution: {integrity: sha512-DE5BaOMANpAiN/Gy/0wWfbSY97sPRjXjwIyQbUGRVhpQeu9AznyK9afkGowQjA7fexT7WHu3GgCVtMzLdh7xDg==, tarball: file:projects/server-pipeline.tgz} + name: '@rush-temp/server-pipeline' + version: 0.0.0 + dependencies: + '@types/jest': 29.5.12 + '@types/node': 20.11.19 + '@types/ws': 8.5.10 + '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3) + '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3) + cross-env: 7.0.3 + elastic-apm-node: 3.26.0 + esbuild: 0.20.1 + eslint: 8.56.0 + eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0)(eslint-plugin-import@2.29.1)(eslint-plugin-n@15.7.0)(eslint-plugin-promise@6.1.1)(eslint@8.56.0)(typescript@5.3.3) + eslint-plugin-import: 2.29.1(eslint@8.56.0) + eslint-plugin-n: 15.7.0(eslint@8.56.0) + eslint-plugin-promise: 6.1.1(eslint@8.56.0) + jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2) + prettier: 3.2.5 + ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3) + ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.3.3) + typescript: 5.3.3 + transitivePeerDependencies: + - '@babel/core' + - '@jest/types' + - '@swc/core' + - '@swc/wasm' + - babel-jest + - babel-plugin-macros + - node-notifier + - supports-color + dev: false + file:projects/server-preference.tgz(esbuild@0.20.1)(ts-node@10.9.2): resolution: {integrity: sha512-YYPJESH2THfyx18g89/ZbECan3phrVyHrCOZ/3CahJaQk6Aj7Ti7eavNZLu36+iO+8S5Tt4Wy65oFZBgbkPG8A==, tarball: file:projects/server-preference.tgz} id: file:projects/server-preference.tgz diff --git a/pods/server/package.json b/pods/server/package.json index 2d0c100c8a..1707a8c2ea 100644 --- a/pods/server/package.json +++ b/pods/server/package.json @@ -51,130 +51,12 @@ "@hcengineering/platform": "^0.6.11", "@hcengineering/server-core": "^0.6.1", "@hcengineering/server-ws": "^0.6.11", - "@hcengineering/server-attachment": "^0.6.1", - "@hcengineering/server-attachment-resources": "^0.6.0", - "@hcengineering/server-collaboration": "^0.6.0", - "@hcengineering/server-collaboration-resources": "^0.6.0", + "@hcengineering/server-token": "^0.6.11", "@hcengineering/server": "^0.6.4", "@hcengineering/server-storage": "^0.6.0", - "@hcengineering/mongo": "^0.6.1", - "@hcengineering/elastic": "^0.6.0", - "elastic-apm-node": "~3.26.0", - "@hcengineering/server-contact": "^0.6.1", - "@hcengineering/server-contact-resources": "^0.6.0", - "@hcengineering/server-notification": "^0.6.1", - "@hcengineering/server-notification-resources": "^0.6.0", - "@hcengineering/server-setting": "^0.6.0", - "@hcengineering/server-setting-resources": "^0.6.0", - "@hcengineering/server-chunter": "^0.6.0", - "@hcengineering/server-chunter-resources": "^0.6.0", - "@hcengineering/server-inventory": "^0.6.1", - "@hcengineering/server-inventory-resources": "^0.6.0", - "@hcengineering/server-lead": "^0.6.0", - "@hcengineering/server-lead-resources": "^0.6.0", - "@hcengineering/server-recruit": "^0.6.0", - "@hcengineering/server-recruit-resources": "^0.6.0", - "@hcengineering/server-task": "^0.6.0", - "@hcengineering/server-task-resources": "^0.6.0", - "@hcengineering/server-tracker": "^0.6.0", - "@hcengineering/server-tracker-resources": "^0.6.0", - "@hcengineering/server-tags": "^0.6.0", - "@hcengineering/server-tags-resources": "^0.6.0", - "@hcengineering/server-calendar": "^0.6.0", - "@hcengineering/server-calendar-resources": "^0.6.0", - "@hcengineering/server-gmail": "^0.6.0", - "@hcengineering/server-gmail-resources": "^0.6.0", - "@hcengineering/server-preference": "^0.6.0", - "@hcengineering/server-telegram": "^0.6.0", - "@hcengineering/server-telegram-resources": "^0.6.0", - "@hcengineering/server-time": "^0.6.0", - "@hcengineering/server-time-resources": "^0.6.0", - "@hcengineering/server-hr": "^0.6.0", - "@hcengineering/server-hr-resources": "^0.6.0", - "@hcengineering/server-request": "^0.6.0", - "@hcengineering/server-request-resources": "^0.6.0", - "@hcengineering/server-view": "^0.6.0", - "@hcengineering/server-view-resources": "^0.6.0", - "@hcengineering/server-activity": "^0.6.0", - "@hcengineering/server-activity-resources": "^0.6.0", - "@hcengineering/server-document": "^0.6.0", - "@hcengineering/server-document-resources": "^0.6.0", - "@hcengineering/server-guest": "^0.6.0", - "@hcengineering/server-guest-resources": "^0.6.0", - "@hcengineering/server-token": "^0.6.11", - "@hcengineering/server-controlled-documents": "^0.1.0", - "@hcengineering/server-controlled-documents-resources": "^0.1.0", - "@hcengineering/server-training": "^0.1.0", - "@hcengineering/server-training-resources": "^0.1.0", - "@hcengineering/middleware": "^0.6.0", - "@hcengineering/minio": "^0.6.0", - "@hcengineering/openai": "^0.6.0", - "@hcengineering/translate": "^0.6.0", - "@hcengineering/login-assets": "^0.6.0", - "@hcengineering/view-assets": "^0.6.11", - "@hcengineering/task-assets": "^0.6.19", - "@hcengineering/chunter-assets": "^0.6.18", - "@hcengineering/recruit-assets": "^0.6.23", - "@hcengineering/setting-assets": "^0.6.15", - "@hcengineering/support-assets": "^0.6.5", - "@hcengineering/contact-assets": "^0.6.13", - "@hcengineering/activity-assets": "^0.6.3", - "@hcengineering/telegram-assets": "^0.6.0", - "@hcengineering/workbench-assets": "^0.6.14", - "@hcengineering/attachment-assets": "^0.6.11", - "@hcengineering/lead-assets": "^0.6.0", - "@hcengineering/drive-assets": "^0.6.0", - "@hcengineering/gmail-assets": "^0.6.0", - "@hcengineering/inventory-assets": "^0.6.11", - "@hcengineering/templates-assets": "^0.6.11", - "@hcengineering/notification-assets": "^0.6.17", - "@hcengineering/preference-assets": "^0.6.0", - "@hcengineering/tags-assets": "^0.6.0", - "@hcengineering/calendar-assets": "^0.6.22", - "@hcengineering/tracker-assets": "^0.6.0", - "@hcengineering/board-assets": "^0.6.19", - "@hcengineering/hr-assets": "^0.6.19", - "@hcengineering/bitrix-assets": "^0.6.0", - "@hcengineering/request-assets": "^0.6.0", - "@hcengineering/document-assets": "^0.6.0", - "@hcengineering/controlled-documents-assets": "^0.1.0", - "@hcengineering/products-assets": "^0.1.0", - "@hcengineering/training-assets": "^0.1.0", - "@hcengineering/login": "^0.6.12", - "@hcengineering/view": "^0.6.13", - "@hcengineering/task": "^0.6.20", - "@hcengineering/chunter": "^0.6.20", - "@hcengineering/recruit": "^0.6.29", - "@hcengineering/setting": "^0.6.17", + "@hcengineering/server-pipeline": "^0.6.0", "@hcengineering/contact": "^0.6.24", - "@hcengineering/activity": "^0.6.0", - "@hcengineering/telegram": "^0.6.21", - "@hcengineering/workbench": "^0.6.16", - "@hcengineering/attachment": "^0.6.14", - "@hcengineering/lead": "^0.6.0", - "@hcengineering/drive": "^0.6.0", - "@hcengineering/server-drive": "^0.6.0", - "@hcengineering/server-drive-resources": "^0.6.0", - "@hcengineering/gmail": "^0.6.22", - "@hcengineering/inventory": "^0.6.11", - "@hcengineering/templates": "^0.6.11", "@hcengineering/notification": "^0.6.23", - "@hcengineering/preference": "^0.6.13", - "@hcengineering/tags": "^0.6.16", - "@hcengineering/calendar": "^0.6.24", - "@hcengineering/tracker": "^0.6.24", - "@hcengineering/board": "^0.6.19", - "@hcengineering/hr": "^0.6.19", - "@hcengineering/bitrix": "^0.6.52", - "@hcengineering/request": "^0.6.14", - "@hcengineering/support": "^0.6.5", - "@hcengineering/document": "^0.6.0", - "@hcengineering/love": "^0.6.0", - "@hcengineering/love-assets": "^0.6.0", - "@hcengineering/server-love": "^0.6.0", - "@hcengineering/server-love-resources": "^0.6.0", - "@hcengineering/controlled-documents": "^0.1.0", - "@hcengineering/products": "^0.1.0", - "@hcengineering/training": "^0.1.0" + "@hcengineering/server-notification": "^0.6.1" } } diff --git a/pods/server/src/server.ts b/pods/server/src/server.ts index e694e7c74c..e21b6b1ad9 100644 --- a/pods/server/src/server.ts +++ b/pods/server/src/server.ts @@ -14,196 +14,16 @@ // limitations under the License. // -import { - coreId, - DOMAIN_BLOB, - DOMAIN_FULLTEXT_BLOB, - DOMAIN_MODEL, - DOMAIN_TRANSIENT, - DOMAIN_TX, - type MeasureContext, - type WorkspaceId, - type BrandingMap, - type Branding -} from '@hcengineering/core' -import { createElasticAdapter, createElasticBackupDataAdapter } from '@hcengineering/elastic' -import { - ConfigurationMiddleware, - LookupMiddleware, - BlobLookupMiddleware, - ModifiedMiddleware, - PrivateMiddleware, - QueryJoinMiddleware, - SpacePermissionsMiddleware, - SpaceSecurityMiddleware -} from '@hcengineering/middleware' -import { createMongoAdapter, createMongoTxAdapter } from '@hcengineering/mongo' -import { OpenAIEmbeddingsStage, openAIId, openAIPluginImpl } from '@hcengineering/openai' -import { addLocation, addStringsLoader, platformId } from '@hcengineering/platform' -import { - BackupClientSession, - buildStorageFromConfig, - createNullAdapter, - createRekoniAdapter, - createStorageDataAdapter, - createYDocAdapter, - getMetricsContext -} from '@hcengineering/server' +import { type BrandingMap } from '@hcengineering/core' +import { BackupClientSession, buildStorageFromConfig, getMetricsContext } from '@hcengineering/server' -import { serverActivityId } from '@hcengineering/server-activity' -import { serverAttachmentId } from '@hcengineering/server-attachment' -import { serverCalendarId } from '@hcengineering/server-calendar' -import { serverChunterId } from '@hcengineering/server-chunter' -import { CollaborativeContentRetrievalStage, serverCollaborationId } from '@hcengineering/server-collaboration' -import { serverContactId } from '@hcengineering/server-contact' -import { - ContentRetrievalStage, - createInMemoryAdapter, - createPipeline, - FullSummaryStage, - FullTextPushStage, - globalIndexer, - IndexedFieldStage, - type ContentTextAdapter, - type DbConfiguration, - type FullTextAdapter, - type FullTextPipelineStage, - type MiddlewareCreator, - type Pipeline, - type StorageAdapter, - type StorageConfiguration, - type ServerStorage -} from '@hcengineering/server-core' -import { serverDocumentId } from '@hcengineering/server-document' -import { serverDriveId } from '@hcengineering/server-drive' -import { serverGmailId } from '@hcengineering/server-gmail' -import { serverGuestId } from '@hcengineering/server-guest' -import { serverHrId } from '@hcengineering/server-hr' -import { serverInventoryId } from '@hcengineering/server-inventory' -import { serverLeadId } from '@hcengineering/server-lead' -import { serverNotificationId } from '@hcengineering/server-notification' -import { serverRecruitId } from '@hcengineering/server-recruit' -import { serverRequestId } from '@hcengineering/server-request' -import { serverSettingId } from '@hcengineering/server-setting' -import { serverTagsId } from '@hcengineering/server-tags' -import { serverTaskId } from '@hcengineering/server-task' -import { serverTelegramId } from '@hcengineering/server-telegram' -import { serverTimeId } from '@hcengineering/server-time' +import { type Pipeline, type StorageConfiguration } from '@hcengineering/server-core' import { type Token } from '@hcengineering/server-token' -import { serverTrackerId } from '@hcengineering/server-tracker' -import { serverViewId } from '@hcengineering/server-view' -import { serverLoveId } from '@hcengineering/server-love' -import { serverTrainingId } from '@hcengineering/server-training' -import { serverDocumentsId } from '@hcengineering/server-controlled-documents' -import { - ClientSession, - start as startJsonRpc, - type PipelineFactory, - type ServerFactory, - type Session -} from '@hcengineering/server-ws' +import { ClientSession, start as startJsonRpc, type ServerFactory, type Session } from '@hcengineering/server-ws' -import { activityId } from '@hcengineering/activity' -import { attachmentId } from '@hcengineering/attachment' -import { bitrixId } from '@hcengineering/bitrix' -import { boardId } from '@hcengineering/board' -import { calendarId } from '@hcengineering/calendar' -import { chunterId } from '@hcengineering/chunter' -import { contactId } from '@hcengineering/contact' -import { documentId } from '@hcengineering/document' -import { driveId } from '@hcengineering/drive' -import { gmailId } from '@hcengineering/gmail' -import { hrId } from '@hcengineering/hr' -import { inventoryId } from '@hcengineering/inventory' -import { leadId } from '@hcengineering/lead' -import { loginId } from '@hcengineering/login' -import { notificationId } from '@hcengineering/notification' -import { preferenceId } from '@hcengineering/preference' -import { recruitId } from '@hcengineering/recruit' -import { requestId } from '@hcengineering/request' -import { settingId } from '@hcengineering/setting' -import { supportId } from '@hcengineering/support' -import { tagsId } from '@hcengineering/tags' -import { taskId } from '@hcengineering/task' -import { telegramId } from '@hcengineering/telegram' -import { templatesId } from '@hcengineering/templates' -import { trackerId } from '@hcengineering/tracker' -import { viewId } from '@hcengineering/view' -import { workbenchId } from '@hcengineering/workbench' -import { loveId } from '@hcengineering/love' -import { documentsId } from '@hcengineering/controlled-documents' -import { productsId } from '@hcengineering/products' -import { trainingId } from '@hcengineering/training' +import { createServerPipeline, registerServerPlugins, registerStringLoaders } from '@hcengineering/server-pipeline' -import coreEng from '@hcengineering/core/lang/en.json' -import loginEng from '@hcengineering/login-assets/lang/en.json' -import platformEng from '@hcengineering/platform/lang/en.json' - -import activityEn from '@hcengineering/activity-assets/lang/en.json' -import attachmentEn from '@hcengineering/attachment-assets/lang/en.json' -import bitrixEn from '@hcengineering/bitrix-assets/lang/en.json' -import boardEn from '@hcengineering/board-assets/lang/en.json' -import calendarEn from '@hcengineering/calendar-assets/lang/en.json' -import chunterEn from '@hcengineering/chunter-assets/lang/en.json' -import contactEn from '@hcengineering/contact-assets/lang/en.json' -import documentEn from '@hcengineering/document-assets/lang/en.json' -import driveEn from '@hcengineering/drive-assets/lang/en.json' -import gmailEn from '@hcengineering/gmail-assets/lang/en.json' -import hrEn from '@hcengineering/hr-assets/lang/en.json' -import inventoryEn from '@hcengineering/inventory-assets/lang/en.json' -import leadEn from '@hcengineering/lead-assets/lang/en.json' -import notificationEn from '@hcengineering/notification-assets/lang/en.json' -import preferenceEn from '@hcengineering/preference-assets/lang/en.json' -import recruitEn from '@hcengineering/recruit-assets/lang/en.json' -import requestEn from '@hcengineering/request-assets/lang/en.json' -import settingEn from '@hcengineering/setting-assets/lang/en.json' -import supportEn from '@hcengineering/support-assets/lang/en.json' -import tagsEn from '@hcengineering/tags-assets/lang/en.json' -import taskEn from '@hcengineering/task-assets/lang/en.json' -import telegramEn from '@hcengineering/telegram-assets/lang/en.json' -import templatesEn from '@hcengineering/templates-assets/lang/en.json' -import trackerEn from '@hcengineering/tracker-assets/lang/en.json' -import viewEn from '@hcengineering/view-assets/lang/en.json' -import workbenchEn from '@hcengineering/workbench-assets/lang/en.json' -import loveEn from '@hcengineering/love-assets/lang/en.json' -import documentsEn from '@hcengineering/controlled-documents-assets/lang/en.json' -import productsEn from '@hcengineering/products-assets/lang/en.json' -import trainingEn from '@hcengineering/training-assets/lang/en.json' - -addStringsLoader(coreId, async (lang: string) => coreEng) -addStringsLoader(loginId, async (lang: string) => loginEng) -addStringsLoader(platformId, async (lang: string) => platformEng) - -addStringsLoader(taskId, async (lang: string) => taskEn) -addStringsLoader(viewId, async (lang: string) => viewEn) -addStringsLoader(chunterId, async (lang: string) => chunterEn) -addStringsLoader(attachmentId, async (lang: string) => attachmentEn) -addStringsLoader(contactId, async (lang: string) => contactEn) -addStringsLoader(recruitId, async (lang: string) => recruitEn) -addStringsLoader(activityId, async (lang: string) => activityEn) -addStringsLoader(settingId, async (lang: string) => settingEn) -addStringsLoader(supportId, async (lang: string) => supportEn) -addStringsLoader(telegramId, async (lang: string) => telegramEn) -addStringsLoader(leadId, async (lang: string) => leadEn) -addStringsLoader(gmailId, async (lang: string) => gmailEn) -addStringsLoader(workbenchId, async (lang: string) => workbenchEn) -addStringsLoader(inventoryId, async (lang: string) => inventoryEn) -addStringsLoader(templatesId, async (lang: string) => templatesEn) -addStringsLoader(notificationId, async (lang: string) => notificationEn) -addStringsLoader(tagsId, async (lang: string) => tagsEn) -addStringsLoader(calendarId, async (lang: string) => calendarEn) -addStringsLoader(trackerId, async (lang: string) => trackerEn) -addStringsLoader(boardId, async (lang: string) => boardEn) -addStringsLoader(preferenceId, async (lang: string) => preferenceEn) -addStringsLoader(hrId, async (lang: string) => hrEn) -addStringsLoader(documentId, async (lang: string) => documentEn) -addStringsLoader(bitrixId, async (lang: string) => bitrixEn) -addStringsLoader(requestId, async (lang: string) => requestEn) -addStringsLoader(loveId, async (lang: string) => loveEn) -addStringsLoader(driveId, async (lang: string) => driveEn) -addStringsLoader(documentsId, async (lang: string) => documentsEn) -addStringsLoader(productsId, async (lang: string) => productsEn) -addStringsLoader(trainingId, async (lang: string) => trainingEn) +registerStringLoaders() /** * @public @@ -227,181 +47,13 @@ export function start ( accountsUrl: string } ): () => Promise { - addLocation(serverActivityId, () => import('@hcengineering/server-activity-resources')) - addLocation(serverAttachmentId, () => import('@hcengineering/server-attachment-resources')) - addLocation(serverCollaborationId, () => import('@hcengineering/server-collaboration-resources')) - addLocation(serverContactId, () => import('@hcengineering/server-contact-resources')) - addLocation(serverNotificationId, () => import('@hcengineering/server-notification-resources')) - addLocation(serverSettingId, () => import('@hcengineering/server-setting-resources')) - addLocation(serverChunterId, () => import('@hcengineering/server-chunter-resources')) - addLocation(serverInventoryId, () => import('@hcengineering/server-inventory-resources')) - addLocation(serverLeadId, () => import('@hcengineering/server-lead-resources')) - addLocation(serverRecruitId, () => import('@hcengineering/server-recruit-resources')) - addLocation(serverTaskId, () => import('@hcengineering/server-task-resources')) - addLocation(serverTrackerId, () => import('@hcengineering/server-tracker-resources')) - addLocation(serverTagsId, () => import('@hcengineering/server-tags-resources')) - addLocation(serverCalendarId, () => import('@hcengineering/server-calendar-resources')) - addLocation(serverGmailId, () => import('@hcengineering/server-gmail-resources')) - addLocation(serverTelegramId, () => import('@hcengineering/server-telegram-resources')) - addLocation(serverRequestId, () => import('@hcengineering/server-request-resources')) - addLocation(serverViewId, () => import('@hcengineering/server-view-resources')) - addLocation(serverHrId, () => import('@hcengineering/server-hr-resources')) - addLocation(serverLoveId, () => import('@hcengineering/server-love-resources')) - addLocation(serverGuestId, () => import('@hcengineering/server-guest-resources')) - addLocation(openAIId, () => Promise.resolve({ default: openAIPluginImpl })) - addLocation(serverDocumentId, () => import('@hcengineering/server-document-resources')) - addLocation(serverTimeId, () => import('@hcengineering/server-time-resources')) - addLocation(serverDriveId, () => import('@hcengineering/server-drive-resources')) - addLocation(serverDocumentsId, () => import('@hcengineering/server-controlled-documents-resources')) - addLocation(serverTrainingId, () => import('@hcengineering/server-training-resources')) - - const middlewares: MiddlewareCreator[] = [ - LookupMiddleware.create, - BlobLookupMiddleware.create, - ModifiedMiddleware.create, - PrivateMiddleware.create, - SpaceSecurityMiddleware.create, - SpacePermissionsMiddleware.create, - ConfigurationMiddleware.create, - QueryJoinMiddleware.create - ] - const metrics = getMetricsContext() + registerServerPlugins() + const externalStorage = buildStorageFromConfig(opt.storageConfig, dbUrl) - function createIndexStages ( - fullText: MeasureContext, - workspace: WorkspaceId, - branding: Branding | null, - adapter: FullTextAdapter, - storage: ServerStorage, - storageAdapter: StorageAdapter, - contentAdapter: ContentTextAdapter - ): FullTextPipelineStage[] { - // Allow 2 workspaces to be indexed in parallel - globalIndexer.allowParallel = opt.indexParallel - globalIndexer.processingSize = opt.indexProcessing - - const stages: FullTextPipelineStage[] = [] - - // Add regular stage to for indexable fields change tracking. - stages.push(new IndexedFieldStage(storage)) - - // Obtain text content from storage(like minio) and use content adapter to convert files to text content. - stages.push(new ContentRetrievalStage(storageAdapter, workspace, fullText.newChild('content', {}), contentAdapter)) - - // Obtain collaborative content - stages.push( - new CollaborativeContentRetrievalStage( - storageAdapter, - workspace, - fullText.newChild('collaborative', {}), - contentAdapter - ) - ) - - // // Add any => english language translation - // const retranslateStage = new LibRetranslateStage(fullText.newChild('retranslate', {}), workspace) - // retranslateStage.clearExcept = stages.map(it => it.stageId) - // for (const st of stages) { - // // Clear retranslation on content change. - // st.updateFields.push((doc, upd) => retranslateStage.update(doc, upd)) - // } - // stages.push(retranslateStage) - - // Summary stage - const summaryStage = new FullSummaryStage(storage) - - stages.push(summaryStage) - - // Push all content to elastic search - const pushStage = new FullTextPushStage(storage, adapter, workspace, branding) - stages.push(pushStage) - - // OpenAI prepare stage - const openAIStage = new OpenAIEmbeddingsStage(adapter, workspace) - // We depend on all available stages. - openAIStage.require = stages.map((it) => it.stageId) - - openAIStage.updateSummary(summaryStage) - - stages.push(openAIStage) - - return stages - } - const pipelineFactory: PipelineFactory = (ctx, workspace, upgrade, broadcast, branding) => { - const wsMetrics = metrics.newChild('🧲 session', {}) - const conf: DbConfiguration = { - domains: { - [DOMAIN_TX]: 'MongoTx', - [DOMAIN_TRANSIENT]: 'InMemory', - [DOMAIN_BLOB]: 'StorageData', - [DOMAIN_FULLTEXT_BLOB]: 'FullTextBlob', - [DOMAIN_MODEL]: 'Null' - }, - metrics: wsMetrics, - defaultAdapter: 'Mongo', - adapters: { - MongoTx: { - factory: createMongoTxAdapter, - url: dbUrl - }, - Mongo: { - factory: createMongoAdapter, - url: dbUrl - }, - Null: { - factory: createNullAdapter, - url: '' - }, - InMemory: { - factory: createInMemoryAdapter, - url: '' - }, - StorageData: { - factory: createStorageDataAdapter, - url: dbUrl - }, - FullTextBlob: { - factory: createElasticBackupDataAdapter, - url: opt.fullTextUrl - } - }, - fulltextAdapter: { - factory: createElasticAdapter, - url: opt.fullTextUrl, - stages: (adapter, storage, storageAdapter, contentAdapter) => - createIndexStages( - wsMetrics.newChild('stages', {}), - workspace, - branding, - adapter, - storage, - storageAdapter, - contentAdapter - ) - }, - serviceAdapters: {}, - contentAdapters: { - Rekoni: { - factory: createRekoniAdapter, - contentType: '*', - url: opt.rekoniUrl - }, - YDoc: { - factory: createYDocAdapter, - contentType: 'application/ydoc', - url: '' - } - }, - defaultContentAdapter: 'Rekoni', - storageFactory: externalStorage, - workspace - } - return createPipeline(ctx, conf, middlewares, upgrade, broadcast, branding) - } - + const pipelineFactory = createServerPipeline(metrics, dbUrl, { ...opt, externalStorage }) const sessionFactory = (token: Token, pipeline: Pipeline): Session => { if (token.extra?.mode === 'backup') { return new BackupClientSession(token, pipeline, opt.brandingMap) diff --git a/rush.json b/rush.json index d53bfac7f2..fae1230ab4 100644 --- a/rush.json +++ b/rush.json @@ -531,6 +531,11 @@ "projectFolder": "server/server-storage", "shouldPublish": false }, + { + "packageName": "@hcengineering/server-pipeline", + "projectFolder": "server/server-pipeline", + "shouldPublish": false + }, { "packageName": "@hcengineering/login", "projectFolder": "plugins/login", diff --git a/server/server-pipeline/.eslintrc.js b/server/server-pipeline/.eslintrc.js new file mode 100644 index 0000000000..ce90fb9646 --- /dev/null +++ b/server/server-pipeline/.eslintrc.js @@ -0,0 +1,7 @@ +module.exports = { + extends: ['./node_modules/@hcengineering/platform-rig/profiles/node/eslint.config.json'], + parserOptions: { + tsconfigRootDir: __dirname, + project: './tsconfig.json' + } +} diff --git a/server/server-pipeline/.npmignore b/server/server-pipeline/.npmignore new file mode 100644 index 0000000000..e3ec093c38 --- /dev/null +++ b/server/server-pipeline/.npmignore @@ -0,0 +1,4 @@ +* +!/lib/** +!CHANGELOG.md +/lib/**/__tests__/ diff --git a/server/server-pipeline/config/rig.json b/server/server-pipeline/config/rig.json new file mode 100644 index 0000000000..78cc5a1733 --- /dev/null +++ b/server/server-pipeline/config/rig.json @@ -0,0 +1,5 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json", + "rigPackageName": "@hcengineering/platform-rig", + "rigProfile": "node" +} diff --git a/server/server-pipeline/jest.config.js b/server/server-pipeline/jest.config.js new file mode 100644 index 0000000000..2cfd408b67 --- /dev/null +++ b/server/server-pipeline/jest.config.js @@ -0,0 +1,7 @@ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'], + roots: ["./src"], + coverageReporters: ["text-summary", "html"] +} diff --git a/server/server-pipeline/package.json b/server/server-pipeline/package.json new file mode 100644 index 0000000000..d4f7d72985 --- /dev/null +++ b/server/server-pipeline/package.json @@ -0,0 +1,171 @@ +{ + "name": "@hcengineering/server-pipeline", + "version": "0.6.0", + "main": "lib/index.js", + "svelte": "src/index.ts", + "types": "types/index.d.ts", + "author": "Anticrm Platform Contributors", + "template": "@hcengineering/node-package", + "license": "EPL-2.0", + "scripts": { + "build": "compile", + "build:watch": "compile", + "format": "format src", + "test": "jest --passWithNoTests --silent --forceExit", + "_phase:build": "compile transpile src", + "_phase:test": "jest --passWithNoTests --silent --forceExit", + "_phase:format": "format src", + "_phase:validate": "compile validate" + }, + "devDependencies": { + "cross-env": "~7.0.3", + "@hcengineering/platform-rig": "^0.6.0", + "@types/node": "~20.11.16", + "@typescript-eslint/eslint-plugin": "^6.11.0", + "eslint-plugin-import": "^2.26.0", + "eslint-plugin-promise": "^6.1.1", + "eslint-plugin-n": "^15.4.0", + "eslint": "^8.54.0", + "@types/ws": "^8.5.3", + "ts-node": "^10.8.0", + "esbuild": "^0.20.0", + "@typescript-eslint/parser": "^6.11.0", + "eslint-config-standard-with-typescript": "^40.0.0", + "prettier": "^3.1.0", + "typescript": "^5.3.3", + "jest": "^29.7.0", + "ts-jest": "^29.1.1", + "@types/jest": "^29.5.5" + }, + "dependencies": { + "@hcengineering/core": "^0.6.32", + "@hcengineering/platform": "^0.6.11", + "@hcengineering/server-core": "^0.6.1", + "@hcengineering/server-ws": "^0.6.11", + "@hcengineering/server-attachment": "^0.6.1", + "@hcengineering/server-attachment-resources": "^0.6.0", + "@hcengineering/server-collaboration": "^0.6.0", + "@hcengineering/server-collaboration-resources": "^0.6.0", + "@hcengineering/server": "^0.6.4", + "@hcengineering/server-storage": "^0.6.0", + "@hcengineering/mongo": "^0.6.1", + "@hcengineering/elastic": "^0.6.0", + "elastic-apm-node": "~3.26.0", + "@hcengineering/server-contact": "^0.6.1", + "@hcengineering/server-contact-resources": "^0.6.0", + "@hcengineering/server-notification": "^0.6.1", + "@hcengineering/server-notification-resources": "^0.6.0", + "@hcengineering/server-setting": "^0.6.0", + "@hcengineering/server-setting-resources": "^0.6.0", + "@hcengineering/server-chunter": "^0.6.0", + "@hcengineering/server-chunter-resources": "^0.6.0", + "@hcengineering/server-inventory": "^0.6.1", + "@hcengineering/server-inventory-resources": "^0.6.0", + "@hcengineering/server-lead": "^0.6.0", + "@hcengineering/server-lead-resources": "^0.6.0", + "@hcengineering/server-recruit": "^0.6.0", + "@hcengineering/server-recruit-resources": "^0.6.0", + "@hcengineering/server-task": "^0.6.0", + "@hcengineering/server-task-resources": "^0.6.0", + "@hcengineering/server-tracker": "^0.6.0", + "@hcengineering/server-tracker-resources": "^0.6.0", + "@hcengineering/server-tags": "^0.6.0", + "@hcengineering/server-tags-resources": "^0.6.0", + "@hcengineering/server-calendar": "^0.6.0", + "@hcengineering/server-calendar-resources": "^0.6.0", + "@hcengineering/server-gmail": "^0.6.0", + "@hcengineering/server-gmail-resources": "^0.6.0", + "@hcengineering/server-preference": "^0.6.0", + "@hcengineering/server-telegram": "^0.6.0", + "@hcengineering/server-telegram-resources": "^0.6.0", + "@hcengineering/server-time": "^0.6.0", + "@hcengineering/server-time-resources": "^0.6.0", + "@hcengineering/server-hr": "^0.6.0", + "@hcengineering/server-hr-resources": "^0.6.0", + "@hcengineering/server-request": "^0.6.0", + "@hcengineering/server-request-resources": "^0.6.0", + "@hcengineering/server-view": "^0.6.0", + "@hcengineering/server-view-resources": "^0.6.0", + "@hcengineering/server-activity": "^0.6.0", + "@hcengineering/server-activity-resources": "^0.6.0", + "@hcengineering/server-document": "^0.6.0", + "@hcengineering/server-document-resources": "^0.6.0", + "@hcengineering/server-guest": "^0.6.0", + "@hcengineering/server-guest-resources": "^0.6.0", + "@hcengineering/server-token": "^0.6.11", + "@hcengineering/server-controlled-documents": "^0.1.0", + "@hcengineering/server-controlled-documents-resources": "^0.1.0", + "@hcengineering/server-training": "^0.1.0", + "@hcengineering/server-training-resources": "^0.1.0", + "@hcengineering/middleware": "^0.6.0", + "@hcengineering/minio": "^0.6.0", + "@hcengineering/openai": "^0.6.0", + "@hcengineering/translate": "^0.6.0", + "@hcengineering/login-assets": "^0.6.0", + "@hcengineering/view-assets": "^0.6.11", + "@hcengineering/task-assets": "^0.6.19", + "@hcengineering/chunter-assets": "^0.6.18", + "@hcengineering/recruit-assets": "^0.6.23", + "@hcengineering/setting-assets": "^0.6.15", + "@hcengineering/support-assets": "^0.6.5", + "@hcengineering/contact-assets": "^0.6.13", + "@hcengineering/activity-assets": "^0.6.3", + "@hcengineering/telegram-assets": "^0.6.0", + "@hcengineering/workbench-assets": "^0.6.14", + "@hcengineering/attachment-assets": "^0.6.11", + "@hcengineering/lead-assets": "^0.6.0", + "@hcengineering/drive-assets": "^0.6.0", + "@hcengineering/gmail-assets": "^0.6.0", + "@hcengineering/inventory-assets": "^0.6.11", + "@hcengineering/templates-assets": "^0.6.11", + "@hcengineering/notification-assets": "^0.6.17", + "@hcengineering/preference-assets": "^0.6.0", + "@hcengineering/tags-assets": "^0.6.0", + "@hcengineering/calendar-assets": "^0.6.22", + "@hcengineering/tracker-assets": "^0.6.0", + "@hcengineering/board-assets": "^0.6.19", + "@hcengineering/hr-assets": "^0.6.19", + "@hcengineering/bitrix-assets": "^0.6.0", + "@hcengineering/request-assets": "^0.6.0", + "@hcengineering/document-assets": "^0.6.0", + "@hcengineering/controlled-documents-assets": "^0.1.0", + "@hcengineering/products-assets": "^0.1.0", + "@hcengineering/training-assets": "^0.1.0", + "@hcengineering/login": "^0.6.12", + "@hcengineering/view": "^0.6.13", + "@hcengineering/task": "^0.6.20", + "@hcengineering/chunter": "^0.6.20", + "@hcengineering/recruit": "^0.6.29", + "@hcengineering/setting": "^0.6.17", + "@hcengineering/contact": "^0.6.24", + "@hcengineering/activity": "^0.6.0", + "@hcengineering/telegram": "^0.6.21", + "@hcengineering/workbench": "^0.6.16", + "@hcengineering/attachment": "^0.6.14", + "@hcengineering/lead": "^0.6.0", + "@hcengineering/drive": "^0.6.0", + "@hcengineering/server-drive": "^0.6.0", + "@hcengineering/server-drive-resources": "^0.6.0", + "@hcengineering/gmail": "^0.6.22", + "@hcengineering/inventory": "^0.6.11", + "@hcengineering/templates": "^0.6.11", + "@hcengineering/notification": "^0.6.23", + "@hcengineering/preference": "^0.6.13", + "@hcengineering/tags": "^0.6.16", + "@hcengineering/calendar": "^0.6.24", + "@hcengineering/tracker": "^0.6.24", + "@hcengineering/board": "^0.6.19", + "@hcengineering/hr": "^0.6.19", + "@hcengineering/bitrix": "^0.6.52", + "@hcengineering/request": "^0.6.14", + "@hcengineering/support": "^0.6.5", + "@hcengineering/document": "^0.6.0", + "@hcengineering/love": "^0.6.0", + "@hcengineering/love-assets": "^0.6.0", + "@hcengineering/server-love": "^0.6.0", + "@hcengineering/server-love-resources": "^0.6.0", + "@hcengineering/controlled-documents": "^0.1.0", + "@hcengineering/products": "^0.1.0", + "@hcengineering/training": "^0.1.0" + } +} diff --git a/server/server-pipeline/src/index.ts b/server/server-pipeline/src/index.ts new file mode 100644 index 0000000000..21ef35d4c3 --- /dev/null +++ b/server/server-pipeline/src/index.ts @@ -0,0 +1,20 @@ +/* eslint-disable @typescript-eslint/unbound-method */ +// +// Copyright © 2022 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +export * from './indexing' +export * from './internationalization' +export * from './pipeline' +export * from './serverPlugins' diff --git a/server/server-pipeline/src/indexing.ts b/server/server-pipeline/src/indexing.ts new file mode 100644 index 0000000000..e79988b1d0 --- /dev/null +++ b/server/server-pipeline/src/indexing.ts @@ -0,0 +1,78 @@ +/* eslint-disable @typescript-eslint/unbound-method */ +import { type Branding, type MeasureContext, type WorkspaceId } from '@hcengineering/core' +import { OpenAIEmbeddingsStage } from '@hcengineering/openai' +import { CollaborativeContentRetrievalStage } from '@hcengineering/server-collaboration' +import { + ContentRetrievalStage, + FullSummaryStage, + FullTextPushStage, + IndexedFieldStage, + globalIndexer, + type ContentTextAdapter, + type FullTextAdapter, + type FullTextPipelineStage, + type ServerStorage, + type StorageAdapter +} from '@hcengineering/server-core' + +export function createIndexStages ( + fullText: MeasureContext, + workspace: WorkspaceId, + branding: Branding | null, + adapter: FullTextAdapter, + storage: ServerStorage, + storageAdapter: StorageAdapter, + contentAdapter: ContentTextAdapter, + indexParallel: number, + indexProcessing: number +): FullTextPipelineStage[] { + // Allow 2 workspaces to be indexed in parallel + globalIndexer.allowParallel = indexParallel + globalIndexer.processingSize = indexProcessing + + const stages: FullTextPipelineStage[] = [] + + // Add regular stage to for indexable fields change tracking. + stages.push(new IndexedFieldStage(storage)) + + // Obtain text content from storage(like minio) and use content adapter to convert files to text content. + stages.push(new ContentRetrievalStage(storageAdapter, workspace, fullText.newChild('content', {}), contentAdapter)) + + // Obtain collaborative content + stages.push( + new CollaborativeContentRetrievalStage( + storageAdapter, + workspace, + fullText.newChild('collaborative', {}), + contentAdapter + ) + ) + + // // Add any => english language translation + // const retranslateStage = new LibRetranslateStage(fullText.newChild('retranslate', {}), workspace) + // retranslateStage.clearExcept = stages.map(it => it.stageId) + // for (const st of stages) { + // // Clear retranslation on content change. + // st.updateFields.push((doc, upd) => retranslateStage.update(doc, upd)) + // } + // stages.push(retranslateStage) + // Summary stage + const summaryStage = new FullSummaryStage(storage) + + stages.push(summaryStage) + + // Push all content to elastic search + const pushStage = new FullTextPushStage(storage, adapter, workspace, branding) + stages.push(pushStage) + + // OpenAI prepare stage + const openAIStage = new OpenAIEmbeddingsStage(adapter, workspace) + // We depend on all available stages. + openAIStage.require = stages.map((it) => it.stageId) + + openAIStage.updateSummary(summaryStage) + + stages.push(openAIStage) + + return stages +} diff --git a/server/server-pipeline/src/internationalization.ts b/server/server-pipeline/src/internationalization.ts new file mode 100644 index 0000000000..0c7d010966 --- /dev/null +++ b/server/server-pipeline/src/internationalization.ts @@ -0,0 +1,103 @@ +import { coreId } from '@hcengineering/core' +import { addStringsLoader, platformId } from '@hcengineering/platform' +import { activityId } from '@hcengineering/activity' +import { attachmentId } from '@hcengineering/attachment' +import { bitrixId } from '@hcengineering/bitrix' +import { boardId } from '@hcengineering/board' +import { calendarId } from '@hcengineering/calendar' +import { chunterId } from '@hcengineering/chunter' +import { contactId } from '@hcengineering/contact' +import { documentsId } from '@hcengineering/controlled-documents' +import { documentId } from '@hcengineering/document' +import { driveId } from '@hcengineering/drive' +import { gmailId } from '@hcengineering/gmail' +import { hrId } from '@hcengineering/hr' +import { inventoryId } from '@hcengineering/inventory' +import { leadId } from '@hcengineering/lead' +import { loginId } from '@hcengineering/login' +import { loveId } from '@hcengineering/love' +import { notificationId } from '@hcengineering/notification' +import { preferenceId } from '@hcengineering/preference' +import { productsId } from '@hcengineering/products' +import { recruitId } from '@hcengineering/recruit' +import { requestId } from '@hcengineering/request' +import { settingId } from '@hcengineering/setting' +import { supportId } from '@hcengineering/support' +import { tagsId } from '@hcengineering/tags' +import { taskId } from '@hcengineering/task' +import { telegramId } from '@hcengineering/telegram' +import { templatesId } from '@hcengineering/templates' +import { trackerId } from '@hcengineering/tracker' +import { trainingId } from '@hcengineering/training' +import { viewId } from '@hcengineering/view' +import { workbenchId } from '@hcengineering/workbench' +import coreEng from '@hcengineering/core/lang/en.json' +import loginEng from '@hcengineering/login-assets/lang/en.json' +import platformEng from '@hcengineering/platform/lang/en.json' +import activityEn from '@hcengineering/activity-assets/lang/en.json' +import attachmentEn from '@hcengineering/attachment-assets/lang/en.json' +import bitrixEn from '@hcengineering/bitrix-assets/lang/en.json' +import boardEn from '@hcengineering/board-assets/lang/en.json' +import calendarEn from '@hcengineering/calendar-assets/lang/en.json' +import chunterEn from '@hcengineering/chunter-assets/lang/en.json' +import contactEn from '@hcengineering/contact-assets/lang/en.json' +import documentsEn from '@hcengineering/controlled-documents-assets/lang/en.json' +import documentEn from '@hcengineering/document-assets/lang/en.json' +import driveEn from '@hcengineering/drive-assets/lang/en.json' +import gmailEn from '@hcengineering/gmail-assets/lang/en.json' +import hrEn from '@hcengineering/hr-assets/lang/en.json' +import inventoryEn from '@hcengineering/inventory-assets/lang/en.json' +import leadEn from '@hcengineering/lead-assets/lang/en.json' +import loveEn from '@hcengineering/love-assets/lang/en.json' +import notificationEn from '@hcengineering/notification-assets/lang/en.json' +import preferenceEn from '@hcengineering/preference-assets/lang/en.json' +import productsEn from '@hcengineering/products-assets/lang/en.json' +import recruitEn from '@hcengineering/recruit-assets/lang/en.json' +import requestEn from '@hcengineering/request-assets/lang/en.json' +import settingEn from '@hcengineering/setting-assets/lang/en.json' +import supportEn from '@hcengineering/support-assets/lang/en.json' +import tagsEn from '@hcengineering/tags-assets/lang/en.json' +import taskEn from '@hcengineering/task-assets/lang/en.json' +import telegramEn from '@hcengineering/telegram-assets/lang/en.json' +import templatesEn from '@hcengineering/templates-assets/lang/en.json' +import trackerEn from '@hcengineering/tracker-assets/lang/en.json' +import trainingEn from '@hcengineering/training-assets/lang/en.json' +import viewEn from '@hcengineering/view-assets/lang/en.json' +import workbenchEn from '@hcengineering/workbench-assets/lang/en.json' + +export function registerStringLoaders (): void { + addStringsLoader(coreId, async (lang: string) => coreEng) + addStringsLoader(loginId, async (lang: string) => loginEng) + addStringsLoader(platformId, async (lang: string) => platformEng) + + addStringsLoader(taskId, async (lang: string) => taskEn) + addStringsLoader(viewId, async (lang: string) => viewEn) + addStringsLoader(chunterId, async (lang: string) => chunterEn) + addStringsLoader(attachmentId, async (lang: string) => attachmentEn) + addStringsLoader(contactId, async (lang: string) => contactEn) + addStringsLoader(recruitId, async (lang: string) => recruitEn) + addStringsLoader(activityId, async (lang: string) => activityEn) + addStringsLoader(settingId, async (lang: string) => settingEn) + addStringsLoader(supportId, async (lang: string) => supportEn) + addStringsLoader(telegramId, async (lang: string) => telegramEn) + addStringsLoader(leadId, async (lang: string) => leadEn) + addStringsLoader(gmailId, async (lang: string) => gmailEn) + addStringsLoader(workbenchId, async (lang: string) => workbenchEn) + addStringsLoader(inventoryId, async (lang: string) => inventoryEn) + addStringsLoader(templatesId, async (lang: string) => templatesEn) + addStringsLoader(notificationId, async (lang: string) => notificationEn) + addStringsLoader(tagsId, async (lang: string) => tagsEn) + addStringsLoader(calendarId, async (lang: string) => calendarEn) + addStringsLoader(trackerId, async (lang: string) => trackerEn) + addStringsLoader(boardId, async (lang: string) => boardEn) + addStringsLoader(preferenceId, async (lang: string) => preferenceEn) + addStringsLoader(hrId, async (lang: string) => hrEn) + addStringsLoader(documentId, async (lang: string) => documentEn) + addStringsLoader(bitrixId, async (lang: string) => bitrixEn) + addStringsLoader(requestId, async (lang: string) => requestEn) + addStringsLoader(loveId, async (lang: string) => loveEn) + addStringsLoader(driveId, async (lang: string) => driveEn) + addStringsLoader(documentsId, async (lang: string) => documentsEn) + addStringsLoader(productsId, async (lang: string) => productsEn) + addStringsLoader(trainingId, async (lang: string) => trainingEn) +} diff --git a/server/server-pipeline/src/pipeline.ts b/server/server-pipeline/src/pipeline.ts new file mode 100644 index 0000000000..da5933f324 --- /dev/null +++ b/server/server-pipeline/src/pipeline.ts @@ -0,0 +1,153 @@ +/* eslint-disable @typescript-eslint/unbound-method */ +import { + DOMAIN_BENCHMARK, + DOMAIN_BLOB, + DOMAIN_FULLTEXT_BLOB, + DOMAIN_MODEL, + DOMAIN_TRANSIENT, + DOMAIN_TX, + type BrandingMap, + type MeasureContext +} from '@hcengineering/core' +import { createElasticAdapter, createElasticBackupDataAdapter } from '@hcengineering/elastic' +import { + BlobLookupMiddleware, + ConfigurationMiddleware, + LookupMiddleware, + ModifiedMiddleware, + PrivateMiddleware, + QueryJoinMiddleware, + SpacePermissionsMiddleware, + SpaceSecurityMiddleware +} from '@hcengineering/middleware' +import { createMongoAdapter, createMongoTxAdapter } from '@hcengineering/mongo' +import { + createNullAdapter, + createRekoniAdapter, + createStorageDataAdapter, + createYDocAdapter +} from '@hcengineering/server' +import { + createBenchmarkAdapter, + createInMemoryAdapter, + createPipeline, + type DbConfiguration, + type MiddlewareCreator, + type StorageAdapter, + type StorageConfiguration +} from '@hcengineering/server-core' +import { type PipelineFactory, type ServerFactory } from '@hcengineering/server-ws' +import { createIndexStages } from './indexing' + +/** + * @public + */ + +export function createServerPipeline ( + metrics: MeasureContext, + dbUrl: string, + opt: { + fullTextUrl: string + storageConfig: StorageConfiguration + rekoniUrl: string + port: number + productId: string + brandingMap: BrandingMap + serverFactory: ServerFactory + + indexProcessing: number // 1000 + indexParallel: number // 2 + + externalStorage: StorageAdapter + }, + extensions?: Partial +): PipelineFactory { + const middlewares: MiddlewareCreator[] = [ + LookupMiddleware.create, + BlobLookupMiddleware.create, + ModifiedMiddleware.create, + PrivateMiddleware.create, + SpaceSecurityMiddleware.create, + SpacePermissionsMiddleware.create, + ConfigurationMiddleware.create, + QueryJoinMiddleware.create + ] + return (ctx, workspace, upgrade, broadcast, branding) => { + const wsMetrics = metrics.newChild('🧲 session', {}) + const conf: DbConfiguration = { + domains: { + [DOMAIN_TX]: 'MongoTx', + [DOMAIN_TRANSIENT]: 'InMemory', + [DOMAIN_BLOB]: 'StorageData', + [DOMAIN_FULLTEXT_BLOB]: 'FullTextBlob', + [DOMAIN_MODEL]: 'Null', + [DOMAIN_BENCHMARK]: 'Benchmark' + }, + metrics: wsMetrics, + defaultAdapter: 'Mongo', + adapters: { + MongoTx: { + factory: createMongoTxAdapter, + url: dbUrl + }, + Mongo: { + factory: createMongoAdapter, + url: dbUrl + }, + Null: { + factory: createNullAdapter, + url: '' + }, + InMemory: { + factory: createInMemoryAdapter, + url: '' + }, + StorageData: { + factory: createStorageDataAdapter, + url: dbUrl + }, + FullTextBlob: { + factory: createElasticBackupDataAdapter, + url: opt.fullTextUrl + }, + Benchmark: { + factory: createBenchmarkAdapter, + url: '' + } + }, + fulltextAdapter: { + factory: createElasticAdapter, + url: opt.fullTextUrl, + stages: (adapter, storage, storageAdapter, contentAdapter) => + createIndexStages( + wsMetrics.newChild('stages', {}), + workspace, + branding, + adapter, + storage, + storageAdapter, + contentAdapter, + opt.indexParallel, + opt.indexProcessing + ) + }, + serviceAdapters: {}, + contentAdapters: { + Rekoni: { + factory: createRekoniAdapter, + contentType: '*', + url: opt.rekoniUrl + }, + YDoc: { + factory: createYDocAdapter, + contentType: 'application/ydoc', + url: '' + } + }, + defaultContentAdapter: 'Rekoni', + storageFactory: opt.externalStorage, + workspace + } + return createPipeline(ctx, conf, middlewares, upgrade, broadcast, branding) + } +} diff --git a/server/server-pipeline/src/serverPlugins.ts b/server/server-pipeline/src/serverPlugins.ts new file mode 100644 index 0000000000..f076aa1457 --- /dev/null +++ b/server/server-pipeline/src/serverPlugins.ts @@ -0,0 +1,58 @@ +import { openAIId, openAIPluginImpl } from '@hcengineering/openai' +import { addLocation } from '@hcengineering/platform' +import { serverActivityId } from '@hcengineering/server-activity' +import { serverAttachmentId } from '@hcengineering/server-attachment' +import { serverCalendarId } from '@hcengineering/server-calendar' +import { serverChunterId } from '@hcengineering/server-chunter' +import { serverCollaborationId } from '@hcengineering/server-collaboration' +import { serverContactId } from '@hcengineering/server-contact' +import { serverDocumentsId } from '@hcengineering/server-controlled-documents' +import { serverDocumentId } from '@hcengineering/server-document' +import { serverDriveId } from '@hcengineering/server-drive' +import { serverGmailId } from '@hcengineering/server-gmail' +import { serverGuestId } from '@hcengineering/server-guest' +import { serverHrId } from '@hcengineering/server-hr' +import { serverInventoryId } from '@hcengineering/server-inventory' +import { serverLeadId } from '@hcengineering/server-lead' +import { serverLoveId } from '@hcengineering/server-love' +import { serverNotificationId } from '@hcengineering/server-notification' +import { serverRecruitId } from '@hcengineering/server-recruit' +import { serverRequestId } from '@hcengineering/server-request' +import { serverSettingId } from '@hcengineering/server-setting' +import { serverTagsId } from '@hcengineering/server-tags' +import { serverTaskId } from '@hcengineering/server-task' +import { serverTelegramId } from '@hcengineering/server-telegram' +import { serverTimeId } from '@hcengineering/server-time' +import { serverTrackerId } from '@hcengineering/server-tracker' +import { serverTrainingId } from '@hcengineering/server-training' +import { serverViewId } from '@hcengineering/server-view' + +export function registerServerPlugins (): void { + addLocation(serverActivityId, () => import('@hcengineering/server-activity-resources')) + addLocation(serverAttachmentId, () => import('@hcengineering/server-attachment-resources')) + addLocation(serverCollaborationId, () => import('@hcengineering/server-collaboration-resources')) + addLocation(serverContactId, () => import('@hcengineering/server-contact-resources')) + addLocation(serverNotificationId, () => import('@hcengineering/server-notification-resources')) + addLocation(serverSettingId, () => import('@hcengineering/server-setting-resources')) + addLocation(serverChunterId, () => import('@hcengineering/server-chunter-resources')) + addLocation(serverInventoryId, () => import('@hcengineering/server-inventory-resources')) + addLocation(serverLeadId, () => import('@hcengineering/server-lead-resources')) + addLocation(serverRecruitId, () => import('@hcengineering/server-recruit-resources')) + addLocation(serverTaskId, () => import('@hcengineering/server-task-resources')) + addLocation(serverTrackerId, () => import('@hcengineering/server-tracker-resources')) + addLocation(serverTagsId, () => import('@hcengineering/server-tags-resources')) + addLocation(serverCalendarId, () => import('@hcengineering/server-calendar-resources')) + addLocation(serverGmailId, () => import('@hcengineering/server-gmail-resources')) + addLocation(serverTelegramId, () => import('@hcengineering/server-telegram-resources')) + addLocation(serverRequestId, () => import('@hcengineering/server-request-resources')) + addLocation(serverViewId, () => import('@hcengineering/server-view-resources')) + addLocation(serverHrId, () => import('@hcengineering/server-hr-resources')) + addLocation(serverLoveId, () => import('@hcengineering/server-love-resources')) + addLocation(serverGuestId, () => import('@hcengineering/server-guest-resources')) + addLocation(openAIId, () => Promise.resolve({ default: openAIPluginImpl })) + addLocation(serverDocumentId, () => import('@hcengineering/server-document-resources')) + addLocation(serverTimeId, () => import('@hcengineering/server-time-resources')) + addLocation(serverDriveId, () => import('@hcengineering/server-drive-resources')) + addLocation(serverDocumentsId, () => import('@hcengineering/server-controlled-documents-resources')) + addLocation(serverTrainingId, () => import('@hcengineering/server-training-resources')) +} diff --git a/server/server-pipeline/tsconfig.json b/server/server-pipeline/tsconfig.json new file mode 100644 index 0000000000..f017cc597c --- /dev/null +++ b/server/server-pipeline/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "./node_modules/@hcengineering/platform-rig/profiles/node/tsconfig.json", + + "compilerOptions": { + "rootDir": "./src", + "outDir": "./lib", + "declarationDir": "./types", + "tsBuildInfoFile": ".build/build.tsbuildinfo" + } +} \ No newline at end of file diff --git a/tests/prepare.sh b/tests/prepare.sh index ffb7cefbaa..ded6e1cece 100755 --- a/tests/prepare.sh +++ b/tests/prepare.sh @@ -20,6 +20,8 @@ fi ./tool.sh create-account user2 -f Kainin -l Dirak -p 1234 ./tool.sh assign-workspace user1 sanity-ws ./tool.sh assign-workspace user2 sanity-ws +./tool.sh set-user-role user1 sanity-ws OWNER +./tool.sh set-user-role user2 sanity-ws OWNER # Make user the workspace maintainer ./tool.sh confirm-email user1 ./tool.sh confirm-email user2 diff --git a/tests/restore-local.sh b/tests/restore-local.sh index 0b2a033168..f61707192a 100755 --- a/tests/restore-local.sh +++ b/tests/restore-local.sh @@ -8,13 +8,15 @@ export ELASTIC_URL=http://localhost:9200 export SERVER_SECRET=secret # Restore workspace contents in mongo/elastic -node ../dev/tool/bundle/bundle.js backup-restore ./sanity-ws sanity-ws +./tool-local.sh backup-restore ./sanity-ws sanity-ws -node ../dev/tool/bundle/bundle.js upgrade-workspace sanity-ws +./tool-local.sh upgrade-workspace sanity-ws # Re-assign user to workspace. -node ../dev/tool/bundle/bundle.js assign-workspace user1 sanity-ws -node ../dev/tool/bundle/bundle.js assign-workspace user2 sanity-ws +./tool-local.sh assign-workspace user1 sanity-ws +./tool-local.sh assign-workspace user2 sanity-ws +./tool-local.sh set-user-role user1 sanity-ws OWNER +./tool-local.sh set-user-role user2 sanity-ws OWNER -node ../dev/tool/bundle/bundle.js configure sanity-ws --enable=* -node ../dev/tool/bundle/bundle.js configure sanity-ws --list \ No newline at end of file +./tool-local.sh configure sanity-ws --enable=* +./tool-local.sh configure sanity-ws --list \ No newline at end of file diff --git a/tests/restore-workspace.sh b/tests/restore-workspace.sh index 812b2e0e67..4eec408b3c 100755 --- a/tests/restore-workspace.sh +++ b/tests/restore-workspace.sh @@ -8,6 +8,8 @@ # Re-assign user to workspace. ./tool.sh assign-workspace user1 sanity-ws ./tool.sh assign-workspace user2 sanity-ws +./tool.sh set-user-role user1 sanity-ws OWNER +./tool.sh set-user-role user2 sanity-ws OWNER ./tool.sh configure sanity-ws --enable=* ./tool.sh configure sanity-ws --list