UBERF-8163: Split ws into server + desktop backup support (#6630)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

.vscode/launch.json

@@ -67,7 +67,7 @@
         "SERVER_SECRET": "secret",
         "ENABLE_CONSOLE": "true",
         "COLLABORATOR_URL": "ws://localhost:3078",
-        "REKONI_URL": "http://localhost:4000",
+        "REKONI_URL": "http://localhost:4004",
         "FRONT_URL": "http://localhost:8080",
         "ACCOUNTS_URL": "http://localhost:3000",
         // "SERVER_PROVIDER":"uweb"
@@ -221,13 +221,14 @@
       "name": "Debug backup tool",
       "type": "node",
       "request": "launch",
-      "args": ["src/index.ts", "backup", "../../../dump/platform2", "platform"],
+      "args": ["src/__start.ts", "backup", "../temp/backup-test", "platform", "--force"],
       "env": {
         "MINIO_ACCESS_KEY": "minioadmin",
         "MINIO_SECRET_KEY": "minioadmin",
         "MINIO_ENDPOINT": "localhost",
+        "SERVER_SECRET": "secret",
         "MONGO_URL": "mongodb://localhost:27017",
-        "ACCOUNTS_URL": "localhost:3000",
+        "ACCOUNTS_URL": "http://localhost:3000",
         "TELEGRAM_DATABASE": "telegram-service",
         "ELASTIC_URL": "http://localhost:9200"
       },
@@ -254,7 +255,7 @@
         "ACCOUNTS_URL": "http://localhost:3000",
         "TELEGRAM_DATABASE": "telegram-service",
         "ELASTIC_URL": "http://localhost:9200",
-        "REKONI_URL": "http://localhost:4000"
+        "REKONI_URL": "http://localhost:4004"
       },
       "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
       "sourceMaps": true,

common/config/rush/command-line.json

@@ -303,7 +303,15 @@
       "description": "Format and autofix linting issues in changed projects",
       "safeForSimultaneousRushProcesses": true,
       "shellCommand": "./common/scripts/each-diff.sh rushx format --force"
-    }
+    },
+    {
+      "commandKind": "global",
+      "name": "desktop",
+      "summary": "Build desktop build",
+      "description": "use to build all docker containers required for platform",
+      "safeForSimultaneousRushProcesses": true,
+      "shellCommand": "rush package -p 20 --to desktop && cd ./desktop-package && rushx dist"
+    },        
   ],
 
   /**

desktop-package/package.json

@@ -21,9 +21,9 @@
     "node-loader": "~2.0.0",
     "cross-env": "~7.0.3",
     "typescript": "^5.3.3",
-    "electron": "^31.3.0",
+    "electron": "^32.1.1",
     "@types/node": "~20.11.16",
-    "electron-builder": "^24.13.3",
+    "electron-builder": "^25.0.5",
     "@electron/notarize": "^2.3.2"
   },
   "dependencies": {

desktop/package.json

@@ -47,7 +47,7 @@
     "typescript": "^5.3.3",
     "ts-node": "^10.8.0",
     "ts-node-dev": "^2.0.0",
-    "electron": "^31.3.0",
+    "electron": "^32.1.1",
     "@types/node": "~20.11.16",
     "copy-webpack-plugin": "^11.0.0",
     "@typescript-eslint/eslint-plugin": "^6.11.0",
@@ -60,7 +60,8 @@
     "prettier": "^3.1.0",
     "esbuild": "^0.20.0",
     "esbuild-loader": "^4.0.3",
-    "svelte-preprocess": "^5.1.3"
+    "svelte-preprocess": "^5.1.3",
+    "@types/ws": "^8.5.11"
   },
   "dependencies": {
     "@hcengineering/platform": "^0.6.11",
@@ -205,14 +206,16 @@
     "@hcengineering/ai-bot-resources": "^0.6.0",
     "electron-squirrel-startup": "~1.0.0",
     "dotenv": "~16.0.0",
-    "electron-context-menu": "^4.0.1",
+    "electron-context-menu": "^4.0.4",
     "electron-windows-badge": "^1.1.0",
     "svelte": "^4.2.12",
     "commander": "^8.1.0",
     "electron-store": "^8.2.0",
     "electron-log": "^5.1.7",
-    "electron-updater": "^6.2.1",
-    "livekit-client": "^2.0.10"
+    "electron-updater": "^6.3.4",
+    "livekit-client": "^2.0.10",
+    "@hcengineering/server-backup": "^0.6.0",
+    "ws": "^8.18.0"
   },
   "productName": "Huly Desktop",
   "description": "Huly Desktop experience",

desktop/src/main/backup.ts

@@ -0,0 +1,106 @@
+import client, { clientId } from '@hcengineering/client'
+import { getWorkspaceId, MeasureMetricsContext, type BackupClient, type Client } from '@hcengineering/core'
+import { addLocation, getResource, setMetadata } from '@hcengineering/platform'
+import WebSocket from 'ws'
+
+import {
+  backup,
+  createFileBackupStorage
+} from '@hcengineering/server-backup'
+import { dialog, type BrowserWindow } from 'electron'
+
+let runningBackup: {
+  notify: (command: string, ...args: any[]) => void
+} | undefined
+
+export async function createClient (
+  transactorUrl: string,
+  token: string
+): Promise<Client & BackupClient> {
+  // We need to override default factory with 'ws' one.
+  // eslint-disable-next-line
+  setMetadata(client.metadata.UseBinaryProtocol, true)
+  setMetadata(client.metadata.UseProtocolCompression, true)
+  setMetadata(client.metadata.ConnectionTimeout, 0)
+  setMetadata(client.metadata.ClientSocketFactory, (url) => {
+    return new WebSocket(url) as any
+  })
+  addLocation(clientId, () => import('@hcengineering/client-resources'))
+
+  const clientFactory = await getResource(client.function.GetClient)
+  return (await clientFactory(token, transactorUrl)) as unknown as Client & BackupClient
+}
+
+async function doBackup (dirName: string, token: string, endpoint: string, workspace: string, notify: (command: string, ...args: any[]) => void, backupHugeFiles: boolean): Promise<void> {
+  notify('backup', 0)
+  const ctx = new MeasureMetricsContext('backup', {})
+
+  const storage = await createFileBackupStorage(dirName)
+  const wsid = getWorkspaceId(workspace)
+  const client = await createClient(endpoint, token)
+  try {
+    ctx.info('do backup', { workspace, endpoint })
+    await backup(ctx, endpoint, wsid, storage, {
+      force: true,
+      recheck: false,
+      skipDomains: [],
+      timeout: 0,
+      connectTimeout: 60 * 1000,
+      blobDownloadLimit: backupHugeFiles ? 10240 : 50,
+      token,
+      skipBlobContentTypes: [],
+      isCanceled: (): boolean => {
+        return runningBackup == null
+      },
+      progress: (value: number): void => {
+        notify('backup', value)
+      },
+      connection: client
+    })
+  } finally {
+    await client.close()
+  }
+
+  notify('backup', 100)
+}
+
+export function startBackup (window: BrowserWindow, token: string, endpoint: string, workspace: string, cmd: (command: string, ...args: any[]) => void): void {
+  void dialog
+    .showOpenDialog(window, {
+      properties: ['openDirectory'],
+      buttonLabel: 'Select a backup folder',
+      message: 'Select a folder for Huly incremental backup.'
+    })
+    .then((response) => {
+      if (!response.canceled && response.filePaths.length > 0) {
+        void dialog
+          .showMessageBox({
+            type: 'info',
+            buttons: ['Backup....', 'Cancel'],
+            defaultId: 0,
+            textWidth: 500,
+            title: 'Huly desktop need a confirmation for a backup process to be started...',
+            message: `Huly Desktop need a confirmation for a backup process to be started....\n\n
+                      An incremental backup will be performed to folder:\n ${response.filePaths[0]}\n
+                      Backup could be canceled and resumed laterly into same location.`,
+            checkboxLabel: 'Include files >= 50mb',
+            checkboxChecked: false
+          })
+          .then((backupKind) => {
+            if (backupKind.response === 1) {
+              return
+            }
+            runningBackup = {
+              notify: cmd
+            }
+            void doBackup(response.filePaths[0], token, endpoint, workspace, cmd, backupKind.checkboxChecked)
+          })
+      }
+    })
+}
+export function cancelBackup (): void {
+  if (runningBackup != null) {
+    runningBackup.notify('backup-cancel')
+  }
+  runningBackup = undefined
+}

desktop/src/main/menu.ts

@@ -13,11 +13,11 @@
 // limitations under the License.
 //
 
-import { Menu, MenuItemConstructorOptions } from 'electron'
+import { Menu, MenuItemConstructorOptions, type BrowserWindow } from 'electron'
 
 const isMac = process.platform === 'darwin'
 
-export const addMenus = (sendCommand: (cmd: string, ...args: any[]) => void): void => {
+export const addMenus = (getWindow: () => BrowserWindow, sendCommand: (cmd: string, ...args: any[]) => void): void => {
   const template: MenuItemConstructorOptions[] = [
     {
       label: 'File',
@@ -25,11 +25,19 @@ export const addMenus = (sendCommand: (cmd: string, ...args: any[]) => void): vo
         {
           label: 'Settings',
           accelerator: 'Meta+,',
-          click: () => sendCommand('open-settings')
+          click: () => { sendCommand('open-settings') }
+        },
+        {
+          label: 'Select workspace',
+          click: () => { sendCommand('select-workspace') }
+        },
+        {
+          label: 'Backup...',
+          click: () => { sendCommand('start-backup') }
         },
         {
           label: 'Logout',
-          click: () => sendCommand('logout')
+          click: () => { sendCommand('logout') }
         },
         { role: isMac ? 'close' : 'quit' }
       ]

desktop/src/main/start.ts

@@ -13,20 +13,21 @@
 // limitations under the License.
 //
 
-import { BrowserWindow, CookiesSetDetails, Notification, app, ipcMain, nativeImage, shell, dialog, systemPreferences, desktopCapturer } from 'electron'
-import contextMenu from 'electron-context-menu'
-import WinBadge from 'electron-windows-badge'
-import Store from 'electron-store'
-import * as path from 'path'
 import { config as dotenvConfig } from 'dotenv'
+import { BrowserWindow, CookiesSetDetails, Notification, app, desktopCapturer, dialog, ipcMain, nativeImage, shell, systemPreferences } from 'electron'
+import contextMenu from 'electron-context-menu'
 import log from 'electron-log'
+import Store from 'electron-store'
 import { ProgressInfo, UpdateInfo } from 'electron-updater'
+import WinBadge from 'electron-windows-badge'
+import * as path from 'path'
 
-import autoUpdater from './updater'
-import { NotificationParams, Config } from '../ui/types'
-import { addMenus } from './menu'
+import { Config, NotificationParams } from '../ui/types'
 import { getOptions } from './args'
+import { cancelBackup, startBackup } from './backup'
+import { addMenus } from './menu'
 import { addPermissionHandlers } from './permissions'
+import autoUpdater from './updater'
 
 let mainWindow: BrowserWindow | undefined
 let winBadge: any
@@ -115,7 +116,6 @@ function hookOpenWindow (window: BrowserWindow): void {
             ]
           }
         })
-        // await childWindow.webContents.openDevTools()
         await childWindow.loadFile(path.join('dist', 'ui', 'index.html'))
         hookOpenWindow(childWindow)
       })()
@@ -165,8 +165,8 @@ const createWindow = async (): Promise<void> => {
   }
 }
 
-addMenus((cmd: string, ...args: any[]) => {
-  mainWindow?.webContents.send(cmd, args)
+addMenus(() => mainWindow as BrowserWindow, (cmd: string, ...args: any[]) => {
+  mainWindow?.webContents.send(cmd, ...args)
 })
 
 contextMenu({
@@ -227,7 +227,7 @@ ipcMain.handle('get-main-config', (event, path) => {
     FRONT_URL,
     INITIAL_URL: process.env.INITIAL_URL ?? '',
     MODEL_VERSION: process.env.MODEL_VERSION ?? '',
-    VERSION: process.env.VERSION ?? '',
+    VERSION: process.env.VERSION ?? ''
   }
   return cfg
 })
@@ -396,3 +396,16 @@ autoUpdater.on('update-downloaded', (info) => {
 
   autoUpdater.quitAndInstall()
 })
+
+ipcMain.on('start-backup', (event, token, endpoint, workspace) => {
+  console.log('start backup', token, endpoint, workspace)
+  if (mainWindow != null) {
+    startBackup(mainWindow, token, endpoint, workspace  , (cmd: string, ...args: any[]) => {
+      mainWindow?.webContents.send(cmd, ...args)
+    })
+  }
+})
+
+ipcMain.on('cancel-backup', (event) => {
+  cancelBackup()
+})

desktop/src/main/updater.ts

@@ -24,7 +24,7 @@ autoUpdater.on('update-available', (info) => {
 autoUpdater.on('update-not-available', (info) => {
   log.info('Update not available.')
 })
-autoUpdater.on('error', (err) => {
+autoUpdater.on('error', (err: any) => {
   log.info('Error in auto-updater. ' + err)
 })
 autoUpdater.on('download-progress', (progressObj) => {

desktop/src/ui/index.ejs

@@ -1,7 +1,7 @@
 <!doctype html>
 <html>
 <script>
-  window.embeddedPlatform=true
+  window.desktopPlatform=true
 </script>
 
 <head>

desktop/src/ui/index.ts

@@ -1,6 +1,6 @@
 import login, { loginId } from '@hcengineering/login'
-import { setMetadata, getMetadata } from '@hcengineering/platform'
-import presentation, { closeClient, setDownloadProgress } from '@hcengineering/presentation'
+import { getEmbeddedLabel, getMetadata, setMetadata } from '@hcengineering/platform'
+import presentation, { closeClient, MessageBox, setDownloadProgress } from '@hcengineering/presentation'
 import { settingId } from '@hcengineering/setting'
 import {
   closePanel,
@@ -11,15 +11,20 @@ import {
   getCurrentResolvedLocation,
   navigate,
   parseLocation,
-  setMetadataLocalStorage
+  pushRootBarProgressComponent,
+  removeRootBarComponent,
+  setMetadataLocalStorage,
+  showPopup
 } from '@hcengineering/ui'
 
-import { workbenchId } from '@hcengineering/workbench'
 import { notificationId } from '@hcengineering/notification'
+import { workbenchId } from '@hcengineering/workbench'
 
+import { isOwnerOrMaintainer } from '@hcengineering/core'
 import { configurePlatform } from './platform'
-import { IPCMainExposed } from './types'
 import { defineScreenShare } from './screenShare'
+import { IPCMainExposed } from './types'
+import settings from '@hcengineering/setting'
 
 defineScreenShare()
 
@@ -41,6 +46,18 @@ window.addEventListener('DOMContentLoaded', () => {
     navigate(loc)
   })
 
+  ipcMain.on('select-workspace', () => {
+    closePopup()
+    closePanel()
+    const loc = getCurrentResolvedLocation()
+    loc.fragment = undefined
+    loc.query = undefined
+    loc.path[0] = loginId
+    loc.path[1] = 'selectWorkspace'
+    loc.path.length = 2
+    navigate(loc)
+  })
+
   ipcMain.on('logout', () => {
     const tokens = fetchMetadataLocalStorage(login.metadata.LoginTokens)
     if (tokens !== null) {
@@ -74,4 +91,36 @@ window.addEventListener('DOMContentLoaded', () => {
   ipcMain.handleUpdateDownloadProgress((progress) => {
     setDownloadProgress(progress)
   })
+
+  ipcMain.on('start-backup', () => {
+    // We need to obtain current token and endpoint and trigger backup
+    const token = getMetadata(presentation.metadata.Token)
+    const endpoint = getMetadata(presentation.metadata.Endpoint)
+    const workspace = getMetadata(presentation.metadata.WorkspaceId)
+    if (isOwnerOrMaintainer()) {
+      if (token != null && endpoint != null && workspace != null) {
+        ipcMain.startBackup(token, endpoint, workspace)
+      }
+    } else {
+      showPopup(MessageBox, {
+        label: settings.string.OwnerOrMainteinerRequired
+      })
+    }
+  })
+
+  ipcMain.on('backup', (evt: any, ...args: any) => {
+    pushRootBarProgressComponent('backup',
+      getEmbeddedLabel('Backup'),
+      () => { return args[0] },
+      () => {
+        ipcMain.cancelBackup()
+      },
+      undefined,
+      undefined,
+      50
+    )
+  })
+  ipcMain.on('backup-cancel', () => {
+    removeRootBarComponent('backup')
+  })
 })

desktop/src/ui/preload.ts

@@ -1,7 +1,7 @@
 // preload.js
 
-import { BrandingMap, Config, IPCMainExposed, NotificationParams } from './types'
 import { contextBridge, ipcRenderer } from 'electron'
+import { BrandingMap, Config, IPCMainExposed, NotificationParams } from './types'
 
 /**
  * @public
@@ -135,6 +135,8 @@ const expose: IPCMainExposed = {
   },
 
   getScreenAccess: () => ipcRenderer.invoke('get-screen-access'),
-  getScreenSources: () => ipcRenderer.invoke('get-screen-sources')
+  getScreenSources: () => ipcRenderer.invoke('get-screen-sources'),
+  cancelBackup: () => { ipcRenderer.send('cancel-backup') },
+  startBackup: (token, endpoint, workspace) => { ipcRenderer.send('start-backup', token, endpoint, workspace) }
 }
 contextBridge.exposeInMainWorld('electron', expose)

desktop/src/ui/types.ts

@@ -80,4 +80,7 @@ export interface IPCMainExposed {
   sendNotification: (notififationParams: NotificationParams) => void
   getScreenAccess: () => Promise<boolean>
   getScreenSources: () => Promise<ScreenSource[]>
+
+  cancelBackup: () => void
+  startBackup: (token: string, endpoint: string, workspace: string) => void
 }

desktop/webpack.config.js

@@ -37,15 +37,15 @@ module.exports = [
             options: {
               target: 'es2021',
               keepNames: true,
-              minify: !prod,
-              sourcemap: !prod
+              minify: prod,
+              sourcemap: true
             }
           }
         }
       ]
     },
     output: {
-      path: __dirname + '/dist',
+      path: path.join(__dirname, 'dist'),
       filename: '[name].js',
       chunkFilename: '[name].js',
       publicPath: '/',
@@ -64,7 +64,6 @@ module.exports = [
     },
     target: 'electron-main',
     plugins: [
-      ...(doValidate ? [new ForkTsCheckerWebpackPlugin()] : []),
       new Dotenv({ path: prod ? '.env' : '.env-dev' }),
       new DefinePlugin({
         'process.env.MODEL_VERSION': JSON.stringify(process.env.MODEL_VERSION),
@@ -98,8 +97,8 @@ module.exports = [
             options: {
               target: 'es2021',
               keepNames: true,
-              minify: !prod,
-              sourcemap: !prod
+              minify: prod,
+              sourcemap: true
             }
           }
         }
@@ -113,7 +112,10 @@ module.exports = [
     },
     resolve: {
       extensions: ['.ts', '.js'],
-      conditionNames: ['svelte', 'browser', 'import']
+      conditionNames: ['svelte', 'browser', 'import'],
+      alias: {
+        ws: path.resolve('node_modules', 'ws/index.js')
+      }
     }
   },
 
@@ -154,15 +156,14 @@ module.exports = [
       ? {
           minimize: true,
           minimizer: [
-            new EsbuildPlugin({
-              target: 'es2021',
-              keepNames: true,
-              minify: !prod,
-              sourcemap: !prod
-            })
+            new EsbuildPlugin({ target: 'es2021' })
           ]
         }
-      : {},
+      : {
+          minimize: false,
+          mangleExports: false,
+          usedExports: false
+        },
     module: {
       rules: [
         {
@@ -171,8 +172,8 @@ module.exports = [
           options: {
             target: 'es2021',
             keepNames: true,
-            minify: !prod,
-            sourcemap: !prod
+            minify: prod,
+            sourcemap: true
           },
           exclude: /node_modules/
         },
@@ -182,13 +183,13 @@ module.exports = [
             loader: 'svelte-loader',
             options: {
               compilerOptions: {
-                dev
+                dev: !prod
               },
               emitCss: true,
-              hotReload: dev,
+              hotReload: !prod,
               preprocess: require('svelte-preprocess')({
                 postcss: true,
-                sourceMap: dev
+                sourceMap: true
               }),
               hotOptions: {
                 // Prevent preserving local component state
@@ -260,21 +261,21 @@ module.exports = [
           }
         },
         {
-          test: /\.(wav|ogg)$/,
+          test: /\.(jpg|png|webp|heic|avif)$/,
           use: {
             loader: 'file-loader',
             options: {
-              'name': 'snd/[contenthash].[ext]',
+              name: 'img/[contenthash].[ext]',
               esModule: false
             }
           }
         },
         {
-          test: /\.(jpg|png|webp|heic|avif)$/,
+          test: /\.(wav|ogg)$/,
           use: {
             loader: 'file-loader',
             options: {
-              name: 'img/[hash:base64:8].[ext]',
+              name: 'snd/[contenthash].[ext]',
               esModule: false
             }
           }
@@ -285,7 +286,7 @@ module.exports = [
             {
               loader: 'file-loader',
               options: {
-                name: 'img/[hash:base64:8].[ext]',
+                name: 'img/[contenthash].[ext]',
                 esModule: false
               }
             },
@@ -304,7 +305,7 @@ module.exports = [
         }
       ]
     },
-    mode: dev ? 'development' : mode,
+    mode,
     plugins: [
       new CopyPlugin({
         patterns: [{ from: 'public', to: 'public' }]
@@ -335,9 +336,9 @@ module.exports = [
       // https://webpack.js.org/configuration/watch/#watchoptionsignored
       // don't use this pattern, if you have a monorepo with linked packages
       ignored: /node_modules/,
-      aggregateTimeout: 500,
-      poll: 1000
+      aggregateTimeout: 100,
+      poll: 250
     },
-    devtool: !dev ? false : 'eval-source-map'
+    devtool: prod ? 'source-map' : 'eval-source-map' // 'inline-source-map',
   }
 ]

packages/ui/src/components/Component.svelte

@@ -16,21 +16,21 @@
   import { getResource } from '@hcengineering/platform'
   import { deepEqual } from 'fast-equals'
   import { SvelteComponent } from 'svelte'
-  import type { AnyComponent } from '../types'
+  import type { AnyComponent, AnySvelteComponent } from '../types'
   import ErrorPresenter from './ErrorPresenter.svelte'
   import Loading from './Loading.svelte'
   import ErrorBoundary from './internal/ErrorBoundary'
 
   // Reference to rendered component instance
   export let innerRef: SvelteComponent | undefined = undefined
-  export let is: AnyComponent
+  export let is: AnyComponent | AnySvelteComponent
   export let props = {}
   export let shrink: boolean = false
   export let showLoading = true
   export let inline: boolean = false
   export let disabled: boolean = false
 
-  let _is: any = is
+  let _is: AnyComponent | AnySvelteComponent = is
   let _props: any = props
 
   $: if (!deepEqual(_is, is)) {
@@ -40,7 +40,12 @@
     _props = props
   }
 
-  $: component = _is != null ? getResource<any>(_is) : Promise.reject(new Error('is not defined'))
+  $: component =
+    _is != null && typeof _is === 'string'
+      ? getResource<any>(_is)
+      : _is == null
+        ? Promise.reject(new Error('is not defined'))
+        : Promise.resolve(_is)
 </script>
 
 {#if _is}

packages/ui/src/components/RootStatusComponent.svelte

@@ -0,0 +1,65 @@
+<script lang="ts">
+  import type { IntlString } from '@hcengineering/platform'
+  import { createEventDispatcher, onMount } from 'svelte'
+  import { IconClose } from '..'
+  import Button from './Button.svelte'
+  import Label from './Label.svelte'
+
+  export let onProgress: (props?: Record<string, any>) => number
+  export let onCancel: ((props?: Record<string, any>) => void) | undefined
+
+  export let interval: number
+
+  export let props: Record<string, any> | undefined
+  export let label: IntlString
+  export let labelProps: Record<string, any> | undefined
+
+  let currentProgress = onProgress(props)
+
+  onMount(() => {
+    const timer = setInterval(() => {
+      currentProgress = onProgress(props)
+      if (currentProgress >= 100) {
+        setTimeout(() => {
+          dispatch('close')
+        }, 1000)
+      }
+    }, interval)
+    return () => {
+      clearInterval(timer)
+    }
+  })
+
+  const dispatch = createEventDispatcher()
+</script>
+
+<div class="flex-row-center container">
+  <Label {label} params={labelProps ?? {}} />
+  {currentProgress}%
+  {#if onCancel !== undefined}
+    <Button
+      icon={IconClose}
+      size={'small'}
+      kind={'ghost'}
+      on:click={() => {
+        if (onCancel !== undefined) {
+          onCancel?.(props)
+        } else {
+          dispatch('close')
+        }
+      }}
+    />
+  {/if}
+</div>
+
+<style lang="scss">
+  .container {
+    padding: 0.125rem 0.125rem 0.125rem 0.5rem;
+    height: 1.625rem;
+    font-weight: 500;
+    background-color: var(--theme-button-pressed);
+    border: 1px solid transparent;
+    border-radius: 0.25rem;
+    cursor: pointer;
+  }
+</style>

packages/ui/src/components/internal/Root.svelte

@@ -6,7 +6,7 @@
   // import { applicationShortcutKey } from '../../utils'
   import { Theme } from '@hcengineering/theme'
   import { IconArrowLeft, IconArrowRight, checkMobile, deviceOptionsStore as deviceInfo } from '../../'
-  import { embeddedPlatform, getCurrentLocation, location, locationStorageKeyId, navigate } from '../../location'
+  import { desktopPlatform, getCurrentLocation, location, locationStorageKeyId, navigate } from '../../location'
   import uiPlugin from '../../plugin'
   import Component from '../Component.svelte'
   import Label from '../Label.svelte'
@@ -137,7 +137,7 @@
   <div id="ui-root">
     <div class="antiStatusBar">
       <div class="flex-row-center h-full content-color gap-3 pl-4">
-        {#if embeddedPlatform}
+        {#if desktopPlatform}
           <div class="history-box flex-row-center gap-3">
             <button
               id="statusbar-back"

packages/ui/src/components/internal/RootBarExtension.svelte

@@ -13,8 +13,18 @@
   })
 </script>
 
-{#each $rootBarExtensions as ext}
+{#each $rootBarExtensions as ext (ext[1].id)}
   {#if ext[0] === position}
-    <Component is={ext[1]} />
+    <div id={ext[1].id} style:margin-right={'1px'}>
+      <Component
+        is={ext[1].component}
+        props={ext[1].props}
+        on:close={() => {
+          rootBarExtensions.update((cur) => {
+            return cur.filter((it) => it[1].id !== ext[1].id)
+          })
+        }}
+      />
+    </div>
   {/if}
 {/each}

packages/ui/src/location.ts

@@ -15,8 +15,8 @@
 
 import { Analytics } from '@hcengineering/analytics'
 import { clone, type Ref, type Space } from '@hcengineering/core'
-import { derived, get, writable } from 'svelte/store'
 import { type Plugin } from '@hcengineering/platform'
+import { derived, get, writable } from 'svelte/store'
 
 import { closePopup } from './popups'
 import { type Location as PlatformLocation } from './types'
@@ -113,15 +113,38 @@ export function getCurrentResolvedLocation (): PlatformLocation {
 
 declare global {
   interface Window {
-    embeddedPlatform?: boolean
+    desktopPlatform?: boolean
   }
 }
-export const embeddedPlatform = window.embeddedPlatform ?? false
+
+function isRunningInElectron (): boolean {
+  // Renderer process
+  if (
+    typeof window !== 'undefined' &&
+    typeof (window as any).process === 'object' &&
+    (window as any).process.type === 'renderer'
+  ) {
+    return true
+  }
+
+  // Detect the user agent when the `nodeIntegration` option is set to true
+  if (
+    typeof navigator === 'object' &&
+    typeof navigator.userAgent === 'string' &&
+    navigator.userAgent.includes('Electron')
+  ) {
+    return true
+  }
+
+  return false
+}
+
+export const desktopPlatform = window.desktopPlatform ?? isRunningInElectron()
 const locationWritable = writable(getRawCurrentLocation())
 
-console.log('embeddedPlatform', window.embeddedPlatform)
+console.log('desktopPlatform', window.desktopPlatform)
 
-if (!embeddedPlatform) {
+if (!desktopPlatform) {
   window.addEventListener('popstate', () => {
     locationWritable.set(getRawCurrentLocation())
   })
@@ -158,7 +181,7 @@ export function setResolvedLocation (location: PlatformLocation): void {
 }
 
 export function getCurrentLocation (): PlatformLocation {
-  if (embeddedPlatform) {
+  if (desktopPlatform) {
     return clone(get(locationWritable))
   }
   return getRawCurrentLocation()
@@ -178,8 +201,8 @@ export function navigate (location: PlatformLocation, replace = false): boolean
   const cur = locationToUrl(getCurrentLocation())
   const url = locationToUrl(location)
   if (cur !== url) {
-    const data = !embeddedPlatform ? null : { location }
-    const _url = !embeddedPlatform ? url : undefined
+    const data = !desktopPlatform ? null : { location }
+    const _url = !desktopPlatform ? url : undefined
     Analytics.navigate(url)
     if (replace) {
       history.replaceState(data, '', _url)

packages/ui/src/utils.ts

@@ -19,8 +19,9 @@ import { setMetadata, translate } from '@hcengineering/platform'
 import autolinker from 'autolinker'
 import { writable } from 'svelte/store'
 import { NotificationPosition, NotificationSeverity, notificationsStore, type Notification } from '.'
-import { deviceSizes, type AnyComponent, type AnySvelteComponent, type WidthType } from './types'
 import ui, { DAY, HOUR, MINUTE } from '..'
+import RootStatusComponent from './components/RootStatusComponent.svelte'
+import { deviceSizes, type AnyComponent, type AnySvelteComponent, type WidthType } from './types'
 
 /**
  * @public
@@ -297,7 +298,18 @@ export class ThrottledCaller {
 
 export const testing = (localStorage.getItem('#platform.testing.enabled') ?? 'false') === 'true'
 
-export const rootBarExtensions = writable<Array<['left' | 'right', AnyComponent]>>([])
+export const rootBarExtensions = writable<
+Array<
+[
+  'left' | 'right',
+  {
+    id: string
+    component: AnyComponent | AnySvelteComponent
+    props?: Record<string, any>
+  }
+]
+>
+>([])
 
 export async function formatDuration (duration: number, language: string): Promise<string> {
   let text = ''
@@ -318,3 +330,65 @@ export async function formatDuration (duration: number, language: string): Promi
   text = text.trim()
   return text
 }
+
+export function pushRootBarComponent (pos: 'left' | 'right', component: AnyComponent): void {
+  rootBarExtensions.update((cur) => {
+    if (cur.find((p) => p[1].component === component) === undefined) {
+      cur.push([
+        pos,
+        {
+          id: component,
+          component
+        }
+      ])
+    }
+    return cur
+  })
+}
+export function removeRootBarComponent (id: string): void {
+  rootBarExtensions.update((cur) => {
+    return cur.filter((p) => p[1].id !== id)
+  })
+}
+
+export function pushRootBarProgressComponent (
+  id: string,
+  label: IntlString,
+  // In case onProgress return value >=100, it will be closed
+  onProgress: (props?: Record<string, any>) => number,
+  onCancel?: (props?: Record<string, any>) => void,
+  props?: Record<string, any>,
+  labelProps?: Record<string, any>,
+  interval?: number
+): void {
+  rootBarExtensions.update((cur) => {
+    const p = cur.find((p) => p[1].id === id)
+    if (p === undefined) {
+      cur.push([
+        'left',
+        {
+          id,
+          component: RootStatusComponent,
+          props: {
+            label,
+            onProgress,
+            onCancel,
+            props,
+            labelProps,
+            interval: interval ?? 100
+          }
+        }
+      ])
+    } else {
+      p[1].props = {
+        label,
+        onProgress,
+        onCancel,
+        props,
+        labelProps,
+        interval: interval ?? 100
+      }
+    }
+    return cur
+  })
+}

plugins/guest-resources/src/connect.ts

@@ -19,7 +19,7 @@ import presentation, {
   setPresentationCookie
 } from '@hcengineering/presentation'
 import {
-  embeddedPlatform,
+  desktopPlatform,
   fetchMetadataLocalStorage,
   getCurrentLocation,
   navigate,
@@ -90,7 +90,7 @@ export async function connect (title: string): Promise<Client | undefined> {
         } else {
           versionError.set(`Front version ${frontVersion} is not in sync with server version ${serverVersion}`)
 
-          if (!embeddedPlatform) {
+          if (!desktopPlatform) {
             setTimeout(() => {
               location.reload()
             }, 5000)

plugins/love-resources/src/components/WorkbenchExtension.svelte

@@ -1,9 +1,10 @@
 <script lang="ts">
-  import { rootBarExtensions } from '@hcengineering/ui'
+  import { pushRootBarComponent, rootBarExtensions } from '@hcengineering/ui'
   import { RemoteParticipant, RemoteTrack, RemoteTrackPublication, RoomEvent, Track } from 'livekit-client'
   import { onDestroy, onMount } from 'svelte'
   import love from '../plugin'
   import { disconnect, isCurrentInstanceConnected, lk } from '../utils'
+  import { generateId } from '@hcengineering/core'
 
   let parentElement: HTMLDivElement
 
@@ -33,12 +34,7 @@
   }
 
   onMount(() => {
-    rootBarExtensions.update((cur) => {
-      if (cur.find((p) => p[1] === love.component.ControlExt) === undefined) {
-        cur.push(['left', love.component.ControlExt])
-      }
-      return cur
-    })
+    pushRootBarComponent('left', love.component.ControlExt)
     lk.on(RoomEvent.TrackSubscribed, handleTrackSubscribed)
     lk.on(RoomEvent.TrackUnsubscribed, handleTrackUnsubscribed)
   })

plugins/setting-assets/lang/en.json

@@ -118,6 +118,7 @@
     "DeleteSpaceType": "Delete space type",
     "DeleteSpaceTypeConfirm": "Are you sure you want to delete this space type?",
     "WorkspaceName": "Workspace name",
-    "Workspace": "Workspace"
+    "Workspace": "Workspace",
+    "OwnerOrMainteinerRequired": "You need to be a workspace Owner or Maintainer"
   }
 }

plugins/setting-assets/lang/es.json

@@ -109,6 +109,7 @@
     "DeleteSpaceType": "Eliminar tipo de espacio",
     "DeleteSpaceTypeConfirm": "¿Estás seguro de que quieres eliminar este tipo de espacio?",
     "WorkspaceName": "Nombre del espacio de trabajo",
-    "Workspace": "Espacio de trabajo"
+    "Workspace": "Espacio de trabajo",
+    "OwnerOrMainteinerRequired": "Necesitas ser Propietario o Mantenedor del espacio de trabajo"
   }
 }

plugins/setting-assets/lang/fr.json

@@ -118,6 +118,7 @@
     "DeleteSpaceType": "Supprimer le type d'espace",
     "DeleteSpaceTypeConfirm": "Êtes-vous sûr de vouloir supprimer ce type d'espace ?",
     "WorkspaceName": "Nom de l'espace de travail",
-    "Workspace": "Espace de travail"
+    "Workspace": "Espace de travail",
+    "OwnerOrMainteinerRequired": "Vous devez être propriétaire ou responsable d'un espace de travail"
   }
 }

plugins/setting-assets/lang/pt.json

@@ -109,6 +109,7 @@
     "DeleteSpaceType": "Eliminar tipo de espaço",
     "DeleteSpaceTypeConfirm": "Tem certeza de que deseja excluir este tipo de espaço?",
     "WorkspaceName": "Nome do espaço de trabalho",
-    "Workspace": "Espaço de trabalho"
+    "Workspace": "Espaço de trabalho",
+    "OwnerOrMainteinerRequired": "Precisa de ser proprietário ou mantenedor do espaço de trabalho"
   }
 }

plugins/setting-assets/lang/ru.json

@@ -119,6 +119,7 @@
     "DeleteSpaceType": "Удалить тип пространства",
     "DeleteSpaceTypeConfirm": "Вы действительно хотите удалить этот тип пространства?",
     "WorkspaceName": "Название рабочего пространства",
-    "Workspace": "Рабочее пространство"
+    "Workspace": "Рабочее пространство",
+    "OwnerOrMainteinerRequired": "Вы должны иметь роль Владельца/Сопровождаюего"
   }
 }

plugins/setting-assets/lang/zh.json

@@ -118,6 +118,7 @@
     "DeleteSpaceType": "删除空间类型",
     "DeleteSpaceTypeConfirm": "您确定要删除此空间类型吗？",
     "WorkspaceName": "工作区名称",
-    "Workspace": "工作区"
+    "Workspace": "工作区",
+    "OwnerOrMainteinerRequired": "您需要成为工作空间所有者或维护者"
   }
 }

plugins/setting/src/index.ts

@@ -202,7 +202,8 @@ export default plugin(settingId, {
     Automations: '' as IntlString,
     Collections: '' as IntlString,
     SpaceTypes: '' as IntlString,
-    Roles: '' as IntlString
+    Roles: '' as IntlString,
+    OwnerOrMainteinerRequired: '' as IntlString
   },
   icon: {
     AccountSettings: '' as Asset,

plugins/uploader-resources/src/components/WorkbenchExtension.svelte

@@ -1,5 +1,5 @@
 <script lang="ts">
-  import { rootBarExtensions } from '@hcengineering/ui'
+  import { pushRootBarComponent } from '@hcengineering/ui'
   import { onMount } from 'svelte'
 
   import uploader from '../plugin'
@@ -7,12 +7,7 @@
   let parentElement: HTMLDivElement
 
   onMount(() => {
-    rootBarExtensions.update((cur) => {
-      if (cur.find((p) => p[1] === uploader.component.FileUploadExt) === undefined) {
-        cur.push(['right', uploader.component.FileUploadExt])
-      }
-      return cur
-    })
+    pushRootBarComponent('right', uploader.component.FileUploadExt)
   })
 </script>
 

plugins/workbench-resources/src/components/Workbench.svelte

@@ -61,8 +61,8 @@
     navigate,
     openPanel,
     popupstore,
+    pushRootBarComponent,
     resolvedLocationStore,
-    rootBarExtensions,
     setResolvedLocation,
     showPopup,
     workbenchSeparators,
@@ -75,8 +75,8 @@
     NavLink,
     accessDeniedStore,
     migrateViewOpttions,
-    updateFocus,
-    parseLinkId
+    parseLinkId,
+    updateFocus
   } from '@hcengineering/view-resources'
   import type { Application, NavigatorModel, SpecialNavModel, ViewConfiguration } from '@hcengineering/workbench'
   import { getContext, onDestroy, onMount, tick } from 'svelte'
@@ -143,12 +143,7 @@
   }
 
   onMount(() => {
-    rootBarExtensions.update((cur) => {
-      if (!cur.find((p) => p[1] === view.component.SearchSelector)) {
-        cur.push(['right', view.component.SearchSelector])
-      }
-      return cur
-    })
+    pushRootBarComponent('right', view.component.SearchSelector)
     void getResource(login.function.GetWorkspaces).then(async (getWorkspaceFn) => {
       $workspacesStore = await getWorkspaceFn()
       await updateWindowTitle(getLocation())

plugins/workbench-resources/src/connect.ts

@@ -27,7 +27,7 @@ import presentation, {
   uiContext
 } from '@hcengineering/presentation'
 import {
-  embeddedPlatform,
+  desktopPlatform,
   fetchMetadataLocalStorage,
   getCurrentLocation,
   locationStorageKeyId,
@@ -83,6 +83,7 @@ export async function connect (title: string): Promise<Client | undefined> {
     setMetadataLocalStorage(login.metadata.LoginTokens, tokens)
     setMetadata(presentation.metadata.Workspace, workspaceLoginInfo.workspace)
     setMetadata(presentation.metadata.WorkspaceId, workspaceLoginInfo.workspaceId)
+    setMetadata(presentation.metadata.Endpoint, workspaceLoginInfo.endpoint)
   }
 
   setMetadata(presentation.metadata.Token, token)
@@ -171,7 +172,7 @@ export async function connect (title: string): Promise<Client | undefined> {
             } else {
               versionError.set(`Front version ${frontVersion} is not in sync with server version ${serverVersion}`)
 
-              if (!embeddedPlatform) {
+              if (!desktopPlatform) {
                 setTimeout(() => {
                   location.reload()
                 }, 5000)

pods/server/package.json

@@ -60,7 +60,6 @@
     "@hcengineering/server-pipeline": "^0.6.0",
     "@hcengineering/mongo": "^0.6.1",
     "@hcengineering/elastic": "^0.6.0",
-    "elastic-apm-node": "~3.26.0",
     "@hcengineering/server-token": "^0.6.11",
     "@hcengineering/middleware": "^0.6.0",
     "@hcengineering/minio": "^0.6.0",

pods/server/src/__start.ts

@@ -4,23 +4,22 @@
 
 // Add this to the VERY top of the first file loaded in your app
 import { Analytics } from '@hcengineering/analytics'
+import { SplitLogger, configureAnalytics } from '@hcengineering/analytics-service'
 import contactPlugin from '@hcengineering/contact'
 import { MeasureMetricsContext, newMetrics, setOperationLogProfiling } from '@hcengineering/core'
 import notification from '@hcengineering/notification'
 import { setMetadata } from '@hcengineering/platform'
 import { getMetricsContext, serverConfigFromEnv } from '@hcengineering/server'
-import { storageConfigFromEnv } from '@hcengineering/server-storage'
+import serverAiBot from '@hcengineering/server-ai-bot'
 import serverCore, { type StorageConfiguration, loadBrandingMap } from '@hcengineering/server-core'
 import serverNotification from '@hcengineering/server-notification'
-import serverToken from '@hcengineering/server-token'
-import { serverFactories } from '@hcengineering/server-ws/src/factories'
-import { SplitLogger, configureAnalytics } from '@hcengineering/analytics-service'
+import { storageConfigFromEnv } from '@hcengineering/server-storage'
 import serverTelegram from '@hcengineering/server-telegram'
-import serverAiBot from '@hcengineering/server-ai-bot'
+import serverToken from '@hcengineering/server-token'
+import { startHttpServer } from '@hcengineering/server-ws'
 import { join } from 'path'
 import { start } from '.'
 import { profileStart, profileStop } from './inspector'
-const serverFactory = serverFactories[(process.env.SERVER_PROVIDER as string) ?? 'ws'] ?? serverFactories.ws
 
 configureAnalytics(process.env.SENTRY_DSN, {})
 Analytics.setTag('application', 'transactor')
@@ -64,7 +63,7 @@ const shutdown = start(config.url, {
   storageConfig,
   rekoniUrl: config.rekoniUrl,
   port: config.serverPort,
-  serverFactory,
+  serverFactory: startHttpServer,
   indexParallel: 2,
   indexProcessing: 500,
   brandingMap: loadBrandingMap(config.brandingPath),

pods/server/src/server.ts

@@ -15,11 +15,11 @@
 //
 
 import { type Branding, type BrandingMap, type WorkspaceIdWithUrl } from '@hcengineering/core'
-import { BackupClientSession, buildStorageFromConfig, getMetricsContext } from '@hcengineering/server'
+import { buildStorageFromConfig, getMetricsContext } from '@hcengineering/server'
 
+import { ClientSession, startSessionManager, type ServerFactory, type Session } from '@hcengineering/server'
 import { type Pipeline, type StorageConfiguration } from '@hcengineering/server-core'
 import { type Token } from '@hcengineering/server-token'
-import { ClientSession, start as startJsonRpc, type ServerFactory, type Session } from '@hcengineering/server-ws'
 
 import { serverAiBotId } from '@hcengineering/server-ai-bot'
 import { createAIBotAdapter } from '@hcengineering/server-ai-bot-resources'
@@ -81,13 +81,10 @@ export function start (
     workspaceId: WorkspaceIdWithUrl,
     branding: Branding | null
   ): Session => {
-    if (token.extra?.mode === 'backup') {
-      return new BackupClientSession(token, pipeline, workspaceId, branding)
-    }
-    return new ClientSession(token, pipeline, workspaceId, branding)
+    return new ClientSession(token, pipeline, workspaceId, branding, token.extra?.mode === 'backup')
   }
 
-  const onClose = startJsonRpc(getMetricsContext(), {
+  const onClose = startSessionManager(getMetricsContext(), {
     pipelineFactory,
     sessionFactory,
     port: opt.port,

rush.json

@@ -1451,11 +1451,6 @@
       "projectFolder": "server-plugins/hr-resources",
       "shouldPublish": false
     },
-    {
-      "packageName": "@hcengineering/apm",
-      "projectFolder": "tools/apm",
-      "shouldPublish": false
-    },
     {
       "packageName": "@hcengineering/minio",
       "projectFolder": "server/minio",

server/account/package.json

@@ -47,7 +47,6 @@
     "@hcengineering/client-resources": "^0.6.27",
     "@hcengineering/client": "^0.6.18",
     "otp-generator": "^4.0.1",
-    "ws": "^8.18.0",
     "@hcengineering/model": "^0.6.11",
     "@hcengineering/server-backup": "^0.6.0",
     "@hcengineering/server-tool": "^0.6.0",

server/backup/src/backup.ts

@@ -54,7 +54,7 @@ const dataBlobSize = 50 * 1024 * 1024
 const dataUploadSize = 2 * 1024 * 1024
 const retrieveChunkSize = 2 * 1024 * 1024
 
-const defaultLevel = 1
+const defaultLevel = 9
 
 /**
  * Blob data from s3 storage
@@ -516,6 +516,10 @@ export async function backup (
     blobDownloadLimit: number
     connection?: CoreClient & BackupClient
     storageAdapter?: StorageAdapter
+    // Return true in case
+    isCanceled?: () => boolean
+    progress?: (progress: number) => void
+    token?: string
   } = {
     force: false,
     recheck: false,
@@ -533,7 +537,11 @@ export async function backup (
     timeout: options.timeout
   })
 
-  let canceled = false
+  let _canceled = false
+  const canceled = (): boolean => {
+    return _canceled || (options.isCanceled?.() ?? false)
+  }
+
   let timer: any
   let ops = 0
 
@@ -542,18 +550,21 @@ export async function backup (
       if (ops === 0) {
         ctx.error('Timeout during backup', { workspace: workspaceId.name, timeout: options.timeout / 1000 })
         ops = 0
-        canceled = true
+        _canceled = true
       }
     }, options.timeout)
   }
 
-  const token = generateToken(systemAccountEmail, workspaceId, {
-    mode: 'backup'
-  })
+  const token =
+    options.token ??
+    generateToken(systemAccountEmail, workspaceId, {
+      mode: 'backup'
+    })
 
   const connection =
     options.connection ??
-    ((await createClient(transactorUrl, token, undefined, options.connectTimeout)) as CoreClient & BackupClient)
+    ((await createClient(transactorUrl, options.token ?? token, undefined, options.connectTimeout)) as CoreClient &
+    BackupClient)
 
   const blobClient = new BlobClient(transactorUrl, token, workspaceId, { storageAdapter: options.storageAdapter })
   ctx.info('starting backup', { workspace: workspaceId.name })
@@ -611,8 +622,10 @@ export async function backup (
       domains: {}
     }
 
+    // Increment snapshot index
+    backupInfo.snapshotsIndex = (backupInfo.snapshotsIndex ?? backupInfo.snapshots.length) + 1
+    let backupIndex = `${backupInfo.snapshotsIndex}`
     backupInfo.snapshots.push(snapshot)
-    let backupIndex = `${backupInfo.snapshotsIndex ?? backupInfo.snapshots.length}`
     while (backupIndex.length < 6) {
       backupIndex = '0' + backupIndex
     }
@@ -717,7 +730,11 @@ export async function backup (
       return { changed, needRetrieveChunks }
     }
 
-    async function processDomain (ctx: MeasureContext, domain: Domain): Promise<void> {
+    async function processDomain (
+      ctx: MeasureContext,
+      domain: Domain,
+      progress: (value: number) => void
+    ): Promise<void> {
       const changes: Snapshot = {
         added: new Map(),
         updated: new Map(),
@@ -745,18 +762,22 @@ export async function backup (
       const digest = await ctx.with('load-digest', {}, (ctx) => loadDigest(ctx, storage, backupInfo.snapshots, domain))
 
       let _pack: Pack | undefined
-      let addedDocuments = 0
+      let addedDocuments = (): number => 0
 
+      progress(0)
       let { changed, needRetrieveChunks } = await ctx.with('load-chunks', { domain }, (ctx) =>
         loadChangesFromServer(ctx, domain, digest, changes)
       )
+      progress(10)
 
       if (needRetrieveChunks.length > 0) {
         ctx.info('dumping domain...', { workspace: workspaceId.name, domain })
       }
 
+      const totalChunks = needRetrieveChunks.flatMap((it) => it.length).reduce((p, c) => p + c, 0)
+      let processed = 0
       while (needRetrieveChunks.length > 0) {
-        if (canceled) {
+        if (canceled()) {
           return
         }
         const needRetrieve = needRetrieveChunks.shift() as Ref<Doc>[]
@@ -779,10 +800,9 @@ export async function backup (
 
         while (docs.length > 0) {
           // Chunk data into small pieces
-          if (addedDocuments > dataBlobSize && _pack !== undefined) {
+          if (addedDocuments() > dataBlobSize && _pack !== undefined) {
             _pack.finalize()
             _pack = undefined
-            addedDocuments = 0
 
             if (changed > 0) {
               snapshot.domains[domain] = domainInfo
@@ -800,7 +820,7 @@ export async function backup (
               processedChanges.updated.clear()
               await storage.writeFile(
                 infoFile,
-                gzipSync(JSON.stringify(backupInfo, undefined, 2), { level: defaultLevel })
+                gzipSync(JSON.stringify(backupInfo, undefined, 2), { level: defaultLevel, memLevel: 9 })
               )
             }
           }
@@ -811,12 +831,24 @@ export async function backup (
             ctx.info('storing from domain', { domain, storageFile, workspace: workspaceId.name })
             domainInfo.storage = [...(domainInfo.storage ?? []), storageFile]
             const dataStream = await storage.write(storageFile)
-            const storageZip = createGzip({ level: defaultLevel, memLevel: 9 })
 
+            const sizePass = new PassThrough()
+            let sz = 0
+            sizePass._transform = (chunk, encoding, cb) => {
+              // No transformation, just pass through data
+              sizePass.push(chunk)
+              sz += chunk.length
+              cb()
+            }
+
+            sizePass.pipe(dataStream)
+
+            const storageZip = createGzip({ level: defaultLevel, memLevel: 9 })
+            addedDocuments = () => sz
             _pack.pipe(storageZip)
-            storageZip.pipe(dataStream)
+            storageZip.pipe(sizePass)
           }
-          if (canceled) {
+          if (canceled()) {
             return
           }
           const d = docs.shift()
@@ -825,6 +857,8 @@ export async function backup (
           }
 
           function processChanges (d: Doc, error: boolean = false): void {
+            processed++
+            progress(10 + (processed / totalChunks) * 90)
             // Move processed document to processedChanges
             if (changes.added.has(d._id)) {
               if (!error) {
@@ -867,8 +901,6 @@ export async function backup (
             }
 
             let blobFiled = false
-            addedDocuments += descrJson.length
-            addedDocuments += blob.size
 
             printDownloaded('', descrJson.length)
             try {
@@ -924,7 +956,6 @@ export async function backup (
             processChanges(d, blobFiled)
           } else {
             const data = JSON.stringify(d)
-            addedDocuments += data.length
             _pack.entry({ name: d._id + '.json' }, data, function (err) {
               if (err != null) throw err
             })
@@ -958,16 +989,20 @@ export async function backup (
       }
     }
 
+    let domainProgress = 0
     for (const domain of domains) {
-      if (canceled) {
+      if (canceled()) {
         break
       }
       await ctx.with('process-domain', { domain }, async (ctx) => {
-        await processDomain(ctx, domain)
+        await processDomain(ctx, domain, (value) => {
+          options.progress?.(Math.round(((domainProgress + value / 100) / domains.length) * 100))
+        })
       })
+      domainProgress++
+      options.progress?.(Math.round((domainProgress / domains.length) * 10000) / 100)
     }
-    if (!canceled) {
-      backupInfo.snapshotsIndex = backupInfo.snapshots.length
+    if (!canceled()) {
       backupInfo.lastTxId = lastTx?._id ?? '0' // We could store last tx, since full backup is complete
       await storage.writeFile(infoFile, gzipSync(JSON.stringify(backupInfo, undefined, 2), { level: defaultLevel }))
     }
@@ -1514,20 +1549,18 @@ export async function compactBackup (
       const rsnapshots = Array.from(oldSnapshots).reverse()
 
       let _pack: Pack | undefined
-      let addedDocuments = 0
+      let addedDocuments: () => number = () => 0
 
       let processed = 0
 
       const blobs = new Map<string, { doc: Doc | undefined, buffer: Buffer | undefined }>()
 
       async function pushDocs (docs: Doc[], size: number, blobData: Record<Ref<Doc>, Buffer>): Promise<void> {
-        addedDocuments += size
         changed += docs.length
         // Chunk data into small pieces
-        if (addedDocuments > dataBlobSize && _pack !== undefined) {
+        if (addedDocuments() > dataBlobSize && _pack !== undefined) {
           _pack.finalize()
           _pack = undefined
-          addedDocuments = 0
 
           if (changed > 0) {
             snapshot.domains[domain] = domainInfo
@@ -1556,10 +1589,22 @@ export async function compactBackup (
           console.log('storing from domain', domain, storageFile)
           domainInfo.storage = [...(domainInfo.storage ?? []), storageFile]
           const dataStream = await storage.write(storageFile)
-          const storageZip = createGzip({ level: defaultLevel })
+          const storageZip = createGzip({ level: defaultLevel, memLevel: 9 })
+
+          const sizePass = new PassThrough()
+          let sz = 0
+          sizePass._transform = (chunk, encoding, cb) => {
+            // No transformation, just pass through data
+            sizePass.push(chunk)
+            sz += chunk.length
+            cb()
+          }
+
+          sizePass.pipe(dataStream)
+          addedDocuments = () => sz
 
           _pack.pipe(storageZip)
-          storageZip.pipe(dataStream)
+          storageZip.pipe(sizePass)
         }
 
         while (docs.length > 0) {
@@ -1576,8 +1621,6 @@ export async function compactBackup (
 
             const data = blobData[blob._id]
             const descrJson = JSON.stringify(d)
-            addedDocuments += descrJson.length
-            addedDocuments += data.length
             _pack.entry({ name: d._id + '.json' }, descrJson, function (err) {
               if (err != null) throw err
             })
@@ -1586,7 +1629,6 @@ export async function compactBackup (
             })
           } else {
             const data = JSON.stringify(d)
-            addedDocuments += data.length
             _pack.entry({ name: d._id + '.json' }, data, function (err) {
               if (err != null) throw err
             })
@@ -1738,8 +1780,6 @@ export async function compactBackup (
         }
       }
     }
-
-    backupInfo.snapshotsIndex = backupInfo.snapshots.length
     await storage.writeFile(infoFile, gzipSync(JSON.stringify(backupInfo, undefined, 2), { level: defaultLevel }))
   } catch (err: any) {
     console.error(err)

server/core/src/content.ts

@@ -25,7 +25,7 @@ class ContentAdapter implements ContentTextAdapter {
     private readonly context: MeasureContext
   ) {}
 
-  async content (name: string, type: string, doc: string | Readable | Buffer): Promise<string> {
+  async content (name: string, type: string, doc: Readable): Promise<string> {
     const adapter = this.adapters.get(type) ?? this.defaultAdapter
     return await adapter.content(name, type, doc)
   }

server/core/src/types.ts

@@ -392,7 +392,7 @@ export type FullTextAdapterFactory = (
  * @public
  */
 export interface ContentTextAdapter {
-  content: (name: string, type: string, doc: Readable | Buffer | string) => Promise<string>
+  content: (name: string, type: string, doc: Readable) => Promise<string>
   metrics: () => MeasureContext
 }
 

server/server-pipeline/package.json

@@ -41,7 +41,6 @@
     "@hcengineering/core": "^0.6.32",
     "@hcengineering/platform": "^0.6.11",
     "@hcengineering/server-core": "^0.6.1",
-    "@hcengineering/server-ws": "^0.6.11",
     "@hcengineering/server-attachment": "^0.6.1",
     "@hcengineering/server-attachment-resources": "^0.6.0",
     "@hcengineering/server-collaboration": "^0.6.0",
@@ -51,7 +50,6 @@
     "@hcengineering/postgres": "^0.6.0",
     "@hcengineering/mongo": "^0.6.1",
     "@hcengineering/elastic": "^0.6.0",
-    "elastic-apm-node": "~3.26.0",
     "@hcengineering/server-contact": "^0.6.1",
     "@hcengineering/server-contact-resources": "^0.6.0",
     "@hcengineering/server-notification": "^0.6.1",

server/server-storage/package.json

@@ -44,12 +44,10 @@
     "@hcengineering/core": "^0.6.32",
     "@hcengineering/platform": "^0.6.11",
     "@hcengineering/server-core": "^0.6.1",
-    "@hcengineering/server-ws": "^0.6.11",
     "@hcengineering/mongo": "^0.6.1",
     "@hcengineering/minio": "^0.6.0",
     "@hcengineering/s3": "^0.6.0",
     "@hcengineering/datalake": "^0.6.0",
-    "elastic-apm-node": "~3.26.0",
     "@hcengineering/server-token": "^0.6.11"
   }
 }

server/server/package.json

@@ -44,14 +44,11 @@
     "@hcengineering/core": "^0.6.32",
     "@hcengineering/platform": "^0.6.11",
     "@hcengineering/server-core": "^0.6.1",
-    "@hcengineering/server-ws": "^0.6.11",
-    "@hcengineering/minio": "^0.6.0",
-    "@hcengineering/elastic": "^0.6.0",
-    "elastic-apm-node": "~3.26.0",
     "@hcengineering/server-token": "^0.6.11",
+    "@hcengineering/rpc": "^0.6.5",
     "@hcengineering/middleware": "^0.6.0",
     "@hcengineering/text": "^0.6.5",
-    "got": "^11.8.3",
-    "@hcengineering/server-storage": "^0.6.0"
+    "@hcengineering/server-storage": "^0.6.0",
+    "@hcengineering/analytics": "^0.6.0"
   }
 }

server/server/src/apm.ts

@@ -1,137 +0,0 @@
-import { MeasureContext, MeasureLogger, ParamType, ParamsType, type FullParamsType } from '@hcengineering/core'
-import apm, { Agent, Span, Transaction } from 'elastic-apm-node'
-
-/**
- * @public
- */
-export function createAPMAgent (apmUrl: string): Agent {
-  const agent: Agent = apm.start({
-    // Override the service name from package.json
-    // Allowed characters: a-z, A-Z, 0-9, -, _, and space
-    serviceName: 'transactor',
-
-    // Use if APM Server requires a secret token
-    secretToken: '',
-
-    // Set the custom APM Server URL (default: http://localhost:8200)
-    serverUrl: apmUrl,
-    logLevel: 'trace'
-  })
-  return agent
-}
-
-/**
- * @public
- */
-export class APMMeasureContext implements MeasureContext {
-  logger: MeasureLogger
-  private readonly transaction?: Transaction | Span
-  private readonly parentTx?: Transaction | Span
-  contextData = {}
-  constructor (
-    private readonly agent: Agent,
-    name: string,
-    params: Record<string, ParamType>,
-    parentTx?: Transaction | Span,
-    noTransaction?: boolean,
-    readonly parent?: MeasureContext
-  ) {
-    this.parentTx = parentTx
-    this.logger = {
-      info: (msg, args) => {
-        agent.logger.info({ message: msg, ...args })
-      },
-      error: (msg, args) => {
-        agent.logger.error({ message: msg, ...args })
-      },
-      warn: (msg, args) => {
-        agent.logger.warn({ message: msg, ...args })
-      },
-      logOperation (operation, time, params) {},
-      close: async () => {}
-    }
-    if (!(noTransaction ?? false)) {
-      if (this.parentTx === undefined) {
-        this.transaction = agent.startTransaction(name) ?? undefined
-      } else {
-        this.transaction = agent.startSpan(name, { childOf: this.parentTx }) ?? undefined
-      }
-      for (const [k, v] of Object.entries(params)) {
-        this.transaction?.setLabel(k, v)
-      }
-    }
-  }
-
-  newChild (name: string, params: Record<string, ParamType>): MeasureContext {
-    return new APMMeasureContext(this.agent, name, params, this.transaction, undefined, this)
-  }
-
-  measure (name: string, value: number): void {}
-
-  async with<T>(
-    name: string,
-    params: ParamsType,
-    op: (ctx: MeasureContext) => T | Promise<T>,
-    fullParams?: FullParamsType | (() => FullParamsType)
-  ): Promise<T> {
-    const c = this.newChild(name, params)
-    try {
-      let value = op(c)
-      if (value instanceof Promise) {
-        value = await value
-      }
-      return value
-    } catch (err: any) {
-      c.error(err)
-      throw err
-    } finally {
-      c.end()
-    }
-  }
-
-  withSync<T>(
-    name: string,
-    params: ParamsType,
-    op: (ctx: MeasureContext) => T,
-    fullParams?: FullParamsType | (() => FullParamsType)
-  ): T {
-    const c = this.newChild(name, params)
-    try {
-      return op(c)
-    } catch (err: any) {
-      c.error(err)
-      throw err
-    } finally {
-      c.end()
-    }
-  }
-
-  async withLog<T>(
-    name: string,
-    params: ParamsType,
-    op: (ctx: MeasureContext) => T | Promise<T>,
-    fullParams?: ParamsType
-  ): Promise<T> {
-    const st = Date.now()
-    const r = await this.with(name, params, op, fullParams)
-    this.logger.logOperation(name, Date.now() - st, { ...params, ...fullParams })
-    return r
-  }
-
-  error (message: string, ...args: any[]): void {
-    this.logger.error(message, args)
-    this.agent.captureError({ message, params: args })
-  }
-
-  info (message: string, ...args: any[]): void {
-    this.logger.info(message, args)
-  }
-
-  warn (message: string, ...args: any[]): void {
-    this.logger.warn(message, args)
-  }
-
-  end (): void {
-    this.transaction?.end()
-  }
-}

server/server/src/backup.ts

@@ -1,81 +0,0 @@
-import { Doc, Domain, Ref, type Branding, type WorkspaceIdWithUrl } from '@hcengineering/core'
-import { PlatformError, unknownError } from '@hcengineering/platform'
-import { BackupClientOps, Pipeline } from '@hcengineering/server-core'
-import { Token } from '@hcengineering/server-token'
-import { ClientSession, Session, type ClientSessionCtx } from '@hcengineering/server-ws'
-
-/**
- * @public
- */
-export interface BackupSession extends Session {
-  loadChunk: (ctx: ClientSessionCtx, domain: Domain, idx?: number, recheck?: boolean) => Promise<void>
-  closeChunk: (ctx: ClientSessionCtx, idx: number) => Promise<void>
-  loadDocs: (ctx: ClientSessionCtx, domain: Domain, docs: Ref<Doc>[]) => Promise<void>
-}
-
-/**
- * @public
- */
-export class BackupClientSession extends ClientSession implements BackupSession {
-  ops: BackupClientOps
-  constructor (
-    protected readonly token: Token,
-    _pipeline: Pipeline,
-    workspaceId: WorkspaceIdWithUrl,
-    branding: Branding | null
-  ) {
-    super(token, _pipeline, workspaceId, branding)
-    if (_pipeline.context.lowLevelStorage === undefined) {
-      throw new PlatformError(unknownError('Low level storage is not available'))
-    }
-    this.ops = new BackupClientOps(_pipeline.context.lowLevelStorage)
-  }
-
-  async loadChunk (_ctx: ClientSessionCtx, domain: Domain, idx?: number, recheck?: boolean): Promise<void> {
-    this.lastRequest = Date.now()
-    try {
-      const result = await this.ops.loadChunk(_ctx.ctx, domain, idx, recheck)
-      await _ctx.sendResponse(result)
-    } catch (err: any) {
-      await _ctx.sendResponse({ error: err.message })
-    }
-  }
-
-  async closeChunk (ctx: ClientSessionCtx, idx: number): Promise<void> {
-    this.lastRequest = Date.now()
-    await this.ops.closeChunk(ctx.ctx, idx)
-    await ctx.sendResponse({})
-  }
-
-  async loadDocs (ctx: ClientSessionCtx, domain: Domain, docs: Ref<Doc>[]): Promise<void> {
-    this.lastRequest = Date.now()
-    try {
-      const result = await this.ops.loadDocs(ctx.ctx, domain, docs)
-      await ctx.sendResponse(result)
-    } catch (err: any) {
-      await ctx.sendResponse({ error: err.message })
-    }
-  }
-
-  async upload (ctx: ClientSessionCtx, domain: Domain, docs: Doc[]): Promise<void> {
-    this.lastRequest = Date.now()
-    try {
-      await this.ops.upload(ctx.ctx, domain, docs)
-    } catch (err: any) {
-      await ctx.sendResponse({ error: err.message })
-      return
-    }
-    await ctx.sendResponse({})
-  }
-
-  async clean (ctx: ClientSessionCtx, domain: Domain, docs: Ref<Doc>[]): Promise<void> {
-    this.lastRequest = Date.now()
-    try {
-      await this.ops.clean(ctx.ctx, domain, docs)
-    } catch (err: any) {
-      await ctx.sendResponse({ error: err.message })
-      return
-    }
-    await ctx.sendResponse({})
-  }
-}

server/server/src/client.ts

@@ -22,6 +22,7 @@ import core, {
   type Class,
   type Doc,
   type DocumentQuery,
+  type Domain,
   type FindOptions,
   type FindResult,
   type MeasureContext,
@@ -33,7 +34,8 @@ import core, {
   type TxCUD,
   type WorkspaceIdWithUrl
 } from '@hcengineering/core'
-import { SessionDataImpl, createBroadcastEvent, type Pipeline } from '@hcengineering/server-core'
+import { PlatformError, unknownError } from '@hcengineering/platform'
+import { BackupClientOps, SessionDataImpl, createBroadcastEvent, type Pipeline } from '@hcengineering/server-core'
 import { type Token } from '@hcengineering/server-token'
 import {
   type ClientSessionCtx,
@@ -43,6 +45,7 @@ import {
   type StatisticsElement
 } from './types'
 import { handleSend } from './utils'
+
 /**
  * @public
  */
@@ -59,11 +62,14 @@ export class ClientSession implements Session {
   mins5: StatisticsElement = { find: 0, tx: 0 }
   measures: { id: string, message: string, time: 0 }[] = []
 
+  ops: BackupClientOps | undefined
+
   constructor (
     protected readonly token: Token,
     protected readonly _pipeline: Pipeline,
     readonly workspaceId: WorkspaceIdWithUrl,
-    readonly branding: Branding | null
+    readonly branding: Branding | null,
+    readonly allowUpload: boolean
   ) {}
 
   getUser (): string {
@@ -211,4 +217,77 @@ export class ClientSession implements Session {
       void handleSend(ctx, socket, { result: tx }, 1024 * 1024, this.binaryMode, this.useCompression)
     }
   }
+
+  getOps (): BackupClientOps {
+    if (this.ops === undefined) {
+      if (this._pipeline.context.lowLevelStorage === undefined) {
+        throw new PlatformError(unknownError('Low level storage is not available'))
+      }
+      this.ops = new BackupClientOps(this._pipeline.context.lowLevelStorage)
+    }
+    return this.ops
+  }
+
+  async loadChunk (_ctx: ClientSessionCtx, domain: Domain, idx?: number, recheck?: boolean): Promise<void> {
+    this.lastRequest = Date.now()
+    try {
+      const result = await this.getOps().loadChunk(_ctx.ctx, domain, idx, recheck)
+      await _ctx.sendResponse(result)
+    } catch (err: any) {
+      await _ctx.sendResponse({ error: err.message })
+    }
+  }
+
+  async closeChunk (ctx: ClientSessionCtx, idx: number): Promise<void> {
+    this.lastRequest = Date.now()
+    await this.getOps().closeChunk(ctx.ctx, idx)
+    await ctx.sendResponse({})
+  }
+
+  async loadDocs (ctx: ClientSessionCtx, domain: Domain, docs: Ref<Doc>[]): Promise<void> {
+    this.lastRequest = Date.now()
+    try {
+      const result = await this.getOps().loadDocs(ctx.ctx, domain, docs)
+      await ctx.sendResponse(result)
+    } catch (err: any) {
+      await ctx.sendResponse({ error: err.message })
+    }
+  }
+
+  async upload (ctx: ClientSessionCtx, domain: Domain, docs: Doc[]): Promise<void> {
+    if (!this.allowUpload) {
+      await ctx.sendResponse({ error: 'Upload not allowed' })
+    }
+    this.lastRequest = Date.now()
+    try {
+      await this.getOps().upload(ctx.ctx, domain, docs)
+    } catch (err: any) {
+      await ctx.sendResponse({ error: err.message })
+      return
+    }
+    await ctx.sendResponse({})
+  }
+
+  async clean (ctx: ClientSessionCtx, domain: Domain, docs: Ref<Doc>[]): Promise<void> {
+    if (!this.allowUpload) {
+      await ctx.sendResponse({ error: 'Clean not allowed' })
+    }
+    this.lastRequest = Date.now()
+    try {
+      await this.getOps().clean(ctx.ctx, domain, docs)
+    } catch (err: any) {
+      await ctx.sendResponse({ error: err.message })
+      return
+    }
+    await ctx.sendResponse({})
+  }
+}
+
+/**
+ * @public
+ */
+export interface BackupSession extends Session {
+  loadChunk: (ctx: ClientSessionCtx, domain: Domain, idx?: number, recheck?: boolean) => Promise<void>
+  closeChunk: (ctx: ClientSessionCtx, idx: number) => Promise<void>
+  loadDocs: (ctx: ClientSessionCtx, domain: Domain, docs: Ref<Doc>[]) => Promise<void>
 }

server/server/src/index.ts

@@ -15,10 +15,14 @@
 //
 
 export { buildStorageFromConfig, createStorageDataAdapter, storageConfigFromEnv } from '@hcengineering/server-storage'
-export * from './apm'
-export * from './backup'
+export * from './client'
 export * from './metrics'
+export * from './nullAdapter'
 export * from './rekoni'
-export * from './server'
+export * from './sessionManager'
 export * from './starter'
+export * from './types'
+export * from './utils'
 export * from './ydoc'
+export * from './stats'
+export * from './blobs'

server/server/src/metrics.ts

@@ -1,8 +1,6 @@
 import { MeasureContext, MeasureMetricsContext, metricsToString, newMetrics } from '@hcengineering/core'
-import { APMMeasureContext, createAPMAgent } from './apm'
 import { writeFile } from 'fs/promises'
 
-const apmUrl = process.env.APM_SERVER_URL
 const metricsFile = process.env.METRICS_FILE
 // const logsRoot = process.env.LOGS_ROOT
 const metricsConsole = (process.env.METRICS_CONSOLE ?? 'false') === 'true'
@@ -21,44 +19,39 @@ export function getMetricsContext (factory?: () => MeasureMetricsContext): Measu
   if (metricsContext !== undefined) {
     return metricsContext
   }
-  if (apmUrl === undefined) {
-    console.info('please provide apm server url for monitoring')
 
-    const metrics = newMetrics()
-    if (factory !== undefined) {
-      metricsContext = factory()
-    } else {
-      metricsContext = new MeasureMetricsContext('System', {}, {}, metrics)
-    }
-
-    if (metricsFile !== undefined || metricsConsole) {
-      console.info('storing measurements into local file', metricsFile)
-      let oldMetricsValue = ''
-
-      const intTimer = setInterval(() => {
-        const val = metricsToString(metrics, 'System', 140)
-        if (val !== oldMetricsValue) {
-          oldMetricsValue = val
-          if (metricsFile !== undefined) {
-            writeFile(metricsFile, val).catch((err) => {
-              console.error(err)
-            })
-          }
-          if (metricsConsole) {
-            console.info('METRICS:', val)
-          }
-        }
-      }, METRICS_UPDATE_INTERVAL)
-
-      const closeTimer = (): void => {
-        clearInterval(intTimer)
-      }
-      process.on('SIGINT', closeTimer)
-      process.on('SIGTERM', closeTimer)
-    }
+  const metrics = newMetrics()
+  if (factory !== undefined) {
+    metricsContext = factory()
   } else {
-    console.log('using APM', apmUrl)
-    metricsContext = new APMMeasureContext(createAPMAgent(apmUrl), 'root', {}, undefined, true)
+    metricsContext = new MeasureMetricsContext('System', {}, {}, metrics)
   }
+
+  if (metricsFile !== undefined || metricsConsole) {
+    console.info('storing measurements into local file', metricsFile)
+    let oldMetricsValue = ''
+
+    const intTimer = setInterval(() => {
+      const val = metricsToString(metrics, 'System', 140)
+      if (val !== oldMetricsValue) {
+        oldMetricsValue = val
+        if (metricsFile !== undefined) {
+          writeFile(metricsFile, val).catch((err) => {
+            console.error(err)
+          })
+        }
+        if (metricsConsole) {
+          console.info('METRICS:', val)
+        }
+      }
+    }, METRICS_UPDATE_INTERVAL)
+
+    const closeTimer = (): void => {
+      clearInterval(intTimer)
+    }
+    process.on('SIGINT', closeTimer)
+    process.on('SIGTERM', closeTimer)
+  }
+
   return metricsContext
 }

server/server/src/rekoni.ts

@@ -1,7 +1,6 @@
 import { MeasureContext, WorkspaceId } from '@hcengineering/core'
 import { ContentTextAdapter } from '@hcengineering/server-core'
 import { generateToken } from '@hcengineering/server-token'
-import got, { HTTPError } from 'got'
 
 /**
  * @public
@@ -15,28 +14,45 @@ export async function createRekoniAdapter (
   return {
     content: async (name: string, type: string, doc): Promise<string> => {
       try {
-        const resContent = await got.post(
-          `${url}/toText?name=${encodeURIComponent(name)}&type=${encodeURIComponent(type)}`,
-          {
-            body: doc,
+        // Node doesn't support Readable with fetch.
+        const chunks: Buffer[] = []
+        let len = 0
+        await new Promise<void>((resolve, reject) => {
+          doc.on('data', (chunk) => {
+            len += (chunk as Buffer).length
+            chunks.push(chunk)
+            if (len > 10 * 1024 * 1024) {
+              reject(new Error('file to big for content processing'))
+            }
+          })
+          doc.on('end', () => {
+            resolve()
+          })
+          doc.on('error', (err) => {
+            reject(err)
+          })
+        })
+
+        const body = Buffer.concat(chunks)
+        const r = await (
+          await fetch(`${url}/toText?name=${encodeURIComponent(name)}&type=${encodeURIComponent(type)}`, {
+            method: 'POST',
+            body,
             // timeout: 15000,
             headers: {
               Authorization: 'Bearer ' + token,
               'Content-type': typeof doc === 'string' ? 'text/plain' : 'application/octet-stream'
             }
-          }
-        )
-        const r = JSON.parse(resContent.body ?? '')
+          } as any)
+        ).json()
         if (r.error !== undefined) {
           throw new Error(r.error)
         }
         return r.content
       } catch (err: any) {
         console.info('Content Processing error', name, type, doc, err.response.body)
-        if (err instanceof HTTPError) {
-          if (err.message === 'Response code 400 (Bad Request)') {
-            return ''
-          }
+        if (err.message === 'Response code 400 (Bad Request)' || err.code === 400) {
+          return ''
         }
         throw err
       }

server/server/src/sessionManager.ts

@@ -214,8 +214,7 @@ class TSessionManager implements SessionManager {
           this.ctx.warn('closing workspace, no users', {
             workspace: workspace.workspaceId.name,
             wsId,
-            upgrade: workspace.upgrade,
-            backup: workspace.backup
+            upgrade: workspace.upgrade
           })
           workspace.closing = this.performWorkspaceCloseCheck(workspace, workspace.workspaceId, wsId)
         }
@@ -473,7 +472,6 @@ class TSessionManager implements SessionManager {
 
     // Mark as upgrade, to prevent any new clients to connect during close
     workspace.upgrade = true
-    workspace.backup = token.extra?.mode === 'backup'
     // If upgrade client is used.
     // Drop all existing clients
     workspace.closing = this.closeAll(wsString, workspace, 0, 'upgrade')
@@ -580,7 +578,6 @@ class TSessionManager implements SessionManager {
     branding: Branding | null
   ): Workspace {
     const upgrade = token.extra?.model === 'upgrade'
-    const backup = token.extra?.mode === 'backup'
     const context = ctx.newChild('🧲 session', {})
     const pipelineCtx = context.newChild('🧲 pipeline-factory', {})
     const workspace: Workspace = {
@@ -598,7 +595,6 @@ class TSessionManager implements SessionManager {
       sessions: new Map(),
       softShutdown: 3,
       upgrade,
-      backup,
       workspaceId: token.workspace,
       workspaceName,
       branding
@@ -986,7 +982,7 @@ class TSessionManager implements SessionManager {
 /**
  * @public
  */
-export function start (
+export function startSessionManager (
   ctx: MeasureContext,
   opt: {
     port: number

server/server/src/types.ts

@@ -116,7 +116,6 @@ export interface Workspace {
   pipeline: Promise<Pipeline>
   sessions: Map<string, { session: Session, socket: ConnectionSocket }>
   upgrade: boolean
-  backup: boolean
 
   closing?: Promise<void>
   softShutdown: number

server/tool/package.json

@@ -46,7 +46,6 @@
     "@hcengineering/contact": "^0.6.24",
     "@hcengineering/client-resources": "^0.6.27",
     "@hcengineering/client": "^0.6.18",
-    "ws": "^8.18.0",
     "@hcengineering/model": "^0.6.11",
     "@hcengineering/rank": "^0.6.4",
     "uuid": "^8.3.2",

server/translate/package.json

@@ -41,7 +41,6 @@
     "@hcengineering/core": "^0.6.32",
     "@hcengineering/platform": "^0.6.11",
     "@hcengineering/server-core": "^0.6.1",
-    "@hcengineering/server": "^0.6.4",
-    "got": "^11.8.3"
+    "@hcengineering/server": "^0.6.4"
   }
 }

server/translate/src/retranslate.ts

@@ -35,7 +35,6 @@ import {
   IndexedDoc
 } from '@hcengineering/server-core'
 
-import got from 'got'
 import { translateStateId, TranslationStage } from './types'
 
 /**
@@ -95,15 +94,17 @@ export class LibRetranslateStage implements TranslationStage {
     let english = false
     try {
       if (text.length > 0) {
-        const langResponse = await got.post(this.endpoint + '/detect', {
-          headers: {
-            'Content-Type': 'application/json'
-          },
-          json: {
-            q: text,
-            api_key: this.token
-          }
-        })
+        const langResponse = await (
+          await fetch(this.endpoint + '/detect', {
+            headers: {
+              'Content-Type': 'application/json'
+            },
+            body: JSON.stringify({
+              q: text,
+              api_key: this.token
+            })
+          })
+        ).json()
         english = JSON.parse(langResponse.body).some((it: any) => it.language === 'en' && it.confidence * 100 > 90)
       }
     } catch (err: any) {
@@ -163,18 +164,20 @@ export class LibRetranslateStage implements TranslationStage {
               try {
                 const st = Date.now()
                 console.log('retranslate:begin: ', doc._id, attr)
-                const translation = await got.post(this.endpoint + '/translate', {
-                  headers: {
-                    'Content-Type': 'application/json'
-                  },
-                  json: {
-                    q: toTranslate,
-                    source: 'auto',
-                    target: 'en',
-                    format: 'text',
-                    api_key: this.token
-                  }
-                })
+                const translation = await (
+                  await fetch(this.endpoint + '/translate', {
+                    headers: {
+                      'Content-Type': 'application/json'
+                    },
+                    body: JSON.stringify({
+                      q: toTranslate,
+                      source: 'auto',
+                      target: 'en',
+                      format: 'text',
+                      api_key: this.token
+                    })
+                  })
+                ).json()
 
                 const response: any = JSON.parse(translation.body)
                 console.log('retranslate:', doc._id, attr, Date.now() - st, response.translatedText.length)

server/ws/package.json

@@ -45,6 +45,7 @@
     "@hcengineering/rpc": "^0.6.5",
     "@hcengineering/server-core": "^0.6.1",
     "@hcengineering/server-token": "^0.6.11",
+    "@hcengineering/server": "^0.6.4",
     "bufferutil": "^4.0.8",
     "cors": "^2.8.5",
     "express": "^4.19.2",

server/ws/src/__tests__/server.test.ts

@@ -18,7 +18,6 @@ import { UNAUTHORIZED } from '@hcengineering/platform'
 import { RPCHandler, type Response } from '@hcengineering/rpc'
 import { generateToken } from '@hcengineering/server-token'
 import WebSocket from 'ws'
-import { start } from '../server'
 
 import {
   getWorkspaceId,
@@ -41,7 +40,7 @@ import {
   type TxResult
 } from '@hcengineering/core'
 import { createDummyStorageAdapter } from '@hcengineering/server-core'
-import { ClientSession } from '../client'
+import { ClientSession, startSessionManager } from '@hcengineering/server'
 import { startHttpServer } from '../server_http'
 import { genMinModel } from './minmodel'
 
@@ -60,7 +59,7 @@ describe('server', () => {
     return { modelDb, hierarchy }
   }
 
-  const cancelOp = start(new MeasureMetricsContext('test', {}), {
+  const cancelOp = startSessionManager(new MeasureMetricsContext('test', {}), {
     pipelineFactory: async () => {
       const { modelDb, hierarchy } = await getModelDb()
       return {
@@ -96,7 +95,7 @@ describe('server', () => {
       }
     },
     sessionFactory: (token, pipeline, workspaceId, branding) =>
-      new ClientSession(token, pipeline, workspaceId, branding),
+      new ClientSession(token, pipeline, workspaceId, branding, true),
     port: 3335,
     brandingMap: {},
     serverFactory: startHttpServer,
@@ -162,7 +161,7 @@ describe('server', () => {
   })
 
   it('reconnect', async () => {
-    const cancelOp = start(new MeasureMetricsContext('test', {}), {
+    const cancelOp = startSessionManager(new MeasureMetricsContext('test', {}), {
       pipelineFactory: async () => {
         const { modelDb, hierarchy } = await getModelDb()
         return {
@@ -208,7 +207,7 @@ describe('server', () => {
         }
       },
       sessionFactory: (token, pipeline, workspaceId, branding) =>
-        new ClientSession(token, pipeline, workspaceId, branding),
+        new ClientSession(token, pipeline, workspaceId, branding, true),
       port: 3336,
       brandingMap: {},
       serverFactory: startHttpServer,

server/ws/src/factories.ts

@@ -1,8 +0,0 @@
-import { startHttpServer } from './server_http'
-import { type ServerFactory } from './types'
-/**
- * @public
- */
-export const serverFactories: Record<string, ServerFactory> = {
-  ws: startHttpServer
-}

server/ws/src/index.ts

@@ -14,7 +14,4 @@
 // limitations under the License.
 //
 
-export * from './client'
-export { start } from './server'
 export * from './server_http'
-export * from './types'

server/ws/src/server_http.ts

@@ -17,24 +17,32 @@ import { Analytics } from '@hcengineering/analytics'
 import { generateId, toWorkspaceString, type MeasureContext } from '@hcengineering/core'
 import { UNAUTHORIZED, unknownStatus } from '@hcengineering/platform'
 import { RPCHandler, type Response } from '@hcengineering/rpc'
+import {
+  doSessionOp,
+  getFile,
+  getFileRange,
+  getStatistics,
+  LOGGING_ENABLED,
+  processRequest,
+  wipeStatistics,
+  type BlobResponse,
+  type ConnectionSocket,
+  type HandleRequestFunction,
+  type SessionManager,
+  type WebsocketData
+} from '@hcengineering/server'
 import { decodeToken, type Token } from '@hcengineering/server-token'
 import cors from 'cors'
 import express, { type Response as ExpressResponse } from 'express'
 import http, { type IncomingMessage } from 'http'
 import os from 'os'
 import { WebSocketServer, type RawData, type WebSocket } from 'ws'
-import { getStatistics, wipeStatistics } from './stats'
-import { LOGGING_ENABLED, type ConnectionSocket, type HandleRequestFunction, type SessionManager } from './types'
 
 import { type PipelineFactory, type StorageAdapter } from '@hcengineering/server-core'
 import 'bufferutil'
 import 'utf-8-validate'
-import { getFile, getFileRange, type BlobResponse } from './blobs'
-import { doSessionOp, processRequest, type WebsocketData } from './utils'
-
-const rpcHandler = new RPCHandler()
 let profiling = false
-
+const rpcHandler = new RPCHandler()
 /**
  * @public
  * @param sessionFactory -

services/ai-bot/pod-ai-bot/package.json

@@ -71,7 +71,6 @@
     "@hcengineering/server-client": "^0.6.0",
     "@hcengineering/server-core": "^0.6.1",
     "@hcengineering/server-token": "^0.6.11",
-    "@hcengineering/server-ws": "^0.6.11",
     "@hcengineering/setting": "^0.6.17",
     "@hcengineering/text": "^0.6.5",
     "cors": "^2.8.5",
@@ -82,7 +81,6 @@
     "mongodb": "^6.9.0",
     "node-fetch": "^2.6.6",
     "openai": "^4.56.0",
-    "js-tiktoken": "^1.0.14",
-    "ws": "^8.18.0"
+    "js-tiktoken": "^1.0.14"
   }
 }

services/analytics-collector/pod-analytics-collector/package.json

@@ -102,7 +102,6 @@
     "dotenv": "~16.0.0",
     "express": "^4.19.2",
     "mongodb": "^6.9.0",
-    "puppeteer": "^22.6.1",
-    "ws": "^8.18.0"
+    "puppeteer": "^22.6.1"
   }
 }

services/calendar/pod-calendar/package.json

@@ -73,7 +73,6 @@
     "googleapis": "^122.0.0",
     "google-auth-library": "^8.0.2",
     "jwt-simple": "^0.5.6",
-    "mongodb": "^6.9.0",
-    "ws": "^8.18.0"
+    "mongodb": "^6.9.0"
   }
 }

services/github/pod-github/package.json

@@ -77,7 +77,6 @@
     "@hcengineering/text": "^0.6.5",
     "@hcengineering/github": "^0.6.0",
     "@hcengineering/attachment": "^0.6.14",
-    "ws": "^8.18.0",
     "dotenv": "~16.0.0",
     "octokit": "^3.1.1",
     "@octokit/webhooks": "^12.0.3",

services/gmail/pod-gmail/package.json

@@ -77,7 +77,6 @@
     "gaxios": "^5.0.1",
     "jwt-simple": "^0.5.6",
     "mongodb": "^6.9.0",
-    "ws": "^8.18.0",
     "uuid": "^8.3.2"
   }
 }

services/love/package.json

@@ -64,7 +64,6 @@
     "@hcengineering/platform": "^0.6.11",
     "@hcengineering/server-client": "^0.6.0",
     "@hcengineering/server-token": "^0.6.11",
-    "ws": "^8.18.0",
     "livekit-server-sdk": "^2.0.10",
     "jwt-simple": "^0.5.6",
     "uuid": "^8.3.2",

services/rekoni/package.json

@@ -64,10 +64,12 @@
     "typescript": "^5.3.3",
     "esbuild": "^0.20.0",
     "eslint-plugin-n": "^15.4.0",
-    "prettier": "^3.1.0"
+    "prettier": "^3.1.0",
+    "@types/morgan": "~1.9.9"
   },
   "dependencies": {
     "@anticrm/skillset": "^0.6.0",
+    "@hcengineering/core": "^0.6.32",
     "@types/email-addresses": "^3.0.0",
     "body-parser": "^1.20.2",
     "cors": "^2.8.5",
@@ -82,7 +84,8 @@
     "mammoth": "^1.6.0",
     "mime-types": "~2.1.34",
     "pdfjs-dist": "2.12.313",
-    "sharp": "~0.32.0"
+    "sharp": "~0.32.0",
+    "morgan": "^1.10.0"
   },
   "description": "Document recognition service"
 }

services/rekoni/src/server.ts

@@ -13,11 +13,13 @@
 // limitations under the License.
 //
 
+import { MeasureMetricsContext, newMetrics } from '@hcengineering/core'
 import bodyParser from 'body-parser'
 import cors from 'cors'
 import express from 'express'
 import formData from 'express-form-data'
 import { type IncomingHttpHeaders, type Server } from 'http'
+import morgan from 'morgan'
 import os from 'os'
 import { type Readable } from 'stream'
 import config from './config'
@@ -40,11 +42,23 @@ const extractToken = (header: IncomingHttpHeaders): any => {
 export const startServer = async (): Promise<void> => {
   const app = express()
 
+  const ctx = new MeasureMetricsContext('rekini', {}, {}, newMetrics())
+
+  class MyStream {
+    write (text: string): void {
+      ctx.info(text)
+    }
+  }
+
+  const myStream = new MyStream()
+
   app.use(cors())
   app.use(express.json({ limit: '50mb' }))
   app.use(express.text({ limit: '50mb' }))
   app.use(express.raw({ limit: '50mb' }))
   app.use(bodyParser.json())
+
+  app.use(morgan('short', { stream: myStream }))
   app.use(
     bodyParser.text({
       type: 'text/plain'

services/sign/pod-sign/package.json

@@ -71,7 +71,6 @@
     "@signpdf/utils": "^3.2.4",
     "node-forge": "^1.3.1",
     "pdf-lib": "^1.12.0",
-    "ws": "^8.18.0",
     "mongodb": "^6.9.0"
   }
 }

services/telegram-bot/pod-telegram-bot/package.json

@@ -82,7 +82,6 @@
     "mongodb": "^6.9.0",
     "node-fetch": "^2.6.6",
     "otp-generator": "^4.0.1",
-    "telegraf": "^4.16.3",
-    "ws": "^8.18.0"
+    "telegraf": "^4.16.3"
   }
 }

services/telegram/pod-telegram/package.json

@@ -78,7 +78,6 @@
     "mime": "^3.0.0",
     "mongodb": "^6.9.0",
     "telegram": "2.22.2",
-    "ws": "^8.18.0",
     "uuid": "^8.3.2"
   }
 }

tools/apm/.eslintrc.js

@@ -1,7 +0,0 @@
-module.exports = {
-  extends: ['./node_modules/@hcengineering/platform-rig/profiles/node/eslint.config.json'],
-  parserOptions: {
-    tsconfigRootDir: __dirname,
-    project: './tsconfig.json'
-  }
-}

tools/apm/config/rig.json

@@ -1,5 +0,0 @@
-{  
-  "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
-  "rigPackageName": "@hcengineering/platform-rig",
-  "rigProfile": "node"
-}

tools/apm/jest.config.js

@@ -1,7 +0,0 @@
-module.exports = {
-  preset: 'ts-jest',
-  testEnvironment: 'node',
-  testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'],
-  roots: ["./src"],
-  coverageReporters: ["text-summary", "html"]
-}

tools/apm/package.json

@@ -1,44 +0,0 @@
-{
-  "name": "@hcengineering/apm",
-  "version": "0.6.0",
-  "main": "lib/index.js",
-  "svelte": "src/index.ts",
-  "types": "types/index.d.ts",
-  "author": "Anticrm Platform Contributors",
-  "template": "@hcengineering/node-package",
-  "license": "EPL-2.0",
-  "scripts": {
-    "build": "compile",
-    "build:watch": "compile",
-    "_phase:bundle": "rushx bundle",
-    "bundle": "mkdir -p bundle && esbuild src/index.ts --bundle --minify --platform=node > apm.js",
-    "format": "format src",
-    "test": "jest --passWithNoTests --silent --forceExit",
-    "_phase:build": "compile transpile src",
-    "_phase:test": "jest --passWithNoTests --silent --forceExit",
-    "_phase:format": "format src",
-    "_phase:validate": "compile validate"
-  },
-  "devDependencies": {
-    "@hcengineering/platform-rig": "^0.6.0",
-    "@typescript-eslint/eslint-plugin": "^6.11.0",
-    "eslint-plugin-import": "^2.26.0",
-    "eslint-plugin-promise": "^6.1.1",
-    "eslint-plugin-n": "^15.4.0",
-    "eslint": "^8.54.0",
-    "@typescript-eslint/parser": "^6.11.0",
-    "eslint-config-standard-with-typescript": "^40.0.0",
-    "prettier": "^3.1.0",
-    "typescript": "^5.3.3",
-    "esbuild": "^0.20.0",
-    "@types/node": "~20.11.16",
-    "jest": "^29.7.0",
-    "ts-jest": "^29.1.1",
-    "@types/jest": "^29.5.5"
-  },
-  "dependencies": {
-    "comment-json": "^4.2.2",
-    "commander": "^8.1.0"
-  },
-  "bin": "./lib/apm.js"
-}

tools/apm/src/index.ts

@@ -1,31 +0,0 @@
-import { program } from 'commander'
-import { join } from 'path'
-import { syncRushFiles } from './sync'
-
-console.info('Anticrm Platform Manager')
-
-program.version('0.6.0')
-
-program
-  .command('rush-sync <root>')
-  .description('Synchronized rush.js files with platform.')
-  .option('-e, --exclude <exclude>', 'List of exclude patterns to override excludes (comma separated)', '')
-  .option('-i, --include <include>', 'List of include patterns to override excludes (comma separated)', '')
-  .option('-s, --source <source>', 'Comma separated list of rush_source.json files', 'rush_source.json')
-  .action(async (root: string, cmd: { include: string, exclude: string, source: string }) => {
-    await syncRushFiles(
-      cmd.source.split(',').map((it) => join(process.cwd(), it.trim())),
-      process.cwd(),
-      root,
-      cmd.include
-        .split(',')
-        .map((it) => it.trim())
-        .filter((it) => it.length > 0),
-      cmd.exclude
-        .split(',')
-        .map((it) => it.trim())
-        .filter((it) => it.length > 0)
-    )
-  })
-
-program.parse(process.argv)

tools/apm/src/sync.ts

@@ -1,69 +0,0 @@
-import { type CommentArray, type CommentObject, parse, stringify } from 'comment-json'
-import { readFile, writeFile } from 'fs/promises'
-import path, { join } from 'path'
-
-interface RushPackage {
-  packageName: string
-  projectFolder: string
-  shouldPublish: boolean
-}
-
-export async function syncRushFiles (
-  root: string[],
-  targetRoot: string,
-  platformRoot: string,
-  include: string[],
-  exclude: string[]
-): Promise<void> {
-  const platformJson: CommentObject = parse(
-    (await readFile(join(process.cwd(), platformRoot, 'rush.json'))).toString()
-  ) as CommentObject
-
-  const rushjs = join(targetRoot, 'rush.json')
-
-  const abs = path.resolve(targetRoot).split(path.sep)
-
-  const projects: RushPackage[] = []
-
-  for (const prj of [join(platformRoot, 'rush.json'), ...root]) {
-    const sPath = path.dirname(path.resolve(prj)).split(path.sep)
-    const diff = path.join(...sPath.slice(abs.length))
-    console.log('processing', diff)
-    const rushJsonSource = parse((await readFile(prj)).toString()) as CommentObject
-    const sprojects = rushJsonSource.projects as unknown as CommentArray<RushPackage>
-    for (const [k, v] of Object.entries(rushJsonSource)) {
-      platformJson[k] = v
-    }
-
-    projects.push(
-      ...sprojects
-        .filter((it) => filterPackage(it, include, exclude))
-        .map((it) => ({
-          ...it,
-          projectFolder: join(diff, it.projectFolder),
-          shouldPublish: diff === '.' ? it.shouldPublish : false
-        }))
-    )
-  }
-
-  platformJson.projects = projects as unknown as CommentArray<CommentObject>
-
-  await writeFile(rushjs, stringify(platformJson, undefined, 2))
-}
-function filterPackage (it: RushPackage, include: string[], exclude: string[]): boolean {
-  const pkgName = it.packageName
-  for (const i of include) {
-    if (pkgName.includes(i)) {
-      console.log('Include', pkgName, i)
-      return true
-    }
-  }
-  for (const i of exclude) {
-    if (pkgName.includes(i)) {
-      console.log('Exclude', pkgName, i)
-      return false
-    }
-  }
-  console.log('Default', pkgName)
-  return true
-}

tools/apm/src/template.ts

@@ -1 +0,0 @@
-export async function createTemplate (root: string, platformRoot: string): Promise<void> {}

tools/apm/tsconfig.json

@@ -1,10 +0,0 @@
-{
-  "extends": "./node_modules/@hcengineering/platform-rig/profiles/node/tsconfig.json",
-
-  "compilerOptions": {
-    "rootDir": "./src",
-    "outDir": "./lib",
-    "declarationDir": "./types",
-    "tsBuildInfoFile": ".build/build.tsbuildinfo"
-  }
-}