UBERF-6161: Storage configuration (#5109)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

server/server/package.json

@@ -41,6 +41,7 @@
     "@hcengineering/server-core": "^0.6.1",
     "@hcengineering/server-ws": "^0.6.11",
     "@hcengineering/mongo": "^0.6.1",
+    "@hcengineering/minio": "^0.6.0",
     "@hcengineering/elastic": "^0.6.0",
     "elastic-apm-node": "~3.26.0",
     "@hcengineering/server-token": "^0.6.7",

server/server/src/index.ts

@@ -21,3 +21,4 @@ export * from './backup'
 export * from './metrics'
 export * from './rekoni'
 export * from './ydoc'
+export * from './starter'

server/server/src/minio.ts

@@ -33,12 +33,13 @@ import core, {
   TxResult,
   WorkspaceId
 } from '@hcengineering/core'
-import { DbAdapter, StorageAdapter, WorkspaceItem } from '@hcengineering/server-core'
+import { DbAdapter, ListBlobResult, StorageAdapter } from '@hcengineering/server-core'
 
 class StorageBlobAdapter implements DbAdapter {
   constructor (
     readonly workspaceId: WorkspaceId,
-    readonly client: StorageAdapter
+    readonly client: StorageAdapter,
+    readonly ctx: MeasureContext
   ) {}
 
   async findAll<T extends Doc>(
@@ -63,18 +64,18 @@ class StorageBlobAdapter implements DbAdapter {
 
   find (domain: Domain): StorageIterator {
     let listReceived = false
-    let items: WorkspaceItem[] = []
+    let items: ListBlobResult[] = []
     let pos = 0
     return {
       next: async () => {
         if (!listReceived) {
-          items = await this.client.list(this.workspaceId)
+          items = await this.client.list(this.ctx, this.workspaceId)
           listReceived = true
         }
         if (pos < items?.length) {
           const item = items[pos]
           const result = {
-            id: item.name,
+            id: item._id,
             hash: item.etag,
             size: item.size
           }
@@ -89,17 +90,20 @@ class StorageBlobAdapter implements DbAdapter {
   async load (domain: Domain, docs: Ref<Doc>[]): Promise<Doc[]> {
     const result: Doc[] = []
     for (const item of docs) {
-      const stat = await this.client.stat(this.workspaceId, item)
-      const chunks: Buffer[] = await this.client.read(this.workspaceId, item)
+      const stat = await this.client.stat(this.ctx, this.workspaceId, item)
+      if (stat === undefined) {
+        throw new Error(`Could not find blob ${item}`)
+      }
+      const chunks: Buffer[] = await this.client.read(this.ctx, this.workspaceId, item)
       const final = Buffer.concat(chunks)
       const dta: BlobData = {
         _id: item as Ref<BlobData>,
         _class: core.class.BlobData,
         name: item as string,
         size: stat.size,
-        type: stat.metaData['content-type'],
+        type: stat.contentType,
         space: 'blob' as Ref<Space>,
-        modifiedOn: stat.lastModified.getTime(),
+        modifiedOn: stat.modifiedOn,
         modifiedBy: core.account.System,
         base64Data: final.toString('base64')
       }
@@ -118,20 +122,19 @@ class StorageBlobAdapter implements DbAdapter {
       const blob = d as unknown as BlobData
       // Remove existing document
       try {
-        await this.client.remove(this.workspaceId, [blob._id])
+        await this.client.remove(this.ctx, this.workspaceId, [blob._id])
       } catch (ee) {
         // ignore error
       }
       const buffer = Buffer.from(blob.base64Data, 'base64')
-      await this.client.put(this.workspaceId, blob._id, buffer, buffer.length, {
-        'Content-Type': blob.type,
-        lastModified: new Date(blob.modifiedOn)
-      })
+      // TODO: Add support of
+      /// lastModified: new Date(blob.modifiedOn)
+      await this.client.put(this.ctx, this.workspaceId, blob._id, buffer, blob.type, buffer.length)
     }
   }
 
   async clean (domain: Domain, docs: Ref<Doc>[]): Promise<void> {
-    await this.client.remove(this.workspaceId, docs)
+    await this.client.remove(this.ctx, this.workspaceId, docs)
   }
 
   async update (domain: Domain, operations: Map<Ref<Doc>, DocumentUpdate<Doc>>): Promise<void> {
@@ -143,6 +146,7 @@ class StorageBlobAdapter implements DbAdapter {
  * @public
  */
 export async function createStorageDataAdapter (
+  ctx: MeasureContext,
   hierarchy: Hierarchy,
   url: string,
   workspaceId: WorkspaceId,
@@ -154,9 +158,9 @@ export async function createStorageDataAdapter (
   }
   // We need to create bucket if it doesn't exist
   if (storage !== undefined) {
-    if (!(await storage.exists(workspaceId))) {
-      await storage.make(workspaceId)
+    if (!(await storage.exists(ctx, workspaceId))) {
+      await storage.make(ctx, workspaceId)
     }
   }
-  return new StorageBlobAdapter(workspaceId, storage)
+  return new StorageBlobAdapter(workspaceId, storage, ctx)
 }

server/server/src/server.ts

@@ -13,13 +13,14 @@
 // limitations under the License.
 //
 
-import { Hierarchy, ModelDb, WorkspaceId } from '@hcengineering/core'
+import { Hierarchy, MeasureContext, ModelDb, WorkspaceId } from '@hcengineering/core'
 import { DbAdapter, DummyDbAdapter } from '@hcengineering/server-core'
 
 /**
  * @public
  */
 export async function createNullAdapter (
+  ctx: MeasureContext,
   hierarchy: Hierarchy,
   url: string,
   workspaceId: WorkspaceId,
@@ -27,12 +28,3 @@ export async function createNullAdapter (
 ): Promise<DbAdapter> {
   return new DummyDbAdapter()
 }
-
-/**
- * @public
- */
-export interface MinioConfig {
-  endPoint: string
-  accessKey: string
-  secretKey: string
-}

server/server/src/starter.ts

@@ -0,0 +1,159 @@
+import { MinioConfig, MinioService } from '@hcengineering/minio'
+import { createRawMongoDBAdapter } from '@hcengineering/mongo'
+import { buildStorage, StorageAdapter, StorageConfiguration } from '@hcengineering/server-core'
+import { serverFactories, ServerFactory } from '@hcengineering/server-ws'
+
+export function storageConfigFromEnv (): StorageConfiguration {
+  const storageConfig: StorageConfiguration = JSON.parse(
+    process.env.STORAGE_CONFIG ?? '{ "default": "", "storages": []}'
+  )
+  if (storageConfig.storages.length === 0 || storageConfig.default === '') {
+    console.info('STORAGE_CONFIG is required for complex configuration, fallback to minio config')
+
+    let minioEndpoint = process.env.MINIO_ENDPOINT
+    if (minioEndpoint === undefined) {
+      console.error('MINIO_ENDPOINT is required')
+      process.exit(1)
+    }
+    const minioAccessKey = process.env.MINIO_ACCESS_KEY
+    if (minioAccessKey === undefined) {
+      console.error('MINIO_ACCESS_KEY is required')
+      process.exit(1)
+    }
+
+    let minioPort = 9000
+    const sp = minioEndpoint.split(':')
+    if (sp.length > 1) {
+      minioEndpoint = sp[0]
+      minioPort = parseInt(sp[1])
+    }
+
+    const minioSecretKey = process.env.MINIO_SECRET_KEY
+    if (minioSecretKey === undefined) {
+      console.error('MINIO_SECRET_KEY is required')
+      process.exit(1)
+    }
+
+    const minioConfig: MinioConfig = {
+      kind: 'minio',
+      name: 'minio',
+      port: minioPort,
+      region: 'us-east-1',
+      useSSL: false,
+      endpoint: minioEndpoint,
+      accessKeyId: minioAccessKey,
+      secretAccessKey: minioSecretKey
+    }
+    storageConfig.storages.push(minioConfig)
+    storageConfig.default = 'minio'
+  }
+  return storageConfig
+}
+
+export function serverConfigFromEnv (): {
+  url: string
+  elasticUrl: string
+  serverSecret: string
+  rekoniUrl: string
+  frontUrl: string
+  sesUrl: string | undefined
+  accountsUrl: string
+  serverPort: number
+  serverFactory: ServerFactory
+  enableCompression: boolean
+  elasticIndexName: string
+} {
+  const serverPort = parseInt(process.env.SERVER_PORT ?? '3333')
+  const serverFactory = serverFactories[(process.env.SERVER_PROVIDER as string) ?? 'ws'] ?? serverFactories.ws
+  const enableCompression = (process.env.ENABLE_COMPRESSION ?? 'true') === 'true'
+
+  const url = process.env.MONGO_URL
+  if (url === undefined) {
+    console.error('please provide mongodb url')
+    process.exit(1)
+  }
+
+  const elasticUrl = process.env.ELASTIC_URL
+  if (elasticUrl === undefined) {
+    console.error('please provide elastic url')
+    process.exit(1)
+  }
+  const elasticIndexName = process.env.ELASTIC_INDEX_NAME
+  if (elasticIndexName === undefined) {
+    console.log('Please provide ELASTIC_INDEX_NAME')
+    process.exit(1)
+  }
+
+  const serverSecret = process.env.SERVER_SECRET
+  if (serverSecret === undefined) {
+    console.log('Please provide server secret')
+    process.exit(1)
+  }
+
+  const rekoniUrl = process.env.REKONI_URL
+  if (rekoniUrl === undefined) {
+    console.log('Please provide REKONI_URL url')
+    process.exit(1)
+  }
+
+  const frontUrl = process.env.FRONT_URL
+  if (frontUrl === undefined) {
+    console.log('Please provide FRONT_URL url')
+    process.exit(1)
+  }
+
+  const sesUrl = process.env.SES_URL
+
+  const accountsUrl = process.env.ACCOUNTS_URL
+  if (accountsUrl === undefined) {
+    console.log('Please provide ACCOUNTS_URL url')
+    process.exit(1)
+  }
+  return {
+    url,
+    elasticUrl,
+    elasticIndexName,
+    serverSecret,
+    rekoniUrl,
+    frontUrl,
+    sesUrl,
+    accountsUrl,
+    serverPort,
+    serverFactory,
+    enableCompression
+  }
+}
+
+// Temporary solution, until migration will be implemented.
+const ONLY_MINIO = true
+
+export function buildStorageFromConfig (config: StorageConfiguration, dbUrl: string): StorageAdapter {
+  if (ONLY_MINIO) {
+    const minioConfig = config.storages.find((it) => it.kind === 'minio') as MinioConfig
+    if (minioConfig === undefined) {
+      throw new Error('minio config is required')
+    }
+
+    return new MinioService({
+      accessKey: minioConfig.accessKeyId,
+      secretKey: minioConfig.accessKeyId,
+      endPoint: minioConfig.endpoint,
+      port: minioConfig.port,
+      useSSL: minioConfig.useSSL
+    })
+  }
+  return buildStorage(config, createRawMongoDBAdapter(dbUrl), (kind, config) => {
+    if (kind === MinioService.config) {
+      const c = config as MinioConfig
+      return new MinioService({
+        accessKey: c.accessKeyId,
+        secretKey: c.accessKeyId,
+        endPoint: c.endpoint,
+        port: c.port,
+        useSSL: c.useSSL
+      })
+    } else {
+      throw new Error('Unsupported storage kind:' + kind)
+    }
+  })
+}