UBERF-6161: Storage configuration (#5109)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

server/core/src/adapter.ts

@@ -33,6 +33,19 @@ import {
 } from '@hcengineering/core'
 import { type StorageAdapter } from './storage'
 
+/**
+ * @public
+ */
+export interface RawDBAdapter {
+  find: <T extends Doc>(
+    workspace: WorkspaceId,
+    domain: Domain,
+    query: DocumentQuery<T>,
+    options?: Omit<FindOptions<T>, 'projection' | 'lookup'>
+  ) => Promise<FindResult<T>>
+  upload: <T extends Doc>(workspace: WorkspaceId, domain: Domain, docs: T[]) => Promise<void>
+}
+
 /**
  * @public
  */
@@ -77,6 +90,7 @@ export interface TxAdapter extends DbAdapter {
  * @public
  */
 export type DbAdapterFactory = (
+  ctx: MeasureContext,
   hierarchy: Hierarchy,
   url: string,
   workspaceId: WorkspaceId,

server/core/src/indexer/content.ts

@@ -100,12 +100,7 @@ export class ContentRetrievalStage implements FullTextPipelineStage {
           // We need retrieve value of attached document content.
           const ref = doc.attributes[docKey(val.name, { _class: val.attributeOf })] as Ref<Doc>
           if (ref !== undefined && ref !== '') {
-            let docInfo: any | undefined
-            try {
-              docInfo = await this.storageAdapter?.stat(this.workspace, ref)
-            } catch (err: any) {
-              // not found.
-            }
+            const docInfo: any | undefined = await this.storageAdapter?.stat(this.metrics, this.workspace, ref)
             if (docInfo !== undefined && docInfo.size < 30 * 1024 * 1024) {
               // We have blob, we need to decode it to string.
               const contentType = ((docInfo.metaData['content-type'] as string) ?? '').split(';')[0]
@@ -116,7 +111,7 @@ export class ContentRetrievalStage implements FullTextPipelineStage {
                 if (doc.attributes[digestKey] !== digest) {
                   ;(update as any)[docUpdKey(digestKey)] = digest
 
-                  const readable = await this.storageAdapter?.get(this.workspace, ref)
+                  const readable = await this.storageAdapter?.get(this.metrics, this.workspace, ref)
 
                   if (readable !== undefined) {
                     let textContent = await this.metrics.with(

server/core/src/mem.ts

@@ -115,6 +115,7 @@ class InMemoryAdapter extends DummyDbAdapter implements DbAdapter {
  * @public
  */
 export async function createInMemoryAdapter (
+  ctx: MeasureContext,
   hierarchy: Hierarchy,
   url: string,
   workspaceId: WorkspaceId

server/core/src/server/aggregator.ts

@@ -0,0 +1,197 @@
+import core, {
+  DOMAIN_BLOB_DATA,
+  generateId,
+  groupByArray,
+  type Blob,
+  type MeasureContext,
+  type Ref,
+  type WorkspaceId
+} from '@hcengineering/core'
+import { type Readable } from 'stream'
+import { type RawDBAdapter } from '../adapter'
+import { type ListBlobResult, type StorageAdapter, type UploadedObjectInfo } from '../storage'
+
+import { v4 as uuid } from 'uuid'
+import { type StorageConfig, type StorageConfiguration } from '../types'
+
+/**
+ * Perform operations on storage adapter and map required information into BinaryDocument into provided DbAdapter storage.
+ */
+export class AggregatorStorageAdapter implements StorageAdapter {
+  constructor (
+    readonly adapters: Map<string, StorageAdapter>,
+    readonly defaultAdapter: string, // Adapter will be used to put new documents into
+    readonly dbAdapter: RawDBAdapter
+  ) {}
+
+  async initialize (ctx: MeasureContext, workspaceId: WorkspaceId): Promise<void> {
+    // We need to initialize internal table if it miss documents.
+  }
+
+  async exists (ctx: MeasureContext, workspaceId: WorkspaceId): Promise<boolean> {
+    for (const a of this.adapters.values()) {
+      if (!(await a.exists(ctx, workspaceId))) {
+        return false
+      }
+    }
+    return true
+  }
+
+  async make (ctx: MeasureContext, workspaceId: WorkspaceId): Promise<void> {
+    for (const a of this.adapters.values()) {
+      if (!(await a.exists(ctx, workspaceId))) {
+        await a.make(ctx, workspaceId)
+      }
+    }
+  }
+
+  async delete (ctx: MeasureContext, workspaceId: WorkspaceId): Promise<void> {
+    for (const a of this.adapters.values()) {
+      if (await a.exists(ctx, workspaceId)) {
+        await a.delete(ctx, workspaceId)
+      }
+    }
+  }
+
+  async remove (ctx: MeasureContext, workspaceId: WorkspaceId, objectNames: string[]): Promise<void> {
+    const docs = await this.dbAdapter.find<Blob>(workspaceId, DOMAIN_BLOB_DATA, {
+      _class: core.class.Blob,
+      _id: { $in: objectNames as Ref<Blob>[] }
+    })
+
+    // Group by provider and delegate into it.
+    const byProvider = groupByArray(docs, (item) => item.provider)
+    for (const [k, docs] of byProvider) {
+      const adapter = this.adapters.get(k)
+      if (adapter !== undefined) {
+        await adapter.remove(
+          ctx,
+          workspaceId,
+          docs.map((it) => it._id)
+        )
+      }
+    }
+  }
+
+  async list (ctx: MeasureContext, workspaceId: WorkspaceId, prefix?: string | undefined): Promise<ListBlobResult[]> {
+    return await this.dbAdapter.find<Blob>(workspaceId, DOMAIN_BLOB_DATA, {
+      _class: core.class.Blob,
+      _id: { $regex: `/^${prefix ?? ''}/i` }
+    })
+  }
+
+  async stat (ctx: MeasureContext, workspaceId: WorkspaceId, name: string): Promise<Blob | undefined> {
+    return (
+      await this.dbAdapter.find<Blob>(
+        workspaceId,
+        DOMAIN_BLOB_DATA,
+        { _class: core.class.Blob, _id: name as Ref<Blob> },
+        { limit: 1 }
+      )
+    ).shift()
+  }
+
+  async get (ctx: MeasureContext, workspaceId: WorkspaceId, name: string): Promise<Readable> {
+    const { provider, stat } = await this.findProvider(workspaceId, ctx, name)
+    return await provider.get(ctx, workspaceId, stat.storageId)
+  }
+
+  private async findProvider (
+    workspaceId: WorkspaceId,
+    ctx: MeasureContext,
+    objectName: string
+  ): Promise<{ provider: StorageAdapter, stat: Blob }> {
+    const stat = (
+      await this.dbAdapter.find<Blob>(
+        workspaceId,
+        DOMAIN_BLOB_DATA,
+        { _class: core.class.Blob, _id: objectName as Ref<Blob> },
+        { limit: 1 }
+      )
+    ).shift()
+    if (stat === undefined) {
+      throw new Error('No such object found')
+    }
+    const provider = this.adapters.get(stat.provider)
+    if (provider === undefined) {
+      throw new Error('No such provider found')
+    }
+    return { provider, stat }
+  }
+
+  async partial (
+    ctx: MeasureContext,
+    workspaceId: WorkspaceId,
+    objectName: string,
+    offset: number,
+    length?: number | undefined
+  ): Promise<Readable> {
+    const { provider, stat } = await this.findProvider(workspaceId, ctx, objectName)
+    return await provider.partial(ctx, workspaceId, stat.storageId, offset, length)
+  }
+
+  async read (ctx: MeasureContext, workspaceId: WorkspaceId, name: string): Promise<Buffer[]> {
+    const { provider, stat } = await this.findProvider(workspaceId, ctx, name)
+    return await provider.read(ctx, workspaceId, stat.storageId)
+  }
+
+  async put (
+    ctx: MeasureContext,
+    workspaceId: WorkspaceId,
+    objectName: string,
+    stream: string | Readable | Buffer,
+    contentType: string,
+    size?: number | undefined
+  ): Promise<UploadedObjectInfo> {
+    const provider = this.adapters.get(this.defaultAdapter)
+    if (provider === undefined) {
+      throw new Error('No such provider found')
+    }
+
+    const storageId = uuid()
+
+    const result = await provider.put(ctx, workspaceId, storageId, stream, contentType, size)
+
+    if (size === undefined || size === 0) {
+      const docStats = await provider.stat(ctx, workspaceId, storageId)
+      if (docStats !== undefined) {
+        if (contentType !== docStats.contentType) {
+          contentType = docStats.contentType
+        }
+        size = docStats.size
+      }
+    }
+
+    const blobDoc: Blob = {
+      _class: core.class.Blob,
+      _id: generateId(),
+      modifiedBy: core.account.System,
+      modifiedOn: Date.now(),
+      space: core.space.Configuration,
+      provider: this.defaultAdapter,
+      storageId,
+      size: size ?? 0,
+      contentType,
+      etag: result.etag,
+      version: result.versionId ?? null
+    }
+
+    await this.dbAdapter.upload<Blob>(workspaceId, DOMAIN_BLOB_DATA, [blobDoc])
+    return result
+  }
+}
+
+/**
+ * @public
+ */
+export function buildStorage (
+  config: StorageConfiguration,
+  dbAdapter: RawDBAdapter,
+  storageFactory: (kind: string, config: StorageConfig) => StorageAdapter
+): StorageAdapter {
+  const adapters = new Map<string, StorageAdapter>()
+  for (const c of config.storages) {
+    adapters.set(c.name, storageFactory(c.kind, c))
+  }
+  return new AggregatorStorageAdapter(adapters, config.default, dbAdapter)
+}

server/core/src/server/index.ts

@@ -35,11 +35,11 @@ import { type DbConfiguration } from '../configuration'
 import { createContentAdapter } from '../content'
 import { FullTextIndex } from '../fulltext'
 import { FullTextIndexPipeline } from '../indexer'
+import { createServiceAdaptersManager } from '../service'
 import { type StorageAdapter } from '../storage'
 import { Triggers } from '../triggers'
 import { type ServerStorageOptions } from '../types'
 import { TServerStorage } from './storage'
-import { createServiceAdaptersManager } from '../service'
 
 /**
  * @public
@@ -58,7 +58,10 @@ export async function createServerStorage (
 
   for (const key in conf.adapters) {
     const adapterConf = conf.adapters[key]
-    adapters.set(key, await adapterConf.factory(hierarchy, adapterConf.url, conf.workspace, modelDb, storageAdapter))
+    adapters.set(
+      key,
+      await adapterConf.factory(ctx, hierarchy, adapterConf.url, conf.workspace, modelDb, storageAdapter)
+    )
   }
 
   const txAdapter = adapters.get(conf.domains[DOMAIN_TX]) as TxAdapter
@@ -187,17 +190,21 @@ export async function createServerStorage (
  */
 export function createNullStorageFactory (): StorageAdapter {
   return {
-    exists: async (workspaceId: WorkspaceId) => {
+    initialize: async (ctx, workspaceId) => {},
+    exists: async (ctx, workspaceId: WorkspaceId) => {
       return false
     },
-    make: async (workspaceId: WorkspaceId) => {},
-    remove: async (workspaceId: WorkspaceId, objectNames: string[]) => {},
-    delete: async (workspaceId: WorkspaceId) => {},
-    list: async (workspaceId: WorkspaceId, prefix?: string) => [],
-    stat: async (workspaceId: WorkspaceId, objectName: string) => ({}) as any,
-    get: async (workspaceId: WorkspaceId, objectName: string) => ({}) as any,
-    put: async (workspaceId: WorkspaceId, objectName: string, stream: any, size?: number, qwe?: any) => ({}) as any,
-    read: async (workspaceId: WorkspaceId, name: string) => ({}) as any,
-    partial: async (workspaceId: WorkspaceId, objectName: string, offset: number, length?: number) => ({}) as any
+    make: async (ctx, workspaceId: WorkspaceId) => {},
+    remove: async (ctx, workspaceId: WorkspaceId, objectNames: string[]) => {},
+    delete: async (ctx, workspaceId: WorkspaceId) => {},
+    list: async (ctx, workspaceId: WorkspaceId, prefix?: string) => [],
+    stat: async (ctx, workspaceId: WorkspaceId, objectName: string) => ({}) as any,
+    get: async (ctx, workspaceId: WorkspaceId, objectName: string) => ({}) as any,
+    put: async (ctx, workspaceId: WorkspaceId, objectName: string, stream: any, contentType: string, size?: number) =>
+      ({}) as any,
+    read: async (ctx, workspaceId: WorkspaceId, name: string) => ({}) as any,
+    partial: async (ctx, workspaceId: WorkspaceId, objectName: string, offset: number, length?: number) => ({}) as any
   }
 }
+
+export { AggregatorStorageAdapter, buildStorage } from './aggregator'

server/core/src/storage.ts

@@ -1,51 +1,11 @@
-import { type WorkspaceId, toWorkspaceString } from '@hcengineering/core'
+import { type Blob, type MeasureContext, type WorkspaceId, toWorkspaceString } from '@hcengineering/core'
 import { type Readable } from 'stream'
 
-export interface MetadataItem {
-  Key: string
-  Value: string
-}
-export type BucketItem =
-  | {
-    name: string
-    size: number
-    etag: string
-    prefix?: never
-    lastModified: Date
-  }
-  | {
-    name?: never
-    etag?: never
-    lastModified?: never
-    prefix: string
-    size: 0
-  }
-
-export interface BucketItemStat {
-  size: number
-  etag: string
-  lastModified: Date
-  metaData: ItemBucketMetadata
-  versionId?: string | null
-}
-
 export interface UploadedObjectInfo {
   etag: string
   versionId: string | null
 }
 
-export interface ItemBucketMetadataList {
-  Items: MetadataItem[]
-}
-export type ItemBucketMetadata = Record<string, any>
-export type BucketItemWithMetadata = BucketItem & {
-  metadata?: ItemBucketMetadata | ItemBucketMetadataList
-}
-/**
- * @public
- */
-export type WorkspaceItem = Required<BucketItem> & { metaData: ItemBucketMetadata }
-
 /**
  * @public
  */
@@ -53,22 +13,33 @@ export function getBucketId (workspaceId: WorkspaceId): string {
   return toWorkspaceString(workspaceId, '.')
 }
 
-export interface StorageAdapter {
-  exists: (workspaceId: WorkspaceId) => Promise<boolean>
+export type ListBlobResult = Omit<Blob, 'contentType' | 'version'>
 
-  make: (workspaceId: WorkspaceId) => Promise<void>
-  remove: (workspaceId: WorkspaceId, objectNames: string[]) => Promise<void>
-  delete: (workspaceId: WorkspaceId) => Promise<void>
-  list: (workspaceId: WorkspaceId, prefix?: string) => Promise<WorkspaceItem[]>
-  stat: (workspaceId: WorkspaceId, objectName: string) => Promise<BucketItemStat>
-  get: (workspaceId: WorkspaceId, objectName: string) => Promise<Readable>
+export interface StorageAdapter {
+  initialize: (ctx: MeasureContext, workspaceId: WorkspaceId) => Promise<void>
+
+  exists: (ctx: MeasureContext, workspaceId: WorkspaceId) => Promise<boolean>
+  make: (ctx: MeasureContext, workspaceId: WorkspaceId) => Promise<void>
+  delete: (ctx: MeasureContext, workspaceId: WorkspaceId) => Promise<void>
+
+  remove: (ctx: MeasureContext, workspaceId: WorkspaceId, objectNames: string[]) => Promise<void>
+  list: (ctx: MeasureContext, workspaceId: WorkspaceId, prefix?: string) => Promise<ListBlobResult[]>
+  stat: (ctx: MeasureContext, workspaceId: WorkspaceId, objectName: string) => Promise<Blob | undefined>
+  get: (ctx: MeasureContext, workspaceId: WorkspaceId, objectName: string) => Promise<Readable>
   put: (
+    ctx: MeasureContext,
     workspaceId: WorkspaceId,
     objectName: string,
     stream: Readable | Buffer | string,
-    size?: number,
-    metaData?: ItemBucketMetadata
+    contentType: string,
+    size?: number
   ) => Promise<UploadedObjectInfo>
-  read: (workspaceId: WorkspaceId, name: string) => Promise<Buffer[]>
-  partial: (workspaceId: WorkspaceId, objectName: string, offset: number, length?: number) => Promise<Readable>
+  read: (ctx: MeasureContext, workspaceId: WorkspaceId, name: string) => Promise<Buffer[]>
+  partial: (
+    ctx: MeasureContext,
+    workspaceId: WorkspaceId,
+    objectName: string,
+    offset: number,
+    length?: number
+  ) => Promise<Readable>
 }

server/core/src/types.ts

@@ -41,9 +41,9 @@ import {
   type WorkspaceIdWithUrl
 } from '@hcengineering/core'
 import type { Asset, Resource } from '@hcengineering/platform'
-import { type StorageAdapter } from './storage'
 import { type Readable } from 'stream'
 import { type ServiceAdaptersManager } from './service'
+import { type StorageAdapter } from './storage'
 
 /**
  * @public
@@ -426,3 +426,13 @@ export interface ServiceAdapterConfig {
   db: string
   url: string
 }
+
+export interface StorageConfig {
+  name: string
+  kind: string
+}
+
+export interface StorageConfiguration {
+  default: string
+  storages: StorageConfig[]
+}