diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 0000000000..53d1c14db3 --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +v22 diff --git a/README.md b/README.md index 9bb3420a7e..a6b1444902 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ This repository includes the following server packages: - [@hcengineering/minio](packages/minio) - MinIO storage provider for object storage - [@hcengineering/s3](packages/s3) - AWS S3 compatible storage provider - [@hcengineering/datalake](packages/datalake) - Data lake storage and management +- [@hcengineering/hulylake](packages/hulylake) - Huly lake storage and management ### Infrastructure diff --git a/common/changes/@hcengineering/hulylake/feat-huylake-storage-adapter_2025-10-09-17-46.json b/common/changes/@hcengineering/hulylake/feat-huylake-storage-adapter_2025-10-09-17-46.json new file mode 100644 index 0000000000..dc69ad1560 --- /dev/null +++ b/common/changes/@hcengineering/hulylake/feat-huylake-storage-adapter_2025-10-09-17-46.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@hcengineering/hulylake", + "comment": "add hulylake storage adapter", + "type": "patch" + } + ], + "packageName": "@hcengineering/hulylake" +} \ No newline at end of file diff --git a/common/changes/@hcengineering/middleware/feat-huylake-storage-adapter_2025-10-09-17-46.json b/common/changes/@hcengineering/middleware/feat-huylake-storage-adapter_2025-10-09-17-46.json new file mode 100644 index 0000000000..d9878f8c74 --- /dev/null +++ b/common/changes/@hcengineering/middleware/feat-huylake-storage-adapter_2025-10-09-17-46.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@hcengineering/middleware", + "comment": "", + "type": "none" + } + ], + "packageName": "@hcengineering/middleware" +} \ No newline at end of file diff --git a/common/changes/@hcengineering/server-storage/feat-huylake-storage-adapter_2025-10-09-17-46.json b/common/changes/@hcengineering/server-storage/feat-huylake-storage-adapter_2025-10-09-17-46.json new file mode 100644 index 0000000000..71fd8d8441 --- /dev/null +++ b/common/changes/@hcengineering/server-storage/feat-huylake-storage-adapter_2025-10-09-17-46.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@hcengineering/server-storage", + "comment": "add hulylake storage adapter", + "type": "patch" + } + ], + "packageName": "@hcengineering/server-storage" +} \ No newline at end of file diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 20509146e2..9345668657 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -156,11 +156,11 @@ importers: specifier: ^0.7.3 version: 0.7.3 '@hcengineering/communication-sdk-types': - specifier: ^0.7.0 - version: 0.7.0 + specifier: ^0.7.3 + version: 0.7.3 '@hcengineering/communication-types': - specifier: ^0.7.0 - version: 0.7.0 + specifier: ^0.7.3 + version: 0.7.3 '@hcengineering/core': specifier: ^0.7.3 version: 0.7.4 @@ -354,6 +354,70 @@ importers: specifier: ^5.8.3 version: 5.9.3 + ../../packages/hulylake: + dependencies: + '@hcengineering/core': + specifier: ^0.7.3 + version: 0.7.4 + '@hcengineering/hulylake-client': + specifier: ^0.7.4 + version: 0.7.4 + '@hcengineering/platform': + specifier: ^0.7.3 + version: 0.7.3 + '@hcengineering/server-core': + specifier: workspace:0.7.0 + version: link:../core + '@hcengineering/server-token': + specifier: ^0.7.0 + version: 0.7.3 + devDependencies: + '@hcengineering/platform-rig': + specifier: ^0.7.10 + version: 0.7.10 + '@types/jest': + specifier: ^29.5.5 + version: 29.5.14 + '@types/node': + specifier: ^22.15.29 + version: 22.18.8 + '@typescript-eslint/eslint-plugin': + specifier: ^6.11.0 + version: 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1)(typescript@5.9.3) + '@typescript-eslint/parser': + specifier: ^6.11.0 + version: 6.21.0(eslint@8.57.1)(typescript@5.9.3) + eslint: + specifier: ^8.54.0 + version: 8.57.1 + eslint-config-standard-with-typescript: + specifier: ^40.0.0 + version: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1)(typescript@5.9.3))(eslint-plugin-import@2.32.0(eslint@8.57.1))(eslint-plugin-n@15.7.0(eslint@8.57.1))(eslint-plugin-promise@6.6.0(eslint@8.57.1))(eslint@8.57.1)(typescript@5.9.3) + eslint-plugin-import: + specifier: ^2.26.0 + version: 2.32.0(eslint@8.57.1) + eslint-plugin-n: + specifier: ^15.4.0 + version: 15.7.0(eslint@8.57.1) + eslint-plugin-promise: + specifier: ^6.1.1 + version: 6.6.0(eslint@8.57.1) + jest: + specifier: ^29.7.0 + version: 29.7.0(@types/node@22.18.8)(ts-node@10.9.2(@types/node@22.18.8)(typescript@5.9.3)) + prettier: + specifier: ^3.1.0 + version: 3.6.2 + ts-jest: + specifier: ^29.1.1 + version: 29.4.4(@babel/core@7.28.4)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.28.4))(jest-util@29.7.0)(jest@29.7.0(@types/node@22.18.8)(ts-node@10.9.2(@types/node@22.18.8)(typescript@5.9.3)))(typescript@5.9.3) + ts-node: + specifier: ^10.8.0 + version: 10.9.2(@types/node@22.18.8)(typescript@5.9.3) + typescript: + specifier: ^5.8.3 + version: 5.9.3 + ../../packages/kafka: dependencies: '@hcengineering/core': @@ -813,6 +877,9 @@ importers: '@hcengineering/datalake': specifier: workspace:0.7.0 version: link:../datalake + '@hcengineering/hulylake': + specifier: workspace:0.7.0 + version: link:../hulylake '@hcengineering/minio': specifier: workspace:0.7.0 version: link:../minio @@ -1444,15 +1511,21 @@ packages: '@hcengineering/client@0.7.3': resolution: {integrity: sha512-kMK8WwK7b1ViZa9HP6xwyX5Es0dqHXCkBmoAtd1SEaU97sGnuUC/vYPcqDJtpm7y9pDhyFfwoHQAk8UYEa0wgQ==} - '@hcengineering/communication-sdk-types@0.7.0': - resolution: {integrity: sha512-N4I8YqEJ6ByzIc4bn3+wmkSr3O79EQArdfaJtt+b4mPGi3sZwYO9XWhql67lpM7kMBuF74TbCiNKCmvDclE+gQ==} + '@hcengineering/communication-sdk-types@0.7.3': + resolution: {integrity: sha512-Noi9J+s0IH5qbnt5wV8WXMClKmCCnvPpquf3GjJ8CuBmmQvUAv0ogIy7mbuC60ZZDJaHHY9ihtrxZY3lNOJsvA==} - '@hcengineering/communication-types@0.7.0': - resolution: {integrity: sha512-4qh3PGYcm6HuczTwXp4iPlsx6ChGnCOQ54fhj/PRWgWT2nuY63jOp9ZWwxrOEblsmOkcWcBrrfdmvddmz+fZBw==} + '@hcengineering/communication-types@0.7.3': + resolution: {integrity: sha512-FeOx3oXgzhTq2abEZY4YZ6aqMTv0QxSx94OAkWvW1JD8l0GyYCf2Qnx1aC9pRIUzrEA61Eri9PLBj/+y00BItw==} '@hcengineering/core@0.7.4': resolution: {integrity: sha512-SPlJEzwa/w4E1XF2qY4d8mxWwYMWu8Zhxrvdlpfs0vytWOInknaMxc7qp6N3YVz13lAFiQf/wnJiETHFGtqmJQ==} + '@hcengineering/core@0.7.5': + resolution: {integrity: sha512-GzbYSoBVzqfqgxWDaqvnlVPASzhIYswRFBOdn+iNXpn4T8iZvXSpkFMxWWX/zI6BgXqOjKS0jR54N3FaZzHvTw==} + + '@hcengineering/hulylake-client@0.7.4': + resolution: {integrity: sha512-MRcyslhQlse72jYgYZsm8amEmxcIxVnWpd2xgcBzgLcTIcegerwwH8oI6gGYK7JTZJ9ERe8QPlsv/M9vWNgf7A==} + '@hcengineering/measurements@0.7.10': resolution: {integrity: sha512-6xsrHorpJkQth4GVtdPoCM5s6USyYtoV9j8tCPhbLUk5yQn+CIvrDZxN60JGBOqvL6hYb6cis2FrAG5NC3HYPw==} @@ -1469,6 +1542,9 @@ packages: '@hcengineering/query@0.7.3': resolution: {integrity: sha512-rfKsi44gexHMUFUraofrDkJURUaYSoBqef4qKKPKPHM42VButYXypfBCom0EYX/c61vJyH5oCEoIWi2RY4UJBQ==} + '@hcengineering/retry@0.7.3': + resolution: {integrity: sha512-uL+drKVnLO9cHCttDyiJnBHlIZh8+iv/qMClCv/BbIM+FRJtMB3TMkUfpKCtxjR5o8D9CG3kCzwnfQNvZBW/Ug==} + '@hcengineering/rpc@0.7.3': resolution: {integrity: sha512-vyDU+bu1frT4zC3z+OWgsgUv8+U2EmYsLrMVSM0IIc28z4PTCIkW7rQ7oXmdsyZ2CjdrCt39I7ZdL65qBCVqiQ==} @@ -5096,14 +5172,14 @@ snapshots: '@hcengineering/core': 0.7.4 '@hcengineering/platform': 0.7.3 - '@hcengineering/communication-sdk-types@0.7.0': + '@hcengineering/communication-sdk-types@0.7.3': dependencies: - '@hcengineering/communication-types': 0.7.0 - '@hcengineering/core': 0.7.4 + '@hcengineering/communication-types': 0.7.3 + '@hcengineering/core': 0.7.5 - '@hcengineering/communication-types@0.7.0': + '@hcengineering/communication-types@0.7.3': dependencies: - '@hcengineering/core': 0.7.4 + '@hcengineering/core': 0.7.5 '@hcengineering/core@0.7.4': dependencies: @@ -5112,6 +5188,18 @@ snapshots: '@hcengineering/platform': 0.7.3 fast-equals: 5.3.2 + '@hcengineering/core@0.7.5': + dependencies: + '@hcengineering/analytics': 0.7.3 + '@hcengineering/measurements': 0.7.10 + '@hcengineering/platform': 0.7.3 + fast-equals: 5.3.2 + + '@hcengineering/hulylake-client@0.7.4': + dependencies: + '@hcengineering/core': 0.7.5 + '@hcengineering/retry': 0.7.3 + '@hcengineering/measurements@0.7.10': {} '@hcengineering/platform-rig@0.7.10': @@ -5145,6 +5233,8 @@ snapshots: '@hcengineering/platform': 0.7.3 fast-equals: 5.3.2 + '@hcengineering/retry@0.7.3': {} + '@hcengineering/rpc@0.7.3': dependencies: '@hcengineering/core': 0.7.4 diff --git a/packages/core/package.json b/packages/core/package.json index 4fe905c277..c5fd7dc27a 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -36,8 +36,8 @@ }, "dependencies": { "@hcengineering/analytics": "^0.7.3", - "@hcengineering/communication-sdk-types": "^0.7.0", - "@hcengineering/communication-types": "^0.7.0", + "@hcengineering/communication-sdk-types": "^0.7.3", + "@hcengineering/communication-types": "^0.7.3", "@hcengineering/core": "^0.7.3", "@hcengineering/platform": "^0.7.3", "@hcengineering/query": "^0.7.3", diff --git a/packages/hulylake/.eslintrc.js b/packages/hulylake/.eslintrc.js new file mode 100644 index 0000000000..ce90fb9646 --- /dev/null +++ b/packages/hulylake/.eslintrc.js @@ -0,0 +1,7 @@ +module.exports = { + extends: ['./node_modules/@hcengineering/platform-rig/profiles/node/eslint.config.json'], + parserOptions: { + tsconfigRootDir: __dirname, + project: './tsconfig.json' + } +} diff --git a/packages/hulylake/.npmignore b/packages/hulylake/.npmignore new file mode 100644 index 0000000000..e3ec093c38 --- /dev/null +++ b/packages/hulylake/.npmignore @@ -0,0 +1,4 @@ +* +!/lib/** +!CHANGELOG.md +/lib/**/__tests__/ diff --git a/packages/hulylake/config/rig.json b/packages/hulylake/config/rig.json new file mode 100644 index 0000000000..78cc5a1733 --- /dev/null +++ b/packages/hulylake/config/rig.json @@ -0,0 +1,5 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json", + "rigPackageName": "@hcengineering/platform-rig", + "rigProfile": "node" +} diff --git a/packages/hulylake/jest.config.js b/packages/hulylake/jest.config.js new file mode 100644 index 0000000000..2cfd408b67 --- /dev/null +++ b/packages/hulylake/jest.config.js @@ -0,0 +1,7 @@ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'], + roots: ["./src"], + coverageReporters: ["text-summary", "html"] +} diff --git a/packages/hulylake/package.json b/packages/hulylake/package.json new file mode 100644 index 0000000000..c56d3cef92 --- /dev/null +++ b/packages/hulylake/package.json @@ -0,0 +1,68 @@ +{ + "name": "@hcengineering/hulylake", + "version": "0.7.0", + "main": "lib/index.js", + "svelte": "src/index.ts", + "types": "types/index.d.ts", + "author": "Anticrm Platform Contributors", + "template": "@hcengineering/node-package", + "license": "EPL-2.0", + "scripts": { + "build": "compile", + "build:watch": "compile", + "test": "jest --passWithNoTests --silent --forceExit", + "format": "format src", + "_phase:build": "compile transpile src", + "_phase:test": "jest --passWithNoTests --silent --forceExit", + "_phase:format": "format src", + "_phase:validate": "compile validate" + }, + "devDependencies": { + "@hcengineering/platform-rig": "^0.7.10", + "@typescript-eslint/eslint-plugin": "^6.11.0", + "eslint-plugin-import": "^2.26.0", + "eslint-plugin-promise": "^6.1.1", + "eslint-plugin-n": "^15.4.0", + "eslint": "^8.54.0", + "@typescript-eslint/parser": "^6.11.0", + "eslint-config-standard-with-typescript": "^40.0.0", + "prettier": "^3.1.0", + "typescript": "^5.8.3", + "@types/node": "^22.15.29", + "jest": "^29.7.0", + "ts-jest": "^29.1.1", + "@types/jest": "^29.5.5", + "ts-node": "^10.8.0" + }, + "dependencies": { + "@hcengineering/core": "^0.7.3", + "@hcengineering/platform": "^0.7.3", + "@hcengineering/server-core": "workspace:0.7.0", + "@hcengineering/server-token": "^0.7.0", + "@hcengineering/hulylake-client": "^0.7.4" + }, + "repository": "https://github.com/hcengineering/huly.server", + "publishConfig": { + "access": "public" + }, + "files": [ + "lib/**/*", + "types/**/*", + "src/**/*", + "!lib/**/*.test.*", + "!lib/**/*.spec.*", + "!lib/__tests__/**/*", + "!lib/__test__/**/*", + "!lib/tests/**/*", + "!types/**/*.test.*", + "!types/**/*.spec.*", + "!types/__tests__/**/*", + "!types/__test__/**/*", + "!types/tests/**/*", + "!src/**/*.test.*", + "!src/**/*.spec.*", + "!src/__tests__/**/*", + "!src/__test__/**/*", + "!src/tests/**/*" + ] +} diff --git a/packages/hulylake/src/__tests__/utils.test.ts b/packages/hulylake/src/__tests__/utils.test.ts new file mode 100644 index 0000000000..ab98910407 --- /dev/null +++ b/packages/hulylake/src/__tests__/utils.test.ts @@ -0,0 +1,40 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { wrapETag, unwrapETag } from '../utils' + +describe('unwrapETag', () => { + it('should unwrap weak validator prefix', () => { + expect(unwrapETag('W/"abc"')).toBe('abc') + }) + + it('should unwrap strong validator prefix', () => { + expect(unwrapETag('"abc"')).toBe('abc') + }) + + it('should unwrap no validator prefix', () => { + expect(unwrapETag('abc')).toBe('abc') + }) +}) + +describe('wrapETag', () => { + it('should wrap strong validator prefix', () => { + expect(wrapETag('abc')).toBe('"abc"') + }) + + it('should wrap weak validator prefix', () => { + expect(wrapETag('abc', true)).toBe('W/"abc"') + }) +}) diff --git a/packages/hulylake/src/error.ts b/packages/hulylake/src/error.ts new file mode 100644 index 0000000000..b5b60262b1 --- /dev/null +++ b/packages/hulylake/src/error.ts @@ -0,0 +1,35 @@ +// +// Copyright © 2024 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +export class NetworkError extends Error { + constructor (message: string) { + super(message) + this.name = 'NetworkError' + } +} + +export class DatalakeError extends Error { + constructor (message: string) { + super(message) + this.name = 'DatalakeError' + } +} + +export class NotFoundError extends DatalakeError { + constructor (message = 'Not Found') { + super(message) + this.name = 'NotFoundError' + } +} diff --git a/packages/hulylake/src/index.ts b/packages/hulylake/src/index.ts new file mode 100644 index 0000000000..feec56529f --- /dev/null +++ b/packages/hulylake/src/index.ts @@ -0,0 +1,256 @@ +// +// Copyright © 2024 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import core, { + type Blob, + type MeasureContext, + type Ref, + type Timestamp, + type WorkspaceIds, + systemAccountUuid, + withContext +} from '@hcengineering/core' +import { type HulylakeClient, type PutOptions, getClient } from '@hcengineering/hulylake-client' +import { getMetadata } from '@hcengineering/platform' +import { + type BlobStorageIterator, + type BucketInfo, + type StorageAdapter, + type StorageConfig, + type StorageConfiguration, + type UploadedObjectInfo +} from '@hcengineering/server-core' +import serverToken, { generateToken } from '@hcengineering/server-token' +import { Readable } from 'stream' +import { NotFoundError } from './error' + +/** + * @public + */ +export interface HulylakeConfig extends StorageConfig { + kind: 'hulylake' +} + +/** + * @public + */ +export function createHulylakeClient (cfg: HulylakeConfig, token: string): HulylakeClient { + const endpoint = Number.isInteger(cfg.port) ? `${cfg.endpoint}:${cfg.port}` : cfg.endpoint + return getClient(endpoint, token) +} + +export const CONFIG_KIND = 'hulylake' + +/** + * @public + */ +export interface HulylakeClientOptions { + retryCount?: number + retryInterval?: number +} + +/** + * @public + */ +export class HulylakeService implements StorageAdapter { + private readonly client: HulylakeClient + private readonly retryCount: number + private readonly retryInterval: number + + constructor ( + readonly cfg: HulylakeConfig, + readonly options: HulylakeClientOptions = {} + ) { + const secret = getMetadata(serverToken.metadata.Secret) + if (secret === undefined) { + console.warn('Server secret not set, hulylake storage adapter initialized with default secret') + } + const token = generateToken(systemAccountUuid, undefined) + this.client = createHulylakeClient(cfg, token) + this.retryCount = options.retryCount ?? 5 + this.retryInterval = options.retryInterval ?? 50 + } + + async initialize (ctx: MeasureContext, wsIds: WorkspaceIds): Promise {} + + async close (): Promise {} + + async exists (ctx: MeasureContext, wsIds: WorkspaceIds): Promise { + // workspace/buckets not supported, assume that always exist + return true + } + + @withContext('make') + async make (ctx: MeasureContext, wsIds: WorkspaceIds): Promise { + // workspace/buckets not supported, assume that always exist + } + + async listBuckets (ctx: MeasureContext): Promise { + return [] + } + + @withContext('remove') + async remove (ctx: MeasureContext, wsIds: WorkspaceIds, objectNames: string[]): Promise { + await Promise.all( + objectNames.map(async (objectName) => { + await this.client.delete(wsIds.uuid, objectName) + }) + ) + } + + @withContext('delete') + async delete (ctx: MeasureContext, wsIds: WorkspaceIds): Promise { + // not supported, just do nothing and pretend we deleted the workspace + } + + @withContext('listStream') + async listStream (ctx: MeasureContext, wsIds: WorkspaceIds): Promise { + throw new Error('not implemented') + } + + @withContext('stat') + async stat (ctx: MeasureContext, wsIds: WorkspaceIds, objectName: string): Promise { + const result = await this.client.head(wsIds.uuid, objectName) + if (result.ok) { + return { + provider: '', + _class: core.class.Blob, + _id: objectName as Ref, + contentType: result.contentType ?? 'application/octet-stream', + size: result.contentLength ?? 0, + etag: result.etag ?? '', + space: core.space.Configuration, + modifiedBy: core.account.System, + modifiedOn: result.lastModified as Timestamp, + version: null + } + } + } + + @withContext('get') + async get (ctx: MeasureContext, wsIds: WorkspaceIds, objectName: string): Promise { + const res = await this.client.get(wsIds.uuid, objectName) + if (res?.body === undefined) { + throw new NotFoundError() + } + + return fromFetchBody(res.body) + } + + @withContext('put') + async put ( + ctx: MeasureContext, + wsIds: WorkspaceIds, + objectName: string, + stream: Readable | Buffer | string, + contentType: string, + size?: number + ): Promise { + size = size ?? (typeof stream === 'string' ? Buffer.byteLength(stream) : undefined) + const body = toFetchBody(stream) + + if (size === undefined) { + throw new Error('Size must be specified for string or stream body') + } + + const params: PutOptions = { + contentLength: size, + contentType + } + + const { etag } = await ctx.with('put', {}, (ctx) => this.client.put(wsIds.uuid, objectName, body, params), { + workspace: wsIds.uuid, + objectName + }) + + return { + etag: etag ?? '', + versionId: '' + } + } + + @withContext('read') + async read (ctx: MeasureContext, wsIds: WorkspaceIds, objectName: string): Promise { + const res = await this.client.get(wsIds.uuid, objectName) + if (res?.body === undefined) { + throw new NotFoundError() + } + + const body = fromFetchBody(res.body) + + const chunks: Buffer[] = [] + for await (const chunk of body) { + chunks.push(chunk) + } + + return chunks + } + + @withContext('partial') + async partial ( + ctx: MeasureContext, + wsIds: WorkspaceIds, + objectName: string, + offset: number, + length?: number + ): Promise { + const res = await this.client.partial(wsIds.uuid, objectName, offset, length) + if (res?.body === undefined) { + throw new NotFoundError() + } + + return fromFetchBody(res.body) + } + + async getUrl (ctx: MeasureContext, wsIds: WorkspaceIds, objectName: string): Promise { + return this.client.objectUrl(wsIds.uuid, objectName) + } +} + +export function processConfigFromEnv (storageConfig: StorageConfiguration): string | undefined { + const endpoint = process.env.HULYLAKE_ENDPOINT + if (endpoint === undefined) { + return 'HULYLAKE_ENDPOINT' + } + + const config: HulylakeConfig = { + kind: 'hulylake', + name: 'hulylake', + endpoint + } + + storageConfig.storages.push(config) + storageConfig.default = 'hulylake' +} + +function toFetchBody (body: Readable | Buffer | string): ReadableStream | ArrayBuffer | string { + if (typeof body === 'string') { + return body + } else if (Buffer.isBuffer(body)) { + return new Uint8Array(body).buffer + } else { + return Readable.toWeb(body) as ReadableStream + } +} + +function fromFetchBody (body: ReadableStream | ArrayBuffer | string): Readable { + if (typeof body === 'string') { + return Readable.from(body) + } else if (body instanceof ArrayBuffer) { + return Readable.from(Buffer.from(body)) + } else { + return Readable.fromWeb(body as any) + } +} diff --git a/packages/hulylake/src/utils.ts b/packages/hulylake/src/utils.ts new file mode 100644 index 0000000000..42d60ff128 --- /dev/null +++ b/packages/hulylake/src/utils.ts @@ -0,0 +1,32 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +export function unwrapETag (etag: string): string { + if (etag.startsWith('W/')) { + etag = etag.substring(2) + } + + if (etag.startsWith('"') && etag.endsWith('"')) { + etag = etag.slice(1, -1) + } + + return etag +} + +export function wrapETag (etag: string, weak: boolean = false): string { + etag = unwrapETag(etag) + const quoted = etag.startsWith('"') ? etag : `"${etag}"` + return weak ? `W/${quoted}` : quoted +} diff --git a/packages/hulylake/tsconfig.json b/packages/hulylake/tsconfig.json new file mode 100644 index 0000000000..c6a877cf6c --- /dev/null +++ b/packages/hulylake/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "./node_modules/@hcengineering/platform-rig/profiles/node/tsconfig.json", + + "compilerOptions": { + "rootDir": "./src", + "outDir": "./lib", + "declarationDir": "./types", + "tsBuildInfoFile": ".build/build.tsbuildinfo" + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "lib", "dist", "types", "bundle"] +} \ No newline at end of file diff --git a/packages/middleware/src/spaceSecurity.ts b/packages/middleware/src/spaceSecurity.ts index d0022d43aa..0ac2df6d15 100644 --- a/packages/middleware/src/spaceSecurity.ts +++ b/packages/middleware/src/spaceSecurity.ts @@ -667,7 +667,7 @@ export class SpaceSecurityMiddleware extends BaseMiddleware implements Middlewar if (clientFilterSpaces !== undefined) { const cfs = clientFilterSpaces findResult = toFindResult( - findResult.filter((it) => cfs.has((it as any)[field])), + findResult.filter((it) => cfs.has(it[field])), findResult.total, findResult.lookupMap ) diff --git a/packages/middleware/src/triggers.ts b/packages/middleware/src/triggers.ts index 51fb2b95f2..dbb445d6ba 100644 --- a/packages/middleware/src/triggers.ts +++ b/packages/middleware/src/triggers.ts @@ -159,7 +159,7 @@ export class TriggersMiddleware extends BaseMiddleware implements Middleware { return addOperation( ctx, 'query-find', - { domain, _class, query: query as any, options: pureOptions as any }, + { domain, _class, query, options: pureOptions as any }, () => // We sure ctx is required to be passed this.context.liveQuery?.queryFind(_class, query) ?? diff --git a/packages/server-storage/package.json b/packages/server-storage/package.json index e59f192a61..563d36db6f 100644 --- a/packages/server-storage/package.json +++ b/packages/server-storage/package.json @@ -61,6 +61,7 @@ "@hcengineering/minio": "workspace:0.7.0", "@hcengineering/s3": "workspace:0.7.0", "@hcengineering/datalake": "workspace:0.7.0", + "@hcengineering/hulylake": "workspace:0.7.0", "@hcengineering/storage": "^0.7.3", "@hcengineering/analytics": "^0.7.3", "@hcengineering/server-token": "^0.7.0" diff --git a/packages/server-storage/src/starter.ts b/packages/server-storage/src/starter.ts index ac33f7b306..6ae8afc1a6 100644 --- a/packages/server-storage/src/starter.ts +++ b/packages/server-storage/src/starter.ts @@ -1,4 +1,5 @@ import { CONFIG_KIND as DATALAKE_CONFIG_KIND, DatalakeService, type DatalakeConfig } from '@hcengineering/datalake' +import { CONFIG_KIND as HULYLAKE_CONFIG_KIND, HulylakeService, type HulylakeConfig } from '@hcengineering/hulylake' import { CONFIG_KIND as MINIO_CONFIG_KIND, MinioConfig, MinioService, addMinioFallback } from '@hcengineering/minio' import { CONFIG_KIND as S3_CONFIG_KIND, S3Service, type S3Config } from '@hcengineering/s3' import { StorageAdapter, StorageConfiguration, type StorageConfig } from '@hcengineering/server-core' @@ -97,6 +98,12 @@ export function createStorageFromConfig (config: StorageConfig): StorageAdapter throw new Error('Endpoint value is not specified') } adapter = new DatalakeService(c) + } else if (kind === HULYLAKE_CONFIG_KIND) { + const c = config as HulylakeConfig + if (c.endpoint == null) { + throw new Error('Endpoint value is not specified') + } + adapter = new HulylakeService(c) } else { throw new Error('Unsupported storage kind:' + kind) } diff --git a/rush.json b/rush.json index e6562a29e5..2e569b6bc2 100644 --- a/rush.json +++ b/rush.json @@ -339,6 +339,7 @@ { "packageName": "@hcengineering/collaboration", "projectFolder": "packages/collaboration", "shouldPublish": true }, { "packageName": "@hcengineering/server-core", "projectFolder": "packages/core", "shouldPublish": true }, { "packageName": "@hcengineering/datalake", "projectFolder": "packages/datalake", "shouldPublish": true }, + { "packageName": "@hcengineering/hulylake", "projectFolder": "packages/hulylake", "shouldPublish": true }, { "packageName": "@hcengineering/elastic", "projectFolder": "packages/elastic", "shouldPublish": true }, { "packageName": "@hcengineering/kafka", "projectFolder": "packages/kafka", "shouldPublish": true }, { "packageName": "@hcengineering/middleware", "projectFolder": "packages/middleware", "shouldPublish": true },