diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 0000000000..53d1c14db3 --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +v22 diff --git a/common/changes/@hcengineering/client-resources/feat-huylake-storage-adapter_2025-10-09-16-19.json b/common/changes/@hcengineering/client-resources/feat-huylake-storage-adapter_2025-10-09-16-19.json new file mode 100644 index 0000000000..e6a10655de --- /dev/null +++ b/common/changes/@hcengineering/client-resources/feat-huylake-storage-adapter_2025-10-09-16-19.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@hcengineering/client-resources", + "comment": "fix formatting", + "type": "patch" + } + ], + "packageName": "@hcengineering/client-resources" +} \ No newline at end of file diff --git a/common/changes/@hcengineering/client/feat-huylake-storage-adapter_2025-10-09-16-19.json b/common/changes/@hcengineering/client/feat-huylake-storage-adapter_2025-10-09-16-19.json new file mode 100644 index 0000000000..009cfc9ed9 --- /dev/null +++ b/common/changes/@hcengineering/client/feat-huylake-storage-adapter_2025-10-09-16-19.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@hcengineering/client", + "comment": "fix formatting", + "type": "patch" + } + ], + "packageName": "@hcengineering/client" +} \ No newline at end of file diff --git a/common/changes/@hcengineering/hulylake-client/feat-huylake-storage-adapter_2025-10-09-16-19.json b/common/changes/@hcengineering/hulylake-client/feat-huylake-storage-adapter_2025-10-09-16-19.json new file mode 100644 index 0000000000..d13cdac07b --- /dev/null +++ b/common/changes/@hcengineering/hulylake-client/feat-huylake-storage-adapter_2025-10-09-16-19.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@hcengineering/hulylake-client", + "comment": "refactoring hulylake client to extract multi-workspace client", + "type": "patch" + } + ], + "packageName": "@hcengineering/hulylake-client" +} \ No newline at end of file diff --git a/packages/client-resources/src/__tests__/connection.test.ts b/packages/client-resources/src/__tests__/connection.test.ts index a080bbf7be..7fb4f195be 100644 --- a/packages/client-resources/src/__tests__/connection.test.ts +++ b/packages/client-resources/src/__tests__/connection.test.ts @@ -261,7 +261,7 @@ describe('MockWebSocket', () => { ws.send(pingConst) - await new Promise(resolve => setTimeout(resolve, 50)) + await new Promise((resolve) => setTimeout(resolve, 50)) expect(receivedMessage).toBe(pongConst) @@ -290,7 +290,7 @@ describe('connect function', () => { mockWebSockets = [] // Give time for all timers to clear - await new Promise(resolve => setTimeout(resolve, 100)) + await new Promise((resolve) => setTimeout(resolve, 100)) }) it('should establish connection', async () => { @@ -311,13 +311,13 @@ describe('connect function', () => { expect(client).toBeDefined() // Wait for connection to establish - await new Promise(resolve => setTimeout(resolve, 100)) + await new Promise((resolve) => setTimeout(resolve, 100)) // Close connection immediately to prevent timers from continuing await client.close() // Wait for close to complete - await new Promise(resolve => setTimeout(resolve, 50)) + await new Promise((resolve) => setTimeout(resolve, 50)) }) it('should handle transactions', async () => { @@ -339,7 +339,7 @@ describe('connect function', () => { connections.push(client) // Wait for connection to establish - await new Promise(resolve => setTimeout(resolve, 150)) + await new Promise((resolve) => setTimeout(resolve, 150)) // Simulate a transaction from server const testTx: TxCreateDoc = { @@ -363,12 +363,12 @@ describe('connect function', () => { mockWs.simulateTransaction(testTx) - await new Promise(resolve => setTimeout(resolve, 100)) + await new Promise((resolve) => setTimeout(resolve, 100)) expect(txReceived).toBeDefined() // Close immediately after test await client.close() - await new Promise(resolve => setTimeout(resolve, 50)) + await new Promise((resolve) => setTimeout(resolve, 50)) }) }) diff --git a/packages/client-resources/src/__tests__/integration.test.ts b/packages/client-resources/src/__tests__/integration.test.ts index 1f1bdc7668..c0f6b2642e 100644 --- a/packages/client-resources/src/__tests__/integration.test.ts +++ b/packages/client-resources/src/__tests__/integration.test.ts @@ -186,7 +186,7 @@ export class MockClientConnection implements ClientConnection { } await this.transactions.tx(tx) - this.handlers.forEach(h => { + this.handlers.forEach((h) => { h(tx) }) @@ -246,7 +246,7 @@ export class MockClientConnection implements ClientConnection { } simulateTransaction (tx: Tx): void { - this.handlers.forEach(h => { + this.handlers.forEach((h) => { h(tx) }) } @@ -285,10 +285,7 @@ export async function createTestClient (initialTxes: Tx[] = []): Promise<{ /** * Helper to create a test transaction */ -export function createTestTx ( - objectClass: Ref> = core.class.Space, - attributes: any = {} -): TxCreateDoc { +export function createTestTx (objectClass: Ref> = core.class.Space, attributes: any = {}): TxCreateDoc { return { _id: generateId(), _class: core.class.TxCreateDoc, @@ -339,7 +336,7 @@ describe('Client-Resources Integration Tests', () => { // The mock connection immediately notifies handlers when we call tx // So notifySpy should have been called - await new Promise(resolve => setTimeout(resolve, 100)) + await new Promise((resolve) => setTimeout(resolve, 100)) expect(notifySpy).toHaveBeenCalled() @@ -525,17 +522,13 @@ describe('Client-Resources Integration Tests', () => { const tx2 = createTestTx(core.class.Space, { name: 'Space2' }) const tx3 = createTestTx(core.class.Space, { name: 'Space3' }) - await Promise.all([ - client.tx(tx1), - client.tx(tx2), - client.tx(tx3) - ]) + await Promise.all([client.tx(tx1), client.tx(tx2), client.tx(tx3)]) connection.simulateTransaction(tx1) connection.simulateTransaction(tx2) connection.simulateTransaction(tx3) - await new Promise(resolve => setTimeout(resolve, 100)) + await new Promise((resolve) => setTimeout(resolve, 100)) expect(notifySpy.mock.calls.length).toBeGreaterThanOrEqual(3) diff --git a/packages/client/src/__tests__/client.test.ts b/packages/client/src/__tests__/client.test.ts index b0646432ff..8e71ac8fb6 100644 --- a/packages/client/src/__tests__/client.test.ts +++ b/packages/client/src/__tests__/client.test.ts @@ -185,7 +185,7 @@ class TestConnection implements ClientConnection { await this.transactions.tx(tx) // Notify handlers - this.handlers.forEach(h => { + this.handlers.forEach((h) => { h(tx) }) @@ -247,7 +247,7 @@ class TestConnection implements ClientConnection { // Simulate receiving transactions from server simulateTransaction (tx: Tx): void { - this.handlers.forEach(h => { + this.handlers.forEach((h) => { h(tx) }) } @@ -336,7 +336,7 @@ describe('Client Core Implementation', () => { testConnection.simulateTransaction(tx) // Wait for async operations - await new Promise(resolve => setTimeout(resolve, 100)) + await new Promise((resolve) => setTimeout(resolve, 100)) expect(notifySpy).toHaveBeenCalled() }) @@ -387,7 +387,11 @@ describe('Client Core Implementation', () => { it('should handle reconnection events', async () => { let eventReceived: ClientConnectEvent | undefined - const onConnectHandler = async (event: ClientConnectEvent, lastTx: string | undefined, data: any): Promise => { + const onConnectHandler = async ( + event: ClientConnectEvent, + lastTx: string | undefined, + data: any + ): Promise => { eventReceived = event } @@ -418,7 +422,7 @@ describe('Client Core Implementation', () => { testConnection.simulateTransaction(workspaceTx) - await new Promise(resolve => setTimeout(resolve, 100)) + await new Promise((resolve) => setTimeout(resolve, 100)) expect(notifySpy).toHaveBeenCalled() }) @@ -466,7 +470,7 @@ describe('Client Core Implementation', () => { // Simulate receiving the same transaction from remote testConnection.simulateTransaction(tx) - await new Promise(resolve => setTimeout(resolve, 100)) + await new Promise((resolve) => setTimeout(resolve, 100)) // Should still notify but skip model update expect(notifySpy).toHaveBeenCalled() diff --git a/packages/hulylake-client/src/client.ts b/packages/hulylake-client/src/client.ts index 956a8f6d23..af88ca5199 100644 --- a/packages/hulylake-client/src/client.ts +++ b/packages/hulylake-client/src/client.ts @@ -17,225 +17,57 @@ import { WorkspaceUuid } from '@hcengineering/core' import { RetryOptions } from '@hcengineering/retry' import { fetchSafe, unwrapContentLength, unwrapEtag, unwrapLastModified } from './utils' -import { HulyHeaders, HulylakeClient, HulyMeta, HulyResponse, JsonPatch, PatchOptions, PutOptions, Body } from './types' +import { + HulyHeaders, + HulylakeClient, + HulylakeWorkspaceClient, + HulyMeta, + HulyResponse, + JsonPatch, + PatchOptions, + PutOptions, + Body +} from './types' -export function getClient (baseUrl: string, workspace: WorkspaceUuid, token: string): HulylakeClient { - return new Client(baseUrl, workspace, token) +export function getWorkspaceClient (baseUrl: string, workspace: WorkspaceUuid, token: string): HulylakeWorkspaceClient { + const client = new Client(baseUrl, token) + return new WorkspaceClient(client, workspace) } -class Client implements HulylakeClient { +export function getClient (baseUrl: string, token: string): HulylakeClient { + return new Client(baseUrl, token) +} + +class WorkspaceClient implements HulylakeWorkspaceClient { constructor ( - private readonly baseUrl: string, - private readonly workspace: WorkspaceUuid, - private readonly token: string - ) { - this.baseUrl = this.baseUrl.endsWith('/') ? this.baseUrl.slice(0, -1) : this.baseUrl + private readonly client: HulylakeClient, + private readonly workspace: WorkspaceUuid + ) {} + + head (key: string, retryOptions?: RetryOptions): Promise> { + return this.client.head(this.workspace, key, retryOptions) } - private objectUrl (key: string): string { - return `${this.baseUrl}/api/${this.workspace}/${encodeURIComponent(key)}` + get (key: string, retryOptions?: RetryOptions): Promise>> { + return this.client.get(this.workspace, key, retryOptions) } - private authHeaders (init?: HeadersInit): Headers { - const headers = new Headers(init) - headers.set('Authorization', `Bearer ${this.token}`) - return headers + put (key: string, body: Body, opts: PutOptions, retryOptions?: RetryOptions): Promise> { + return this.client.put(this.workspace, key, body, opts, retryOptions) } - private applyHeaders (h: Headers, headers?: HulyHeaders): void { - if (headers != null) for (const [k, v] of Object.entries(headers)) h.set(`huly-header-${k}`, v) + patch (key: string, body: Body, opts: PatchOptions, retryOptions?: RetryOptions): Promise> { + return this.client.patch(this.workspace, key, body, opts, retryOptions) } - private applyMeta (h: Headers, meta?: HulyMeta): void { - if (meta != null) for (const [k, v] of Object.entries(meta)) h.set(`huly-meta-${k}`, v) - } - - public async status (): Promise { - try { - const res = await fetchSafe(`${this.baseUrl}/status`) - return res.ok - } catch { - return false - } - } - - public async head (key: string, retryOptions?: RetryOptions): Promise> { - const res = await fetchSafe( - this.objectUrl(key), - { - method: 'HEAD', - headers: this.authHeaders() - }, - retryOptions - ) - - return { - ok: res.ok, - status: res.status, - etag: unwrapEtag(res.headers.get('ETag')), - lastModified: unwrapLastModified(res.headers.get('Last-Modified')), - contentLength: unwrapContentLength(res.headers.get('Content-Length')), - headers: res.headers - } - } - - public async get (key: string, retryOptions?: RetryOptions): Promise>> { - try { - const res = await fetchSafe( - this.objectUrl(key), - { - method: 'GET', - headers: this.authHeaders() - }, - retryOptions - ) - - let body: ReadableStream | undefined - - if (res.ok) { - body = res.body ?? undefined - } - - return { - ok: res.ok, - status: res.status, - etag: unwrapEtag(res.headers.get('ETag')), - headers: res.headers, - body - } - } catch (err: any) { - if (err.name === 'NotFoundError') { - return { - ok: false, - status: 404, - etag: undefined, - headers: new Headers(), - body: undefined - } - } - throw err - } - } - - public async put ( - key: string, - body: Body, - opts: PutOptions = {}, - retryOptions?: RetryOptions - ): Promise> { - const { mergeStrategy, headers, meta } = opts - const contentType = 'contentType' in opts ? opts.contentType : undefined - - const h = this.authHeaders() - - if (mergeStrategy != null) { - h.set('Huly-Merge-Strategy', mergeStrategy) - } - - if (contentType != null) { - h.set('Content-Type', contentType) - } else if (mergeStrategy === 'jsonpatch') { - h.set('Content-Type', 'application/json') - } - - this.applyHeaders(h, headers) - this.applyMeta(h, meta) - - const res = await fetchSafe( - this.objectUrl(key), - { - method: 'PUT', - headers: h, - body: body as any - }, - retryOptions - ) - - return { - ok: res.ok, - status: res.status, - etag: unwrapEtag(res.headers.get('ETag')), - lastModified: unwrapLastModified(res.headers.get('Last-Modified')), - contentLength: unwrapContentLength(res.headers.get('Content-Length')), - headers: res.headers - } - } - - public async patch ( - key: string, - body: Body, - opts: PatchOptions = {}, - retryOptions?: RetryOptions - ): Promise> { - const { contentType, headers, meta } = opts - - const h = this.authHeaders() - - if (contentType != null) { - h.set('Content-Type', contentType) - } - - this.applyHeaders(h, headers) - this.applyMeta(h, meta) - - const res = await fetchSafe( - this.objectUrl(key), - { - method: 'PATCH', - headers: h, - body: body as any - }, - retryOptions - ) - - return { - ok: res.ok, - status: res.status, - etag: unwrapEtag(res.headers.get('ETag')), - lastModified: unwrapLastModified(res.headers.get('Last-Modified')), - contentLength: unwrapContentLength(res.headers.get('Content-Length')), - headers: res.headers - } + delete (key: string, retryOptions?: RetryOptions): Promise> { + return this.client.delete(this.workspace, key, retryOptions) } public async getJson(key: string, retryOptions?: RetryOptions): Promise> { - try { - const res = await fetchSafe( - this.objectUrl(key), - { - method: 'GET', - headers: this.authHeaders() - }, - retryOptions - ) - - let body: T | undefined - - if (res.ok) { - body = (await res.json()) as T - } - - return { - ok: res.ok, - status: res.status, - etag: unwrapEtag(res.headers.get('ETag')), - lastModified: unwrapLastModified(res.headers.get('Last-Modified')), - contentLength: unwrapContentLength(res.headers.get('Content-Length')), - headers: res.headers, - body - } - } catch (err: any) { - if (err.name === 'NotFoundError') { - return { - ok: false, - status: 404, - etag: undefined, - headers: new Headers(), - body: undefined - } - } - throw err - } + const res = await this.client.get(this.workspace, key, retryOptions) + const body = res.ok && res.body != null ? ((await new Response(res.body).json()) as T) : undefined + return { ...res, body } } public async putJson( @@ -261,3 +93,257 @@ class Client implements HulylakeClient { ) } } + +class Client implements HulylakeClient { + constructor ( + private readonly baseUrl: string, + private readonly token: string + ) { + this.baseUrl = this.baseUrl.endsWith('/') ? this.baseUrl.slice(0, -1) : this.baseUrl + } + + private authHeaders (init?: HeadersInit): Headers { + const headers = new Headers(init) + headers.set('Authorization', `Bearer ${this.token}`) + return headers + } + + private applyHeaders (h: Headers, headers?: HulyHeaders): void { + if (headers != null) for (const [k, v] of Object.entries(headers)) h.set(`huly-header-${k}`, v) + } + + private applyMeta (h: Headers, meta?: HulyMeta): void { + if (meta != null) for (const [k, v] of Object.entries(meta)) h.set(`huly-meta-${k}`, v) + } + + public async status (): Promise { + try { + const res = await fetchSafe(`${this.baseUrl}/status`) + return res.ok + } catch { + return false + } + } + + public objectUrl (workspace: string, key: string): string { + return `${this.baseUrl}/api/${workspace}/${encodeURIComponent(key)}` + } + + public async head (workspace: string, key: string, retryOptions?: RetryOptions): Promise> { + const res = await fetchSafe( + this.objectUrl(workspace, key), + { + method: 'HEAD', + headers: this.authHeaders() + }, + retryOptions + ) + + return { + ok: res.ok, + status: res.status, + etag: unwrapEtag(res.headers.get('ETag')), + lastModified: unwrapLastModified(res.headers.get('Last-Modified')), + contentType: res.headers.get('Content-Type') ?? 'application/octet-stream', + contentLength: unwrapContentLength(res.headers.get('Content-Length')), + headers: res.headers + } + } + + public async get ( + workspace: string, + key: string, + retryOptions?: RetryOptions + ): Promise>> { + try { + const res = await fetchSafe( + this.objectUrl(workspace, key), + { + method: 'GET', + headers: this.authHeaders() + }, + retryOptions + ) + + return { + ok: res.ok, + status: res.status, + etag: unwrapEtag(res.headers.get('ETag')), + headers: res.headers, + lastModified: unwrapLastModified(res.headers.get('Last-Modified')), + contentType: res.headers.get('Content-Type') ?? 'application/octet-stream', + contentLength: unwrapContentLength(res.headers.get('Content-Length')), + body: res.ok ? (res.body ?? undefined) : undefined + } + } catch (err: any) { + if (err.name === 'NotFoundError') { + return { + ok: false, + status: 404, + etag: undefined, + headers: new Headers(), + body: undefined + } + } + throw err + } + } + + public async partial ( + workspace: string, + key: string, + offset: number, + length?: number, + retryOptions?: RetryOptions + ): Promise>> { + try { + const res = await fetchSafe( + this.objectUrl(workspace, key), + { + method: 'GET', + headers: this.authHeaders({ + Range: length !== undefined ? `bytes=${offset}-${offset + length - 1}` : `bytes=${offset}` + }) + }, + retryOptions + ) + + return { + ok: res.ok, + status: res.status, + etag: unwrapEtag(res.headers.get('ETag')), + headers: res.headers, + body: res.ok ? (res.body ?? undefined) : undefined + } + } catch (err: any) { + if (err.name === 'NotFoundError') { + return { + ok: false, + status: 404, + etag: undefined, + headers: new Headers(), + body: undefined + } + } + throw err + } + } + + public async put ( + workspace: string, + key: string, + body: Body, + opts: PutOptions = {}, + retryOptions?: RetryOptions + ): Promise> { + const { mergeStrategy, headers, meta } = opts + const contentType = 'contentType' in opts ? opts.contentType : undefined + const contentLength = 'contentLength' in opts ? opts.contentLength : undefined + + const h = this.authHeaders() + + if (mergeStrategy != null) { + h.set('Huly-Merge-Strategy', mergeStrategy) + } + + if (contentType != null) { + h.set('Content-Type', contentType) + } else if (mergeStrategy === 'jsonpatch') { + h.set('Content-Type', 'application/json') + } + + if (contentLength != null) { + h.set('Content-Length', contentLength.toString()) + } + + this.applyHeaders(h, headers) + this.applyMeta(h, meta) + + const duplex = body instanceof ReadableStream ? 'half' : undefined + + const res = await fetchSafe( + this.objectUrl(workspace, key), + { + method: 'PUT', + headers: h, + body, + // @ts-expect-error must present for ReadableStream but it is not in the interface + duplex + }, + retryOptions + ) + + return { + ok: res.ok, + status: res.status, + etag: unwrapEtag(res.headers.get('ETag')), + lastModified: unwrapLastModified(res.headers.get('Last-Modified')), + contentLength: unwrapContentLength(res.headers.get('Content-Length')), + headers: res.headers + } + } + + public async patch ( + workspace: string, + key: string, + body: Body, + opts: PatchOptions = {}, + retryOptions?: RetryOptions + ): Promise> { + const { contentType, contentLength, headers, meta } = opts + + const h = this.authHeaders() + + if (contentType != null) { + h.set('Content-Type', contentType) + } + + if (contentLength != null) { + h.set('Content-Length', contentLength.toString()) + } + + this.applyHeaders(h, headers) + this.applyMeta(h, meta) + + const duplex = body instanceof ReadableStream ? 'half' : undefined + + const res = await fetchSafe( + this.objectUrl(workspace, key), + { + method: 'PATCH', + headers: h, + body, + // @ts-expect-error must present for ReadableStream but it is not in the interface + duplex + }, + retryOptions + ) + + return { + ok: res.ok, + status: res.status, + etag: unwrapEtag(res.headers.get('ETag')), + lastModified: unwrapLastModified(res.headers.get('Last-Modified')), + contentLength: unwrapContentLength(res.headers.get('Content-Length')), + contentType: res.headers.get('Content-Type') ?? undefined, + headers: res.headers + } + } + + public async delete (workspace: string, key: string, retryOptions?: RetryOptions): Promise> { + const res = await fetchSafe( + this.objectUrl(workspace, key), + { + method: 'DELETE', + headers: this.authHeaders() + }, + retryOptions + ) + + return { + ok: res.ok, + status: res.status, + headers: res.headers + } + } +} diff --git a/packages/hulylake-client/src/types.ts b/packages/hulylake-client/src/types.ts index bce6daf6f0..c9c53eb0d6 100644 --- a/packages/hulylake-client/src/types.ts +++ b/packages/hulylake-client/src/types.ts @@ -16,10 +16,44 @@ import { RetryOptions } from '@hcengineering/retry' export interface HulylakeClient { + head: (workspace: string, key: string, retryOptions?: RetryOptions) => Promise> + get: ( + workspace: string, + key: string, + retryOptions?: RetryOptions + ) => Promise>> + partial: ( + workspace: string, + key: string, + offset: number, + length?: number, + retryOptions?: RetryOptions + ) => Promise>> + put: ( + workspace: string, + key: string, + body: Body, + opts: PutOptions, + retryOptions?: RetryOptions + ) => Promise> + patch: ( + workspace: string, + key: string, + body: Body, + opts: PatchOptions, + retryOptions?: RetryOptions + ) => Promise> + delete: (workspace: string, key: string, retryOptions?: RetryOptions) => Promise> + + objectUrl: (workspace: string, key: string) => string +} + +export interface HulylakeWorkspaceClient { head: (key: string, retryOptions?: RetryOptions) => Promise> get: (key: string, retryOptions?: RetryOptions) => Promise>> put: (key: string, body: Body, opts: PutOptions, retryOptions?: RetryOptions) => Promise> patch: (key: string, body: Body, opts: PatchOptions, retryOptions?: RetryOptions) => Promise> + delete: (key: string, retryOptions?: RetryOptions) => Promise> getJson: (key: string, retryOptions?: RetryOptions) => Promise> putJson: ( @@ -36,7 +70,7 @@ export interface HulylakeClient { ) => Promise> } -export type Body = ArrayBuffer | Blob | string +export type Body = ReadableStream | ArrayBuffer | Blob | string export type MergeStrategy = 'concatenate' | 'jsonpatch' export type HulyHeaders = Record export type HulyMeta = Record @@ -44,17 +78,20 @@ export type HulyMeta = Record export type PutOptions = | { mergeStrategy?: 'concatenate' + contentLength?: number contentType?: string headers?: HulyHeaders meta?: HulyMeta } | { mergeStrategy: 'jsonpatch' + contentLength?: number headers?: HulyHeaders meta?: HulyMeta } export interface PatchOptions { + contentLength?: number contentType?: string headers?: HulyHeaders meta?: HulyMeta @@ -75,6 +112,7 @@ export interface HulyResponse { ok: boolean status: number etag?: string + contentType?: string contentLength?: number lastModified?: number headers: Headers diff --git a/packages/hulylake-client/src/utils.ts b/packages/hulylake-client/src/utils.ts index 5202756d51..1d93750e86 100644 --- a/packages/hulylake-client/src/utils.ts +++ b/packages/hulylake-client/src/utils.ts @@ -29,12 +29,12 @@ async function innerFetchSafe (url: string | URL, init?: RequestInit): Promise {