feat: adjust hulylake client for storage adapter

Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com>
This commit is contained in:
Alexander Onnikov
2025-10-09 23:19:47 +07:00
parent 4cc3dde93f
commit 138adaf021
10 changed files with 386 additions and 234 deletions
+1
View File
@@ -0,0 +1 @@
v22
@@ -0,0 +1,10 @@
{
"changes": [
{
"packageName": "@hcengineering/client-resources",
"comment": "fix formatting",
"type": "patch"
}
],
"packageName": "@hcengineering/client-resources"
}
@@ -0,0 +1,10 @@
{
"changes": [
{
"packageName": "@hcengineering/client",
"comment": "fix formatting",
"type": "patch"
}
],
"packageName": "@hcengineering/client"
}
@@ -0,0 +1,10 @@
{
"changes": [
{
"packageName": "@hcengineering/hulylake-client",
"comment": "refactoring hulylake client to extract multi-workspace client",
"type": "patch"
}
],
"packageName": "@hcengineering/hulylake-client"
}
@@ -261,7 +261,7 @@ describe('MockWebSocket', () => {
ws.send(pingConst)
await new Promise(resolve => setTimeout(resolve, 50))
await new Promise((resolve) => setTimeout(resolve, 50))
expect(receivedMessage).toBe(pongConst)
@@ -290,7 +290,7 @@ describe('connect function', () => {
mockWebSockets = []
// Give time for all timers to clear
await new Promise(resolve => setTimeout(resolve, 100))
await new Promise((resolve) => setTimeout(resolve, 100))
})
it('should establish connection', async () => {
@@ -311,13 +311,13 @@ describe('connect function', () => {
expect(client).toBeDefined()
// Wait for connection to establish
await new Promise(resolve => setTimeout(resolve, 100))
await new Promise((resolve) => setTimeout(resolve, 100))
// Close connection immediately to prevent timers from continuing
await client.close()
// Wait for close to complete
await new Promise(resolve => setTimeout(resolve, 50))
await new Promise((resolve) => setTimeout(resolve, 50))
})
it('should handle transactions', async () => {
@@ -339,7 +339,7 @@ describe('connect function', () => {
connections.push(client)
// Wait for connection to establish
await new Promise(resolve => setTimeout(resolve, 150))
await new Promise((resolve) => setTimeout(resolve, 150))
// Simulate a transaction from server
const testTx: TxCreateDoc<Doc> = {
@@ -363,12 +363,12 @@ describe('connect function', () => {
mockWs.simulateTransaction(testTx)
await new Promise(resolve => setTimeout(resolve, 100))
await new Promise((resolve) => setTimeout(resolve, 100))
expect(txReceived).toBeDefined()
// Close immediately after test
await client.close()
await new Promise(resolve => setTimeout(resolve, 50))
await new Promise((resolve) => setTimeout(resolve, 50))
})
})
@@ -186,7 +186,7 @@ export class MockClientConnection implements ClientConnection {
}
await this.transactions.tx(tx)
this.handlers.forEach(h => {
this.handlers.forEach((h) => {
h(tx)
})
@@ -246,7 +246,7 @@ export class MockClientConnection implements ClientConnection {
}
simulateTransaction (tx: Tx): void {
this.handlers.forEach(h => {
this.handlers.forEach((h) => {
h(tx)
})
}
@@ -285,10 +285,7 @@ export async function createTestClient (initialTxes: Tx[] = []): Promise<{
/**
* Helper to create a test transaction
*/
export function createTestTx (
objectClass: Ref<Class<Doc>> = core.class.Space,
attributes: any = {}
): TxCreateDoc<Doc> {
export function createTestTx (objectClass: Ref<Class<Doc>> = core.class.Space, attributes: any = {}): TxCreateDoc<Doc> {
return {
_id: generateId(),
_class: core.class.TxCreateDoc,
@@ -339,7 +336,7 @@ describe('Client-Resources Integration Tests', () => {
// The mock connection immediately notifies handlers when we call tx
// So notifySpy should have been called
await new Promise(resolve => setTimeout(resolve, 100))
await new Promise((resolve) => setTimeout(resolve, 100))
expect(notifySpy).toHaveBeenCalled()
@@ -525,17 +522,13 @@ describe('Client-Resources Integration Tests', () => {
const tx2 = createTestTx(core.class.Space, { name: 'Space2' })
const tx3 = createTestTx(core.class.Space, { name: 'Space3' })
await Promise.all([
client.tx(tx1),
client.tx(tx2),
client.tx(tx3)
])
await Promise.all([client.tx(tx1), client.tx(tx2), client.tx(tx3)])
connection.simulateTransaction(tx1)
connection.simulateTransaction(tx2)
connection.simulateTransaction(tx3)
await new Promise(resolve => setTimeout(resolve, 100))
await new Promise((resolve) => setTimeout(resolve, 100))
expect(notifySpy.mock.calls.length).toBeGreaterThanOrEqual(3)
+10 -6
View File
@@ -185,7 +185,7 @@ class TestConnection implements ClientConnection {
await this.transactions.tx(tx)
// Notify handlers
this.handlers.forEach(h => {
this.handlers.forEach((h) => {
h(tx)
})
@@ -247,7 +247,7 @@ class TestConnection implements ClientConnection {
// Simulate receiving transactions from server
simulateTransaction (tx: Tx): void {
this.handlers.forEach(h => {
this.handlers.forEach((h) => {
h(tx)
})
}
@@ -336,7 +336,7 @@ describe('Client Core Implementation', () => {
testConnection.simulateTransaction(tx)
// Wait for async operations
await new Promise(resolve => setTimeout(resolve, 100))
await new Promise((resolve) => setTimeout(resolve, 100))
expect(notifySpy).toHaveBeenCalled()
})
@@ -387,7 +387,11 @@ describe('Client Core Implementation', () => {
it('should handle reconnection events', async () => {
let eventReceived: ClientConnectEvent | undefined
const onConnectHandler = async (event: ClientConnectEvent, lastTx: string | undefined, data: any): Promise<void> => {
const onConnectHandler = async (
event: ClientConnectEvent,
lastTx: string | undefined,
data: any
): Promise<void> => {
eventReceived = event
}
@@ -418,7 +422,7 @@ describe('Client Core Implementation', () => {
testConnection.simulateTransaction(workspaceTx)
await new Promise(resolve => setTimeout(resolve, 100))
await new Promise((resolve) => setTimeout(resolve, 100))
expect(notifySpy).toHaveBeenCalled()
})
@@ -466,7 +470,7 @@ describe('Client Core Implementation', () => {
// Simulate receiving the same transaction from remote
testConnection.simulateTransaction(tx)
await new Promise(resolve => setTimeout(resolve, 100))
await new Promise((resolve) => setTimeout(resolve, 100))
// Should still notify but skip model update
expect(notifySpy).toHaveBeenCalled()
+290 -204
View File
@@ -17,225 +17,57 @@ import { WorkspaceUuid } from '@hcengineering/core'
import { RetryOptions } from '@hcengineering/retry'
import { fetchSafe, unwrapContentLength, unwrapEtag, unwrapLastModified } from './utils'
import { HulyHeaders, HulylakeClient, HulyMeta, HulyResponse, JsonPatch, PatchOptions, PutOptions, Body } from './types'
import {
HulyHeaders,
HulylakeClient,
HulylakeWorkspaceClient,
HulyMeta,
HulyResponse,
JsonPatch,
PatchOptions,
PutOptions,
Body
} from './types'
export function getClient (baseUrl: string, workspace: WorkspaceUuid, token: string): HulylakeClient {
return new Client(baseUrl, workspace, token)
export function getWorkspaceClient (baseUrl: string, workspace: WorkspaceUuid, token: string): HulylakeWorkspaceClient {
const client = new Client(baseUrl, token)
return new WorkspaceClient(client, workspace)
}
class Client implements HulylakeClient {
export function getClient (baseUrl: string, token: string): HulylakeClient {
return new Client(baseUrl, token)
}
class WorkspaceClient implements HulylakeWorkspaceClient {
constructor (
private readonly baseUrl: string,
private readonly workspace: WorkspaceUuid,
private readonly token: string
) {
this.baseUrl = this.baseUrl.endsWith('/') ? this.baseUrl.slice(0, -1) : this.baseUrl
private readonly client: HulylakeClient,
private readonly workspace: WorkspaceUuid
) {}
head (key: string, retryOptions?: RetryOptions): Promise<HulyResponse<void>> {
return this.client.head(this.workspace, key, retryOptions)
}
private objectUrl (key: string): string {
return `${this.baseUrl}/api/${this.workspace}/${encodeURIComponent(key)}`
get (key: string, retryOptions?: RetryOptions): Promise<HulyResponse<ReadableStream<Uint8Array>>> {
return this.client.get(this.workspace, key, retryOptions)
}
private authHeaders (init?: HeadersInit): Headers {
const headers = new Headers(init)
headers.set('Authorization', `Bearer ${this.token}`)
return headers
put (key: string, body: Body, opts: PutOptions, retryOptions?: RetryOptions): Promise<HulyResponse<void>> {
return this.client.put(this.workspace, key, body, opts, retryOptions)
}
private applyHeaders (h: Headers, headers?: HulyHeaders): void {
if (headers != null) for (const [k, v] of Object.entries(headers)) h.set(`huly-header-${k}`, v)
patch (key: string, body: Body, opts: PatchOptions, retryOptions?: RetryOptions): Promise<HulyResponse<void>> {
return this.client.patch(this.workspace, key, body, opts, retryOptions)
}
private applyMeta (h: Headers, meta?: HulyMeta): void {
if (meta != null) for (const [k, v] of Object.entries(meta)) h.set(`huly-meta-${k}`, v)
}
public async status (): Promise<boolean> {
try {
const res = await fetchSafe(`${this.baseUrl}/status`)
return res.ok
} catch {
return false
}
}
public async head (key: string, retryOptions?: RetryOptions): Promise<HulyResponse<void>> {
const res = await fetchSafe(
this.objectUrl(key),
{
method: 'HEAD',
headers: this.authHeaders()
},
retryOptions
)
return {
ok: res.ok,
status: res.status,
etag: unwrapEtag(res.headers.get('ETag')),
lastModified: unwrapLastModified(res.headers.get('Last-Modified')),
contentLength: unwrapContentLength(res.headers.get('Content-Length')),
headers: res.headers
}
}
public async get (key: string, retryOptions?: RetryOptions): Promise<HulyResponse<ReadableStream<Uint8Array>>> {
try {
const res = await fetchSafe(
this.objectUrl(key),
{
method: 'GET',
headers: this.authHeaders()
},
retryOptions
)
let body: ReadableStream<Uint8Array> | undefined
if (res.ok) {
body = res.body ?? undefined
}
return {
ok: res.ok,
status: res.status,
etag: unwrapEtag(res.headers.get('ETag')),
headers: res.headers,
body
}
} catch (err: any) {
if (err.name === 'NotFoundError') {
return {
ok: false,
status: 404,
etag: undefined,
headers: new Headers(),
body: undefined
}
}
throw err
}
}
public async put (
key: string,
body: Body,
opts: PutOptions = {},
retryOptions?: RetryOptions
): Promise<HulyResponse<void>> {
const { mergeStrategy, headers, meta } = opts
const contentType = 'contentType' in opts ? opts.contentType : undefined
const h = this.authHeaders()
if (mergeStrategy != null) {
h.set('Huly-Merge-Strategy', mergeStrategy)
}
if (contentType != null) {
h.set('Content-Type', contentType)
} else if (mergeStrategy === 'jsonpatch') {
h.set('Content-Type', 'application/json')
}
this.applyHeaders(h, headers)
this.applyMeta(h, meta)
const res = await fetchSafe(
this.objectUrl(key),
{
method: 'PUT',
headers: h,
body: body as any
},
retryOptions
)
return {
ok: res.ok,
status: res.status,
etag: unwrapEtag(res.headers.get('ETag')),
lastModified: unwrapLastModified(res.headers.get('Last-Modified')),
contentLength: unwrapContentLength(res.headers.get('Content-Length')),
headers: res.headers
}
}
public async patch (
key: string,
body: Body,
opts: PatchOptions = {},
retryOptions?: RetryOptions
): Promise<HulyResponse<void>> {
const { contentType, headers, meta } = opts
const h = this.authHeaders()
if (contentType != null) {
h.set('Content-Type', contentType)
}
this.applyHeaders(h, headers)
this.applyMeta(h, meta)
const res = await fetchSafe(
this.objectUrl(key),
{
method: 'PATCH',
headers: h,
body: body as any
},
retryOptions
)
return {
ok: res.ok,
status: res.status,
etag: unwrapEtag(res.headers.get('ETag')),
lastModified: unwrapLastModified(res.headers.get('Last-Modified')),
contentLength: unwrapContentLength(res.headers.get('Content-Length')),
headers: res.headers
}
delete (key: string, retryOptions?: RetryOptions): Promise<HulyResponse<void>> {
return this.client.delete(this.workspace, key, retryOptions)
}
public async getJson<T>(key: string, retryOptions?: RetryOptions): Promise<HulyResponse<T>> {
try {
const res = await fetchSafe(
this.objectUrl(key),
{
method: 'GET',
headers: this.authHeaders()
},
retryOptions
)
let body: T | undefined
if (res.ok) {
body = (await res.json()) as T
}
return {
ok: res.ok,
status: res.status,
etag: unwrapEtag(res.headers.get('ETag')),
lastModified: unwrapLastModified(res.headers.get('Last-Modified')),
contentLength: unwrapContentLength(res.headers.get('Content-Length')),
headers: res.headers,
body
}
} catch (err: any) {
if (err.name === 'NotFoundError') {
return {
ok: false,
status: 404,
etag: undefined,
headers: new Headers(),
body: undefined
}
}
throw err
}
const res = await this.client.get(this.workspace, key, retryOptions)
const body = res.ok && res.body != null ? ((await new Response(res.body).json()) as T) : undefined
return { ...res, body }
}
public async putJson<T extends object>(
@@ -261,3 +93,257 @@ class Client implements HulylakeClient {
)
}
}
class Client implements HulylakeClient {
constructor (
private readonly baseUrl: string,
private readonly token: string
) {
this.baseUrl = this.baseUrl.endsWith('/') ? this.baseUrl.slice(0, -1) : this.baseUrl
}
private authHeaders (init?: HeadersInit): Headers {
const headers = new Headers(init)
headers.set('Authorization', `Bearer ${this.token}`)
return headers
}
private applyHeaders (h: Headers, headers?: HulyHeaders): void {
if (headers != null) for (const [k, v] of Object.entries(headers)) h.set(`huly-header-${k}`, v)
}
private applyMeta (h: Headers, meta?: HulyMeta): void {
if (meta != null) for (const [k, v] of Object.entries(meta)) h.set(`huly-meta-${k}`, v)
}
public async status (): Promise<boolean> {
try {
const res = await fetchSafe(`${this.baseUrl}/status`)
return res.ok
} catch {
return false
}
}
public objectUrl (workspace: string, key: string): string {
return `${this.baseUrl}/api/${workspace}/${encodeURIComponent(key)}`
}
public async head (workspace: string, key: string, retryOptions?: RetryOptions): Promise<HulyResponse<void>> {
const res = await fetchSafe(
this.objectUrl(workspace, key),
{
method: 'HEAD',
headers: this.authHeaders()
},
retryOptions
)
return {
ok: res.ok,
status: res.status,
etag: unwrapEtag(res.headers.get('ETag')),
lastModified: unwrapLastModified(res.headers.get('Last-Modified')),
contentType: res.headers.get('Content-Type') ?? 'application/octet-stream',
contentLength: unwrapContentLength(res.headers.get('Content-Length')),
headers: res.headers
}
}
public async get (
workspace: string,
key: string,
retryOptions?: RetryOptions
): Promise<HulyResponse<ReadableStream<Uint8Array>>> {
try {
const res = await fetchSafe(
this.objectUrl(workspace, key),
{
method: 'GET',
headers: this.authHeaders()
},
retryOptions
)
return {
ok: res.ok,
status: res.status,
etag: unwrapEtag(res.headers.get('ETag')),
headers: res.headers,
lastModified: unwrapLastModified(res.headers.get('Last-Modified')),
contentType: res.headers.get('Content-Type') ?? 'application/octet-stream',
contentLength: unwrapContentLength(res.headers.get('Content-Length')),
body: res.ok ? (res.body ?? undefined) : undefined
}
} catch (err: any) {
if (err.name === 'NotFoundError') {
return {
ok: false,
status: 404,
etag: undefined,
headers: new Headers(),
body: undefined
}
}
throw err
}
}
public async partial (
workspace: string,
key: string,
offset: number,
length?: number,
retryOptions?: RetryOptions
): Promise<HulyResponse<ReadableStream<Uint8Array>>> {
try {
const res = await fetchSafe(
this.objectUrl(workspace, key),
{
method: 'GET',
headers: this.authHeaders({
Range: length !== undefined ? `bytes=${offset}-${offset + length - 1}` : `bytes=${offset}`
})
},
retryOptions
)
return {
ok: res.ok,
status: res.status,
etag: unwrapEtag(res.headers.get('ETag')),
headers: res.headers,
body: res.ok ? (res.body ?? undefined) : undefined
}
} catch (err: any) {
if (err.name === 'NotFoundError') {
return {
ok: false,
status: 404,
etag: undefined,
headers: new Headers(),
body: undefined
}
}
throw err
}
}
public async put (
workspace: string,
key: string,
body: Body,
opts: PutOptions = {},
retryOptions?: RetryOptions
): Promise<HulyResponse<void>> {
const { mergeStrategy, headers, meta } = opts
const contentType = 'contentType' in opts ? opts.contentType : undefined
const contentLength = 'contentLength' in opts ? opts.contentLength : undefined
const h = this.authHeaders()
if (mergeStrategy != null) {
h.set('Huly-Merge-Strategy', mergeStrategy)
}
if (contentType != null) {
h.set('Content-Type', contentType)
} else if (mergeStrategy === 'jsonpatch') {
h.set('Content-Type', 'application/json')
}
if (contentLength != null) {
h.set('Content-Length', contentLength.toString())
}
this.applyHeaders(h, headers)
this.applyMeta(h, meta)
const duplex = body instanceof ReadableStream ? 'half' : undefined
const res = await fetchSafe(
this.objectUrl(workspace, key),
{
method: 'PUT',
headers: h,
body,
// @ts-expect-error must present for ReadableStream but it is not in the interface
duplex
},
retryOptions
)
return {
ok: res.ok,
status: res.status,
etag: unwrapEtag(res.headers.get('ETag')),
lastModified: unwrapLastModified(res.headers.get('Last-Modified')),
contentLength: unwrapContentLength(res.headers.get('Content-Length')),
headers: res.headers
}
}
public async patch (
workspace: string,
key: string,
body: Body,
opts: PatchOptions = {},
retryOptions?: RetryOptions
): Promise<HulyResponse<void>> {
const { contentType, contentLength, headers, meta } = opts
const h = this.authHeaders()
if (contentType != null) {
h.set('Content-Type', contentType)
}
if (contentLength != null) {
h.set('Content-Length', contentLength.toString())
}
this.applyHeaders(h, headers)
this.applyMeta(h, meta)
const duplex = body instanceof ReadableStream ? 'half' : undefined
const res = await fetchSafe(
this.objectUrl(workspace, key),
{
method: 'PATCH',
headers: h,
body,
// @ts-expect-error must present for ReadableStream but it is not in the interface
duplex
},
retryOptions
)
return {
ok: res.ok,
status: res.status,
etag: unwrapEtag(res.headers.get('ETag')),
lastModified: unwrapLastModified(res.headers.get('Last-Modified')),
contentLength: unwrapContentLength(res.headers.get('Content-Length')),
contentType: res.headers.get('Content-Type') ?? undefined,
headers: res.headers
}
}
public async delete (workspace: string, key: string, retryOptions?: RetryOptions): Promise<HulyResponse<void>> {
const res = await fetchSafe(
this.objectUrl(workspace, key),
{
method: 'DELETE',
headers: this.authHeaders()
},
retryOptions
)
return {
ok: res.ok,
status: res.status,
headers: res.headers
}
}
}
+39 -1
View File
@@ -16,10 +16,44 @@
import { RetryOptions } from '@hcengineering/retry'
export interface HulylakeClient {
head: (workspace: string, key: string, retryOptions?: RetryOptions) => Promise<HulyResponse<void>>
get: (
workspace: string,
key: string,
retryOptions?: RetryOptions
) => Promise<HulyResponse<ReadableStream<Uint8Array>>>
partial: (
workspace: string,
key: string,
offset: number,
length?: number,
retryOptions?: RetryOptions
) => Promise<HulyResponse<ReadableStream<Uint8Array>>>
put: (
workspace: string,
key: string,
body: Body,
opts: PutOptions,
retryOptions?: RetryOptions
) => Promise<HulyResponse<void>>
patch: (
workspace: string,
key: string,
body: Body,
opts: PatchOptions,
retryOptions?: RetryOptions
) => Promise<HulyResponse<void>>
delete: (workspace: string, key: string, retryOptions?: RetryOptions) => Promise<HulyResponse<void>>
objectUrl: (workspace: string, key: string) => string
}
export interface HulylakeWorkspaceClient {
head: (key: string, retryOptions?: RetryOptions) => Promise<HulyResponse<void>>
get: (key: string, retryOptions?: RetryOptions) => Promise<HulyResponse<ReadableStream<Uint8Array>>>
put: (key: string, body: Body, opts: PutOptions, retryOptions?: RetryOptions) => Promise<HulyResponse<void>>
patch: (key: string, body: Body, opts: PatchOptions, retryOptions?: RetryOptions) => Promise<HulyResponse<void>>
delete: (key: string, retryOptions?: RetryOptions) => Promise<HulyResponse<void>>
getJson: <T>(key: string, retryOptions?: RetryOptions) => Promise<HulyResponse<T>>
putJson: <T extends object>(
@@ -36,7 +70,7 @@ export interface HulylakeClient {
) => Promise<HulyResponse<void>>
}
export type Body = ArrayBuffer | Blob | string
export type Body = ReadableStream | ArrayBuffer | Blob | string
export type MergeStrategy = 'concatenate' | 'jsonpatch'
export type HulyHeaders = Record<string, string>
export type HulyMeta = Record<string, string>
@@ -44,17 +78,20 @@ export type HulyMeta = Record<string, string>
export type PutOptions =
| {
mergeStrategy?: 'concatenate'
contentLength?: number
contentType?: string
headers?: HulyHeaders
meta?: HulyMeta
}
| {
mergeStrategy: 'jsonpatch'
contentLength?: number
headers?: HulyHeaders
meta?: HulyMeta
}
export interface PatchOptions {
contentLength?: number
contentType?: string
headers?: HulyHeaders
meta?: HulyMeta
@@ -75,6 +112,7 @@ export interface HulyResponse<Body = ArrayBuffer | string | any> {
ok: boolean
status: number
etag?: string
contentType?: string
contentLength?: number
lastModified?: number
headers: Headers
+3 -3
View File
@@ -29,12 +29,12 @@ async function innerFetchSafe (url: string | URL, init?: RequestInit): Promise<R
return response
}
const text = await response.text()
if (response.status === 404) {
return response
} else {
throw new HulylakeError(text)
}
const text = await response.text()
throw new HulylakeError(text)
}
export async function fetchSafe (url: string | URL, init?: RequestInit, retryOptions?: RetryOptions): Promise<Response> {