diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 183553c29c..d437f58336 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -135,7 +135,7 @@ importers: version: file:projects/analytics-service.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@22.15.29)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@22.15.29)(typescript@5.3.3)) '@rush-temp/api-client': specifier: file:./projects/api-client.tgz - version: file:projects/api-client.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(encoding@0.1.13)(esbuild@0.24.2)(utf-8-validate@6.0.4) + version: file:projects/api-client.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(esbuild@0.24.2)(utf-8-validate@6.0.4) '@rush-temp/api-tests': specifier: file:./projects/api-tests.tgz version: file:projects/api-tests.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@22.15.29)(typescript@5.3.3)) @@ -255,7 +255,7 @@ importers: version: file:projects/core.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@22.15.29)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(ts-node@10.9.2(@types/node@22.15.29)(typescript@5.3.3)) '@rush-temp/datalake': specifier: file:./projects/datalake.tgz - version: file:projects/datalake.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(encoding@0.1.13)(esbuild@0.24.2) + version: file:projects/datalake.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2) '@rush-temp/desktop': specifier: file:./projects/desktop.tgz version: file:projects/desktop.tgz(@babel/core@7.23.9)(bufferutil@4.0.8)(sass@1.71.1)(utf-8-validate@6.0.4) @@ -1570,9 +1570,6 @@ importers: '@types/node-cron': specifier: ^3.0.11 version: 3.0.11 - '@types/node-fetch': - specifier: ~2.6.2 - version: 2.6.12 '@types/nodemailer': specifier: ^6.4.17 version: 6.4.17 @@ -1981,9 +1978,6 @@ importers: node-cron: specifier: ^3.0.3 version: 3.0.3 - node-fetch: - specifier: ^2.6.6 - version: 2.7.0(encoding@0.1.13) node-forge: specifier: ^1.3.1 version: 1.3.1 @@ -16749,11 +16743,10 @@ snapshots: - supports-color - ts-node - '@rush-temp/api-client@file:projects/api-client.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(encoding@0.1.13)(esbuild@0.24.2)(utf-8-validate@6.0.4)': + '@rush-temp/api-client@file:projects/api-client.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(bufferutil@4.0.8)(esbuild@0.24.2)(utf-8-validate@6.0.4)': dependencies: '@types/jest': 29.5.12 '@types/node': 22.15.29 - '@types/node-fetch': 2.6.12 '@types/snappyjs': 0.7.1 '@types/ws': 8.5.11 '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.6.2) @@ -16763,9 +16756,7 @@ snapshots: eslint-plugin-import: 2.29.1(eslint@8.56.0) eslint-plugin-n: 15.7.0(eslint@8.56.0) eslint-plugin-promise: 6.1.1(eslint@8.56.0) - form-data: 4.0.0 - jest: 29.7.0(@types/node@22.15.29)(ts-node@10.9.2(@types/node@22.15.29)(typescript@5.3.3)) - node-fetch: 2.7.0(encoding@0.1.13) + jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) prettier: 3.2.5 simplytyped: 3.3.0(typescript@5.6.2) snappyjs: 0.7.0 @@ -16782,7 +16773,6 @@ snapshots: - babel-jest - babel-plugin-macros - bufferutil - - encoding - esbuild - node-notifier - supports-color @@ -18024,11 +18014,10 @@ snapshots: - supports-color - ts-node - '@rush-temp/datalake@file:projects/datalake.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(encoding@0.1.13)(esbuild@0.24.2)': + '@rush-temp/datalake@file:projects/datalake.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)': dependencies: '@types/jest': 29.5.12 '@types/node': 22.15.29 - '@types/node-fetch': 2.6.12 '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3) '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3) eslint: 8.56.0 @@ -18036,9 +18025,7 @@ snapshots: eslint-plugin-import: 2.29.1(eslint@8.56.0) eslint-plugin-n: 15.7.0(eslint@8.56.0) eslint-plugin-promise: 6.1.1(eslint@8.56.0) - form-data: 4.0.0 - jest: 29.7.0(@types/node@22.15.29)(ts-node@10.9.2(@types/node@22.15.29)(typescript@5.3.3)) - node-fetch: 2.7.0(encoding@0.1.13) + jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) prettier: 3.2.5 ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@22.15.29)(ts-node@10.9.2(@types/node@22.15.29)(typescript@5.3.3)))(typescript@5.3.3) ts-node: 10.9.2(@types/node@22.15.29)(typescript@5.3.3) @@ -18050,7 +18037,6 @@ snapshots: - '@swc/wasm' - babel-jest - babel-plugin-macros - - encoding - esbuild - node-notifier - supports-color diff --git a/packages/api-client/package.json b/packages/api-client/package.json index 0d40e2af2c..ea7e97c1cb 100644 --- a/packages/api-client/package.json +++ b/packages/api-client/package.json @@ -47,7 +47,6 @@ "@types/node": "^22.15.29", "@types/jest": "^29.5.5", "@types/ws": "^8.5.11", - "@types/node-fetch": "~2.6.2", "@types/snappyjs": "^0.7.1" }, "dependencies": { @@ -59,8 +58,6 @@ "@hcengineering/platform": "^0.6.11", "@hcengineering/text": "^0.6.5", "@hcengineering/text-markdown": "^0.6.0", - "form-data": "^4.0.0", - "node-fetch": "^2.6.6", "snappyjs": "^0.7.0" }, "repository": "https://github.com/hcengineering/platform", diff --git a/packages/api-client/src/storage/client.ts b/packages/api-client/src/storage/client.ts index 71726e4b65..e0c068a6cf 100644 --- a/packages/api-client/src/storage/client.ts +++ b/packages/api-client/src/storage/client.ts @@ -14,14 +14,12 @@ // import core, { concatLink, WorkspaceUuid, Blob, Ref } from '@hcengineering/core' -import FormData from 'form-data' import { Readable } from 'stream' import { StorageClient } from './types' import { loadServerConfig, ServerConfig } from '../config' import { NetworkError, NotFoundError, StorageError } from './error' import { AuthOptions } from '../types' import { getWorkspaceToken } from '../utils' -import nodeFetch from 'node-fetch' interface ObjectMetadata { name: string @@ -102,16 +100,13 @@ export class StorageClientImpl implements StorageClient { } async put (objectName: string, stream: Readable | Buffer | string, contentType: string, size?: number): Promise { + const buffer = await toBuffer(stream) + const file = new File([buffer], objectName, { type: contentType }) const formData = new FormData() - const options: FormData.AppendOptions = { - filename: objectName, - contentType, - knownLength: size - } - formData.append('file', stream, options) + formData.append('file', file) let response try { - response = await nodeFetch(this.uploadUrl, { + response = await fetch(this.uploadUrl, { method: 'POST', body: formData, headers: { ...this.headers } @@ -168,6 +163,22 @@ export class StorageClientImpl implements StorageClient { } } +async function toBuffer (data: Buffer | string | Readable): Promise { + if (Buffer.isBuffer(data)) { + return data + } else if (typeof data === 'string') { + return Buffer.from(data) + } else if (data instanceof Readable) { + const chunks: Buffer[] = [] + for await (const chunk of data) { + chunks.push(chunk) + } + return Buffer.concat(chunks as any) + } else { + throw new TypeError('Unsupported data type') + } +} + async function wrappedFetch (url: string | URL, init?: RequestInit): Promise { let response: Response try { diff --git a/server/datalake/package.json b/server/datalake/package.json index b7d4dc7305..406acf5701 100644 --- a/server/datalake/package.json +++ b/server/datalake/package.json @@ -32,15 +32,12 @@ "jest": "^29.7.0", "ts-jest": "^29.1.1", "@types/jest": "^29.5.5", - "@types/node-fetch": "~2.6.2", "ts-node": "^10.8.0" }, "dependencies": { "@hcengineering/core": "^0.6.32", "@hcengineering/platform": "^0.6.11", "@hcengineering/server-core": "^0.6.1", - "@hcengineering/server-token": "^0.6.11", - "node-fetch": "^2.6.6", - "form-data": "^4.0.0" + "@hcengineering/server-token": "^0.6.11" } } diff --git a/server/datalake/src/client.ts b/server/datalake/src/client.ts index ea8374a8ac..012f994bf2 100644 --- a/server/datalake/src/client.ts +++ b/server/datalake/src/client.ts @@ -14,8 +14,6 @@ // import { type MeasureContext, type WorkspaceUuid, concatLink } from '@hcengineering/core' -import FormData from 'form-data' -import fetch, { type RequestInfo, type RequestInit, type Response } from 'node-fetch' import { Readable } from 'stream' import { DatalakeError, NetworkError, NotFoundError } from './error' @@ -130,7 +128,7 @@ export class DatalakeClient { throw new DatalakeError('Missing response body') } - return Readable.from(response.body) + return Readable.fromWeb(response.body as any) } async getPartialObject ( @@ -162,7 +160,7 @@ export class DatalakeClient { throw new DatalakeError('Missing response body') } - return Readable.from(response.body) + return Readable.fromWeb(response.body as any) } async statObject ( @@ -258,21 +256,18 @@ export class DatalakeClient { const path = `/upload/form-data/${workspace}` const url = concatLink(this.endpoint, path) + const buffer = await toBuffer(stream) + const file = new File([buffer], objectName, { type: params.type, lastModified: params.lastModified }) + const form = new FormData() - const options: FormData.AppendOptions = { - filename: objectName, - contentType: params.type, - knownLength: params.size, - header: { - 'Last-Modified': params.lastModified - } - } - form.append('file', stream, options) + form.append('file', file) const response = await fetchSafe(ctx, url, { method: 'POST', - body: form, - headers: { ...this.headers } + body: form as unknown as BodyInit, + headers: { + ...this.headers + } }) const result = (await response.json()) as BlobUploadResult[] @@ -404,13 +399,15 @@ export class DatalakeClient { objectName: string, multipart: MultipartUpload, partNumber: number, - body: Readable | Buffer | string + data: Readable | Buffer | string ): Promise { const path = `/upload/multipart/${workspace}/${encodeURIComponent(objectName)}/part` const url = new URL(concatLink(this.endpoint, path)) url.searchParams.set('uploadId', multipart.uploadId) url.searchParams.set('partNumber', partNumber.toString()) + const body = data instanceof Readable ? (Readable.toWeb(data) as ReadableStream) : data + try { const response = await fetchSafe(ctx, url, { method: 'PUT', @@ -470,6 +467,22 @@ export class DatalakeClient { } } +async function toBuffer (data: Buffer | string | Readable): Promise { + if (Buffer.isBuffer(data)) { + return data + } else if (typeof data === 'string') { + return Buffer.from(data) + } else if (data instanceof Readable) { + const chunks: Buffer[] = [] + for await (const chunk of data) { + chunks.push(chunk) + } + return Buffer.concat(chunks as any) + } else { + throw new TypeError('Unsupported data type') + } +} + async function * getChunks (data: Buffer | string | Readable, chunkSize: number): AsyncGenerator { if (Buffer.isBuffer(data)) { let offset = 0 @@ -497,7 +510,7 @@ async function * getChunks (data: Buffer | string | Readable, chunkSize: number) } } -async function fetchSafe (ctx: MeasureContext, url: RequestInfo, init?: RequestInit): Promise { +async function fetchSafe (ctx: MeasureContext, url: string | URL, init?: RequestInit): Promise { let response try { response = await fetch(url, init)