From 808670b8cf6bfe7d3b1eee502d35b799977edfea Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Wed, 15 Oct 2025 11:50:36 +0700 Subject: [PATCH] Rework client storage access (#10081) * add generic uploader interface Signed-off-by: Alexander Onnikov * fix: refactor xhr upload Signed-off-by: Alexander Onnikov * enhance xhr upload Signed-off-by: Alexander Onnikov * refactor uploader Signed-off-by: Alexander Onnikov * fix: restore files url Signed-off-by: Alexander Onnikov * move storage client to core Signed-off-by: Alexander Onnikov * fix: revert front cookie Signed-off-by: Alexander Onnikov --------- Signed-off-by: Alexander Onnikov --- .vscode/launch.json | 2 +- common/config/rush/pnpm-lock.yaml | 13 +- desktop/src/ui/platform.ts | 8 +- desktop/src/ui/types.ts | 5 +- dev/docker-compose.yaml | 4 +- dev/prod/public/config-dev.json | 1 + dev/prod/public/config-huly.json | 1 - dev/prod/public/config-worker.json | 5 - dev/prod/public/config.json | 6 +- dev/prod/src/platform.ts | 11 +- packages/presentation/package.json | 1 + packages/presentation/src/file.ts | 279 +++-------------- packages/presentation/src/plugin.ts | 10 +- packages/presentation/src/preview.ts | 103 +------ .../src/components/AttachmentPopup.svelte | 5 +- .../src/components/AttachmentRefInput.svelte | 6 +- .../AttachmentStyleBoxCollabEditor.svelte | 12 +- .../src/components/AttachmentStyledBox.svelte | 4 +- .../src/components/Attachments.svelte | 3 +- .../src/components/Photos.svelte | 5 +- plugins/attachment-resources/src/utils.ts | 15 +- plugins/attachment/src/index.ts | 2 +- .../src/components/Description.svelte | 4 +- .../src/components/FilePlaceholder.svelte | 5 +- .../components/message/MessageInput.svelte | 8 +- .../src/components/EditableAvatar.svelte | 3 +- .../components/document/EditDocContent.svelte | 5 +- .../src/components/EditDoc.svelte | 5 +- .../src/components/FilePanel.svelte | 9 +- plugins/drive-resources/src/utils.ts | 9 +- .../settings/CreateCustomEmojiPopup.svelte | 2 +- .../src/components/NewMessage.svelte | 5 +- .../src/components/NewMessages.svelte | 5 +- .../src/components/RecordingPopup.svelte | 2 + .../src/components/CreateCandidate.svelte | 3 +- .../src/components/FileUploadStatusBar.svelte | 2 +- .../components/FileUploadStatusPopup.svelte | 109 +++---- plugins/uploader-resources/src/store.ts | 27 +- plugins/uploader-resources/src/utils.ts | 290 ++++++++++-------- plugins/uploader/src/types.ts | 38 ++- server/front/readme.md | 13 - server/front/src/index.ts | 6 +- server/front/src/starter.ts | 18 +- tests/docker-compose.override.yaml | 3 +- 44 files changed, 402 insertions(+), 670 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index b9f801d119..093e6e3c59 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -833,7 +833,7 @@ "SECRET": "secret", "DB_URL": "postgresql://root@localhost:26257/defaultdb?sslmode=disable", "BUCKETS": "blobs,eu|http://localhost:9000?accessKey=minioadmin&secretKey=minioadmin", - "ACCOUNTS_URL": "http://localhost:3000", + "ACCOUNTS_URL": "http://huly.local:3000", "STATS_URL": "http://huly.local:4900", "QUEUE_CONFIG": "localhost:19092" }, diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index c223c0074e..f4cea99fe3 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -145,6 +145,9 @@ importers: '@hcengineering/storage': specifier: ^0.7.5 version: 0.7.5 + '@hcengineering/storage-client': + specifier: ^0.7.6 + version: 0.7.6 '@hcengineering/text': specifier: ^0.7.5 version: 0.7.5(prosemirror-inputrules@1.4.0)(prosemirror-model@1.24.1)(prosemirror-state@1.4.3)(prosemirror-view@1.37.2) @@ -3241,6 +3244,9 @@ packages: '@hcengineering/server@0.7.6': resolution: {integrity: sha512-plu4EphWdFkS4h8OXKWzE1fYq+/+S9c5kuFtpIdrHoG6YA08JMOsQPwNWAJ2yUHTTwcLYMB/URxQNLdH/SYCnA==} + '@hcengineering/storage-client@0.7.6': + resolution: {integrity: sha512-+WgE5buPwcP2aTc0BiokM1rgDA0RXALL0GckVXb0haZnKJFHSAd8wT34Yf2F20Cwc0sSaRGeS57H9Vz2YokHGw==} + '@hcengineering/storage@0.7.5': resolution: {integrity: sha512-E7PiVruvqVhxUDqMEmsTOKu9knVeElyyU796wbRE5SEFOd9MY0p2lLr7vcwmbc0U/NPA+JD5NWHZwoTmdvBcgg==} @@ -5701,7 +5707,7 @@ packages: version: 0.0.0 '@rush-temp/presentation@file:projects/presentation.tgz': - resolution: {integrity: sha512-BeyUrrJQ0fcJqY271gqIIulRq1jKDnzEL+wNhbGbDvAXm0JUIcTsbX88LjYHLcRHaJ7k223iGhdctQpiOEY5Bg==, tarball: file:projects/presentation.tgz} + resolution: {integrity: sha512-Bc77dEEqJ5ktzx8GnxQItwE6jfyGmWlGS7PCPBmyelM88/BK55DnZ6EkkSJJqVVT6SeirFTEVa7cP+yAlNEmKQ==, tarball: file:projects/presentation.tgz} version: 0.0.0 '@rush-temp/print-assets@file:projects/print-assets.tgz': @@ -15878,6 +15884,10 @@ snapshots: '@hcengineering/server-token': 0.7.5 utf-8-validate: 6.0.4 + '@hcengineering/storage-client@0.7.6': + dependencies: + '@hcengineering/core': 0.7.7 + '@hcengineering/storage@0.7.5': dependencies: '@hcengineering/core': 0.7.7 @@ -28972,6 +28982,7 @@ snapshots: '@hcengineering/platform-rig': 0.7.19(@babel/core@7.23.9)(postcss-load-config@4.0.2(postcss@8.5.3)(ts-node@10.9.2(@swc/core@1.13.5)(@types/node@22.15.29)(typescript@5.9.3)))(postcss@8.5.3)(sass@1.93.2)(ts-node@10.9.2(@swc/core@1.13.5)(@types/node@22.15.29)(typescript@5.9.3)) '@hcengineering/query': 0.7.6 '@hcengineering/retry': 0.7.5 + '@hcengineering/storage-client': 0.7.6 '@hcengineering/text': 0.7.5(prosemirror-inputrules@1.4.0)(prosemirror-model@1.24.1)(prosemirror-state@1.4.3)(prosemirror-view@1.37.2) '@testing-library/jest-dom': 6.6.3 '@types/jest': 29.5.12 diff --git a/desktop/src/ui/platform.ts b/desktop/src/ui/platform.ts index 1f2e747293..3f41a17982 100644 --- a/desktop/src/ui/platform.ts +++ b/desktop/src/ui/platform.ts @@ -131,7 +131,7 @@ import '@hcengineering/ai-assistant-assets' import analyticsCollector, { analyticsCollectorId } from '@hcengineering/analytics-collector' import { coreId } from '@hcengineering/core' import love, { loveId } from '@hcengineering/love' -import presentation, { parsePreviewConfig, parseUploadConfig, presentationId } from '@hcengineering/presentation' +import presentation, { createFileStorage, presentationId } from '@hcengineering/presentation' import print, { printId } from '@hcengineering/print' import sign from '@hcengineering/sign' import textEditor, { textEditorId } from '@hcengineering/text-editor' @@ -288,11 +288,11 @@ export async function configurePlatform (onWorkbenchConnect?: () => Promise -} - -interface FileUploadError { - key: string - error: string -} - -interface FileUploadSuccess { - key: string - id: string -} - -type FileUploadResult = FileUploadSuccess | FileUploadError - -const defaultUploadUrl = '/files' -const defaultFilesUrl = '/files/:workspace/:filename?file=:blobId&workspace=:workspace' - -function parseInt (value: string, fallback: number): number { - const number = Number.parseInt(value) - return Number.isInteger(number) ? number : fallback -} - -export function parseUploadConfig (config: string, uploadUrl: string): UploadConfig { - const uploadConfig: UploadConfig = { - 'form-data': { url: uploadUrl }, - 'signed-url': undefined - } - - if (config !== undefined) { - const configs = config.split(';') - for (const c of configs) { - if (c === '') { - continue - } - - const [key, size, url] = c.split('|') - - if (url === undefined || url === '') { - throw new Error(`Bad upload config: ${c}`) - } - - if (key === 'form-data') { - uploadConfig['form-data'] = { url } - } else if (key === 'signed-url') { - uploadConfig['signed-url'] = { - url, - size: parseInt(size, 0) * 1024 * 1024 - } - } else { - throw new Error(`Unknown upload config key: ${key}`) - } - } - } - - return uploadConfig -} - -export function getFilesUrl (): string { - const filesUrl = getMetadata(plugin.metadata.FilesURL) ?? defaultFilesUrl - const frontUrl = getMetadata(plugin.metadata.FrontUrl) ?? window.location.origin - - return filesUrl.includes('://') ? filesUrl : concatLink(frontUrl, filesUrl) -} +import { getFileMetadata } from './filetypes' export function getCurrentWorkspaceUuid (): WorkspaceUuid { const workspaceUuid = getMetadata(plugin.metadata.WorkspaceUuid) ?? '' return workspaceUuid as WorkspaceUuid } -/** - * @public - */ +function getToken (): string { + return getMetadata(plugin.metadata.Token) ?? '' +} + +/** @public */ export function generateFileId (): string { return uuid() } -/** - * @public - */ -export function getUploadUrl (): string { - const template = getMetadata(plugin.metadata.UploadURL) ?? defaultUploadUrl - - return template.replaceAll(':workspace', encodeURIComponent(getCurrentWorkspaceUuid())) +/** @public */ +export function createFileStorage (uploadUrl: string, datalakeUrl?: string, hulylakeUrl?: string): FileStorage { + return createStorageClient({ uploadUrl, datalakeUrl, hulylakeUrl }) } -function getUploadConfig (): UploadConfig { - return getMetadata(plugin.metadata.UploadConfig) ?? { 'form-data': { url: getUploadUrl() } } -} - -function getFileUploadMethod (blob: Blob): { method: FileUploadMethod, url: string } { - const config = getUploadConfig() - - const signedUrl = config['signed-url'] - if (signedUrl !== undefined && signedUrl.size < blob.size) { - return { method: 'signed-url', url: signedUrl.url } +/** @public */ +export function getFileStorage (): FileStorage { + const storage = getMetadata(plugin.metadata.FileStorage) + if (storage === undefined) { + throw new Error('Missing file storage metadata') } - return { method: 'form-data', url: config['form-data'].url } + return storage } -/** - * @public - */ -export function getFileUploadParams (blobId: string, blob: Blob): FileUploadParams { - const workspaceId = encodeURIComponent(getCurrentWorkspaceUuid()) - const fileId = encodeURIComponent(blobId) - - const { method, url: urlTemplate } = getFileUploadMethod(blob) - - const url = urlTemplate.replaceAll(':workspace', workspaceId).replaceAll(':blobId', fileId) - - const headers: Record = - method !== 'signed-url' - ? { - Authorization: 'Bearer ' + (getMetadata(plugin.metadata.Token) as string) - } - : {} - - return { method, url, headers } -} - -/** - * @public - */ +/** @public */ export function getFileUrl (file: string, filename?: string): string { if (file.includes('://')) { return file } - const template = getFilesUrl() - return template - .replaceAll(':filename', encodeURIComponent(filename ?? file)) - .replaceAll(':workspace', encodeURIComponent(getCurrentWorkspaceUuid())) - .replaceAll(':blobId', encodeURIComponent(file)) + const workspace = getCurrentWorkspaceUuid() + + const storage = getFileStorage() + return storage.getFileUrl(workspace, file, filename) } -/** - * @public - */ -export async function uploadFile (file: File, uuid?: Ref): Promise> { +/** @public */ +export async function uploadFile ( + file: File, + uuid?: Ref +): Promise<{ uuid: Ref, metadata: Record }> { uuid ??= generateFileId() as Ref - const params = getFileUploadParams(uuid, file) + const token = getToken() + const workspace = getCurrentWorkspaceUuid() - if (params.method === 'signed-url') { - await uploadFileWithSignedUrl(file, uuid, params.url) - } else { - await uploadFileWithFormData(file, uuid, params.url) - } + const storage = getFileStorage() + await storage.uploadFile(token, workspace, uuid, file) - return uuid + const metadata = (await getFileMetadata(file, uuid)) ?? {} + + return { uuid, metadata } } -/** - * @public - */ -export async function deleteFile (id: string): Promise { - const fileUrl = getFileUrl(id) +/** @public */ +export async function deleteFile (file: string): Promise { + const token = getToken() + const workspace = getCurrentWorkspaceUuid() - const resp = await fetch(fileUrl, { - method: 'DELETE', - headers: { - Authorization: 'Bearer ' + (getMetadata(plugin.metadata.Token) as string) - } - }) - - if (!resp.ok) { - throw new Error('Failed to delete file') - } -} - -async function uploadFileWithFormData (file: File, uuid: string, uploadUrl: string): Promise { - const data = new FormData() - data.append('file', file, uuid) - - const resp = await fetch(uploadUrl, { - method: 'POST', - headers: { - Authorization: 'Bearer ' + (getMetadata(plugin.metadata.Token) as string) - }, - body: data - }) - - if (!resp.ok) { - if (resp.status === 413) { - throw new PlatformError(new Status(Severity.ERROR, plugin.status.FileTooLarge, {})) - } else { - throw Error(`Failed to upload file: ${resp.statusText}`) - } - } - - const result = (await resp.json()) as FileUploadResult[] - if (result.length !== 1) { - throw Error('Bad upload response') - } - - if ('error' in result[0]) { - throw Error(`Failed to upload file: ${result[0].error}`) - } -} - -async function uploadFileWithSignedUrl (file: File, uuid: string, uploadUrl: string): Promise { - const response = await fetch(uploadUrl, { - method: 'POST', - headers: { - Authorization: 'Bearer ' + (getMetadata(plugin.metadata.Token) as string) - } - }) - - if (response.ok) { - throw Error(`Failed to genearte signed upload URL: ${response.statusText}`) - } - - const signedUrl = await response.text() - if (signedUrl === undefined || signedUrl === '') { - throw Error('Missing signed upload URL') - } - - try { - const response = await fetch(signedUrl, { - body: file, - method: 'PUT', - headers: { - 'Content-Type': file.type, - 'Content-Length': file.size.toString() - // 'x-amz-meta-last-modified': file.lastModified.toString() - } - }) - - if (!response.ok) { - throw Error(`Failed to upload file: ${response.statusText}`) - } - - // confirm we uploaded file - await fetch(uploadUrl, { - method: 'PUT', - headers: { - Authorization: 'Bearer ' + (getMetadata(plugin.metadata.Token) as string) - } - }) - } catch (err) { - // abort the upload - await fetch(uploadUrl, { - method: 'DELETE', - headers: { - Authorization: 'Bearer ' + (getMetadata(plugin.metadata.Token) as string) - } - }) - } + const storage = getFileStorage() + await storage.deleteFile(token, workspace, file) } export async function getJsonOrEmpty (file: string, name: string): Promise { diff --git a/packages/presentation/src/plugin.ts b/packages/presentation/src/plugin.ts index b52c3d7893..952c76f0bc 100644 --- a/packages/presentation/src/plugin.ts +++ b/packages/presentation/src/plugin.ts @@ -38,10 +38,9 @@ import { } from '@hcengineering/core' import type { Asset, IntlString, Metadata, Plugin, StatusCode } from '@hcengineering/platform' import { plugin } from '@hcengineering/platform' +import { type FileStorage } from '@hcengineering/storage-client' import { type ComponentExtensionId } from '@hcengineering/ui/src/types' -import { type UploadConfig } from './file' import { type PresentationMiddlewareFactory } from './pipeline' -import type { PreviewConfig } from './preview' import { type ComponentPointExtension, type DocCreateExtension, @@ -51,6 +50,8 @@ import { type ObjectSearchCategory } from './types' +export type { FileStorage } from '@hcengineering/storage-client' + /** * @public */ @@ -165,7 +166,7 @@ export default plugin(presentationId, { FrontVersion: '' as Metadata, Draft: '' as Metadata>, UploadURL: '' as Metadata, - FilesURL: '' as Metadata, + DatalakeUrl: '' as Metadata, CollaboratorUrl: '' as Metadata, Token: '' as Metadata, Endpoint: '' as Metadata, @@ -173,8 +174,7 @@ export default plugin(presentationId, { WorkspaceDataId: '' as Metadata, FrontUrl: '' as Asset, LinkPreviewUrl: '' as Metadata, - UploadConfig: '' as Metadata, - PreviewConfig: '' as Metadata, + FileStorage: '' as Metadata, ClientHook: '' as Metadata, SessionId: '' as Metadata, StatsUrl: '' as Metadata, diff --git a/packages/presentation/src/preview.ts b/packages/presentation/src/preview.ts index 2c0e9ca17a..15cdc2f572 100644 --- a/packages/presentation/src/preview.ts +++ b/packages/presentation/src/preview.ts @@ -3,9 +3,11 @@ import { concatLink } from '@hcengineering/core' import { getMetadata } from '@hcengineering/platform' import { withRetry } from '@hcengineering/retry' -import { getFileUrl, getCurrentWorkspaceUuid } from './file' +import { getFileUrl, getCurrentWorkspaceUuid, getFileStorage } from './file' import presentation from './plugin' +const frontImagePreviewUrl = '/files/:workspace?file=:blobId&size=:size' + export interface PreviewMetadata { thumbnail?: { width: number @@ -14,11 +16,6 @@ export interface PreviewMetadata { } } -export interface PreviewConfig { - image: string - video: string -} - export interface VideoMeta { hls?: HLSMeta } @@ -28,50 +25,6 @@ export interface HLSMeta { source?: string } -const defaultImagePreview = (): string => `/files/${getCurrentWorkspaceUuid()}?file=:blobId&size=:size` - -/** - * - * PREVIEW_CONFIG env variable format. - * - image - an Url with :workspace, :blobId, :downloadFile, :size placeholders. - * - video - an Url with :workspace, :blobId placeholders. - */ -export function parsePreviewConfig (config?: string): PreviewConfig | undefined { - if (config === undefined) { - return - } - - const previewConfig = { image: defaultImagePreview(), video: '' } - - const configs = config.split(';') - for (const c of configs) { - if (c.includes('|')) { - const [key, value] = c.split('|') - if (key === 'image') { - previewConfig.image = value - } else if (key === 'video') { - previewConfig.video = value - } else { - throw new Error(`Unknown preview config key: ${key}`) - } - } else { - // fallback to image-only config for compatibility - previewConfig.image = c - } - } - - return Object.freeze(previewConfig) -} - -export function getPreviewConfig (): PreviewConfig { - return ( - (getMetadata(presentation.metadata.PreviewConfig) as PreviewConfig) ?? { - image: defaultImagePreview(), - video: '' - } - ) -} - export async function getBlobRef ( file: Ref, name?: string, @@ -92,15 +45,10 @@ export async function getBlobSrcSet (file: Ref, width?: number, height?: n } export function getSrcSet (_blob: Ref, width?: number, height?: number): string { - return blobToSrcSet(getPreviewConfig(), _blob, width, height) + return blobToSrcSet(_blob, width, height) } -function blobToSrcSet ( - cfg: PreviewConfig, - blob: Ref, - width: number | undefined, - height: number | undefined -): string { +function blobToSrcSet (blob: Ref, width: number | undefined, height: number | undefined): string { if (blob.includes('://')) { return '' } @@ -124,15 +72,8 @@ function blobToSrcSet ( } } - let url = cfg.image.replaceAll(':workspace', workspace) - const downloadUrl = getFileUrl(blob) - const frontUrl = getMetadata(presentation.metadata.FrontUrl) ?? window.location.origin - if (!url.includes('://')) { - url = concatLink(frontUrl ?? '', url) - } - url = url.replaceAll(':downloadFile', encodeURIComponent(downloadUrl)) - url = url.replaceAll(':blobId', name) + const url = concatLink(frontUrl, frontImagePreviewUrl).replaceAll(':workspace', workspace).replaceAll(':blobId', name) let result = '' if (width !== undefined) { @@ -198,33 +139,21 @@ function formatImageSize (url: string, width: number, height: number, dpr: numbe * @deprecated, please use Blob direct operations. */ export function getFileSrcSet (_blob: Ref, width?: number, height?: number): string { - return blobToSrcSet(getPreviewConfig(), _blob, width, height) + return blobToSrcSet(_blob, width, height) } /** * @public */ export async function getVideoMeta (file: string, filename?: string): Promise { - const cfg = getPreviewConfig() - - let url = cfg.video - .replaceAll(':workspace', encodeURIComponent(getCurrentWorkspaceUuid())) - .replaceAll(':blobId', encodeURIComponent(file)) - - if (url === '') { - return undefined - } - - const token = getMetadata(presentation.metadata.Token) ?? '' - const frontUrl = getMetadata(presentation.metadata.FrontUrl) ?? window.location.origin - if (!url.includes('://')) { - url = concatLink(frontUrl ?? '', url) - } - try { - const response = await fetch(url, { headers: { Authorization: `Bearer ${token}` } }) - if (response.ok) { - return (await response.json()) as VideoMeta - } - } catch {} + const token = getMetadata(presentation.metadata.Token) ?? '' + const workspace = getCurrentWorkspaceUuid() + + const storage = getFileStorage() + const meta = await storage.getFileMeta(token, workspace, file) + return meta as VideoMeta + } catch { + return {} + } } diff --git a/plugins/attachment-resources/src/components/AttachmentPopup.svelte b/plugins/attachment-resources/src/components/AttachmentPopup.svelte index 6c17b87ec5..799ccce04f 100644 --- a/plugins/attachment-resources/src/components/AttachmentPopup.svelte +++ b/plugins/attachment-resources/src/components/AttachmentPopup.svelte @@ -15,7 +15,7 @@ -->