From f5c3c5720ad2c8c735164c34be57efd12a163106 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Fri, 10 Jan 2025 13:44:43 +0700 Subject: [PATCH 1/7] remove datalake worker (#7626) --- common/config/rush/pnpm-lock.yaml | 51 +-- rush.json | 5 - workers/datalake/.eslintrc.js | 7 - workers/datalake/config/rig.json | 5 - workers/datalake/jest.config.js | 7 - workers/datalake/package.json | 43 --- workers/datalake/schema/Dockerfile | 15 - workers/datalake/schema/README.md | 5 - workers/datalake/schema/datalake.sql | 31 -- workers/datalake/schema/init.sh | 9 - workers/datalake/schema/logs.yaml | 67 ---- workers/datalake/schema/optimizations.sql | 11 - workers/datalake/src/blob.ts | 397 --------------------- workers/datalake/src/const.ts | 19 - workers/datalake/src/cors.ts | 103 ------ workers/datalake/src/db.ts | 195 ---------- workers/datalake/src/encodings.ts | 35 -- workers/datalake/src/hash.ts | 25 -- workers/datalake/src/image.ts | 58 --- workers/datalake/src/index.ts | 145 -------- workers/datalake/src/metrics.ts | 188 ---------- workers/datalake/src/multipart.ts | 152 -------- workers/datalake/src/s3.ts | 77 ---- workers/datalake/src/sign.ts | 137 ------- workers/datalake/src/storage.ts | 76 ---- workers/datalake/src/types.ts | 43 --- workers/datalake/src/video.ts | 117 ------ workers/datalake/tsconfig.json | 12 - workers/datalake/worker-configuration.d.ts | 31 -- workers/datalake/wrangler.toml | 110 ------ 30 files changed, 2 insertions(+), 2174 deletions(-) delete mode 100644 workers/datalake/.eslintrc.js delete mode 100644 workers/datalake/config/rig.json delete mode 100644 workers/datalake/jest.config.js delete mode 100644 workers/datalake/package.json delete mode 100644 workers/datalake/schema/Dockerfile delete mode 100644 workers/datalake/schema/README.md delete mode 100644 workers/datalake/schema/datalake.sql delete mode 100644 workers/datalake/schema/init.sh delete mode 100644 workers/datalake/schema/logs.yaml delete mode 100644 workers/datalake/schema/optimizations.sql delete mode 100644 workers/datalake/src/blob.ts delete mode 100644 workers/datalake/src/const.ts delete mode 100644 workers/datalake/src/cors.ts delete mode 100644 workers/datalake/src/db.ts delete mode 100644 workers/datalake/src/encodings.ts delete mode 100644 workers/datalake/src/hash.ts delete mode 100644 workers/datalake/src/image.ts delete mode 100644 workers/datalake/src/index.ts delete mode 100644 workers/datalake/src/metrics.ts delete mode 100644 workers/datalake/src/multipart.ts delete mode 100644 workers/datalake/src/s3.ts delete mode 100644 workers/datalake/src/sign.ts delete mode 100644 workers/datalake/src/storage.ts delete mode 100644 workers/datalake/src/types.ts delete mode 100644 workers/datalake/src/video.ts delete mode 100644 workers/datalake/tsconfig.json delete mode 100644 workers/datalake/worker-configuration.d.ts delete mode 100644 workers/datalake/wrangler.toml diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index e95b91172a..a5d4a5d887 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -158,9 +158,6 @@ dependencies: '@rush-temp/cloud-branding': specifier: file:./projects/cloud-branding.tgz version: file:projects/cloud-branding.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.24.2)(ts-node@10.9.2)(utf-8-validate@6.0.4) - '@rush-temp/cloud-datalake': - specifier: file:./projects/cloud-datalake.tgz - version: file:projects/cloud-datalake.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.24.2)(ts-node@10.9.2)(utf-8-validate@6.0.4) '@rush-temp/cloud-transactor': specifier: file:./projects/cloud-transactor.tgz version: file:projects/cloud-transactor.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(ts-node@10.9.2)(utf-8-validate@6.0.4) @@ -1442,9 +1439,6 @@ dependencies: aws-sdk: specifier: ^2.1423.0 version: 2.1664.0 - aws4fetch: - specifier: ^1.0.20 - version: 1.0.20 base64-js: specifier: ^1.5.1 version: 1.5.1 @@ -8652,10 +8646,6 @@ packages: xml2js: 0.6.2 dev: false - /aws4fetch@1.0.20: - resolution: {integrity: sha512-/djoAN709iY65ETD6LKCtyyEI04XIBP5xVvfmNxsEP0uJB5tyaGBztSryRr4HqMStr9R06PisQE7m9zDTXKu6g==} - dev: false - /axobject-query@4.0.0: resolution: {integrity: sha512-+60uv1hiVFhHZeO+Lz0RYzsVHy5Wr1ayX0mwda9KPDVLNJgZ1T9Ny7VmFbLDzxsH0D87I86vgj3gFrjTJUYznw==} dependencies: @@ -23000,43 +22990,6 @@ packages: - utf-8-validate dev: false - file:projects/cloud-datalake.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.24.2)(ts-node@10.9.2)(utf-8-validate@6.0.4): - resolution: {integrity: sha512-pOjup2gTfDH4Qq9r4LTeVnf+nDQvlTSUOxLtT+bKnrHM0jQQ705v6wYtmkpWD7CtEwwCvS9CH+iPZwB5ZNpzqQ==, tarball: file:projects/cloud-datalake.tgz} - id: file:projects/cloud-datalake.tgz - name: '@rush-temp/cloud-datalake' - version: 0.0.0 - dependencies: - '@cloudflare/workers-types': 4.20241022.0 - '@types/jest': 29.5.12 - '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.6.2) - '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2) - aws4fetch: 1.0.20 - eslint: 8.56.0 - eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0)(eslint-plugin-import@2.29.1)(eslint-plugin-n@15.7.0)(eslint-plugin-promise@6.1.1)(eslint@8.56.0)(typescript@5.6.2) - eslint-plugin-import: 2.29.1(eslint@8.56.0) - eslint-plugin-n: 15.7.0(eslint@8.56.0) - eslint-plugin-promise: 6.1.1(eslint@8.56.0) - itty-router: 5.0.18 - jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2) - postgres: 3.4.5 - prettier: 3.2.5 - ts-jest: 29.1.2(esbuild@0.24.2)(jest@29.7.0)(typescript@5.6.2) - typescript: 5.6.2 - wrangler: 3.97.0(@cloudflare/workers-types@4.20241022.0)(bufferutil@4.0.8)(utf-8-validate@6.0.4) - transitivePeerDependencies: - - '@babel/core' - - '@jest/types' - - '@types/node' - - babel-jest - - babel-plugin-macros - - bufferutil - - esbuild - - node-notifier - - supports-color - - ts-node - - utf-8-validate - dev: false - file:projects/cloud-transactor.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(ts-node@10.9.2)(utf-8-validate@6.0.4): resolution: {integrity: sha512-6kBv/3KfTsOvsDrfVXQvQwtLM72ltx4oPHc3cJ9xtG82XLeQLLpeJmCy6CNDVOtXaqnagwF5TKm/zCw4sc/dFQ==, tarball: file:projects/cloud-transactor.tgz} id: file:projects/cloud-transactor.tgz @@ -27298,7 +27251,7 @@ packages: dev: false file:projects/pod-ai-bot.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.4)(zod@3.23.8): - resolution: {integrity: sha512-OPo+KhRKsPQhO1eqOIgL30ef9s/TAnEf21bAwzxLLcB70AIwMyjfPRzzJHDOIbVjCzA8t4YFQ5MWWoLUveylFg==, tarball: file:projects/pod-ai-bot.tgz} + resolution: {integrity: sha512-R2ByQWsnvxxBGk/Ql7z8CK8NRb4XXREC0rndthdpfqw1dhwofuHiHpU2sb7L+OTHiz0do16L5gOLb7R1CbMecw==, tarball: file:projects/pod-ai-bot.tgz} id: file:projects/pod-ai-bot.tgz name: '@rush-temp/pod-ai-bot' version: 0.0.0 @@ -27773,7 +27726,7 @@ packages: dev: false file:projects/pod-love.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.4): - resolution: {integrity: sha512-Ibi2id+aFeeH3lGo2z5GW3Wg3ZWha/5VTWYxXxC0IQr/ivvk+LFlWShEuCyG8XYvIoy6OVAmafgpfOKeUrQjsg==, tarball: file:projects/pod-love.tgz} + resolution: {integrity: sha512-G70rzmJHk/en1EdQHEs2HEK9/D2IY0i/iF0WZtU2FWLtSzlJb5UC0hrpVfxD/aWcnfmmqcLQu+b2siYrnUFSmQ==, tarball: file:projects/pod-love.tgz} id: file:projects/pod-love.tgz name: '@rush-temp/pod-love' version: 0.0.0 diff --git a/rush.json b/rush.json index 12f052a2a5..c0f405aa5a 100644 --- a/rush.json +++ b/rush.json @@ -2176,11 +2176,6 @@ "projectFolder": "workers/branding", "shouldPublish": false }, - { - "packageName": "@hcengineering/cloud-datalake", - "projectFolder": "workers/datalake", - "shouldPublish": false - }, { "packageName": "@hcengineering/scripts", "projectFolder": "common/scripts", diff --git a/workers/datalake/.eslintrc.js b/workers/datalake/.eslintrc.js deleted file mode 100644 index ce90fb9646..0000000000 --- a/workers/datalake/.eslintrc.js +++ /dev/null @@ -1,7 +0,0 @@ -module.exports = { - extends: ['./node_modules/@hcengineering/platform-rig/profiles/node/eslint.config.json'], - parserOptions: { - tsconfigRootDir: __dirname, - project: './tsconfig.json' - } -} diff --git a/workers/datalake/config/rig.json b/workers/datalake/config/rig.json deleted file mode 100644 index 78cc5a1733..0000000000 --- a/workers/datalake/config/rig.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json", - "rigPackageName": "@hcengineering/platform-rig", - "rigProfile": "node" -} diff --git a/workers/datalake/jest.config.js b/workers/datalake/jest.config.js deleted file mode 100644 index 2cfd408b67..0000000000 --- a/workers/datalake/jest.config.js +++ /dev/null @@ -1,7 +0,0 @@ -module.exports = { - preset: 'ts-jest', - testEnvironment: 'node', - testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'], - roots: ["./src"], - coverageReporters: ["text-summary", "html"] -} diff --git a/workers/datalake/package.json b/workers/datalake/package.json deleted file mode 100644 index bdcea6e74d..0000000000 --- a/workers/datalake/package.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "name": "@hcengineering/cloud-datalake", - "version": "0.6.0", - "main": "lib/index.js", - "types": "types/index.d.ts", - "template": "@hcengineering/cloud-package", - "scripts": { - "deploy": "wrangler deploy", - "dev": "wrangler dev --port 4021", - "start": "wrangler dev --port 4021", - "cf-typegen": "wrangler types", - "build": "compile", - "build:watch": "compile", - "test": "jest --passWithNoTests --silent --forceExit", - "format": "format src", - "_phase:build": "compile transpile src", - "_phase:test": "jest --passWithNoTests --silent --forceExit", - "_phase:format": "format src", - "_phase:validate": "compile validate" - }, - "devDependencies": { - "@hcengineering/platform-rig": "^0.6.0", - "@cloudflare/workers-types": "^4.20241022.0", - "typescript": "^5.3.3", - "wrangler": "^3.97.0", - "jest": "^29.7.0", - "prettier": "^3.1.0", - "ts-jest": "^29.1.1", - "@typescript-eslint/eslint-plugin": "^6.11.0", - "@typescript-eslint/parser": "^6.11.0", - "eslint-config-standard-with-typescript": "^40.0.0", - "eslint-plugin-import": "^2.26.0", - "eslint-plugin-n": "^15.4.0", - "eslint-plugin-promise": "^6.1.1", - "eslint": "^8.54.0", - "@types/jest": "^29.5.5" - }, - "dependencies": { - "aws4fetch": "^1.0.20", - "itty-router": "^5.0.18", - "postgres": "^3.4.5" - } -} diff --git a/workers/datalake/schema/Dockerfile b/workers/datalake/schema/Dockerfile deleted file mode 100644 index 91028cbe01..0000000000 --- a/workers/datalake/schema/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM cockroachdb/cockroach:latest - -ADD init.sh /cockroach/ -RUN chmod a+x /cockroach/init.sh - -ADD logs.yaml /cockroach/ -ADD optimizations.sql /cockroach/ -ADD datalake.sql /cockroach/ - -WORKDIR /cockroach/ - -EXPOSE 8080 -EXPOSE 26257 - -ENTRYPOINT ["/cockroach/init.sh"] diff --git a/workers/datalake/schema/README.md b/workers/datalake/schema/README.md deleted file mode 100644 index 41919c774f..0000000000 --- a/workers/datalake/schema/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Running Cockroach DB in Docker - -```bash -docker run -d -p 8080:8080 -p 26257:26257 cockroach:dev -name cockroach -``` diff --git a/workers/datalake/schema/datalake.sql b/workers/datalake/schema/datalake.sql deleted file mode 100644 index d9b36f1b0b..0000000000 --- a/workers/datalake/schema/datalake.sql +++ /dev/null @@ -1,31 +0,0 @@ - -CREATE SCHEMA IF NOT EXISTS blob; - -DROP TABLE IF EXISTS blob.blob; -DROP TABLE IF EXISTS blob.data; -DROP TYPE IF EXISTS blob.location; - --- B L O B - -CREATE TYPE blob.location AS ENUM ('weur', 'eeur', 'wnam', 'enam', 'apac'); - -\echo "Creating blob.data..." -CREATE TABLE blob.data ( - hash UUID NOT NULL, - location blob.location NOT NULL, - size INT8 NOT NULL, - filename UUID NOT NULL, - type STRING(255) NOT NULL, - CONSTRAINT pk_data PRIMARY KEY (hash, location) -); - -\echo "Creating blob.blob..." -CREATE TABLE blob.blob ( - workspace STRING(255) NOT NULL, - name STRING(255) NOT NULL, - hash UUID NOT NULL, - location blob.location NOT NULL, - deleted BOOL NOT NULL, - CONSTRAINT pk_blob PRIMARY KEY (workspace, name), - CONSTRAINT fk_data FOREIGN KEY (hash, location) REFERENCES blob.data (hash, location) -); diff --git a/workers/datalake/schema/init.sh b/workers/datalake/schema/init.sh deleted file mode 100644 index a4a7abebed..0000000000 --- a/workers/datalake/schema/init.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -./cockroach start-single-node --insecure --log-config-file=logs.yaml --cache=.25 --background --store=type=mem,size=50% -./cockroach sql --insecure --file optimizations.sql -./cockroach sql --insecure --execute="CREATE DATABASE datalake;" -./cockroach sql --insecure --database=datalake --file datalake.sql - -cd /cockroach/cockroach-data/logs -tail -f cockroach.log diff --git a/workers/datalake/schema/logs.yaml b/workers/datalake/schema/logs.yaml deleted file mode 100644 index 98bcd88e9c..0000000000 --- a/workers/datalake/schema/logs.yaml +++ /dev/null @@ -1,67 +0,0 @@ -file-defaults: - max-file-size: 10MiB - max-group-size: 100MiB - file-permissions: 644 - buffered-writes: true - filter: WARNING - format: crdb-v2 - redact: false - redactable: true - exit-on-error: true - auditable: false - dir: cockroach-data/logs -fluent-defaults: - filter: WARNING - format: json-fluent-compact - redact: false - redactable: true - exit-on-error: false - auditable: false -http-defaults: - method: POST - unsafe-tls: false - timeout: 0s - disable-keep-alives: false - filter: WARNING - format: json-compact - redact: false - redactable: true - exit-on-error: false - auditable: false -sinks: - file-groups: - default: - channels: - WARNING: all - health: - channels: [HEALTH] - pebble: - channels: [STORAGE] - security: - channels: [PRIVILEGES, USER_ADMIN] - auditable: true - sql-audit: - channels: [SENSITIVE_ACCESS] - auditable: true - sql-auth: - channels: [SESSIONS] - auditable: true - sql-exec: - channels: [SQL_EXEC] - sql-slow: - channels: [SQL_PERF] - sql-slow-internal-only: - channels: [SQL_INTERNAL_PERF] - telemetry: - channels: [TELEMETRY] - max-file-size: 100KiB - max-group-size: 1.0MiB - stderr: - channels: all - filter: NONE - redact: false - redactable: true - exit-on-error: true -capture-stray-errors: - enable: true - max-group-size: 100MiB diff --git a/workers/datalake/schema/optimizations.sql b/workers/datalake/schema/optimizations.sql deleted file mode 100644 index 9c1d14c779..0000000000 --- a/workers/datalake/schema/optimizations.sql +++ /dev/null @@ -1,11 +0,0 @@ --- see https://www.cockroachlabs.com/docs/v21.2/local-testing.html#use-a-local-single-node-cluster-with-in-memory-storage -SET CLUSTER SETTING kv.raft_log.disable_synchronization_unsafe = true; -SET CLUSTER SETTING kv.range_merge.queue_interval = '50ms'; -SET CLUSTER SETTING jobs.registry.interval.gc = '30s'; -SET CLUSTER SETTING jobs.registry.interval.cancel = '180s'; -SET CLUSTER SETTING jobs.retention_time = '15s'; ---SET CLUSTER SETTING schemachanger.backfiller.buffer_increment = '128 KiB'; -SET CLUSTER SETTING sql.stats.automatic_collection.enabled = false; -SET CLUSTER SETTING kv.range_split.by_load_merge_delay = '5s'; -ALTER RANGE default CONFIGURE ZONE USING "gc.ttlseconds" = 600; -ALTER DATABASE system CONFIGURE ZONE USING "gc.ttlseconds" = 600; diff --git a/workers/datalake/src/blob.ts b/workers/datalake/src/blob.ts deleted file mode 100644 index fd4782cb6e..0000000000 --- a/workers/datalake/src/blob.ts +++ /dev/null @@ -1,397 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { error, json } from 'itty-router' -import { type BlobDB, withPostgres } from './db' -import { cacheControl, hashLimit } from './const' -import { toUUID } from './encodings' -import { getSha256 } from './hash' -import { selectStorage } from './storage' -import { type BlobRequest, type WorkspaceRequest, type UUID } from './types' -import { copyVideo, deleteVideo } from './video' -import { type MetricsContext, LoggedCache } from './metrics' - -export interface BlobMetadata { - lastModified: number - type: string - size: number - name: string - etag: string -} - -export function getBlobURL (request: Request, workspace: string, name: string): string { - const path = `/blob/${workspace}/${name}` - return new URL(path, request.url).toString() -} - -export async function handleBlobList ( - request: WorkspaceRequest, - env: Env, - ctx: ExecutionContext, - metrics: MetricsContext -): Promise { - const { workspace } = request - const cursor = extractStrParam(request.query.cursor) - const limit = extractIntParam(request.query.limit) - - const response = await withPostgres(env, ctx, metrics, (db) => { - return db.listBlobs(workspace, cursor, limit) - }) - - const blobs = response.blobs.map((blob) => { - const { name, size, type, hash } = blob - return { name, size, type, etag: hash } - }) - - return json({ blobs, cursor: response.cursor }) -} - -export async function handleBlobGet ( - request: BlobRequest, - env: Env, - ctx: ExecutionContext, - metrics: MetricsContext -): Promise { - const { workspace, name } = request - - const cache = new LoggedCache(caches.default, metrics) - - const cacheControl = request.headers.get('Cache-Control') ?? '' - if (!cacheControl.includes('no-cache')) { - const cached = await cache.match(request) - if (cached !== undefined) { - return cached - } - } - - const { bucket } = selectStorage(env, workspace) - - const blob = await withPostgres(env, ctx, metrics, (db) => { - return db.getBlob({ workspace, name }) - }) - if (blob === null || blob.deleted) { - return error(404) - } - - const range = request.headers.has('Range') ? request.headers : undefined - const object = await bucket.get(blob.filename, { range }) - if (object === null) { - return error(404) - } - - const headers = r2MetadataHeaders(blob.hash, object) - if (range !== undefined && object?.range !== undefined) { - headers.set('Content-Range', rangeHeader(object.range, object.size)) - } - - const length = object?.range !== undefined && 'length' in object.range ? object?.range?.length : undefined - const status = length !== undefined && length < object.size ? 206 : 200 - if (length !== undefined && length < object.size) { - // for partial content use etag returned by R2 - headers.set('ETag', object.httpEtag) - } - - const response = new Response(object?.body, { headers, status }) - - if (response.status === 200) { - if (!cacheControl.includes('no-store')) { - const clone = metrics.withSync('response.clone', () => response.clone()) - ctx.waitUntil(cache.put(request, clone)) - } - } - - return response -} - -export async function handleBlobHead ( - request: BlobRequest, - env: Env, - ctx: ExecutionContext, - metrics: MetricsContext -): Promise { - const { workspace, name } = request - - const { bucket } = selectStorage(env, workspace) - - const blob = await withPostgres(env, ctx, metrics, (db) => { - return db.getBlob({ workspace, name }) - }) - if (blob === null || blob.deleted) { - return error(404) - } - - const head = await bucket.head(blob.filename) - if (head?.httpMetadata === undefined) { - return error(404) - } - - const headers = r2MetadataHeaders(blob.hash, head) - return new Response(null, { headers, status: 200 }) -} - -export async function handleBlobDelete ( - request: BlobRequest, - env: Env, - ctx: ExecutionContext, - metrics: MetricsContext -): Promise { - const { workspace, name } = request - - try { - await withPostgres(env, ctx, metrics, (db) => { - return Promise.all([db.deleteBlob({ workspace, name }), deleteVideo(env, workspace, name)]) - }) - - return new Response(null, { status: 204 }) - } catch (err: any) { - const message = err instanceof Error ? err.message : String(err) - console.error({ error: 'failed to delete blob:' + message }) - return error(500) - } -} - -export async function handleUploadFormData ( - request: WorkspaceRequest, - env: Env, - ctx: ExecutionContext, - metrics: MetricsContext -): Promise { - const contentType = request.headers.get('Content-Type') - if (contentType === null || !contentType.includes('multipart/form-data')) { - console.error({ error: 'expected multipart/form-data' }) - return error(400, 'expected multipart/form-data') - } - - const { workspace } = request - - let formData: FormData - try { - formData = await metrics.with('request.formData', () => request.formData()) - } catch (err: any) { - const message = err instanceof Error ? err.message : String(err) - console.error({ error: 'failed to parse form data', message }) - return error(400, 'failed to parse form data') - } - - const files: [File, key: string][] = [] - formData.forEach((value: any, key: string) => { - if (typeof value === 'object') files.push([value, key]) - }) - - const result = await Promise.all( - files.map(async ([file, key]) => { - const { name, type, lastModified } = file - try { - const metadata = await withPostgres(env, ctx, metrics, (db) => { - return saveBlob(env, db, file.stream(), file.size, type, workspace, name, lastModified) - }) - - // TODO this probably should happen via queue, let it be here for now - if (type.startsWith('video/')) { - const blobURL = getBlobURL(request, workspace, name) - await copyVideo(env, blobURL, workspace, name) - } - - return { key, metadata } - } catch (err: any) { - const error = err instanceof Error ? err.message : String(err) - console.error('failed to upload blob:', error) - return { key, error } - } - }) - ) - - return json(result) -} - -export async function saveBlob ( - env: Env, - db: BlobDB, - stream: ReadableStream, - size: number, - type: string, - workspace: string, - name: string, - lastModified: number -): Promise { - const { location, bucket } = selectStorage(env, workspace) - - const httpMetadata = { contentType: type, cacheControl, lastModified } - const filename = getUniqueFilename() - - const blob = await db.getBlob({ workspace, name }) - - if (size <= hashLimit) { - const [hashStream, uploadStream] = stream.tee() - - const hash = await getSha256(hashStream) - - // Check if we have the same blob already - if (blob?.hash === hash && blob?.type === type) { - return { type, size, lastModified, name, etag: hash } - } - - const data = await db.getData({ hash, location }) - - if (data !== null) { - // Lucky boy, nothing to upload, use existing blob - await db.createBlob({ workspace, name, hash, location }) - - return { type, size, lastModified, name, etag: data.hash } - } else { - await bucket.put(filename, uploadStream, { httpMetadata }) - - await db.createData({ hash, location, filename, type, size }) - await db.createBlob({ workspace, name, hash, location }) - - return { type, size, lastModified, name, etag: hash } - } - } else { - // For large files we cannot calculate checksum beforehead - // upload file with unique filename and then obtain checksum - const { hash } = await uploadLargeFile(bucket, stream, filename, { httpMetadata }) - const data = await db.getData({ hash, location }) - if (data !== null) { - // We found an existing blob with the same hash - // we can safely remove the existing blob from storage - await Promise.all([bucket.delete(filename), db.createBlob({ workspace, name, hash, location })]) - - return { type, size, lastModified, name, etag: hash } - } else { - // Otherwise register a new hash and blob - await db.createData({ hash, location, filename, type, size }) - await db.createBlob({ workspace, name, hash, location }) - - return { type, size, lastModified, name, etag: hash } - } - } -} - -export async function handleBlobUploaded ( - env: Env, - ctx: ExecutionContext, - metrics: MetricsContext, - workspace: string, - name: string, - filename: UUID -): Promise { - const { location, bucket } = selectStorage(env, workspace) - - const object = await bucket.head(filename) - if (object?.httpMetadata === undefined) { - throw Error('blob not found') - } - - const hash = object.checksums.md5 !== undefined ? digestToUUID(object.checksums.md5) : (crypto.randomUUID() as UUID) - const size = object.size - const type = object.httpMetadata?.contentType ?? 'application/octet-stream' - - await withPostgres(env, ctx, metrics, async (db) => { - const data = await db.getData({ hash, location }) - if (data !== null) { - await Promise.all([bucket.delete(filename), db.createBlob({ workspace, name, hash, location })]) - } else { - await db.createData({ hash, location, filename, type, size }) - await db.createBlob({ workspace, name, hash, location }) - } - }) - - return { type, size, name, etag: hash, lastModified: object.uploaded.getTime() } -} - -async function uploadLargeFile ( - bucket: R2Bucket, - stream: ReadableStream, - filename: string, - options: R2PutOptions -): Promise<{ hash: UUID }> { - const digestStream = new crypto.DigestStream('SHA-256') - - const [digestFS, uploadFS] = stream.tee() - - const digestPromise = digestFS.pipeTo(digestStream) - const uploadPromise = bucket.put(filename, uploadFS, options) - - await Promise.all([digestPromise, uploadPromise]) - - const hash = digestToUUID(await digestStream.digest) - - return { hash } -} - -function getUniqueFilename (): UUID { - return crypto.randomUUID() as UUID -} - -function digestToUUID (digest: ArrayBuffer): UUID { - return toUUID(new Uint8Array(digest)) -} - -function rangeHeader (range: R2Range, size: number): string { - const offset = 'offset' in range ? range.offset : undefined - const length = 'length' in range ? range.length : undefined - const suffix = 'suffix' in range ? range.suffix : undefined - - const start = suffix !== undefined ? size - suffix : offset ?? 0 - const end = suffix !== undefined ? size : length !== undefined ? start + length : size - - return `bytes ${start}-${end - 1}/${size}` -} - -function r2MetadataHeaders (hash: string, head: R2Object): Headers { - return head.httpMetadata !== undefined - ? new Headers({ - 'Accept-Ranges': 'bytes', - 'Content-Length': head.size.toString(), - 'Content-Type': head.httpMetadata.contentType ?? '', - 'Content-Security-Policy': "default-src 'none';", - 'Cache-Control': head.httpMetadata.cacheControl ?? cacheControl, - 'Last-Modified': head.uploaded.toUTCString(), - ETag: hash - }) - : new Headers({ - 'Accept-Ranges': 'bytes', - 'Content-Length': head.size.toString(), - 'Content-Security-Policy': "default-src 'none';", - 'Cache-Control': cacheControl, - 'Last-Modified': head.uploaded.toUTCString(), - ETag: hash - }) -} - -function extractStrParam (value: string | string[] | undefined): string | undefined { - if (Array.isArray(value)) { - return value[0] - } - - return value -} - -function extractIntParam (value: string | string[] | undefined): number | undefined { - if (value === undefined) { - return undefined - } - - if (Array.isArray(value)) { - value = value[0] - } - - const intValue = Number.parseInt(value) - if (Number.isInteger(intValue)) { - return intValue - } - - return undefined -} diff --git a/workers/datalake/src/const.ts b/workers/datalake/src/const.ts deleted file mode 100644 index f267ea4594..0000000000 --- a/workers/datalake/src/const.ts +++ /dev/null @@ -1,19 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -export const expires = 86400 -export const cacheControl = `public,max-age=${expires}` - -export const hashLimit = 1 * 1024 * 1024 diff --git a/workers/datalake/src/cors.ts b/workers/datalake/src/cors.ts deleted file mode 100644 index 978a2d31dd..0000000000 --- a/workers/datalake/src/cors.ts +++ /dev/null @@ -1,103 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { type IRequest } from 'itty-router' - -// This is a copy of cors.ts from itty-router with following issues fixed: -// - https://github.com/kwhitley/itty-router/issues/242 -// - https://github.com/kwhitley/itty-router/issues/249 -export interface CorsOptions { - credentials?: true - origin?: boolean | string | string[] | RegExp | ((origin: string) => string | undefined) - maxAge?: number - allowMethods?: string | string[] - allowHeaders?: any - exposeHeaders?: string | string[] -} - -export type Preflight = (request: IRequest) => Response | undefined -export type Corsify = (response: Response, request?: IRequest) => Response | undefined - -export interface CorsPair { - preflight: Preflight - corsify: Corsify -} - -// Create CORS function with default options. -export const cors = (options: CorsOptions = {}): CorsPair => { - // Destructure and set defaults for options. - const { origin = '*', credentials = false, allowMethods = '*', allowHeaders, exposeHeaders, maxAge } = options - - const getAccessControlOrigin = (request?: Request): string | null | undefined => { - const requestOrigin = request?.headers.get('origin') // may be null if no request passed - if (requestOrigin === undefined || requestOrigin === null) return requestOrigin - - if (origin === true) return requestOrigin - if (origin instanceof RegExp) return origin.test(requestOrigin) ? requestOrigin : undefined - if (Array.isArray(origin)) return origin.includes(requestOrigin) ? requestOrigin : undefined - if (origin instanceof Function) return origin(requestOrigin) ?? undefined - - return origin === '*' && credentials ? requestOrigin : (origin as string) - } - - const appendHeadersAndReturn = (response: Response, headers: Record): Response => { - for (const [key, value] of Object.entries(headers)) { - if (value !== undefined && value !== null && value !== '') { - response.headers.append(key, value) - } - } - return response - } - - const preflight = (request: Request): Response | undefined => { - if (request.method === 'OPTIONS') { - const response = new Response(null, { status: 204 }) - - const allowMethodsHeader = Array.isArray(allowMethods) ? allowMethods.join(',') : allowMethods - const allowHeadersHeader = Array.isArray(allowHeaders) ? allowHeaders.join(',') : allowHeaders - const exposeHeadersHeader = Array.isArray(exposeHeaders) ? exposeHeaders.join(',') : exposeHeaders - - return appendHeadersAndReturn(response, { - 'access-control-allow-origin': getAccessControlOrigin(request), - 'access-control-allow-methods': allowMethodsHeader, - 'access-control-expose-headers': exposeHeadersHeader, - 'access-control-allow-headers': allowHeadersHeader ?? request.headers.get('access-control-request-headers'), - 'access-control-max-age': maxAge, - 'access-control-allow-credentials': credentials - }) - } // otherwise ignore - } - - const corsify = (response: Response, request?: Request): Response | undefined => { - // ignore if already has CORS headers - if (response?.headers?.has('access-control-allow-origin') || response.status === 101) { - return response - } - - const responseCopy = new Response(response.body, { - status: response.status, - statusText: response.statusText, - headers: response.headers - }) - - return appendHeadersAndReturn(responseCopy, { - 'access-control-allow-origin': getAccessControlOrigin(request), - 'access-control-allow-credentials': credentials - }) - } - - // Return corsify and preflight methods. - return { corsify, preflight } -} diff --git a/workers/datalake/src/db.ts b/workers/datalake/src/db.ts deleted file mode 100644 index be5a03f99e..0000000000 --- a/workers/datalake/src/db.ts +++ /dev/null @@ -1,195 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import postgres from 'postgres' -import { type MetricsContext } from './metrics' -import { type Location, type UUID } from './types' - -export interface BlobDataId { - hash: UUID - location: Location -} - -export interface BlobDataRecord extends BlobDataId { - filename: UUID - size: number - type: string -} - -export interface BlobId { - workspace: string - name: string -} - -export interface BlobRecord extends BlobId { - hash: UUID - location: Location - deleted: boolean -} - -export type BlobWithDataRecord = BlobRecord & BlobDataRecord - -export interface ListBlobResult { - cursor: string | undefined - blobs: BlobWithDataRecord[] -} - -export async function withPostgres ( - env: Env, - ctx: ExecutionContext, - metrics: MetricsContext, - fn: (db: BlobDB) => Promise -): Promise { - const sql = metrics.withSync('db.connect', () => { - return postgres(env.HYPERDRIVE.connectionString, { - connection: { - application_name: 'datalake' - }, - fetch_types: false - }) - }) - - const db = new LoggedDB(new PostgresDB(sql), metrics) - - try { - return await fn(db) - } finally { - metrics.withSync('db.disconnect', () => { - ctx.waitUntil(sql.end({ timeout: 0 })) - }) - } -} - -export interface BlobDB { - getData: (dataId: BlobDataId) => Promise - createData: (data: BlobDataRecord) => Promise - listBlobs: (workspace: string, cursor?: string, limit?: number) => Promise - getBlob: (blobId: BlobId) => Promise - createBlob: (blob: Omit) => Promise - deleteBlob: (blob: BlobId) => Promise -} - -export class PostgresDB implements BlobDB { - constructor (private readonly sql: postgres.Sql) {} - - async getData (dataId: BlobDataId): Promise { - const { hash, location } = dataId - const rows = await this.sql` - SELECT hash, location, filename, size, type - FROM blob.data - WHERE hash = ${hash} AND location = ${location} - ` - return rows.length > 0 ? rows[0] : null - } - - async createData (data: BlobDataRecord): Promise { - const { hash, location, filename, size, type } = data - - await this.sql` - UPSERT INTO blob.data (hash, location, filename, size, type) - VALUES (${hash}, ${location}, ${filename}, ${size}, ${type}) - ` - } - - async getBlob (blobId: BlobId): Promise { - const { workspace, name } = blobId - - try { - const rows = await this.sql` - SELECT b.workspace, b.name, b.hash, b.location, b.deleted, d.filename, d.size, d.type - FROM blob.blob AS b - JOIN blob.data AS d ON b.hash = d.hash AND b.location = d.location - WHERE b.workspace = ${workspace} AND b.name = ${name} - ` - - if (rows.length > 0) { - return rows[0] - } - } catch (err) { - console.error(err) - } - - return null - } - - async listBlobs (workspace: string, cursor?: string, limit?: number): Promise { - cursor = cursor ?? '' - limit = Math.min(limit ?? 100, 1000) - - const rows = await this.sql` - SELECT b.workspace, b.name, b.hash, b.location, b.deleted, d.filename, d.size, d.type - FROM blob.blob AS b - JOIN blob.data AS d ON b.hash = d.hash AND b.location = d.location - WHERE b.workspace = ${workspace} AND b.name > ${cursor} AND b.deleted = false - ORDER BY b.workspace, b.name - LIMIT ${limit} - ` - - return { - cursor: rows.length > 0 ? rows[rows.length - 1].name : undefined, - blobs: rows - } - } - - async createBlob (blob: Omit): Promise { - const { workspace, name, hash, location } = blob - - await this.sql` - UPSERT INTO blob.blob (workspace, name, hash, location, deleted) - VALUES (${workspace}, ${name}, ${hash}, ${location}, false) - ` - } - - async deleteBlob (blob: BlobId): Promise { - const { workspace, name } = blob - - await this.sql` - UPDATE blob.blob - SET deleted = true - WHERE workspace = ${workspace} AND name = ${name} - ` - } -} - -export class LoggedDB implements BlobDB { - constructor ( - private readonly db: BlobDB, - private readonly ctx: MetricsContext - ) {} - - async getData (dataId: BlobDataId): Promise { - return await this.ctx.with('db.getData', () => this.db.getData(dataId)) - } - - async createData (data: BlobDataRecord): Promise { - await this.ctx.with('db.createData', () => this.db.createData(data)) - } - - async getBlob (blobId: BlobId): Promise { - return await this.ctx.with('db.getBlob', () => this.db.getBlob(blobId)) - } - - async listBlobs (workspace: string, cursor?: string, limit?: number): Promise { - return await this.ctx.with('db.listBlobs', () => this.db.listBlobs(workspace, cursor, limit)) - } - - async createBlob (blob: Omit): Promise { - await this.ctx.with('db.createBlob', () => this.db.createBlob(blob)) - } - - async deleteBlob (blob: BlobId): Promise { - await this.ctx.with('db.deleteBlob', () => this.db.deleteBlob(blob)) - } -} diff --git a/workers/datalake/src/encodings.ts b/workers/datalake/src/encodings.ts deleted file mode 100644 index 9fc707cece..0000000000 --- a/workers/datalake/src/encodings.ts +++ /dev/null @@ -1,35 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { type UUID } from './types' - -export const toUUID = (buffer: Uint8Array): UUID => { - const hex = toHex(buffer) - const hex32 = hex.slice(0, 32).padStart(32, '0') - return formatHexAsUUID(hex32) -} - -export const toHex = (buffer: Uint8Array): string => { - return Array.from(buffer) - .map((b) => b.toString(16).padStart(2, '0')) - .join('') -} - -export function formatHexAsUUID (hexString: string): UUID { - if (hexString.length !== 32) { - throw new Error('Hex string must be exactly 32 characters long.') - } - return hexString.replace(/^(.{8})(.{4})(.{4})(.{4})(.{12})$/, '$1-$2-$3-$4-$5') as UUID -} diff --git a/workers/datalake/src/hash.ts b/workers/datalake/src/hash.ts deleted file mode 100644 index 10c8b93ff3..0000000000 --- a/workers/datalake/src/hash.ts +++ /dev/null @@ -1,25 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { toUUID } from './encodings' -import { type UUID } from './types' - -export async function getSha256 (stream: ReadableStream): Promise { - const digestStream = new crypto.DigestStream('SHA-256') - await stream.pipeTo(digestStream) - const digest = await digestStream.digest - - return toUUID(new Uint8Array(digest)) -} diff --git a/workers/datalake/src/image.ts b/workers/datalake/src/image.ts deleted file mode 100644 index e39646a3fc..0000000000 --- a/workers/datalake/src/image.ts +++ /dev/null @@ -1,58 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { getBlobURL } from './blob' -import { type MetricsContext } from './metrics' -import { type BlobRequest } from './types' - -const prefferedImageFormats = ['webp', 'avif', 'jpeg', 'png'] - -export async function handleImageGet ( - request: BlobRequest, - env: Env, - ctx: ExecutionContext, - metrics: MetricsContext -): Promise { - const { - workspace, - name, - params: { transform } - } = request - - const Accept = request.headers.get('Accept') ?? 'image/*' - const image: Record = {} - - // select format based on Accept header - const formats = Accept.split(',') - for (const format of formats) { - const [type] = format.split(';') - const [clazz, kind] = type.split('/') - if (clazz === 'image' && prefferedImageFormats.includes(kind)) { - image.format = kind - break - } - } - - // apply transforms - transform.split(',').reduce((acc, param) => { - const [key, value] = param.split('=') - acc[key] = value - return acc - }, image) - - const blobURL = getBlobURL(request, workspace, name) - const imageRequest = new Request(blobURL, { headers: { Accept } }) - return await metrics.with('image.transform', () => fetch(imageRequest, { cf: { image, cacheTtl: 3600 } })) -} diff --git a/workers/datalake/src/index.ts b/workers/datalake/src/index.ts deleted file mode 100644 index 012b547795..0000000000 --- a/workers/datalake/src/index.ts +++ /dev/null @@ -1,145 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { WorkerEntrypoint } from 'cloudflare:workers' -import { type IRequest, type IRequestStrict, type RequestHandler, Router, error, html } from 'itty-router' - -import { handleBlobDelete, handleBlobGet, handleBlobHead, handleBlobList, handleUploadFormData } from './blob' -import { cors } from './cors' -import { LoggedKVNamespace, LoggedR2Bucket, MetricsContext } from './metrics' -import { handleImageGet } from './image' -import { handleS3Blob } from './s3' -import { handleVideoMetaGet } from './video' -import { handleSignAbort, handleSignComplete, handleSignCreate } from './sign' -import { - handleMultipartUploadStart, - handleMultipartUploadPart, - handleMultipartUploadComplete, - handleMultipartUploadAbort -} from './multipart' -import { type BlobRequest, type WorkspaceRequest } from './types' - -const { preflight, corsify } = cors({ - maxAge: 86400 -}) - -const router = Router({ - before: [preflight], - finally: [corsify] -}) - -const withWorkspace: RequestHandler = (request: WorkspaceRequest) => { - if (request.params.workspace === undefined || request.params.workspace === '') { - return error(400, 'Missing workspace') - } - request.workspace = decodeURIComponent(request.params.workspace) -} - -const withBlob: RequestHandler = (request: BlobRequest) => { - if (request.params.workspace === undefined || request.params.workspace === '') { - return error(400, 'Missing workspace') - } - if (request.params.name === undefined || request.params.name === '') { - return error(400, 'Missing blob name') - } - request.workspace = decodeURIComponent(request.params.workspace) - request.name = decodeURIComponent(request.params.name) -} - -router - .get('/blob/:workspace', withWorkspace, handleBlobList) - .get('/blob/:workspace/:name', withBlob, handleBlobGet) - .get('/blob/:workspace/:name/:filename', withBlob, handleBlobGet) - .head('/blob/:workspace/:name', withBlob, handleBlobHead) - .head('/blob/:workspace/:name/:filename', withBlob, handleBlobHead) - .delete('/blob/:workspace/:name', withBlob, handleBlobDelete) - // Image - .get('/image/:transform/:workspace/:name', withBlob, handleImageGet) - // Video - .get('/video/:workspace/:name/meta', withBlob, handleVideoMetaGet) - // Form Data - .post('/upload/form-data/:workspace', withWorkspace, handleUploadFormData) - // Signed URL - .post('/upload/signed-url/:workspace/:name', withBlob, handleSignCreate) - .put('/upload/signed-url/:workspace/:name', withBlob, handleSignComplete) - .delete('/upload/signed-url/:workspace/:name', withBlob, handleSignAbort) - // Multipart - .post('/upload/multipart/:workspace/:name', withBlob, handleMultipartUploadStart) - .post('/upload/multipart/:workspace/:name/part', withBlob, handleMultipartUploadPart) - .post('/upload/multipart/:workspace/:name/complete', withBlob, handleMultipartUploadComplete) - .post('/upload/multipart/:workspace/:name/abort', withBlob, handleMultipartUploadAbort) - // S3 - .post('/upload/s3/:workspace/:name', withBlob, handleS3Blob) - .all('/', () => - html( - `Huly® Datalake™ https://huly.io - © 2024 Huly Labs` - ) - ) - .all('*', () => error(404)) - -export default class DatalakeWorker extends WorkerEntrypoint { - async fetch (request: IRequest): Promise { - const start = performance.now() - const context = new MetricsContext() - - const env = { - ...this.env, - datalake_blobs: new LoggedKVNamespace(this.env.datalake_blobs, context), - DATALAKE_APAC: new LoggedR2Bucket(this.env.DATALAKE_APAC, context), - DATALAKE_EEUR: new LoggedR2Bucket(this.env.DATALAKE_EEUR, context), - DATALAKE_WEUR: new LoggedR2Bucket(this.env.DATALAKE_WEUR, context), - DATALAKE_ENAM: new LoggedR2Bucket(this.env.DATALAKE_ENAM, context), - DATALAKE_WNAM: new LoggedR2Bucket(this.env.DATALAKE_WNAM, context) - } - - try { - return await router.fetch(request, env, this.ctx, context).catch(error) - } finally { - const total = performance.now() - start - const ops = context.metrics - const url = `${request.method} ${request.url}` - const message = `total=${total} ` + context.toString() - console.log({ message, total, ops, url }) - } - } - - async getBlob (workspace: string, name: string): Promise { - const request = new Request(`https://datalake/blob/${workspace}/${name}`) - const response = await router.fetch(request) - - if (!response.ok) { - console.error({ error: 'datalake error: ' + response.statusText, workspace, name }) - throw new Error(`Failed to fetch blob: ${response.statusText}`) - } - - return await response.arrayBuffer() - } - - async putBlob (workspace: string, name: string, data: ArrayBuffer | Blob | string, type: string): Promise { - const request = new Request(`https://datalake/upload/form-data/${workspace}`) - - const body = new FormData() - const blob = new Blob([data], { type }) - body.set('file', blob, name) - - const response = await router.fetch(request, { method: 'POST', body }) - - if (!response.ok) { - console.error({ error: 'datalake error: ' + response.statusText, workspace, name }) - throw new Error(`Failed to fetch blob: ${response.statusText}`) - } - } -} diff --git a/workers/datalake/src/metrics.ts b/workers/datalake/src/metrics.ts deleted file mode 100644 index 589bbb396f..0000000000 --- a/workers/datalake/src/metrics.ts +++ /dev/null @@ -1,188 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the 'License'); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an 'AS IS' BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -export interface MetricsData { - name: string - time: number -} - -export class MetricsContext { - metrics: Array = [] - - async with(name: string, fn: () => Promise): Promise { - const start = performance.now() - try { - return await fn() - } finally { - const time = performance.now() - start - this.metrics.push({ name, time }) - } - } - - withSync(name: string, fn: () => T): T { - const start = performance.now() - try { - return fn() - } finally { - const time = performance.now() - start - this.metrics.push({ name, time }) - } - } - - toString (): string { - return this.metrics.map((p) => `${p.name}=${p.time}`).join(' ') - } -} - -export class LoggedR2Bucket implements R2Bucket { - constructor ( - private readonly bucket: R2Bucket, - private readonly ctx: MetricsContext - ) {} - - async head (key: string): Promise { - return await this.ctx.with('r2.head', () => this.bucket.head(key)) - } - - async get ( - key: string, - options?: R2GetOptions & { - onlyIf?: R2Conditional | Headers - } - ): Promise { - return await this.ctx.with('r2.get', () => this.bucket.get(key, options)) - } - - async put ( - key: string, - value: ReadableStream | ArrayBuffer | ArrayBufferView | string | null | Blob, - options?: R2PutOptions & { - onlyIf?: R2Conditional | Headers - } - ): Promise { - return await this.ctx.with('r2.put', () => this.bucket.put(key, value, options)) - } - - async createMultipartUpload (key: string, options?: R2MultipartOptions): Promise { - return await this.ctx.with('r2.createMultipartUpload', () => this.bucket.createMultipartUpload(key, options)) - } - - resumeMultipartUpload (key: string, uploadId: string): R2MultipartUpload { - return this.ctx.withSync('r2.resumeMultipartUpload', () => this.bucket.resumeMultipartUpload(key, uploadId)) - } - - async delete (keys: string | string[]): Promise { - await this.ctx.with('r2.delete', () => this.bucket.delete(keys)) - } - - async list (options?: R2ListOptions): Promise { - return await this.ctx.with('r2.list', () => this.bucket.list(options)) - } -} - -export class LoggedKVNamespace implements KVNamespace { - constructor ( - private readonly kv: KVNamespace, - private readonly ctx: MetricsContext - ) {} - - get (key: string, options?: Partial>): Promise - get (key: string, type: 'text'): Promise - get(key: string, type: 'json'): Promise - get (key: string, type: 'arrayBuffer'): Promise - get (key: string, type: 'stream'): Promise - get (key: string, options?: KVNamespaceGetOptions<'text'>): Promise - get(key: string, options?: KVNamespaceGetOptions<'json'>): Promise - get (key: string, options?: KVNamespaceGetOptions<'arrayBuffer'>): Promise - get (key: string, options?: KVNamespaceGetOptions<'stream'>): Promise - async get (key: string, options?: any): Promise { - return await this.ctx.with('kv.get', () => this.kv.get(key, options)) - } - - getWithMetadata( - key: string, - options?: Partial> - ): Promise> - getWithMetadata( - key: string, - type: 'text' - ): Promise> - getWithMetadata( - key: string, - type: 'json' - ): Promise> - getWithMetadata( - key: string, - type: 'arrayBuffer' - ): Promise> - getWithMetadata( - key: string, - type: 'stream' - ): Promise> - getWithMetadata( - key: string, - options?: KVNamespaceGetOptions<'text'> - ): Promise> - getWithMetadata( - key: string, - options?: KVNamespaceGetOptions<'json'> - ): Promise> - getWithMetadata( - key: string, - options?: KVNamespaceGetOptions<'arrayBuffer'> - ): Promise> - getWithMetadata( - key: string, - options?: KVNamespaceGetOptions<'stream'> - ): Promise> - async getWithMetadata (key: string, options?: any): Promise { - return await this.ctx.with('kv.getWithMetadata', () => this.kv.getWithMetadata(key, options)) - } - - async list(options?: KVNamespaceListOptions): Promise> { - return await this.ctx.with('kv.list', () => this.kv.list(options)) - } - - async put ( - key: string, - value: string | ArrayBuffer | ArrayBufferView | ReadableStream, - options?: KVNamespacePutOptions - ): Promise { - await this.ctx.with('kv.put', () => this.kv.put(key, value)) - } - - async delete (key: string): Promise { - await this.ctx.with('kv.delete', () => this.kv.delete(key)) - } -} - -export class LoggedCache implements Cache { - constructor ( - private readonly cache: Cache, - private readonly ctx: MetricsContext - ) {} - - async match (request: RequestInfo, options?: CacheQueryOptions): Promise { - return await this.ctx.with('cache.match', () => this.cache.match(request, options)) - } - - async delete (request: RequestInfo, options?: CacheQueryOptions): Promise { - return await this.ctx.with('cache.delete', () => this.cache.delete(request, options)) - } - - async put (request: RequestInfo, response: Response): Promise { - await this.ctx.with('cache.put', () => this.cache.put(request, response)) - } -} diff --git a/workers/datalake/src/multipart.ts b/workers/datalake/src/multipart.ts deleted file mode 100644 index 6818705cf0..0000000000 --- a/workers/datalake/src/multipart.ts +++ /dev/null @@ -1,152 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { error, json } from 'itty-router' -import { type BlobMetadata } from './blob' -import { withPostgres } from './db' -import { cacheControl } from './const' -import { toUUID } from './encodings' -import { type MetricsContext } from './metrics' -import { selectStorage } from './storage' -import { type BlobRequest, type UUID } from './types' - -export interface MultipartUpload { - key: string - uploadId: string -} - -export interface MultipartUploadPart { - partNumber: number - etag: string -} - -export interface MultipartUploadCompleteRequest { - parts: MultipartUploadPart[] -} - -export async function handleMultipartUploadStart ( - request: BlobRequest, - env: Env, - ctx: ExecutionContext -): Promise { - const { workspace } = request - - const { bucket } = selectStorage(env, workspace) - - const contentType = request.headers.get('content-type') ?? 'application/octet-stream' - const lastModifiedHeader = request.headers.get('last-modified') - const lastModified = lastModifiedHeader !== null ? new Date(lastModifiedHeader).getTime() : Date.now() - const httpMetadata = { contentType, cacheControl, lastModified } - const uuid = crypto.randomUUID() as UUID - - const multipart = await bucket.createMultipartUpload(uuid, { httpMetadata }) - return json({ key: multipart.key, uploadId: multipart.uploadId }) -} - -export async function handleMultipartUploadPart ( - request: BlobRequest, - env: Env, - ctx: ExecutionContext -): Promise { - const { workspace } = request - - const multipartKey = request.query?.key - const multipartUploadId = request.query?.uploadId - const partNumber = request.query?.partNumber - if (typeof multipartKey !== 'string' || typeof multipartUploadId !== 'string' || typeof partNumber !== 'string') { - return error(400, 'missing key or uploadId or partNumber') - } - - if (request.body === null) { - return error(400, 'missing body') - } - - const { bucket } = selectStorage(env, workspace) - - const upload = bucket.resumeMultipartUpload(multipartKey, multipartUploadId) - const part = await upload.uploadPart(Number.parseInt(partNumber), request.body) - - return json({ partNumber: part.partNumber, etag: part.etag }) -} - -export async function handleMultipartUploadComplete ( - request: BlobRequest, - env: Env, - ctx: ExecutionContext, - metrics: MetricsContext -): Promise { - const { workspace, name } = request - - const multipartKey = request.query?.key - const multipartUploadId = request.query?.uploadId - if (typeof multipartKey !== 'string' || typeof multipartUploadId !== 'string') { - return error(400, 'missing key or uploadId') - } - - const { parts } = await request.json() - - const { bucket, location } = selectStorage(env, workspace) - - const upload = bucket.resumeMultipartUpload(multipartKey, multipartUploadId) - const object = await upload.complete(parts) - - const hash = - object.checksums.md5 !== undefined ? toUUID(new Uint8Array(object.checksums.md5)) : (crypto.randomUUID() as UUID) - const type = object.httpMetadata?.contentType ?? 'application/octet-stream' - const size = object.size ?? 0 - const filename = multipartKey as UUID - - await withPostgres(env, ctx, metrics, async (db) => { - const data = await db.getData({ hash, location }) - if (data !== null) { - // blob already exists - await Promise.all([bucket.delete(filename), db.createBlob({ workspace, name, hash, location })]) - } else { - // Otherwise register a new hash and blob - await db.createData({ hash, location, filename, type, size }) - await db.createBlob({ workspace, name, hash, location }) - } - }) - - const metadata: BlobMetadata = { - type, - size, - name, - etag: hash, - lastModified: object.uploaded.getTime() - } - - return json(metadata) -} - -export async function handleMultipartUploadAbort ( - request: BlobRequest, - env: Env, - ctx: ExecutionContext -): Promise { - const { workspace } = request - - const multipartKey = request.query?.key - const multipartUploadId = request.query?.uploadId - if (typeof multipartKey !== 'string' || typeof multipartUploadId !== 'string') { - return error(400, 'missing key or uploadId') - } - - const { bucket } = selectStorage(env, workspace) - - const upload = bucket.resumeMultipartUpload(multipartKey, multipartUploadId) - await upload.abort() - return new Response(null, { status: 204 }) -} diff --git a/workers/datalake/src/s3.ts b/workers/datalake/src/s3.ts deleted file mode 100644 index 2f1e1d1f47..0000000000 --- a/workers/datalake/src/s3.ts +++ /dev/null @@ -1,77 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { AwsClient } from 'aws4fetch' -import { error, json } from 'itty-router' -import { withPostgres } from './db' -import { saveBlob } from './blob' -import { type MetricsContext } from './metrics' -import { type BlobRequest } from './types' - -export interface S3UploadPayload { - url: string - region: string - accessKeyId: string - secretAccessKey: string -} - -function getS3Client (payload: S3UploadPayload): AwsClient { - return new AwsClient({ - service: 's3', - region: payload.region, - accessKeyId: payload.accessKeyId, - secretAccessKey: payload.secretAccessKey - }) -} - -export async function handleS3Blob ( - request: BlobRequest, - env: Env, - ctx: ExecutionContext, - metrics: MetricsContext -): Promise { - const { workspace, name } = request - - const payload = await request.json() - - const client = getS3Client(payload) - - return await withPostgres(env, ctx, metrics, async (db) => { - // Ensure the blob does not exist - const blob = await db.getBlob({ workspace, name }) - if (blob !== null) { - return new Response(null, { status: 200 }) - } - - const object = await client.fetch(payload.url) - if (!object.ok || object.status !== 200) { - return error(object.status) - } - - if (object.body === null) { - return error(400) - } - - const contentType = object.headers.get('content-type') ?? 'application/octet-stream' - const contentLengthHeader = object.headers.get('content-length') ?? '0' - const lastModifiedHeader = object.headers.get('last-modified') - - const contentLength = Number.parseInt(contentLengthHeader) - const lastModified = lastModifiedHeader !== null ? new Date(lastModifiedHeader).getTime() : Date.now() - - const result = await saveBlob(env, db, object.body, contentLength, contentType, workspace, name, lastModified) - return json(result) - }) -} diff --git a/workers/datalake/src/sign.ts b/workers/datalake/src/sign.ts deleted file mode 100644 index 8a68f99b61..0000000000 --- a/workers/datalake/src/sign.ts +++ /dev/null @@ -1,137 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { AwsClient } from 'aws4fetch' -import { error, json } from 'itty-router' - -import { type BlobMetadata, handleBlobUploaded } from './blob' -import { type MetricsContext } from './metrics' -import { type Storage, selectStorage } from './storage' -import { type BlobRequest, type UUID } from './types' - -const S3_SIGNED_LINK_TTL = 3600 - -interface SignBlobInfo { - uuid: UUID -} - -function signBlobKey (workspace: string, name: string): string { - return `s/${workspace}/${name}` -} - -function getS3Client (storage: Storage): AwsClient { - return new AwsClient({ - service: 's3', - region: 'auto', - accessKeyId: storage.bucketAccessKey, - secretAccessKey: storage.bucketSecretKey - }) -} - -export async function handleSignCreate ( - request: BlobRequest, - env: Env, - ctx: ExecutionContext, - metrics: MetricsContext -): Promise { - const { workspace, name } = request - const storage = selectStorage(env, workspace) - const accountId = env.R2_ACCOUNT_ID - - const key = signBlobKey(workspace, name) - const uuid = crypto.randomUUID() as UUID - - // Generate R2 object link - const url = new URL(`https://${storage.bucketName}.${accountId}.r2.cloudflarestorage.com`) - url.pathname = uuid - url.searchParams.set('X-Amz-Expires', S3_SIGNED_LINK_TTL.toString()) - - // Sign R2 object link - let signed: Request - try { - const client = getS3Client(storage) - - signed = await metrics.with('s3.sign', () => { - return client.sign(new Request(url, { method: 'PUT' }), { aws: { signQuery: true } }) - }) - } catch (err: any) { - console.error({ error: 'failed to generate signed url', message: `${err}` }) - return error(500, 'failed to generate signed url') - } - - // Save upload details - const s3BlobInfo: SignBlobInfo = { uuid } - await env.datalake_blobs.put(key, JSON.stringify(s3BlobInfo), { expirationTtl: S3_SIGNED_LINK_TTL }) - - const headers = new Headers({ - Expires: new Date(Date.now() + S3_SIGNED_LINK_TTL * 1000).toISOString() - }) - return new Response(signed.url, { status: 200, headers }) -} - -export async function handleSignComplete ( - request: BlobRequest, - env: Env, - ctx: ExecutionContext, - metrics: MetricsContext -): Promise { - const { workspace, name } = request - - const { bucket } = selectStorage(env, workspace) - const key = signBlobKey(workspace, name) - - // Ensure we generated presigned URL earlier - // TODO what if we came after expiration date? - const signBlobInfo = await env.datalake_blobs.get(key, { type: 'json' }) - if (signBlobInfo === null) { - console.error({ error: 'blob sign info not found', workspace, name }) - return error(404) - } - - // Ensure the blob has been uploaded - const { uuid } = signBlobInfo - const head = await bucket.get(uuid) - if (head === null) { - console.error({ error: 'blob not found', workspace, name, uuid }) - return error(400) - } - - let metadata: BlobMetadata - try { - metadata = await handleBlobUploaded(env, ctx, metrics, workspace, name, uuid) - } catch (err) { - const message = err instanceof Error ? err.message : String(err) - console.error({ error: message, workspace, name, uuid }) - return error(500, 'failed to upload blob') - } - - await env.datalake_blobs.delete(key) - - return json(metadata) -} - -export async function handleSignAbort (request: BlobRequest, env: Env, ctx: ExecutionContext): Promise { - const { workspace, name } = request - - const key = signBlobKey(workspace, name) - - // Check if the blob has been uploaded - const s3BlobInfo = await env.datalake_blobs.get(key, { type: 'json' }) - if (s3BlobInfo !== null) { - await env.datalake_blobs.delete(key) - } - - return new Response(null, { status: 204 }) -} diff --git a/workers/datalake/src/storage.ts b/workers/datalake/src/storage.ts deleted file mode 100644 index 777ab03317..0000000000 --- a/workers/datalake/src/storage.ts +++ /dev/null @@ -1,76 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the 'License'); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an 'AS IS' BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { type Location } from './types' - -export interface Storage { - location: Location - bucket: R2Bucket - - bucketName: string - bucketAccessKey: string - bucketSecretKey: string -} - -export function selectStorage (env: Env, workspace: string): Storage { - const location = selectLocation(env, workspace) - switch (location) { - case 'apac': - return { - location, - bucket: env.DATALAKE_APAC, - bucketName: env.DATALAKE_APAC_BUCKET_NAME, - bucketAccessKey: env.DATALAKE_APAC_ACCESS_KEY, - bucketSecretKey: env.DATALAKE_APAC_SECRET_KEY - } - case 'eeur': - return { - location, - bucket: env.DATALAKE_EEUR, - bucketName: env.DATALAKE_EEUR_BUCKET_NAME, - bucketAccessKey: env.DATALAKE_EEUR_ACCESS_KEY, - bucketSecretKey: env.DATALAKE_EEUR_SECRET_KEY - } - case 'weur': - return { - location, - bucket: env.DATALAKE_WEUR, - bucketName: env.DATALAKE_WEUR_BUCKET_NAME, - bucketAccessKey: env.DATALAKE_WEUR_ACCESS_KEY, - bucketSecretKey: env.DATALAKE_WEUR_SECRET_KEY - } - case 'enam': - return { - location, - bucket: env.DATALAKE_ENAM, - bucketName: env.DATALAKE_ENAM_BUCKET_NAME, - bucketAccessKey: env.DATALAKE_ENAM_ACCESS_KEY, - bucketSecretKey: env.DATALAKE_ENAM_SECRET_KEY - } - case 'wnam': - return { - location, - bucket: env.DATALAKE_WNAM, - bucketName: env.DATALAKE_WNAM_BUCKET_NAME, - bucketAccessKey: env.DATALAKE_WNAM_ACCESS_KEY, - bucketSecretKey: env.DATALAKE_WNAM_SECRET_KEY - } - } -} - -function selectLocation (env: Env, workspace: string): Location { - // TODO select location based on workspace - return 'weur' -} diff --git a/workers/datalake/src/types.ts b/workers/datalake/src/types.ts deleted file mode 100644 index acad59f013..0000000000 --- a/workers/datalake/src/types.ts +++ /dev/null @@ -1,43 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { type IRequestStrict } from 'itty-router' - -export type Location = 'weur' | 'eeur' | 'wnam' | 'enam' | 'apac' - -export type UUID = string & { __uuid: true } - -export type WorkspaceRequest = { - workspace: string -} & IRequestStrict - -export type BlobRequest = { - workspace: string - name: string -} & IRequestStrict - -export interface CloudflareResponse { - success: boolean - errors: any - messages: any - result: any -} - -export interface StreamUploadResponse extends CloudflareResponse { - result: { - uid: string - uploadURL: string - } -} diff --git a/workers/datalake/src/video.ts b/workers/datalake/src/video.ts deleted file mode 100644 index b8effcbffa..0000000000 --- a/workers/datalake/src/video.ts +++ /dev/null @@ -1,117 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { error, json } from 'itty-router' - -import { type BlobRequest, type CloudflareResponse, type StreamUploadResponse } from './types' - -export type StreamUploadState = 'ready' | 'error' | 'inprogress' | 'queued' | 'downloading' | 'pendingupload' - -// https://developers.cloudflare.com/api/operations/stream-videos-list-videos#response-body -export interface StreamDetailsResponse extends CloudflareResponse { - result: { - uid: string - thumbnail: string - status: { - state: StreamUploadState - } - playback: { - hls: string - dash: string - } - } -} - -interface StreamBlobInfo { - streamId: string -} - -function streamBlobKey (workspace: string, name: string): string { - return `v/${workspace}/${name}` -} - -export async function handleVideoMetaGet (request: BlobRequest, env: Env, ctx: ExecutionContext): Promise { - const { workspace, name } = request - - const key = streamBlobKey(workspace, name) - - const streamInfo = await env.datalake_blobs.get(key, { type: 'json' }) - if (streamInfo === null) { - return error(404) - } - - const url = `https://api.cloudflare.com/client/v4/accounts/${env.STREAMS_ACCOUNT_ID}/stream/${streamInfo.streamId}` - const streamRequest = new Request(url, { - headers: { - Authorization: `Bearer ${env.STREAMS_AUTH_KEY}`, - 'Content-Type': 'application/json' - } - }) - - const streamResponse = await fetch(streamRequest) - const stream = await streamResponse.json() - - if (stream.success) { - return json({ - status: stream.result.status.state, - thumbnail: stream.result.thumbnail, - hls: stream.result.playback.hls - }) - } else { - return error(500, { errors: stream.errors }) - } -} - -export async function copyVideo (env: Env, source: string, workspace: string, name: string): Promise { - const key = streamBlobKey(workspace, name) - - const url = `https://api.cloudflare.com/client/v4/accounts/${env.STREAMS_ACCOUNT_ID}/stream/copy` - const request = new Request(url, { - method: 'POST', - headers: { - Authorization: `Bearer ${env.STREAMS_AUTH_KEY}`, - 'Content-Type': 'application/json' - }, - body: JSON.stringify({ url: source, meta: { name } }) - }) - - const response = await fetch(request) - const upload = await response.json() - - if (upload.success) { - const streamInfo: StreamBlobInfo = { - streamId: upload.result.uid - } - await env.datalake_blobs.put(key, JSON.stringify(streamInfo)) - } -} - -export async function deleteVideo (env: Env, workspace: string, name: string): Promise { - const key = streamBlobKey(workspace, name) - - const streamInfo = await env.datalake_blobs.get(key, { type: 'json' }) - if (streamInfo !== null) { - const url = `https://api.cloudflare.com/client/v4/accounts/${env.STREAMS_ACCOUNT_ID}/stream/${streamInfo.streamId}` - const request = new Request(url, { - method: 'DELETE', - headers: { - Authorization: `Bearer ${env.STREAMS_AUTH_KEY}`, - 'Content-Type': 'application/json' - } - }) - - await Promise.all([fetch(request), env.datalake_blobs.delete(key)]) - } -} diff --git a/workers/datalake/tsconfig.json b/workers/datalake/tsconfig.json deleted file mode 100644 index da8672e6cb..0000000000 --- a/workers/datalake/tsconfig.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "extends": "./node_modules/@hcengineering/platform-rig/profiles/default/tsconfig.json", - - "compilerOptions": { - "rootDir": "./src", - "outDir": "./lib", - "declarationDir": "./types", - "tsBuildInfoFile": ".build/build.tsbuildinfo", - "types": ["@cloudflare/workers-types", "jest"], - "lib": ["esnext"] - } -} \ No newline at end of file diff --git a/workers/datalake/worker-configuration.d.ts b/workers/datalake/worker-configuration.d.ts deleted file mode 100644 index 532c6991ce..0000000000 --- a/workers/datalake/worker-configuration.d.ts +++ /dev/null @@ -1,31 +0,0 @@ -// Generated by Wrangler on Sat Jul 06 2024 18:52:21 GMT+0200 (Central European Summer Time) -// by running `wrangler types` - -interface Env { - datalake_blobs: KVNamespace; - DATALAKE_APAC: R2Bucket; - DATALAKE_EEUR: R2Bucket; - DATALAKE_WEUR: R2Bucket; - DATALAKE_ENAM: R2Bucket; - DATALAKE_WNAM: R2Bucket; - HYPERDRIVE: Hyperdrive; - STREAMS_ACCOUNT_ID: string; - STREAMS_AUTH_KEY: string; - R2_ACCOUNT_ID: string; - DB_URL: string; - DATALAKE_APAC_ACCESS_KEY: string; - DATALAKE_APAC_SECRET_KEY: string; - DATALAKE_APAC_BUCKET_NAME: string; - DATALAKE_EEUR_ACCESS_KEY: string; - DATALAKE_EEUR_SECRET_KEY: string; - DATALAKE_EEUR_BUCKET_NAME: string; - DATALAKE_WEUR_ACCESS_KEY: string; - DATALAKE_WEUR_SECRET_KEY: string; - DATALAKE_WEUR_BUCKET_NAME: string; - DATALAKE_ENAM_ACCESS_KEY: string; - DATALAKE_ENAM_SECRET_KEY: string; - DATALAKE_ENAM_BUCKET_NAME: string; - DATALAKE_WNAM_ACCESS_KEY: string; - DATALAKE_WNAM_SECRET_KEY: string; - DATALAKE_WNAM_BUCKET_NAME: string; -} diff --git a/workers/datalake/wrangler.toml b/workers/datalake/wrangler.toml deleted file mode 100644 index 1008edbbe5..0000000000 --- a/workers/datalake/wrangler.toml +++ /dev/null @@ -1,110 +0,0 @@ -#:schema node_modules/wrangler/config-schema.json -name = "datalake-worker" -main = "src/index.ts" -compatibility_date = "2024-09-23" -compatibility_flags = ["nodejs_compat"] -keep_vars = true - -kv_namespaces = [ - { binding = "datalake_blobs", id = "64144eb146fd45febc928d44419ebb39", preview_id = "31c6f6e76e7e4524a59f87a4f381de82" } -] - -r2_buckets = [ - { binding = "DATALAKE_APAC", bucket_name = "datalake-eu-west", preview_bucket_name = "dev-datalake-eu-west" }, - { binding = "DATALAKE_EEUR", bucket_name = "datalake-eu-west", preview_bucket_name = "dev-datalake-eu-west" }, - { binding = "DATALAKE_WEUR", bucket_name = "datalake-eu-west", preview_bucket_name = "dev-datalake-eu-west" }, - { binding = "DATALAKE_ENAM", bucket_name = "datalake-eu-west", preview_bucket_name = "dev-datalake-eu-west" }, - { binding = "DATALAKE_WNAM", bucket_name = "datalake-eu-west", preview_bucket_name = "dev-datalake-eu-west" } -] - -[[hyperdrive]] -binding = "HYPERDRIVE" -id = "055e968f3067414eaa30467d8a9c5021" -localConnectionString = "postgresql://root:roach@localhost:26257/datalake" - -[observability] -enabled = true -head_sampling_rate = 1 - -[vars] -DATALAKE_EEUR_BUCKET_NAME = "datalake-eeur" -# DATALAKE_EEUR_ACCESS_KEY = "" -# DATALAKE_EEUR_SECRET_KEY = "" -DATALAKE_WEUR_BUCKET_NAME = "datalake-weur" -# DATALAKE_WEUR_ACCESS_KEY = "" -# DATALAKE_WEUR_SECRET_KEY = "" -DATALAKE_APAC_BUCKET_NAME = "datalake-apac" -# DATALAKE_APAC_ACCESS_KEY = "" -# DATALAKE_APAC_SECRET_KEY = "" -DATALAKE_ENAM_BUCKET_NAME = "datalake-enam" -# DATALAKE_ENAM_ACCESS_KEY = "" -# DATALAKE_ENAM_SECRET_KEY = "" -DATALAKE_WNAM_BUCKET_NAME = "datalake-wnam" -# DATALAKE_WNAM_ACCESS_KEY = "" -# DATALAKE_WNAM_SECRET_KEY = "" - -# STREAMS_ACCOUNT_ID = "" -# STREAMS_AUTH_KEY = "" -# R2_ACCOUNT_ID = "" - - -[env.staging] -name = "datalake-worker-staging" - -kv_namespaces = [ - { binding = "datalake_blobs", id = "d52f8b6a9a1e495a8d6625d14d8f3188" } -] - -r2_buckets = [ - { binding = "DATALAKE_APAC", bucket_name = "datalake-staging-wnam" }, - { binding = "DATALAKE_EEUR", bucket_name = "datalake-staging-wnam" }, - { binding = "DATALAKE_WEUR", bucket_name = "datalake-staging-weur" }, - { binding = "DATALAKE_ENAM", bucket_name = "datalake-staging-wnam" }, - { binding = "DATALAKE_WNAM", bucket_name = "datalake-staging-wnam" } -] - -hyperdrive = [ - { binding = "HYPERDRIVE", id = "1f713bb2ec1b464cb663983feca4c89e" } -] - -[env.staging.vars] -DATALAKE_EEUR_BUCKET_NAME = "datalake-staging-weur" -DATALAKE_WEUR_BUCKET_NAME = "datalake-staging-wnam" -DATALAKE_APAC_BUCKET_NAME = "datalake-staging-wnam" -DATALAKE_ENAM_BUCKET_NAME = "datalake-staging-wnam" -DATALAKE_WNAM_BUCKET_NAME = "datalake-staging-wnam" - -[env.staging.observability] -enabled = true -head_sampling_rate = 1 - - -[env.dev] -name = "datalake-worker-dev" - -kv_namespaces = [ - { binding = "datalake_blobs", id = "5acdcea18a504ab3a2d73556314b0047" } -] - -r2_buckets = [ - { binding = "DATALAKE_APAC", bucket_name = "datalake-dev" }, - { binding = "DATALAKE_EEUR", bucket_name = "datalake-dev" }, - { binding = "DATALAKE_WEUR", bucket_name = "datalake-dev" }, - { binding = "DATALAKE_ENAM", bucket_name = "datalake-dev" }, - { binding = "DATALAKE_WNAM", bucket_name = "datalake-dev" } -] - -hyperdrive = [ - { binding = "HYPERDRIVE", id = "055e968f3067414eaa30467d8a9c5021", localConnectionString = "postgresql://root:roach@localhost:26257/datalake" } -] - -[env.dev.vars] -DATALAKE_EEUR_BUCKET_NAME = "datalake-dev" -DATALAKE_WEUR_BUCKET_NAME = "datalake-dev" -DATALAKE_APAC_BUCKET_NAME = "datalake-dev" -DATALAKE_ENAM_BUCKET_NAME = "datalake-dev" -DATALAKE_WNAM_BUCKET_NAME = "datalake-dev" - -[env.dev.observability] -enabled = true -head_sampling_rate = 1 From 24b66777eaea442a5e323ca4c57b4e9decf6a345 Mon Sep 17 00:00:00 2001 From: Victor Ilyushchenko Date: Fri, 10 Jan 2025 12:32:30 +0300 Subject: [PATCH 2/7] Text & Table cell color styling features (#7624) Signed-off-by: Victor Ilyushchenko --- common/config/rush/pnpm-lock.yaml | 12 +- models/text-editor/src/index.ts | 29 +++- models/text-editor/src/plugin.ts | 6 +- packages/text/package.json | 3 +- packages/text/src/index.ts | 3 + packages/text/src/kits/server-kit.ts | 7 +- packages/text/src/marks/colors.ts | 142 ++++++++++++++++++ packages/theme/styles/_colors.scss | 44 ++++++ packages/theme/styles/prose.scss | 2 +- plugins/text-editor-assets/assets/icons.svg | 7 + plugins/text-editor-assets/lang/en.json | 4 +- plugins/text-editor-assets/lang/ru.json | 4 +- plugins/text-editor-assets/src/index.ts | 4 +- .../src/components/extension/colors.ts | 100 ++++++++++++ .../extension/popups/ColorPicker.svelte | 65 ++++++++ .../extension/table/TableNodeView.svelte | 2 +- .../extension/table/TableToolbar.svelte | 27 +++- plugins/text-editor-resources/src/index.ts | 7 +- .../src/kits/editor-kit.ts | 8 +- plugins/text-editor/src/plugin.ts | 13 +- plugins/text-editor/src/types.ts | 2 +- 21 files changed, 469 insertions(+), 22 deletions(-) create mode 100644 packages/text/src/marks/colors.ts create mode 100644 plugins/text-editor-resources/src/components/extension/colors.ts create mode 100644 plugins/text-editor-resources/src/components/extension/popups/ColorPicker.svelte diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index a5d4a5d887..cda5be48d4 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -6254,6 +6254,14 @@ packages: '@tiptap/core': 2.6.6(@tiptap/pm@2.6.6) dev: false + /@tiptap/extension-text-style@2.11.0(@tiptap/core@2.6.6): + resolution: {integrity: sha512-vuA16wMZ6J3fboL7FObwV2f5uN9Vg0WYmqU7971vxzJyaRj9VE1eeH8Kh5fq4RgwDzc13MZGvZZV4HcE1R8o8A==} + peerDependencies: + '@tiptap/core': ^2.7.0 + dependencies: + '@tiptap/core': 2.6.6(@tiptap/pm@2.6.6) + dev: false + /@tiptap/extension-text@2.6.6(@tiptap/core@2.6.6): resolution: {integrity: sha512-e84uILnRzNzcwK1DVQNpXVmBG1Cq3BJipTOIDl1LHifOok7MBjhI/X+/NR0bd3N2t6gmDTWi63+4GuJ5EeDmsg==} peerDependencies: @@ -32495,6 +32503,7 @@ packages: '@tiptap/extension-table-header': 2.6.6(@tiptap/core@2.6.6) '@tiptap/extension-table-row': 2.6.6(@tiptap/core@2.6.6) '@tiptap/extension-text-align': 2.11.0(@tiptap/core@2.6.6) + '@tiptap/extension-text-style': 2.11.0(@tiptap/core@2.6.6) '@tiptap/extension-typography': 2.6.6(@tiptap/core@2.6.6) '@tiptap/extension-underline': 2.6.6(@tiptap/core@2.6.6) '@tiptap/pm': 2.6.6 @@ -32639,7 +32648,7 @@ packages: dev: false file:projects/text.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.24.2)(ts-node@10.9.2)(utf-8-validate@6.0.4): - resolution: {integrity: sha512-zmM6GDzeNuMbMA7iWwhg8hBc43fylNW7cEmk0fSWw9+G9NqD2MLv2dVb5dnAcPAioj9WBdDakWcrLAAHUBJyMw==, tarball: file:projects/text.tgz} + resolution: {integrity: sha512-p2TRL6FgtqVBooDf+E5A4HB+45VS03IvNxudhC9gCotewS726ZLOxl2XdfeuYB2rHYbzdmmiG2zb8GbPwqTkOw==, tarball: file:projects/text.tgz} id: file:projects/text.tgz name: '@rush-temp/text' version: 0.0.0 @@ -32660,6 +32669,7 @@ packages: '@tiptap/extension-task-item': 2.6.6(@tiptap/core@2.6.6)(@tiptap/pm@2.6.6) '@tiptap/extension-task-list': 2.6.6(@tiptap/core@2.6.6) '@tiptap/extension-text-align': 2.11.0(@tiptap/core@2.6.6) + '@tiptap/extension-text-style': 2.11.0(@tiptap/core@2.6.6) '@tiptap/extension-typography': 2.6.6(@tiptap/core@2.6.6) '@tiptap/extension-underline': 2.6.6(@tiptap/core@2.6.6) '@tiptap/html': 2.6.6(@tiptap/core@2.6.6)(@tiptap/pm@2.6.6) diff --git a/models/text-editor/src/index.ts b/models/text-editor/src/index.ts index 028616d9c3..82557e4dfb 100644 --- a/models/text-editor/src/index.ts +++ b/models/text-editor/src/index.ts @@ -247,6 +247,18 @@ export function createModel (builder: Builder): void { index: 20 }) + builder.createDoc(textEditor.class.TextEditorAction, core.space.Model, { + action: textEditor.function.SetTextColor, + icon: textEditor.icon.TextStyle, + visibilityTester: textEditor.function.IsTextStylingEnabled, + isActive: { + name: 'textStyle' + }, + label: textEditor.string.SetTextColor, + category: 20, + index: 25 + }) + // Link category builder.createDoc(textEditor.class.TextEditorAction, core.space.Model, { action: textEditor.function.FormatLink, @@ -338,23 +350,36 @@ export function createModel (builder: Builder): void { index: 10 }) + // Table cell category + builder.createDoc(textEditor.class.TextEditorAction, core.space.Model, { + kind: 'table', + action: textEditor.function.SetBackgroundColor, + icon: textEditor.icon.Brush, + visibilityTester: textEditor.function.IsTableToolbarContext, + label: textEditor.string.SetCellHighlightColor, + category: 65, + index: 5 + }) + // Table category builder.createDoc(textEditor.class.TextEditorAction, core.space.Model, { + kind: 'table', action: textEditor.function.SelectTable, icon: textEditor.icon.SelectTable, visibilityTester: textEditor.function.IsTableToolbarContext, label: textEditor.string.SelectTable, category: 70, - index: 5 + index: 15 }) builder.createDoc(textEditor.class.TextEditorAction, core.space.Model, { + kind: 'table', action: textEditor.function.OpenTableOptions, icon: textEditor.icon.TableProps, visibilityTester: textEditor.function.IsTableToolbarContext, label: textEditor.string.TableOptions, category: 70, - index: 10 + index: 20 }) // Image align category diff --git a/models/text-editor/src/plugin.ts b/models/text-editor/src/plugin.ts index 5540a9240a..2039564663 100644 --- a/models/text-editor/src/plugin.ts +++ b/models/text-editor/src/plugin.ts @@ -36,9 +36,13 @@ export default mergeIds(textEditorId, textEditor, { IsTableToolbarContext: '' as Resource, IsEditableNote: '' as Resource, IsEditable: '' as Resource, + IsTextStylingEnabled: '' as Resource, IsHeadingVisible: '' as Resource, CreateInlineComment: '' as Resource, - ShouldShowCreateInlineCommentAction: '' as Resource + ShouldShowCreateInlineCommentAction: '' as Resource, + + SetBackgroundColor: '' as Resource, + SetTextColor: '' as Resource } }) diff --git a/packages/text/package.json b/packages/text/package.json index 0bf657add2..0147c859eb 100644 --- a/packages/text/package.json +++ b/packages/text/package.json @@ -64,7 +64,8 @@ "prosemirror-codemark": "^0.4.2", "markdown-it": "^14.0.0", "fast-equals": "^5.0.1", - "@tiptap/extension-text-align": "~2.11.0" + "@tiptap/extension-text-align": "~2.11.0", + "@tiptap/extension-text-style": "~2.11.0" }, "repository": "https://github.com/hcengineering/platform", "publishConfig": { diff --git a/packages/text/src/index.ts b/packages/text/src/index.ts index 87456d522c..d543c4f264 100644 --- a/packages/text/src/index.ts +++ b/packages/text/src/index.ts @@ -22,6 +22,7 @@ export * from './markup/utils' export * from './nodes' // export * from './ydoc' export * from './marks/code' +export * from './marks/colors' export * from './marks/noteBase' export * from './markdown' export * from './markdown/serializer' @@ -31,3 +32,5 @@ export * from './markdown/node' export * from './kits/default-kit' export * from './kits/server-kit' + +export { TextStyle, type TextStyleOptions } from '@tiptap/extension-text-style' diff --git a/packages/text/src/kits/server-kit.ts b/packages/text/src/kits/server-kit.ts index b6a7345bec..0226c673d5 100644 --- a/packages/text/src/kits/server-kit.ts +++ b/packages/text/src/kits/server-kit.ts @@ -34,6 +34,8 @@ import { CodeExtension, codeOptions } from '../marks/code' import { NoteBaseExtension } from '../marks/noteBase' import { CommentNode } from '../nodes/comment' import TextAlign from '@tiptap/extension-text-align' +import TextStyle from '@tiptap/extension-text-style' +import { BackgroundColor, TextColor } from '../marks/colors' const headingLevels: Level[] = [1, 2, 3, 4, 5, 6] @@ -97,7 +99,10 @@ export const ServerKit = Extension.create({ ReferenceNode, CommentNode, NodeUuid, - NoteBaseExtension + NoteBaseExtension, + TextStyle.configure({}), + TextColor.configure({}), + BackgroundColor.configure({ types: ['tableCell'] }) ] } }) diff --git a/packages/text/src/marks/colors.ts b/packages/text/src/marks/colors.ts new file mode 100644 index 0000000000..d228507a2d --- /dev/null +++ b/packages/text/src/marks/colors.ts @@ -0,0 +1,142 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { Extension } from '@tiptap/core' +import '@tiptap/extension-text-style' + +export interface BackgroundColorOptions { + types: string[] +} + +declare module '@tiptap/core' { + interface Commands { + colors: { + setTextColor: (color: string) => ReturnType + unsetTextColor: () => ReturnType + setBackgroundColor: (color: string) => ReturnType + unsetBackgroundColor: () => ReturnType + } + } +} + +export const BackgroundColor = Extension.create({ + name: 'backgroundColor', + + addOptions () { + return { + types: [] + } + }, + + addGlobalAttributes () { + return [ + { + types: this.options.types, + attributes: { + backgroundColor: { + parseHTML: (element) => { + return element.getAttribute('data-background-color') ?? undefined + }, + renderHTML: (attributes) => { + if (typeof attributes.backgroundColor !== 'string') { + return {} + } + + return { + 'data-background-color': attributes.backgroundColor, + style: `background-color: ${attributes.backgroundColor}` + } + } + } + } + } + ] + }, + + addCommands () { + return { + setBackgroundColor: + (backgroundColor: string) => + ({ commands }) => { + return this.options.types + .map((type) => commands.updateAttributes(type, { backgroundColor })) + .every((response) => response) + }, + + unsetBackgroundColor: + () => + ({ commands }) => { + return this.options.types + .map((type) => commands.resetAttributes(type, 'backgroundColor')) + .every((response) => response) + } + } + } +}) + +export interface TextColorOptions { + types: string[] +} + +export const TextColor = Extension.create({ + name: 'textColor', + + addOptions () { + return { + types: ['textStyle'] + } + }, + + addGlobalAttributes () { + return [ + { + types: this.options.types, + attributes: { + color: { + parseHTML: (element) => { + return element.getAttribute('data-color') ?? undefined + }, + renderHTML: (attributes) => { + if (typeof attributes.color !== 'string') { + return {} + } + + return { + 'data-color': attributes.color, + style: `color: ${attributes.color}` + } + } + } + } + } + ] + }, + + addCommands () { + return { + setTextColor: + (color: string) => + ({ chain }) => { + return chain().setMark('textStyle', { color }).run() + }, + + unsetTextColor: + () => + ({ chain }) => { + return chain().setMark('textStyle', { color: null }).removeEmptyTextStyle().run() + } + } + } +}) diff --git a/packages/theme/styles/_colors.scss b/packages/theme/styles/_colors.scss index bd095464bd..6e9132a7e8 100644 --- a/packages/theme/styles/_colors.scss +++ b/packages/theme/styles/_colors.scss @@ -248,6 +248,28 @@ --theme-text-editor-note-anchor-bg-primary: #688797; --theme-text-editor-note-anchor-bg-primary-light: #747C81; + --text-editor-table-border-color: hsl(220, 6%, 40%); + + --theme-text-editor-palette-text-gray: rgba(155, 155, 155, 1); + --theme-text-editor-palette-text-brown: rgba(186, 133, 111, 1); + --theme-text-editor-palette-text-orange: rgba(199, 125, 72, 1); + --theme-text-editor-palette-text-yellow: rgba(202, 152, 73, 1); + --theme-text-editor-palette-text-green: rgba(82, 158, 114, 1); + --theme-text-editor-palette-text-blue: rgba(94, 135, 201, 1); + --theme-text-editor-palette-text-purple: rgba(157, 104, 211, 1); + --theme-text-editor-palette-text-pink: rgba(209, 87, 150, 1); + --theme-text-editor-palette-text-red: rgba(223, 84, 82, 1); + + --theme-text-editor-palette-bg-gray: rgba(47, 47, 47, 1); + --theme-text-editor-palette-bg-brown: rgba(74, 50, 40, 1); + --theme-text-editor-palette-bg-orange: rgba(92, 59, 35, 1); + --theme-text-editor-palette-bg-yellow: rgba(86, 67, 40, 1); + --theme-text-editor-palette-bg-green: rgba(36, 61, 48, 1); + --theme-text-editor-palette-bg-blue: rgba(20, 58, 78, 1); + --theme-text-editor-palette-bg-purple: rgba(60, 45, 73, 1); + --theme-text-editor-palette-bg-pink: rgba(78, 44, 60, 1); + --theme-text-editor-palette-bg-red:rgba(82, 46, 42, 1); + --accent-bg-color: #27282b; --accent-shadow: rgb(0 0 0 / 10%) 0px 2px 4px; @@ -507,6 +529,28 @@ --theme-text-editor-note-anchor-bg-primary: #AAC5E9; --theme-text-editor-note-anchor-bg-primary-light: #D5E5F5; + --text-editor-table-border-color: #c9cbcd; + + --theme-text-editor-palette-text-gray: rgba(120, 119, 116, 1); + --theme-text-editor-palette-text-brown: rgba(159, 107, 83, 1); + --theme-text-editor-palette-text-orange: rgba(217, 115, 13, 1); + --theme-text-editor-palette-text-yellow: rgba(203, 145, 47, 1); + --theme-text-editor-palette-text-green: rgba(68, 131, 97, 1); + --theme-text-editor-palette-text-blue: rgba(51, 126, 169, 1); + --theme-text-editor-palette-text-purple: rgba(144, 101, 176, 1); + --theme-text-editor-palette-text-pink: rgba(193, 76, 138, 1); + --theme-text-editor-palette-text-red: rgba(212, 76, 71, 1); + + --theme-text-editor-palette-bg-gray: rgba(241, 241, 239, 1); + --theme-text-editor-palette-bg-brown: rgba(244, 238, 238, 1); + --theme-text-editor-palette-bg-orange: rgba(251, 236, 221, 1); + --theme-text-editor-palette-bg-yellow: rgba(251, 243, 219, 1); + --theme-text-editor-palette-bg-green: rgba(237, 243, 236, 1); + --theme-text-editor-palette-bg-blue: rgba(231, 243, 248, 1); + --theme-text-editor-palette-bg-purple: rgba(244, 240, 247, 0.8); + --theme-text-editor-palette-bg-pink: rgba(249, 238, 243, 0.8); + --theme-text-editor-palette-bg-red: rgba(253, 235, 236, 1); + --accent-bg-color: #eff0f2; // HZ --accent-shadow: rgb(0 0 0 / 10%) 0px 2px 4px; // Dark diff --git a/packages/theme/styles/prose.scss b/packages/theme/styles/prose.scss index 927528fa0e..9b2a92fa36 100644 --- a/packages/theme/styles/prose.scss +++ b/packages/theme/styles/prose.scss @@ -38,7 +38,7 @@ table.proseTable { th { min-width: 1rem; height: 2rem; - border: 1px solid var(--button-border-hover); + border: 1px solid var(--text-editor-table-border-color); padding: .25rem .5rem; vertical-align: top; box-sizing: border-box; diff --git a/plugins/text-editor-assets/assets/icons.svg b/plugins/text-editor-assets/assets/icons.svg index 56514789e4..4546048f49 100644 --- a/plugins/text-editor-assets/assets/icons.svg +++ b/plugins/text-editor-assets/assets/icons.svg @@ -214,4 +214,11 @@ + + + + + + + \ No newline at end of file diff --git a/plugins/text-editor-assets/lang/en.json b/plugins/text-editor-assets/lang/en.json index 8677c3e156..805f0b3eba 100644 --- a/plugins/text-editor-assets/lang/en.json +++ b/plugins/text-editor-assets/lang/en.json @@ -66,6 +66,8 @@ "MermaidDiargram": "Diagram", "Comment": "Comment", "AddComment": "Add a comment", - "AddCommentPlaceholder": "Add a comment..." + "AddCommentPlaceholder": "Add a comment...", + "SetCellHighlightColor": "Set cell color", + "SetTextColor": "Set text color" } } \ No newline at end of file diff --git a/plugins/text-editor-assets/lang/ru.json b/plugins/text-editor-assets/lang/ru.json index 704ab81366..a39503cef8 100644 --- a/plugins/text-editor-assets/lang/ru.json +++ b/plugins/text-editor-assets/lang/ru.json @@ -66,6 +66,8 @@ "MermaidDiargram": "Диаграмма", "Comment": "Комментарий", "AddComment": "Добавить комментарий", - "AddCommentPlaceholder": "Добавьте комментарий..." + "AddCommentPlaceholder": "Добавьте комментарий...", + "SetCellHighlightColor": "Изменить цвет ячеек", + "SetTextColor": "Изменить цвет текста" } } \ No newline at end of file diff --git a/plugins/text-editor-assets/src/index.ts b/plugins/text-editor-assets/src/index.ts index 0a4173dbd5..c1e7e3cb18 100644 --- a/plugins/text-editor-assets/src/index.ts +++ b/plugins/text-editor-assets/src/index.ts @@ -43,5 +43,7 @@ loadMetadata(textEditor.icon, { Comment: `${icons}#comment`, SelectTable: `${icons}#move`, MergeCells: `${icons}#union`, - SplitCells: `${icons}#divide` + SplitCells: `${icons}#divide`, + Brush: `${icons}#brush`, + TextStyle: `${icons}#textStyle` }) diff --git a/plugins/text-editor-resources/src/components/extension/colors.ts b/plugins/text-editor-resources/src/components/extension/colors.ts new file mode 100644 index 0000000000..aa162e140a --- /dev/null +++ b/plugins/text-editor-resources/src/components/extension/colors.ts @@ -0,0 +1,100 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { getEventPositionElement, showPopup } from '@hcengineering/ui' +import { type Editor } from '@tiptap/core' +import ColorPicker from './popups/ColorPicker.svelte' +import { type ActionContext } from '@hcengineering/text-editor' + +export interface BackgroundColorOptions { + types: string[] +} + +interface ColorSpec { + color: string + preview?: string +} + +function colorVar (tag: string, prefix = 'text'): string { + return `var(--theme-text-editor-palette-${prefix}-${tag})` +} + +function colorSpec (tag: string, prefix = 'text'): ColorSpec { + const color = colorVar(tag, prefix) + return { color, preview: colorVar(tag) } +} + +const palette = { + background: [ + { color: 'transparent' }, + colorSpec('gray', 'bg'), + colorSpec('brown', 'bg'), + colorSpec('orange', 'bg'), + colorSpec('yellow', 'bg'), + colorSpec('green', 'bg'), + colorSpec('blue', 'bg'), + colorSpec('purple', 'bg'), + colorSpec('pink', 'bg'), + colorSpec('red', 'bg') + ], + text: [ + { color: 'var(--theme-text-color-primary)' }, + colorSpec('gray'), + colorSpec('brown'), + colorSpec('orange'), + colorSpec('yellow'), + colorSpec('green'), + colorSpec('blue'), + colorSpec('purple'), + colorSpec('pink'), + colorSpec('red') + ] +} + +export async function openBackgroundColorOptions (editor: Editor, event: MouseEvent): Promise { + await new Promise((resolve) => { + showPopup(ColorPicker, { palette: palette.background }, getEventPositionElement(event), (val) => { + const color: string | undefined = val?.color + if (color === undefined) return + + if (color === 'transparent') { + editor.commands.unsetBackgroundColor() + } else { + editor.commands.setBackgroundColor(color) + } + resolve() + }) + }) +} + +export async function openTextColorOptions (editor: Editor, event: MouseEvent): Promise { + await new Promise((resolve) => { + showPopup(ColorPicker, { palette: palette.text }, getEventPositionElement(event), (val) => { + const color: string | undefined = val?.color + if (color === undefined) return + + if (color === 'var(--theme-text-color-primary)') { + editor.commands.unsetTextColor() + } else { + editor.commands.setTextColor(color) + } + resolve() + }) + }) +} + +export async function isTextStylingEnabled (editor: Editor, context: ActionContext): Promise { + return editor.isEditable && editor.commands.setTextColor !== undefined +} diff --git a/plugins/text-editor-resources/src/components/extension/popups/ColorPicker.svelte b/plugins/text-editor-resources/src/components/extension/popups/ColorPicker.svelte new file mode 100644 index 0000000000..83cf73cd92 --- /dev/null +++ b/plugins/text-editor-resources/src/components/extension/popups/ColorPicker.svelte @@ -0,0 +1,65 @@ + + + +
+
+ {#each palette as k} + + +
{ + handleSubmit(k) + }} + /> + {/each} +
+
+ + diff --git a/plugins/text-editor-resources/src/components/extension/table/TableNodeView.svelte b/plugins/text-editor-resources/src/components/extension/table/TableNodeView.svelte index fb164977d2..b4da3b4912 100644 --- a/plugins/text-editor-resources/src/components/extension/table/TableNodeView.svelte +++ b/plugins/text-editor-resources/src/components/extension/table/TableNodeView.svelte @@ -200,7 +200,7 @@ .table-toolbar-container { position: absolute; top: -1.5rem; - right: 0; + right: var(--table-offscreen-spacing); z-index: 200; } diff --git a/plugins/text-editor-resources/src/components/extension/table/TableToolbar.svelte b/plugins/text-editor-resources/src/components/extension/table/TableToolbar.svelte index b2fc07a074..406f778eda 100644 --- a/plugins/text-editor-resources/src/components/extension/table/TableToolbar.svelte +++ b/plugins/text-editor-resources/src/components/extension/table/TableToolbar.svelte @@ -50,15 +50,34 @@ actions = out } - $: actionsQuery.query(textEditor.class.TextEditorAction, { category: 70 }, (result) => { + $: actionsQuery.query(textEditor.class.TextEditorAction, { kind: 'table' }, (result) => { void updateActions([...result], actionCtx) }) + + $: categories = actions.reduce<[number, TextEditorAction][][]>((acc, action) => { + const { category, index } = action + if (acc[category] === undefined) acc[category] = [] + acc[category].push([index, action]) + return acc + }, []) + + $: categories.forEach((category) => { + category.sort((a, b) => a[0] - b[0]) + })
- {#each actions as action} - - {/each} +
+ {#each Object.values(categories) as category, index} + {#if index > 0} +
+ {/if} + + {#each category as [_, action]} + + {/each} + {/each} +