qfix: remove cf workers to fix ci/build on develop branch (#8226)

Signed-off-by: denis-tingaikin <denis.tingajkin@xored.com>
This commit is contained in:
Denis Tingaikin
2025-03-14 01:08:21 +07:00
committed by GitHub
parent 31bf564ffc
commit 06605ed619
26 changed files with 3 additions and 2307 deletions
+3 -847
View File
File diff suppressed because it is too large Load Diff
-10
View File
@@ -2228,11 +2228,6 @@
"projectFolder": "plugins/openai",
"shouldPublish": false
},
{
"packageName": "@hcengineering/cloud-branding",
"projectFolder": "workers/branding",
"shouldPublish": false
},
{
"packageName": "@hcengineering/scripts",
"projectFolder": "common/scripts",
@@ -2278,11 +2273,6 @@
"projectFolder": "plugins/survey-resources",
"shouldPublish": false
},
{
"packageName": "@hcengineering/cloud-transactor",
"projectFolder": "workers/transactor",
"shouldPublish": false
},
{
"packageName": "@hcengineering/card",
"projectFolder": "plugins/card",
-1
View File
@@ -1 +0,0 @@
**/.dev.vars
View File
-7
View File
@@ -1,7 +0,0 @@
module.exports = {
extends: ['./node_modules/@hcengineering/platform-rig/profiles/node/eslint.config.json'],
parserOptions: {
tsconfigRootDir: __dirname,
project: './tsconfig.json'
}
}
-5
View File
@@ -1,5 +0,0 @@
{
"$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
"rigPackageName": "@hcengineering/platform-rig",
"rigProfile": "node"
}
-7
View File
@@ -1,7 +0,0 @@
module.exports = {
preset: 'ts-jest',
testEnvironment: 'node',
testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'],
roots: ["./src"],
coverageReporters: ["text-summary", "html"]
}
-43
View File
@@ -1,43 +0,0 @@
{
"name": "@hcengineering/cloud-branding",
"version": "0.6.0",
"main": "lib/index.js",
"types": "types/index.d.ts",
"template": "@hcengineering/cloud-package",
"scripts": {
"deploy": "wrangler deploy",
"dev": "wrangler dev --port 4021",
"start": "wrangler dev --port 4021",
"cf-typegen": "wrangler types",
"build": "compile",
"build:watch": "compile",
"test": "jest --passWithNoTests --silent --forceExit",
"format": "format src",
"_phase:build": "compile transpile src",
"_phase:test": "jest --passWithNoTests --silent --forceExit",
"_phase:format": "format src",
"_phase:validate": "compile validate"
},
"devDependencies": {
"@hcengineering/platform-rig": "^0.6.0",
"@cloudflare/workers-types": "^4.20241022.0",
"typescript": "^5.3.3",
"wrangler": "^3.108.1",
"jest": "^29.7.0",
"prettier": "^3.1.0",
"ts-jest": "^29.1.1",
"@hcengineering/core": "^0.6.32",
"@typescript-eslint/eslint-plugin": "^6.11.0",
"@typescript-eslint/parser": "^6.11.0",
"eslint-config-standard-with-typescript": "^40.0.0",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-n": "^15.4.0",
"eslint-plugin-promise": "^6.1.1",
"eslint": "^8.54.0",
"@types/jest": "^29.5.5"
},
"dependencies": {
"itty-router": "^5.0.18"
},
"private": true
}
-18
View File
@@ -1,18 +0,0 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
export interface Env {
KV: KVNamespace
}
-81
View File
@@ -1,81 +0,0 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { Router, error, cors } from 'itty-router'
import { WorkerEntrypoint } from 'cloudflare:workers'
import type { Branding, BrandingMap } from '@hcengineering/core'
import type { Env } from './env'
const { preflight, corsify } = cors({
maxAge: 86400
})
const router = Router<Request>({
before: [preflight],
catch: error,
finally: [(r: any) => r ?? error(404), corsify]
})
const BRANDING_KEY_PREFIX = 'branding/'
async function getBrandings (env: Env): Promise<BrandingMap> {
// Note: up to 100 keys can be listed at once. If more, use cursor.
const keys = (await env.KV.list({ prefix: BRANDING_KEY_PREFIX })).keys
const values = await Promise.all(keys.map(async (key) => await env.KV.get<Branding>(key.name, { type: 'json' })))
return keys.reduce<BrandingMap>((acc, key, idx) => {
if (values[idx] === null) return acc
acc[getBrandingHost(key.name)] = values[idx]
return acc
}, {})
}
function getBrandingKey (host: string): string {
return `${BRANDING_KEY_PREFIX}${host}`
}
function getBrandingHost (key: string): string {
return key.slice(BRANDING_KEY_PREFIX.length)
}
router.get('/brandings', async (req, ctx: { env: Env, exCtx: ExecutionContext }) => {
return new Response(JSON.stringify(await getBrandings(ctx.env)), { status: 200 })
})
// TODO later
// router.get('/branding/{host}')
// router.post('/branding/{host}') [with auth]
// router.put('/branding/{host}') [with auth]
export default class BrandingWorker extends WorkerEntrypoint<Env> {
async fetch (request: Request): Promise<Response> {
return await router.fetch(request, { env: this.env })
}
async getBranding (host: string): Promise<Branding | null> {
const value = await this.env.KV.get<Branding>(getBrandingKey(host), { type: 'json' })
if (value == null) {
return null
}
return value
}
async getBrandings (): Promise<BrandingMap> {
return await getBrandings(this.env)
}
}
-12
View File
@@ -1,12 +0,0 @@
{
"extends": "./node_modules/@hcengineering/platform-rig/profiles/default/tsconfig.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./lib",
"declarationDir": "./types",
"tsBuildInfoFile": ".build/build.tsbuildinfo",
"types": ["@cloudflare/workers-types", "jest"],
"lib": ["esnext"]
}
}
-14
View File
@@ -1,14 +0,0 @@
#:schema node_modules/wrangler/config-schema.json
name = "branding-worker"
main = "src/index.ts"
compatibility_date = "2024-07-01"
keep_vars = true
[build]
command = "npm run build"
watch_dir = "./src"
[[kv_namespaces]]
binding = "KV"
id = "c020474a-1223-402b-be0c-cd122bab9599"
preview_id = "c320577a-5563-457b-be0c-cd152bab5599"
-95
View File
@@ -1,95 +0,0 @@
//
// Copyright © 2024 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the 'License');
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an 'AS IS' BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
import { type Env } from './env'
export async function withMetrics<T> (name: string, fn: (ctx: MetricsContext) => Promise<T>): Promise<T> {
const ctx = new MetricsContext()
const start = performance.now()
try {
return await fn(ctx)
} finally {
const total = performance.now() - start
const ops = ctx.metrics
const message = `${name} total=${total} ` + ctx.toString()
console.log({ message, total, ops })
}
}
export interface MetricsData {
op: string
time: number
}
export class MetricsContext {
metrics: Array<MetricsData> = []
debug (...data: any[]): void {
console.debug(...data)
}
log (...data: any[]): void {
console.log(...data)
}
error (...data: any[]): void {
console.error(...data)
}
async with<T>(op: string, fn: () => Promise<T>): Promise<T> {
const start = performance.now()
try {
return await fn()
} finally {
const time = performance.now() - start
this.metrics.push({ op, time })
}
}
withSync<T>(op: string, fn: () => T): T {
const start = performance.now()
try {
return fn()
} finally {
const time = performance.now() - start
this.metrics.push({ op, time })
}
}
toString (): string {
return this.metrics.map((p) => `${p.op}=${p.time}`).join(' ')
}
}
export class LoggedDatalake {
constructor (
private readonly datalake: Env['DATALAKE'],
private readonly ctx: MetricsContext
) {}
async getBlob (workspace: string, name: string): Promise<ArrayBuffer> {
return await this.ctx.with('datalake.getBlob', () => {
return this.datalake.getBlob(workspace, name)
})
}
async putBlob (workspace: string, name: string, data: ArrayBuffer | Blob | string, type: string): Promise<void> {
await this.ctx.with('datalake.putBlob', () => {
return this.datalake.putBlob(workspace, name, data, type)
})
}
}
-9
View File
@@ -1,9 +0,0 @@
# http://editorconfig.org
root = true
[*]
indent_style = space
end_of_line = lf
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
-7
View File
@@ -1,7 +0,0 @@
module.exports = {
extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/eslint.config.json'],
parserOptions: {
tsconfigRootDir: __dirname,
project: './tsconfig.json'
}
}
-173
View File
@@ -1,173 +0,0 @@
# Logs
logs
_.log
npm-debug.log_
yarn-debug.log*
yarn-error.log*
lerna-debug.log*
.pnpm-debug.log*
# Diagnostic reports (https://nodejs.org/api/report.html)
report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
# Runtime data
pids
_.pid
_.seed
\*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
\*.lcov
# nyc test coverage
.nyc_output
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
jspm_packages/
# Snowpack dependency directory (https://snowpack.dev/)
web_modules/
# TypeScript cache
\*.tsbuildinfo
# Optional npm cache directory
.npm
# Optional eslint cache
.eslintcache
# Optional stylelint cache
.stylelintcache
# Microbundle cache
.rpt2_cache/
.rts2_cache_cjs/
.rts2_cache_es/
.rts2_cache_umd/
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
\*.tgz
# Yarn Integrity file
.yarn-integrity
# dotenv environment variable files
.env
.env.development.local
.env.test.local
.env.production.local
.env.local
# parcel-bundler cache (https://parceljs.org/)
.cache
.parcel-cache
# Next.js build output
.next
out
# Nuxt.js build / generate output
.nuxt
dist
# Gatsby files
.cache/
# Comment in the public line in if your project uses Gatsby and not Next.js
# https://nextjs.org/blog/next-9-1#public-directory-support
# public
# vuepress build output
.vuepress/dist
# vuepress v2.x temp and cache directory
.temp
.cache
# Docusaurus cache and generated files
.docusaurus
# Serverless directories
.serverless/
# FuseBox cache
.fusebox/
# DynamoDB Local files
.dynamodb/
# TernJS port file
.tern-port
# Stores VSCode versions used for testing VSCode extensions
.vscode-test
# yarn v2
.yarn/cache
.yarn/unplugged
.yarn/build-state.yml
.yarn/install-state.gz
.pnp.\*
# wrangler project
.dev.vars
.wrangler/
src/model.json
-23
View File
@@ -1,23 +0,0 @@
#!/bin/bash
files=$(cat ./dist/index.js|grep node_modules | grep //)
declare -a file_info
# Iterate over each line in $files
while IFS= read -r line; do
file=${line##*//}
size=$(ls -l $file 2>/dev/null | awk '{print $5}')
# echo "Processing: $file $size"
if [ ! -z "$size" ]; then
# Store size and path together
file_info+=("$size:$file")
fi
done <<< "$files"
# Sort the array by size (numerically, in descending order) and print
printf '%s\n' "${file_info[@]}" | sort -t: -k1,1nr | while IFS=: read -r size path; do
if [ $(($size/1024)) -ne 0 ]; then
echo "Size: $(($size/1024)) KB - $path"
fi
done
-4
View File
@@ -1,4 +0,0 @@
{
"$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
"rigPackageName": "@hcengineering/platform-rig"
}
-66
View File
@@ -1,66 +0,0 @@
{
"name": "@hcengineering/cloud-transactor",
"version": "0.0.0",
"private": true,
"template": "cloud",
"scripts": {
"deploy": "wrangler deploy",
"dev": "wrangler dev --port 3335 --remote",
"dev-local": "wrangler dev --port 3335 --local --upstream-protocol=http",
"start": "wrangler dev --port 3335",
"logs": "npx wrangler tail --format pretty",
"cf-typegen": "wrangler types",
"get-model": "mkdir -p bundle && esbuild src/get-model.ts --bundle --keep-names --external:*.node --platform=node --define:process.env.MODEL_VERSION=$(node ../../common/scripts/show_version.js) --define:process.env.VERSION=$(node ../../common/scripts/show_tag.js) --define:process.env.GIT_REVISION=$(../../common/scripts/git_version.sh) --outfile=bundle/bundle.js --log-level=error && node ./bundle/bundle.js > ./src/model.json",
"bundle": "rushx get-model && wrangler deploy --dry-run --outdir dist",
"build": "compile",
"build:watch": "compile",
"test": "jest --passWithNoTests --silent --forceExit",
"format": "format src",
"_phase:bundle": "rushx get-model",
"_phase:build": "compile transpile src",
"_phase:test": "jest --passWithNoTests --silent --forceExit",
"_phase:format": "format src",
"_phase:validate": "rushx bundle && compile validate"
},
"devDependencies": {
"@cloudflare/workers-types": "^4.20241022.0",
"@hcengineering/model-all": "^0.6.0",
"@hcengineering/platform-rig": "^0.6.0",
"@types/jest": "^29.5.5",
"@typescript-eslint/eslint-plugin": "^6.11.0",
"@typescript-eslint/parser": "^6.11.0",
"eslint-config-standard-with-typescript": "^40.0.0",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-promise": "^6.1.1",
"eslint-plugin-n": "^15.4.0",
"eslint": "^8.54.0",
"jest": "^29.7.0",
"prettier": "^3.1.0",
"ts-jest": "^29.1.1",
"typescript": "^5.3.3",
"wrangler": "^3.108.1",
"esbuild": "^0.24.2",
"@types/snappyjs": "^0.7.1"
},
"dependencies": {
"@hcengineering/core": "^0.6.32",
"@hcengineering/middleware": "^0.6.0",
"@hcengineering/platform-rig": "^0.6.0",
"@hcengineering/rpc": "^0.6.5",
"@hcengineering/platform": "^0.6.11",
"@hcengineering/postgres": "^0.6.0",
"@hcengineering/server": "^0.6.4",
"@hcengineering/server-core": "^0.6.1",
"@hcengineering/server-client": "^0.6.0",
"@hcengineering/server-pipeline": "^0.6.0",
"@hcengineering/server-token": "^0.6.11",
"@hcengineering/contact": "^0.6.24",
"@hcengineering/storage": "^0.6.0",
"@hcengineering/server-notification": "^0.6.1",
"@hcengineering/notification": "^0.6.23",
"@hcengineering/server-ai-bot": "^0.6.0",
"@hcengineering/server-telegram": "^0.6.0",
"itty-router": "^5.0.18",
"snappyjs": "^0.7.0"
}
}
-1
View File
@@ -1 +0,0 @@
import '@hcengineering/model-all/src/show'
-90
View File
@@ -1,90 +0,0 @@
// Copyright © 2024 Huly Labs.
import { Class, Doc, DocumentQuery, FindOptions, Ref, Tx } from '@hcengineering/core'
import { decodeToken } from '@hcengineering/server-token'
import { RpcTarget, WorkerEntrypoint } from 'cloudflare:workers'
import { Router, error, html } from 'itty-router'
import type { Transactor } from './transactor'
export { Transactor } from './transactor'
export interface Env {
TRANSACTOR: DurableObjectNamespace<Transactor>
HYPERDRIVE: Hyperdrive
SERVER_SECRET: string
ACCOUNTS_URL: string
}
export default {
async fetch (request: Request, env: Env, ctx: ExecutionContext): Promise<Response> {
const router = Router()
router
.get('/:token', async ({ params, headers }) => {
if (headers.get('Upgrade') !== 'websocket') {
return new Response('Expected header Upgrade: websocket', { status: 426 })
}
try {
const { account, workspace } = decodeToken(params.token, true, env.SERVER_SECRET)
console.log({ message: 'connecting', account, workspace })
const id = env.TRANSACTOR.idFromName(workspace)
const stub = env.TRANSACTOR.get(id)
return await stub.fetch(request)
} catch (err: any) {
console.error({ message: 'Request failed:', err, errMessage: err.message, stack: err.stack })
return new Response('Invalid', { status: 401 })
}
})
// TODO: Add statistics using storage
.all('/', () =>
html(
`Huly&reg; Transactor&trade; <a href="https://huly.io">https://huly.io</a>
&copy; 2024 <a href="https://hulylabs.com">Huly Labs</a>`
)
)
.all('*', () => error(404))
return await router.fetch(request).catch(error)
}
} satisfies ExportedHandler<Env>
class TransactorRpcTarget extends RpcTarget {
constructor (
private readonly token: string,
private readonly workspaceId: string,
private readonly transactor: DurableObjectStub<Transactor>
) {
super()
}
async findAll (_class: Ref<Class<Doc>>, query?: DocumentQuery<Doc>, options?: FindOptions<Doc>): Promise<any> {
return (this.transactor as any).findAll(this.token, this.workspaceId, _class, query, options)
}
async tx (tx: Tx): Promise<any> {
return (this.transactor as any).tx(this.token, this.workspaceId, tx)
}
async getModel (): Promise<any> {
return (this.transactor as any).getModel(this.token)
}
async getAccount (): Promise<any> {
return (this.transactor as any).getAccount(this.token, this.workspaceId)
}
}
export class TransactorRpc extends WorkerEntrypoint<Env> {
async openRpc (token: string, workspaceId: string): Promise<TransactorRpcTarget> {
const decodedToken = decodeToken(token, true, this.env.SERVER_SECRET)
const id = this.env.TRANSACTOR.idFromName(decodedToken.workspace)
const stub = this.env.TRANSACTOR.get(id)
return new TransactorRpcTarget(token, workspaceId, stub)
}
}
-39
View File
@@ -1,39 +0,0 @@
//
// Copyright © 2025 Hardcore Engineering Inc.
//
import type { MeasureLogger, ParamsType } from '@hcengineering/core'
export class CloudFlareLogger implements MeasureLogger {
error (message: string, obj?: Record<string, any>): void {
const errMsg: Record<string, any> = {}
// Check if obj has error inside, so we could send it to Analytics
for (const v of Object.values(obj ?? {})) {
if (v instanceof Error) {
// Analytics.handleError(v)
errMsg.error = v.message
errMsg.stack = v.stack
}
}
console.error({ message, ...obj, ...errMsg })
}
info (message: string, obj?: Record<string, any>): void {
console.info({ message, ...obj })
}
warn (message: string, obj?: Record<string, any>): void {
console.warn({ message, ...obj })
}
logOperation (operation: string, time: number, params: ParamsType): void {
console.info({ time, ...params, message: operation })
}
childLogger (name: string, params: Record<string, string>): MeasureLogger {
return this
}
async close (): Promise<void> {}
}
-582
View File
@@ -1,582 +0,0 @@
// Copyright © 2024 Huly Labs.
import {
generateId,
NoMetricsContext,
platformNow,
platformNowDiff,
WorkspaceUuid,
type Account,
type Class,
type Doc,
type DocumentQuery,
type FindOptions,
type MeasureContext,
type Ref,
type Tx
} from '@hcengineering/core'
import { setMetadata } from '@hcengineering/platform'
import { RPCHandler } from '@hcengineering/rpc'
import { ClientSession, createSessionManager, doSessionOp, type WebsocketData } from '@hcengineering/server'
import serverCore, {
createDummyStorageAdapter,
loadBrandingMap,
pingConst,
pongConst,
Session,
Workspace,
type ConnectionSocket,
type Pipeline,
type PipelineFactory,
type SessionManager
} from '@hcengineering/server-core'
import serverPlugin, { decodeToken, type Token } from '@hcengineering/server-token'
import { DurableObject } from 'cloudflare:workers'
import { compress, uncompress } from 'snappyjs'
import { promisify } from 'util'
import { gzip } from 'zlib'
// Approach usefull only for separate build, after model-all bundle phase is executed.
import {
createPostgreeDestroyAdapter,
createPostgresAdapter,
createPostgresTxAdapter,
registerGreenDecoder,
registerGreenUrl,
setDBExtraOptions
} from '@hcengineering/postgres'
import {
createServerPipeline,
isAdapterSecurity,
registerAdapterFactory,
registerDestroyFactory,
registerServerPlugins,
registerStringLoaders,
registerTxAdapterFactory,
setAdapterSecurity
} from '@hcengineering/server-pipeline'
import { CloudFlareLogger } from './logger'
import model from './model.json'
// import { configureAnalytics } from '@hcengineering/analytics-service'
// import { Analytics } from '@hcengineering/analytics'
import contactPlugin from '@hcengineering/contact'
import serverAiBot from '@hcengineering/server-ai-bot'
import serverNotification from '@hcengineering/server-notification'
import serverTelegram from '@hcengineering/server-telegram'
export const PREFERRED_SAVE_SIZE = 500
export const PREFERRED_SAVE_INTERVAL = 30 * 1000
export class Transactor extends DurableObject<Env> {
private workspace = '' as WorkspaceUuid
private sessionManager!: SessionManager
private readonly measureCtx: MeasureContext
private readonly pipelineFactory: PipelineFactory
private readonly accountsUrl: string
private readonly sessions = new Map<WebSocket, WebsocketData>()
private readonly contextVars: Record<string, any> = {}
constructor (ctx: DurableObjectState, env: Env) {
super(ctx, env)
setDBExtraOptions({
ssl: false,
max: env.USE_GREEN === 'true' ? 2 : 5, // Cloud flare limit an concurrent connection to be 6 total
connection: {
application_name: 'cloud-transactor'
}
})
// this.ctx.setHibernatableWebSocketEventTimeout(60 * 1000)
this.ctx.setWebSocketAutoResponse(new WebSocketRequestResponsePair(pingConst, pongConst))
// configureAnalytics(env.SENTRY_DSN, {})
// Analytics.setTag('application', 'transactor')
const lastNameFirst = process.env.LAST_NAME_FIRST === 'true'
setMetadata(contactPlugin.metadata.LastNameFirst, lastNameFirst)
setMetadata(serverCore.metadata.FrontUrl, env.FRONT_URL)
setMetadata(serverCore.metadata.FilesUrl, env.FILES_URL)
setMetadata(serverNotification.metadata.SesUrl, env.SES_URL ?? '')
setMetadata(serverNotification.metadata.SesAuthToken, env.SES_AUTH_TOKEN)
setMetadata(serverTelegram.metadata.BotUrl, process.env.TELEGRAM_BOT_URL)
setMetadata(serverAiBot.metadata.EndpointURL, process.env.AI_BOT_URL)
registerTxAdapterFactory('postgresql', createPostgresTxAdapter, true)
registerAdapterFactory('postgresql', createPostgresAdapter, true)
registerDestroyFactory('postgresql', createPostgreeDestroyAdapter, true)
setAdapterSecurity('postgresql', true)
if (env.USE_GREEN === 'true') {
registerGreenUrl(env.GREEN_URL)
registerGreenDecoder('snappy', async (data) => uncompress(data))
}
registerStringLoaders()
registerServerPlugins()
this.accountsUrl = env.ACCOUNTS_URL ?? 'http://127.0.0.1:3000'
this.measureCtx = new NoMetricsContext(new CloudFlareLogger())
// initStatisticsContext('ctr-' + ctx.id.toString(), {
// statsUrl: this.env.STATS_URL ?? 'http://127.0.0.1:4900',
// serviceName: () => 'cloud-transactor: ' + this.workspace,
// factory: () => new MeasureMetricsContext('transactor', {}, {}, newMetrics(), new CloudFlareLogger())
// })
setMetadata(serverPlugin.metadata.Secret, env.SERVER_SECRET ?? 'secret')
console.log({ message: 'Connecting DB', mode: env.DB_URL !== '' ? 'Direct ' : 'Hyperdrive' })
console.log({ message: 'use stats', url: this.env.STATS_URL })
console.log({ message: 'use fulltext', url: this.env.FULLTEXT_URL })
const dbUrl = env.DB_MODE === 'direct' ? env.DB_URL ?? '' : env.HYPERDRIVE.connectionString
// TODO:
const storage = createDummyStorageAdapter()
this.pipelineFactory = async (ctx, ws, upgrade, broadcast, branding) => {
const pipeline = createServerPipeline(this.measureCtx, dbUrl, model, {
externalStorage: storage,
adapterSecurity: isAdapterSecurity(dbUrl),
disableTriggers: false,
fulltextUrl: env.FULLTEXT_URL,
extraLogging: true,
pipelineContextVars: this.contextVars
})
return await pipeline(ctx, ws, upgrade, broadcast, branding)
}
void this.ctx
.blockConcurrencyWhile(async () => {
const wakeUps = ((await ctx.storage.get('wakeUps')) as number) ?? 0
console.log({ message: `wakeup ${wakeUps}`, connections: ctx.getWebSockets().length })
await ctx.storage.put('wakeUps', wakeUps + 1)
this.sessionManager = createSessionManager(
this.measureCtx,
(token: Token, workspace: Workspace, account: Account) => new ClientSession(token, workspace, account, false),
loadBrandingMap(), // TODO: Support branding map
{
pingTimeout: 10000,
reconnectTimeout: 3000
},
undefined,
this.accountsUrl,
env.ENABLE_COMPRESSION === 'true',
false
)
})
.catch((err) => {
console.error({ message: 'Failed to init transactor', err })
})
}
async fetch (request: Request): Promise<Response> {
const { 0: client, 1: server } = new WebSocketPair()
const url = new URL(request.url ?? '')
const token = url.pathname.substring(1)
try {
const payload = decodeToken(token ?? '')
const sessionId = url.searchParams.get('sessionId')
// By design, all fetches to this durable object will be for the same workspace
if (this.workspace === '') {
this.workspace = payload.workspace
}
if (!(await this.handleSession(server, request, payload, token, sessionId))) {
return new Response(null, { status: 404 })
}
return new Response(null, { status: 101, webSocket: client })
} catch (err: any) {
console.error({ message: 'Failed to handle request', errMsg: err.message, errStack: err.stack })
return new Response(null, { status: 404 })
}
}
async webSocketMessage (ws: WebSocket, message: ArrayBuffer | string): Promise<void> {
const session = this.sessions.get(ws)
if (session === undefined) {
return
}
const cs = session.connectionSocket
if (cs === undefined) {
return
}
try {
doSessionOp(
session,
(s, buff) => {
s.context.measure('receive-data', buff?.length ?? 0)
// processRequest(s.session, cs, s.context, s.workspaceId, buff, handleRequest)
const request = cs.readRequest(buff, s.session.binaryMode)
const st = platformNow()
const r = this.sessionManager.handleRequest(this.measureCtx, s.session, cs, request, this.workspace)
this.ctx.waitUntil(
r.finally(() => {
const time = platformNowDiff(st)
console.log({
message: `handle-request: ${request.method} time: ${time}`,
method: request.method,
params: request.params,
workspace: s.workspaceId,
user: s.session.getUser(),
time
})
})
)
},
typeof message === 'string' ? Buffer.from(message) : Buffer.from(message)
)
} catch (err: any) {
console.error({ message: 'Failed to handle message:', errMsg: err.message, err, stack: err.stack })
}
}
async webSocketError (ws: WebSocket, error: unknown): Promise<void> {
const session = this.sessions.get(ws)
console.error({ message: 'error:', error, account: session?.payload?.account })
await this.handleClose(ws, 1011, 'error')
}
async webSocketClose (ws: WebSocket, code: number, reason: string, wasClean: boolean): Promise<void> {
const session = this.sessions.get(ws)
console.warn({ message: 'closed', reason, code, wasClean, account: session?.payload?.account })
}
async alarm (): Promise<void> {
const memoryUsage = process.memoryUsage()
console.log({
message: 'Resource usage',
memoryUsed: memoryUsage.rss,
heapTotal: memoryUsage.heapTotal,
heapUsed: memoryUsage.heapUsed
})
}
async handleSession (
ws: WebSocket,
request: Request,
token: Token,
rawToken: string,
sessionId: string | null
): Promise<boolean> {
const data = {
remoteAddress: request.headers.get('CF-Connecting-IP') ?? '',
userAgent: request.headers.get('user-agent') ?? '',
language: request.headers.get('accept-language') ?? '',
account: token.account,
mode: token.extra?.mode,
model: token.extra?.model
}
console.log({
message: 'New session attempt',
remoteAddress: data.remoteAddress,
userAgent: data.userAgent,
account: data.account
})
this.ctx.acceptWebSocket(ws)
const cs = this.createWebsocketClientSocket(ws, data)
try {
const session = await this.sessionManager.addSession(
this.measureCtx,
cs,
token,
rawToken,
this.pipelineFactory,
sessionId ?? undefined
)
const webSocketData: WebsocketData = {
connectionSocket: cs,
payload: token,
token: rawToken,
session,
url: ''
}
if ('error' in session) {
console.error({ message: 'Failed to establish session:', error: session.error })
if (session.terminate === true) {
ws.close(4003, 'Session establishment failed')
}
throw session.error
}
if ('upgrade' in session) {
await cs.send(
this.measureCtx,
{ id: -1, result: { state: 'upgrading', stats: (session as any).upgradeInfo } },
false,
false
)
ws.close(4003, 'Session establishment failed')
return true
}
console.log({
message: 'Session established successfully:',
sessionId: session.session.sessionId,
workspaceId: token.workspace,
user: token.account
})
this.sessions.set(ws, webSocketData)
} catch (err: any) {
console.error({ message: 'Failed to establish session:', err })
ws.close(4003, 'Session establishment failed')
throw err
}
return true
}
createWebsocketClientSocket (
ws: WebSocket,
data: {
remoteAddress: string
userAgent: string
language: string
account: string
mode: any
model: any
}
): ConnectionSocket {
const rpcHandler = new RPCHandler()
const cs: ConnectionSocket = {
id: generateId(),
isClosed: false,
close: () => {
cs.isClosed = true
console.warn({ message: 'close socket', id: cs.id })
ws.close()
},
checkState: () => {
if (ws.readyState === WebSocket.CLOSED || ws.readyState === WebSocket.CLOSING) {
return false
}
return true
},
backpressure: async (ctx) => {},
isBackpressure: () => false,
readRequest: (buffer: Buffer, binary: boolean) => {
if (buffer.length === pingConst.length) {
if (buffer.toString() === pingConst) {
return { method: pingConst, params: [], id: -1, time: Date.now() }
}
}
return rpcHandler.readRequest(buffer, binary)
},
data: () => data,
send: async (ctx: MeasureContext, msg, binary, _compression) => {
let smsg = rpcHandler.serialize(msg, binary)
ctx.measure('send-data', smsg.length)
if (ws.readyState !== WebSocket.OPEN || cs.isClosed) {
return
}
if (_compression) {
smsg = compress(smsg)
}
try {
ws.send(smsg)
} catch (err: any) {
console.error({ message: 'failed to send', err })
}
},
sendPong: () => {
if (ws.readyState !== WebSocket.OPEN || cs.isClosed) {
return
}
try {
ws.send(pongConst)
} catch (err: any) {
console.error({ message: 'failed to send', err })
}
}
}
return cs
}
async broadcastMessage (message: Uint8Array, origin?: any): Promise<void> {
console.log({ message: 'broadcast' })
const wss = this.ctx.getWebSockets().filter((ws) => ws.readyState === WebSocket.OPEN)
await Promise.all(
wss.map(async (ws) => {
await this.sendMessage(ws, message)
})
)
}
async sendMessage (ws: WebSocket, message: Uint8Array): Promise<void> {
try {
ws.send(message)
} catch (error) {
console.error({ message: 'Failed to send message:', error })
await this.handleClose(ws, 1011, 'error')
}
}
async handleClose (ws: WebSocket, code: number, reason?: string): Promise<void> {
try {
console.log({ message: 'Closing connection with code', code, reason })
ws.close(code, reason)
} catch (err) {
console.error({ message: 'Failed to close WebSocket:', err })
}
const session = this.sessions.get(ws)
if (session !== undefined) {
this.sessions.delete(ws)
console.log({ message: 'Cleaning up session for', account: session.payload.account })
await this.sessionManager.close(this.measureCtx, session.connectionSocket as ConnectionSocket, this.workspace)
}
}
private createDummyClientSocket (): ConnectionSocket {
const cs: ConnectionSocket = {
id: generateId(),
isClosed: false,
close: () => {
cs.isClosed = true
},
checkState: () => {
return !cs.isClosed
},
readRequest: (buffer: Buffer, binary: boolean) => {
return {} as any
},
data: () => {
return {}
},
isBackpressure: () => false,
backpressure: async (ctx) => {},
send: async (ctx: MeasureContext, msg, binary, compression) => {},
sendPong: () => {}
}
return cs
}
private async makeRpcSession (rawToken: string, cs: ConnectionSocket): Promise<Session> {
const token = decodeToken(rawToken ?? '')
const session = await this.sessionManager.addSession(
this.measureCtx,
cs,
token,
rawToken,
this.pipelineFactory,
generateId()
)
if ('error' in session) {
throw session.error
}
if ('upgrade' in session) {
throw new Error('Workspace is upgrading')
}
if (!('session' in session) || session.session === undefined) {
throw new Error('No session')
}
// By design, all fetches to this durable object will be for the same workspace
if (this.workspace === '') {
this.workspace = token.workspace
}
return session.session
}
private async getRpcPipeline (rawToken: string, cs: ConnectionSocket): Promise<Pipeline> {
const session = await this.makeRpcSession(rawToken, cs)
const pipeline =
session.workspace.pipeline instanceof Promise ? await session.workspace.pipeline : session.workspace.pipeline
const opContext = this.sessionManager.createOpContext(
this.measureCtx,
this.measureCtx,
pipeline,
undefined,
session,
cs
)
session.includeSessionContext(opContext)
return pipeline
}
async findAll (
rawToken: string,
workspaceId: string,
_class: Ref<Class<Doc>>,
query?: DocumentQuery<Doc>,
options?: FindOptions<Doc>
): Promise<any> {
let result
const cs = this.createDummyClientSocket()
try {
const pipeline = await this.getRpcPipeline(rawToken, cs)
result = await pipeline.findAll(this.measureCtx, _class, query ?? {}, options ?? {})
} catch (error: any) {
console.error(error)
result = { error: `${error}` }
} finally {
await this.sessionManager.close(this.measureCtx, cs, this.workspace)
}
return result
}
async tx (rawToken: string, workspaceId: string, tx: Tx): Promise<any> {
let result
const cs = this.createDummyClientSocket()
try {
const pipeline = await this.getRpcPipeline(rawToken, cs)
result = await pipeline.tx(this.measureCtx, [tx])
} catch (error: any) {
console.error(error)
result = { error: `${error}` }
} finally {
await this.sessionManager.close(this.measureCtx, cs, this.workspace)
}
return result
}
async getModel (rawToken: string): Promise<any> {
let result: Tx[] = []
const cs = this.createDummyClientSocket()
try {
const pipeline = await this.getRpcPipeline(rawToken, cs)
const ret = await pipeline.loadModel(this.measureCtx, 0)
if (Array.isArray(ret)) {
result = ret
} else {
result = ret.transactions
}
} catch (error: any) {
console.error(error)
return { error: `${error}` }
} finally {
await this.sessionManager.close(this.measureCtx, cs, this.workspace)
}
const encoder = new TextEncoder()
const buffer = encoder.encode(JSON.stringify(result))
const gzipAsync = promisify(gzip)
const compressed = await gzipAsync(buffer)
return compressed
}
async getAccount (rawToken: string): Promise<any> {
const cs = this.createDummyClientSocket()
try {
const session = await this.makeRpcSession(rawToken, cs)
return session.getRawAccount()
} catch (error: any) {
return { error: `${error}` }
} finally {
await this.sessionManager.close(this.measureCtx, cs, this.workspace)
}
}
}
-11
View File
@@ -1,11 +0,0 @@
{
"extends": "./node_modules/@hcengineering/platform-rig/profiles/default/tsconfig.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./lib",
"declarationDir": "./types",
"tsBuildInfoFile": ".build/build.tsbuildinfo",
"types": ["@cloudflare/workers-types", "jest"]
}
}
-34
View File
@@ -1,34 +0,0 @@
// Generated by Wrangler
// After adding bindings to `wrangler.toml`, regenerate this interface via `npm run cf-typegen`
interface Env {
TRANSACTOR: DurableObjectNamespace
SERVER_SECRET: string
HYPERDRIVE: Hyperdrive
ACCOUNTS_URL: string
DB_URL: string | undefined
STATS_URL: string | undefined
ENABLE_COMPRESSION: string | undefined
FULLTEXT_URL: string | undefined
DB_MODE: 'hyperdrive' | 'direct' | undefined
FRONT_URL: string
FILES_URL?: string
SES_URL?: string
SES_AUTH_TOKEN?: string
TELEGRAM_BOT_URL: string
AI_BOT_URL?: string
LAST_NAME_FIRST?: string
GREEN_URL?: string
USE_GREEN?: string
}
-128
View File
@@ -1,128 +0,0 @@
#:schema node_modules/wrangler/config-schema.json
name = "cloud-transactor"
main = "src/index.ts"
compatibility_date = "2024-11-11"
compatibility_flags = ["nodejs_compat"]
keep_vars = true
[observability.logs]
enabled = true
head_sampling_rate = 1 # optional. default = 1.
# Automatically place your workloads in an optimal location to minimize latency.
# If you are running back-end logic in a Worker, running it closer to your back-end infrastructure
# rather than the end user may result in better performance.
# Docs: https://developers.cloudflare.com/workers/configuration/smart-placement/#smart-placement
[placement]
mode = "smart"
# Variable bindings. These are arbitrary, plaintext strings (similar to environment variables)
# Docs:
# - https://developers.cloudflare.com/workers/wrangler/configuration/#environment-variables
# Note: Use secrets to store sensitive data.
# - https://developers.cloudflare.com/workers/configuration/secrets/
[vars]
# ACCOUNTS_URL = "http://127.0.0.1:3000"
# SERVER_SECRET = "secret"
DB_MODE='hyperdrive'
# FRONT_URL
ENABLE_COMPRESSION=true
# ACCOUNTS_URL
# FULLTEXT_URL
# STATS_URL
# PUSH_PUBLIC_KEY
# PUSH_PRIVATE_KEY
# SENTRY_DSN
# TELEGRAM_BOT_URL
# AI_BOT_URL
# LAST_NAME_FIRST
USE_GREEN='true'
# Bind the Workers AI model catalog. Run machine learning models, powered by serverless GPUs, on Cloudflares global network
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#workers-ai
# [ai]
# binding = "AI"
# Bind an Analytics Engine dataset. Use Analytics Engine to write analytics within your Pages Function.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#analytics-engine-datasets
# [[analytics_engine_datasets]]
# binding = "MY_DATASET"
# Bind a headless browser instance running on Cloudflare's global network.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#browser-rendering
# [browser]
# binding = "MY_BROWSER"
# Bind a D1 database. D1 is Cloudflares native serverless SQL database.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#d1-databases
# [[d1_databases]]
# binding = "MY_DB"
# database_name = "my-database"
# database_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
# Bind a dispatch namespace. Use Workers for Platforms to deploy serverless functions programmatically on behalf of your customers.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#dispatch-namespace-bindings-workers-for-platforms
# [[dispatch_namespaces]]
# binding = "MY_DISPATCHER"
# namespace = "my-namespace"
# Bind a Durable Object. Durable objects are a scale-to-zero compute primitive based on the actor model.
# Durable Objects can live for as long as needed. Use these when you need a long-running "server", such as in realtime apps.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#durable-objects
[[durable_objects.bindings]]
name = "TRANSACTOR"
class_name = "Transactor"
# Durable Object migrations.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#migrations
[[migrations]]
tag = "v1"
new_classes = ["Transactor"]
# Bind a Hyperdrive configuration. Use to accelerate access to your existing databases from Cloudflare Workers.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#hyperdrive
[[hyperdrive]]
binding = "HYPERDRIVE"
id = "1f713bb2ec1b464cb663983feca4c89e"
localConnectionString = "postgresql://postgree:example@huly.local:26257/defaultdb?sslmode=disable"
# Bind a KV Namespace. Use KV as persistent storage for small key-value pairs.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#kv-namespaces
#[[kv_namespaces]]
#binding = "transactor_model"
#id = ""
# Bind an mTLS certificate. Use to present a client certificate when communicating with another service.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#mtls-certificates
# [[mtls_certificates]]
# binding = "MY_CERTIFICATE"
# certificate_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
# Bind a Queue producer. Use this binding to schedule an arbitrary task that may be processed later by a Queue consumer.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#queues
# [[queues.producers]]
# binding = "MY_QUEUE"
# queue = "my-queue"
# Bind a Queue consumer. Queue Consumers can retrieve tasks scheduled by Producers to act on them.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#queues
# [[queues.consumers]]
# queue = "my-queue"
# Bind an R2 Bucket. Use R2 to store arbitrarily large blobs of data, such as files.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#r2-buckets
# [[r2_buckets]]
# binding = "MY_BUCKET"
# bucket_name = "my-bucket"
# Bind another Worker service. Use this binding to call another Worker without network overhead.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#service-bindings
# [[services]]
# binding = "MY_SERVICE"
# service = "my-service"
# Bind a Vectorize index. Use to store and query vector embeddings for semantic search, classification and other vector search use-cases.
# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/#vectorize-indexes
# [[vectorize]]
# binding = "MY_INDEX"
# index_name = "my-index"