diff --git a/services/datalake/pod-datalake/src/datalake/db.ts b/services/datalake/pod-datalake/src/datalake/db.ts index 2ad1c8758b..7e94d60a63 100644 --- a/services/datalake/pod-datalake/src/datalake/db.ts +++ b/services/datalake/pod-datalake/src/datalake/db.ts @@ -572,7 +572,7 @@ export function escape (value: any): string { } function getMigrations (): [string, string][] { - return [migrationV1()] + return [migrationV1(), migrationV2()] } function migrationV1 (): [string, string] { @@ -611,3 +611,22 @@ function migrationV1 (): [string, string] { ` return ['init_tables_01', sql] } + +function migrationV2 (): [string, string] { + const sql = ` + ALTER TABLE blob.meta DROP CONSTRAINT IF EXISTS fk_blob; + + UPDATE blob.blob + SET workspace = w.uuid + FROM global_account.workspace w + WHERE workspace = w.data_id; + + UPDATE blob.meta + SET workspace = w.uuid + FROM global_account.workspace w + WHERE workspace = w.data_id; + + ALTER TABLE blob.meta ADD CONSTRAINT fk_blob_meta FOREIGN KEY (workspace, name) REFERENCES blob.blob (workspace, name); + ` + return ['migrate_workspaces_02', sql] +} diff --git a/services/datalake/pod-datalake/src/server.ts b/services/datalake/pod-datalake/src/server.ts index 69a0fac15b..1c0011ad8c 100644 --- a/services/datalake/pod-datalake/src/server.ts +++ b/services/datalake/pod-datalake/src/server.ts @@ -14,6 +14,7 @@ // import { Analytics } from '@hcengineering/analytics' +import { getClient } from '@hcengineering/account-client' import { MeasureContext, Tx, metricsAggregate } from '@hcengineering/core' import { PlatformQueue, QueueTopic, getCPUInfo, getMemoryInfo } from '@hcengineering/server-core' import { decodeToken, TokenError } from '@hcengineering/server-token' @@ -115,6 +116,8 @@ export async function createServer ( queue: PlatformQueue, config: Config ): Promise<{ app: Express, close: () => void }> { + await ensureAccountReady(ctx, config) + const buckets: Array<{ location: Location, bucket: S3Bucket }> = [] for (const bucket of config.Buckets) { const location = bucket.location as Location @@ -348,3 +351,13 @@ export function listen (e: Express, port: number, host?: string): Server { return server } + +async function ensureAccountReady (ctx: MeasureContext, config: Config): Promise { + const client = getClient(config.AccountsUrl) + try { + await client.getRegionInfo() + } catch (err: any) { + ctx.error('Accounts service not ready', { err }) + throw new Error('Accounts service not ready') + } +} diff --git a/tests/.env b/tests/.env index 1672ce40ac..7c1f474386 100644 --- a/tests/.env +++ b/tests/.env @@ -1,4 +1,5 @@ -STORAGE_CONFIG="datalake|http://datalake:4031" +STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin" +DATALAKE_STORAGE_CONFIG="datalake|http://datalake:4031" BACKUP_STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin" BACKUP_BUCKET_NAME=dev-backups diff --git a/tests/docker-compose.override.yaml b/tests/docker-compose.override.yaml index ea00a636cb..8f2fa5f539 100644 --- a/tests/docker-compose.override.yaml +++ b/tests/docker-compose.override.yaml @@ -2,13 +2,49 @@ services: account: environment: - DB_URL=${DB_PG_URL} + - STORAGE_CONFIG=${DATALAKE_STORAGE_CONFIG} - PROCEED_V7_MONGO=false transactor: environment: - DB_URL=${DB_PG_URL} + - STORAGE_CONFIG=${DATALAKE_STORAGE_CONFIG} workspace: environment: - DB_URL=${DB_PG_URL} + - STORAGE_CONFIG=${DATALAKE_STORAGE_CONFIG} fulltext: environment: - DB_URL=${DB_PG_URL} + - STORAGE_CONFIG=${DATALAKE_STORAGE_CONFIG} + front: + environment: + - STORAGE_CONFIG=${DATALAKE_STORAGE_CONFIG} + - FILES_URL=http://localhost:4031/blob/:workspace/:blobId/:filename + - UPLOAD_URL=http://localhost:4031/upload/form-data/:workspace + - PREVIEW_CONFIG=http://localhost:4031/image/fit=cover,width=:width,height=:height,dpr=:dpr/:workspace/:blobId + collaborator: + environment: + - STORAGE_CONFIG=${DATALAKE_STORAGE_CONFIG} + datalake: + image: hardcoreeng/datalake + depends_on: + minio: + condition: service_healthy + cockroach: + condition: service_started + stats: + condition: service_started + account: + condition: service_started + ports: + - 4031:4031 + environment: + - PORT=4031 + - SECRET=secret + - ACCOUNTS_URL=http://account:3003 + - STATS_URL=http://stats:4901 + - STREAM_URL=http://localhost:1081/recording + - DB_URL=${DB_PG_URL} + - BUCKETS=blobs,eu|http://minio:9000?accessKey=minioadmin&secretKey=minioadmin + - QUEUE_CONFIG=${QUEUE_CONFIG} + restart: unless-stopped \ No newline at end of file diff --git a/tests/docker-compose.yaml b/tests/docker-compose.yaml index e71ea44f0f..cb409daf1e 100644 --- a/tests/docker-compose.yaml +++ b/tests/docker-compose.yaml @@ -174,9 +174,7 @@ services: - SERVER_SECRET=secret - ACCOUNTS_URL_INTERNAL=http://account:3003 - ACCOUNTS_URL=http://localhost:3003 - - FILES_URL=http://localhost:4031/blob/:workspace/:blobId/:filename - - UPLOAD_URL=http://localhost:4031/upload/form-data/:workspace - - PREVIEW_CONFIG=http://localhost:4031/image/fit=cover,width=:width,height=:height,dpr=:dpr/:workspace/:blobId + - UPLOAD_URL=/files - ELASTIC_URL=http://elastic:9200 - GMAIL_URL=http://localhost:8088 - CALENDAR_URL=http://localhost:8095 @@ -278,28 +276,6 @@ services: - REKONI_URL=http://rekoni:4007 - ACCOUNTS_URL=http://account:3003 - STATS_URL=http://stats:4901 - datalake: - image: hardcoreeng/datalake - depends_on: - minio: - condition: service_healthy - cockroach: - condition: service_started - stats: - condition: service_started - account: - condition: service_started - ports: - - 4031:4031 - environment: - - PORT=4031 - - SECRET=secret - - ACCOUNTS_URL=http://account:3003 - - STATS_URL=http://stats:4901 - - STREAM_URL=http://localhost:1081/recording - - DB_URL=${DB_PG_URL} - - BUCKETS=blobs,eu|http://minio:9000?accessKey=minioadmin&secretKey=minioadmin - restart: unless-stopped stats: image: hardcoreeng/stats ports: diff --git a/tests/tool.sh b/tests/tool.sh index e69ffec165..54b9b383e3 100755 --- a/tests/tool.sh +++ b/tests/tool.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash export MODEL_VERSION=$(node ../common/scripts/show_version.js) -export STORAGE_CONFIG="datalake|http://localhost:4031" +export STORAGE_CONFIG="minio|localhost:9002?accessKey=minioadmin&secretKey=minioadmin" export ACCOUNTS_URL=http://localhost:3003 export TRANSACTOR_URL=ws://localhost:3334 export ACCOUNT_DB_URL=mongodb://localhost:27018 diff --git a/ws-tests/.env b/ws-tests/.env index 57f3cbfcd8..865d521e8a 100644 --- a/ws-tests/.env +++ b/ws-tests/.env @@ -1,7 +1,7 @@ -STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin" +STORAGE_CONFIG="datalake|http://huly.local:4031" BACKUP_STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin" BACKUP_BUCKET_NAME=ws-dev-backups DB_URL=mongodb://huly.local:27018 MONGO_URL=mongodb://huly.local:27018 -DB_EU_URL=postgresql://root@huly.local:26258/defaultdb?sslmode=disable +DB_EU_URL=postgresql://root@cockroach:26257/defaultdb?sslmode=disable QUEUE_CONFIG=redpanda:9093 diff --git a/ws-tests/docker-compose.yaml b/ws-tests/docker-compose.yaml index 168258cfe0..9291ec9575 100644 --- a/ws-tests/docker-compose.yaml +++ b/ws-tests/docker-compose.yaml @@ -5,11 +5,9 @@ services: extra_hosts: - 'huly.local:host-gateway' environment: - - STREAM_ENDPOINT_URL=s3://huly.local:9000 + - STREAM_ENDPOINT_URL=datalake://huly.local:4031 - STREAM_INSECURE=true - STREAM_SERVER_SECRET=secret - - AWS_ACCESS_KEY_ID=minioadmin - - AWS_SECRET_ACCESS_KEY=minioadmin ports: - 1081:1080 restart: unless-stopped @@ -95,11 +93,38 @@ services: extra_hosts: - 'huly.local:host-gateway' expose: - - 9002 - - 9003 + - 9000 + - 9001 ports: - 9002:9000 - 9003:9001 + healthcheck: + test: ['CMD', 'mc', 'ready', 'local'] + interval: 5s + retries: 10 + datalake: + image: hardcoreeng/datalake + depends_on: + minio: + condition: service_healthy + cockroach: + condition: service_started + stats: + condition: service_started + account: + condition: service_started + ports: + - 4031:4031 + environment: + - PORT=4031 + - SECRET=secret + - ACCOUNTS_URL=http://account:3003 + - STATS_URL=http://stats:4901 + - STREAM_URL=http://localhost:1081/recording + - DB_URL=${DB_EU_URL} + - BUCKETS=blobs,eu|http://minio:9000?accessKey=minioadmin&secretKey=minioadmin + - QUEUE_CONFIG=${QUEUE_CONFIG} + restart: unless-stopped elastic: image: 'elasticsearch:7.14.2' expose: