From f03ffa7aa4bc238e35847c3d8cce79614e9b18d0 Mon Sep 17 00:00:00 2001 From: Alexey Zinoviev Date: Sat, 15 Mar 2025 17:50:55 +0400 Subject: [PATCH] QFIX: account migration mongo to cr (#8237) --- dev/tool/src/db.ts | 6 +- server/account/src/__tests__/postgres.test.ts | 6 +- server/account/src/collections/postgres.ts | 64 +++++++++---------- 3 files changed, 42 insertions(+), 34 deletions(-) diff --git a/dev/tool/src/db.ts b/dev/tool/src/db.ts index 5f9b9dfcfc..8eba0fb95a 100644 --- a/dev/tool/src/db.ts +++ b/dev/tool/src/db.ts @@ -198,7 +198,7 @@ export async function moveAccountDbFromMongoToPG ( } if (person.lastName == null) { - person.lastName = 'n/a' + person.lastName = '' } await pgDb.person.insertOne(person) @@ -254,6 +254,7 @@ export async function moveAccountDbFromMongoToPG ( const exists = await pgDb.socialId.findOne({ key: socialId.key }) if (exists == null) { delete (socialId as any).key + delete (socialId as any).id await pgDb.socialId.insertOne(socialId) socialIdsCount++ @@ -281,6 +282,9 @@ export async function moveAccountDbFromMongoToPG ( time: accountEvent.time }) if (exists == null) { + const account = await pgDb.account.findOne({ uuid: accountEvent.accountUuid }) + if (account == null) continue // Not a big deal if we don't move the event for non-existing account + await pgDb.accountEvent.insertOne(accountEvent) eventsCount++ if (eventsCount % 100 === 0) { diff --git a/server/account/src/__tests__/postgres.test.ts b/server/account/src/__tests__/postgres.test.ts index 99aa2861bf..cbfe8131a4 100644 --- a/server/account/src/__tests__/postgres.test.ts +++ b/server/account/src/__tests__/postgres.test.ts @@ -383,7 +383,11 @@ describe('PostgresAccountDB', () => { expect(mockClient.begin).toHaveBeenCalled() expect(mockClient).toHaveBeenCalledWith( - ['INSERT INTO _account_applied_migrations (identifier, ddl) VALUES (', ', ', ') ON CONFLICT DO NOTHING'], + 'global_account' // First call with schema name + ) + expect(mockClient).toHaveBeenCalledWith( + ['INSERT INTO ', '._account_applied_migrations (identifier, ddl) VALUES (', ', ', ') ON CONFLICT DO NOTHING'], + expect.anything(), 'test_migration', 'CREATE TABLE test' ) diff --git a/server/account/src/collections/postgres.ts b/server/account/src/collections/postgres.ts index 4dd210bb67..0ae2cf8eaf 100644 --- a/server/account/src/collections/postgres.ts +++ b/server/account/src/collections/postgres.ts @@ -399,7 +399,7 @@ export class PostgresAccountDB implements AccountDB { async migrate (name: string, ddl: string): Promise { await this.client.begin(async (client) => { const res = - await client`INSERT INTO _account_applied_migrations (identifier, ddl) VALUES (${name}, ${ddl}) ON CONFLICT DO NOTHING` + await client`INSERT INTO ${this.client(this.ns)}._account_applied_migrations (identifier, ddl) VALUES (${name}, ${ddl}) ON CONFLICT DO NOTHING` if (res.count === 1) { console.log(`Applying migration: ${name}`) @@ -413,7 +413,9 @@ export class PostgresAccountDB implements AccountDB { async _init (): Promise { await this.client.unsafe( ` - CREATE TABLE IF NOT EXISTS _account_applied_migrations ( + CREATE SCHEMA IF NOT EXISTS ${this.ns}; + + CREATE TABLE IF NOT EXISTS ${this.ns}._account_applied_migrations ( identifier VARCHAR(255) NOT NULL PRIMARY KEY , ddl TEXT NOT NULL , applied_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() @@ -629,8 +631,6 @@ export class PostgresAccountDB implements AccountDB { return [ 'account_db_v1_global_init', ` - CREATE SCHEMA IF NOT EXISTS global_account; - /* ======= FUNCTIONS ======= */ CREATE OR REPLACE FUNCTION current_epoch_ms() @@ -639,12 +639,12 @@ export class PostgresAccountDB implements AccountDB { $$ LANGUAGE SQL; /* ======= T Y P E S ======= */ - CREATE TYPE IF NOT EXISTS global_account.social_id_type AS ENUM ('email', 'github', 'google', 'phone', 'oidc', 'huly', 'telegram'); - CREATE TYPE IF NOT EXISTS global_account.location AS ENUM ('kv', 'weur', 'eeur', 'wnam', 'enam', 'apac'); - CREATE TYPE IF NOT EXISTS global_account.workspace_role AS ENUM ('OWNER', 'MAINTAINER', 'USER', 'GUEST', 'DOCGUEST'); + CREATE TYPE IF NOT EXISTS ${this.ns}.social_id_type AS ENUM ('email', 'github', 'google', 'phone', 'oidc', 'huly', 'telegram'); + CREATE TYPE IF NOT EXISTS ${this.ns}.location AS ENUM ('kv', 'weur', 'eeur', 'wnam', 'enam', 'apac'); + CREATE TYPE IF NOT EXISTS ${this.ns}.workspace_role AS ENUM ('OWNER', 'MAINTAINER', 'USER', 'GUEST', 'DOCGUEST'); /* ======= P E R S O N ======= */ - CREATE TABLE IF NOT EXISTS global_account.person ( + CREATE TABLE IF NOT EXISTS ${this.ns}.person ( uuid UUID NOT NULL DEFAULT gen_random_uuid(), first_name STRING NOT NULL, last_name STRING NOT NULL, @@ -654,34 +654,34 @@ export class PostgresAccountDB implements AccountDB { ); /* ======= A C C O U N T ======= */ - CREATE TABLE IF NOT EXISTS global_account.account ( + CREATE TABLE IF NOT EXISTS ${this.ns}.account ( uuid UUID NOT NULL, timezone STRING, locale STRING, CONSTRAINT account_pk PRIMARY KEY (uuid), - CONSTRAINT account_person_fk FOREIGN KEY (uuid) REFERENCES global_account.person(uuid) + CONSTRAINT account_person_fk FOREIGN KEY (uuid) REFERENCES ${this.ns}.person(uuid) ); - CREATE TABLE IF NOT EXISTS global_account.account_passwords ( + CREATE TABLE IF NOT EXISTS ${this.ns}.account_passwords ( account_uuid UUID NOT NULL, hash BYTES NOT NULL, salt BYTES NOT NULL, CONSTRAINT account_auth_pk PRIMARY KEY (account_uuid), - CONSTRAINT account_passwords_account_fk FOREIGN KEY (account_uuid) REFERENCES global_account.account(uuid) + CONSTRAINT account_passwords_account_fk FOREIGN KEY (account_uuid) REFERENCES ${this.ns}.account(uuid) ); - CREATE TABLE IF NOT EXISTS global_account.account_events ( + CREATE TABLE IF NOT EXISTS ${this.ns}.account_events ( account_uuid UUID NOT NULL, event_type STRING NOT NULL, time BIGINT NOT NULL DEFAULT current_epoch_ms(), data JSONB, CONSTRAINT account_events_pk PRIMARY KEY (account_uuid, event_type, time), - CONSTRAINT account_events_account_fk FOREIGN KEY (account_uuid) REFERENCES global_account.account(uuid) + CONSTRAINT account_events_account_fk FOREIGN KEY (account_uuid) REFERENCES ${this.ns}.account(uuid) ); /* ======= S O C I A L I D S ======= */ - CREATE TABLE IF NOT EXISTS global_account.social_id ( - type global_account.social_id_type NOT NULL, + CREATE TABLE IF NOT EXISTS ${this.ns}.social_id ( + type ${this.ns}.social_id_type NOT NULL, value STRING NOT NULL, key STRING AS (CONCAT(type::STRING, ':', value)) STORED, person_uuid UUID NOT NULL, @@ -690,28 +690,28 @@ export class PostgresAccountDB implements AccountDB { CONSTRAINT social_id_pk PRIMARY KEY (type, value), CONSTRAINT social_id_key_unique UNIQUE (key), INDEX social_id_account_idx (person_uuid), - CONSTRAINT social_id_person_fk FOREIGN KEY (person_uuid) REFERENCES global_account.person(uuid) + CONSTRAINT social_id_person_fk FOREIGN KEY (person_uuid) REFERENCES ${this.ns}.person(uuid) ); /* ======= W O R K S P A C E ======= */ - CREATE TABLE IF NOT EXISTS global_account.workspace ( + CREATE TABLE IF NOT EXISTS ${this.ns}.workspace ( uuid UUID NOT NULL DEFAULT gen_random_uuid(), name STRING NOT NULL, url STRING NOT NULL, data_id STRING, branding STRING, - location global_account.location, + location ${this.ns}.location, region STRING, created_by UUID, -- account uuid created_on BIGINT NOT NULL DEFAULT current_epoch_ms(), billing_account UUID, CONSTRAINT workspace_pk PRIMARY KEY (uuid), CONSTRAINT workspace_url_unique UNIQUE (url), - CONSTRAINT workspace_created_by_fk FOREIGN KEY (created_by) REFERENCES global_account.account(uuid), - CONSTRAINT workspace_billing_account_fk FOREIGN KEY (billing_account) REFERENCES global_account.account(uuid) + CONSTRAINT workspace_created_by_fk FOREIGN KEY (created_by) REFERENCES ${this.ns}.account(uuid), + CONSTRAINT workspace_billing_account_fk FOREIGN KEY (billing_account) REFERENCES ${this.ns}.account(uuid) ); - CREATE TABLE IF NOT EXISTS global_account.workspace_status ( + CREATE TABLE IF NOT EXISTS ${this.ns}.workspace_status ( workspace_uuid UUID NOT NULL, mode STRING, processing_progress INT2 DEFAULT 0, @@ -725,16 +725,16 @@ export class PostgresAccountDB implements AccountDB { processing_message STRING, backup_info JSONB, CONSTRAINT workspace_status_pk PRIMARY KEY (workspace_uuid), - CONSTRAINT workspace_status_workspace_fk FOREIGN KEY (workspace_uuid) REFERENCES global_account.workspace(uuid) + CONSTRAINT workspace_status_workspace_fk FOREIGN KEY (workspace_uuid) REFERENCES ${this.ns}.workspace(uuid) ); - CREATE TABLE IF NOT EXISTS global_account.workspace_members ( + CREATE TABLE IF NOT EXISTS ${this.ns}.workspace_members ( workspace_uuid UUID NOT NULL, account_uuid UUID NOT NULL, - role global_account.workspace_role NOT NULL DEFAULT 'USER', + role ${this.ns}.workspace_role NOT NULL DEFAULT 'USER', CONSTRAINT workspace_assignment_pk PRIMARY KEY (workspace_uuid, account_uuid), - CONSTRAINT members_workspace_fk FOREIGN KEY (workspace_uuid) REFERENCES global_account.workspace(uuid), - CONSTRAINT members_account_fk FOREIGN KEY (account_uuid) REFERENCES global_account.account(uuid) + CONSTRAINT members_workspace_fk FOREIGN KEY (workspace_uuid) REFERENCES ${this.ns}.workspace(uuid), + CONSTRAINT members_account_fk FOREIGN KEY (account_uuid) REFERENCES ${this.ns}.account(uuid) ); /* ========================================================================================== */ @@ -742,28 +742,28 @@ export class PostgresAccountDB implements AccountDB { /* ===================== */ /* ======= O T P ======= */ - CREATE TABLE IF NOT EXISTS global_account.otp ( + CREATE TABLE IF NOT EXISTS ${this.ns}.otp ( social_id STRING NOT NULL, code STRING NOT NULL, expires_on BIGINT NOT NULL, created_on BIGINT NOT NULL DEFAULT current_epoch_ms(), CONSTRAINT otp_pk PRIMARY KEY (social_id, code), - CONSTRAINT otp_social_id_fk FOREIGN KEY (social_id) REFERENCES global_account.social_id(key) + CONSTRAINT otp_social_id_fk FOREIGN KEY (social_id) REFERENCES ${this.ns}.social_id(key) ); /* ======= I N V I T E ======= */ - CREATE TABLE IF NOT EXISTS global_account.invite ( + CREATE TABLE IF NOT EXISTS ${this.ns}.invite ( id INT8 NOT NULL DEFAULT unique_rowid(), workspace_uuid UUID NOT NULL, expires_on BIGINT NOT NULL, email_pattern STRING, remaining_uses INT2, - role global_account.workspace_role NOT NULL DEFAULT 'USER', + role ${this.ns}.workspace_role NOT NULL DEFAULT 'USER', migrated_from STRING, CONSTRAINT invite_pk PRIMARY KEY (id), INDEX workspace_invite_idx (workspace_uuid), INDEX migrated_from_idx (migrated_from), - CONSTRAINT invite_workspace_fk FOREIGN KEY (workspace_uuid) REFERENCES global_account.workspace(uuid) + CONSTRAINT invite_workspace_fk FOREIGN KEY (workspace_uuid) REFERENCES ${this.ns}.workspace(uuid) ); ` ]