WEBDIENSTE/huly-platform
1
0
Fork 0
mirror of https://github.com/hcengineering/platform.git synced 2026-08-17 18:05:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

Postgress support (#10331)

Browse Source 
* Pure PostgreSQL

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

* Fix formatting

* Optimize a tests

+ Optimize a stream docker build

* Fix migrations

Signed-off-by: Artem Savchenko <armisav@gmail.com>

* Fix datalake image

Signed-off-by: Artem Savchenko <armisav@gmail.com>

* Fix formatting

Signed-off-by: Artem Savchenko <armisav@gmail.com>

---------

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
Signed-off-by: Artem Savchenko <armisav@gmail.com>
Co-authored-by: Andrey Sobolev <haiodo@gmail.com>
This commit is contained in:
Artyom Savchenko
2025-12-26 12:11:38 +07:00
committed by GitHub
co-authored by Andrey Sobolev
parent d69af65ea1
commit c8aec52f13
19 changed files with 950 additions and 276 deletions
Download Patch File Download Diff File Expand all files Collapse all files
foundations/stream/Dockerfile
+2 -2
View File
@@ -21,8 +21,6 @@ ENV GOBIN=/bin
ENV PATH=$PATH:$GOBIN
ARG BUILDARCH=amd64
COPY . ./
FROM base AS linter
RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.5.0
@@ -31,6 +29,8 @@ RUN golangci-lint run --verbose
FROM base AS builder
COPY . ./
RUN set -xe && GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /go/bin/stream ./cmd/stream
FROM alpine
server/account-service/src/index.ts
+16 -6
View File
@@ -18,7 +18,7 @@ import accountRu from '@hcengineering/account/lang/ru.json'
import { Analytics } from '@hcengineering/analytics'
import { registerProviders } from '@hcengineering/auth-providers'
import { metricsAggregate, type Branding, type BrandingMap, type MeasureContext } from '@hcengineering/core'
import platform, { Severity, Status, addStringsLoader, setMetadata } from '@hcengineering/platform'
import platform, { Severity, Status, addStringsLoader, setMetadata, unknownStatus } from '@hcengineering/platform'
import serverToken, { decodeToken, decodeTokenVerbose, generateToken } from '@hcengineering/server-token'
import cors from '@koa/cors'
import type Cookies from 'cookies'
@@ -424,15 +424,25 @@ export function serveAccount (measureCtx: MeasureContext, brandings: BrandingMap
error: new Status(Severity.ERROR, platform.status.UnknownMethod, { method: request.method })
}
ctx.body = JSON.stringify(response)
ctx.res.writeHead(400, KEEP_ALIVE_HEADERS)
ctx.res.end(JSON.stringify(response))
return
}
const result = await method(_ctx, db, branding, request, token, meta)
try {
const result = await method(_ctx, db, branding, request, token, meta)
const body = JSON.stringify(result)
ctx.res.writeHead(200, KEEP_ALIVE_HEADERS)
ctx.res.end(body)
const body = JSON.stringify(result)
ctx.res.writeHead(200, KEEP_ALIVE_HEADERS)
ctx.res.end(body)
} catch (err: any) {
const response = {
id: request.id,
error: unknownStatus(err.message)
}
ctx.res.writeHead(400, KEEP_ALIVE_HEADERS)
ctx.res.end(JSON.stringify(response))
}
},
{ method: request.method }
)
server/account/src/__tests__/postgres-real.test.ts
+278
View File
@@ -0,0 +1,278 @@
/**
* A set of tests against a real PostgreSQL database, for both CorockachDB and pure.
*/
import { generateUuid, SocialIdType, type AccountUuid, type PersonId } from '@hcengineering/core'
import { getDBClient, shutdownPostgres, type PostgresClientReference } from '@hcengineering/postgres'
import { PostgresAccountDB } from '../collections/postgres/postgres'
import { type SocialId } from '../types'
import { createAccount, getDbFlavor, normalizeValue } from '../utils'
jest.setTimeout(90000)
describe('real-account', () => {
// It should create a DB and test on it for every execution, and drop it after it.
//
// Use environment variable or default to localhost CockroachDB
const cockroachDB: string = process.env.DB_URL ?? 'postgresql://root@localhost:26258/defaultdb?sslmode=disable'
const postgresDB: string = process.env.POSTGRES_URL ?? 'postgresql://postgres:postgres@localhost:5433/postgres'
let crDbUri = cockroachDB
let pgDbUri = postgresDB
// Administrative client for creating/dropping test databases
// This connects to 'defaultdb' and is used ONLY for DB admin operations
let adminClientCRRef: PostgresClientReference
let adminClientPGRef: PostgresClientReference
let dbUuid: string
let crClient: PostgresClientReference
let pgClient: PostgresClientReference
let crAccount: PostgresAccountDB
let pgAccount: PostgresAccountDB
const users = [
{
name: 'user1',
uuid: generateUuid() as AccountUuid,
email: 'user1@example.com',
firstName: 'Jon',
lastName: 'Doe'
},
{
name: 'user2',
uuid: generateUuid() as AccountUuid,
email: 'user2@example.com',
firstName: 'Pavel',
lastName: 'Siaro'
}
]
async function addSocialId (
account: PostgresAccountDB,
user: (typeof users)[0],
type: SocialIdType,
value: string
): Promise<PersonId> {
const normalizedValue = normalizeValue(value)
const newSocialId = {
type,
value: normalizedValue,
personUuid: user.uuid
}
return await account.socialId.insertOne(newSocialId)
}
async function prepareAccounts (account: PostgresAccountDB): Promise<void> {
for (const user of users) {
const ex = await account.account.findOne({ uuid: user.uuid })
if (ex == null) {
await account.person.insertOne({ uuid: user.uuid, firstName: user.firstName, lastName: user.lastName })
await createAccount(account, user.uuid, true)
await addSocialId(account, user, SocialIdType.EMAIL, user.email)
}
}
}
beforeAll(() => {
// Get admin client for database creation/deletion
// This client stays connected to 'defaultdb' for admin operations only
adminClientCRRef = getDBClient(cockroachDB)
adminClientPGRef = getDBClient(postgresDB)
})
afterAll(async () => {
adminClientCRRef.close()
adminClientPGRef.close()
await shutdownPostgres()
})
beforeEach(async () => {
// Create a unique database for each test to ensure isolation
dbUuid = 'accountdb' + Date.now().toString()
crDbUri = cockroachDB.replace('/defaultdb', '/' + dbUuid)
const c = postgresDB.split('/')
c[c.length - 1] = dbUuid
pgDbUri = c.join('/')
try {
// Use admin client to create the test database
await Promise.all([initCockroachDB(adminClientCRRef, dbUuid), initPostgreSQL(adminClientPGRef, dbUuid)])
} catch (err) {
console.error('Failed to create test database:', err)
throw err
}
crClient = getDBClient(crDbUri)
const crPGClient = await crClient.getClient()
pgClient = getDBClient(pgDbUri)
const pgPGClient = await pgClient.getClient()
// Initial DB's
crAccount = new PostgresAccountDB(crPGClient, dbUuid)
pgAccount = new PostgresAccountDB(pgPGClient, dbUuid, await getDbFlavor(pgPGClient))
await Promise.all([migrateCockroachDB(crAccount, crDbUri), migratePostgreSQL(pgAccount, pgDbUri)])
await Promise.all([prepareAccounts(pgAccount), prepareAccounts(crAccount)])
})
afterEach(async () => {
try {
pgClient.close()
crClient.close()
// Use admin client to drop the test database
const adminClient = await adminClientCRRef.getClient()
await adminClient`DROP DATABASE IF EXISTS ${adminClient(dbUuid)} CASCADE`
const adminClientPG = await adminClientPGRef.getClient()
await adminClientPG`DROP DATABASE IF EXISTS ${adminClient(dbUuid)}`
} catch (err) {
console.error('Cleanup error:', err)
}
})
it('Check accounts', async () => {
const user1 = await crAccount.account.findOne({ uuid: users[0].uuid })
expect(user1).not.toBeNull()
expect(user1).toBeDefined()
const user1PG = await pgAccount.account.findOne({ uuid: users[0].uuid })
expect(user1).not.toBeNull()
expect(user1PG).toBeDefined()
})
it('Check social ids', async () => {
const user1 = await crAccount.account.findOne({ uuid: users[0].uuid })
expect(user1).not.toBeNull()
expect(user1).toBeDefined()
const socialIds = await crAccount.socialId.find({ personUuid: user1?.uuid })
expect(socialIds).not.toBeNull()
expect(socialIds).toBeDefined()
expect(socialIds.length).toEqual(2)
const user1PG = await pgAccount.account.findOne({ uuid: users[0].uuid })
expect(user1).not.toBeNull()
expect(user1PG).toBeDefined()
const socialIdsPG = await pgAccount.socialId.find({ personUuid: user1PG?.uuid })
expect(socialIdsPG).not.toBeNull()
expect(socialIdsPG).toBeDefined()
expect(socialIdsPG.length).toEqual(2)
const em = socialIdsPG.find((it) => it.type === SocialIdType.EMAIL) as SocialId
expect(em).toBeDefined()
expect(em.key).toEqual('email:user1@example.com')
})
it('List accounts', async () => {
const users = await crAccount.listAccounts()
expect(users.length).toBe(2)
const usersPG = await pgAccount.listAccounts()
expect(usersPG.length).toBe(2)
})
it('check invites', async () => {
const wsUuid = await crAccount.createWorkspace(
{
url: 'test-ws',
name: 'test-ws',
allowGuestSignUp: true,
allowReadOnlyGuest: true
},
{
isDisabled: false,
mode: 'active',
versionMajor: 0,
versionMinor: 7,
versionPatch: 0
}
)
const inviteLink = await crAccount.invite.insertOne({
workspaceUuid: wsUuid,
expiresOn: new Date(Date.now() + 1000 * 60 * 60 * 24 * 30).getTime()
})
expect(inviteLink).toBeDefined()
const wsUuidPG = await pgAccount.createWorkspace(
{
url: 'test-ws',
name: 'test-ws',
allowGuestSignUp: true,
allowReadOnlyGuest: true
},
{
isDisabled: false,
mode: 'active',
versionMajor: 0,
versionMinor: 7,
versionPatch: 0
}
)
const inviteLinkPG = await pgAccount.invite.insertOne({
workspaceUuid: wsUuidPG,
expiresOn: new Date(Date.now() + 1000 * 60 * 60 * 24 * 30).getTime()
})
expect(inviteLinkPG).toBeDefined()
})
})
async function migratePostgreSQL (pgAccount: PostgresAccountDB, pgDbUri: string): Promise<void> {
let error = false
do {
try {
await pgAccount.init()
error = false
} catch (e) {
console.error('Error while initializing postgres account db', e, pgDbUri)
error = true
await new Promise((resolve) => setTimeout(resolve, 1000))
}
} while (error)
}
async function migrateCockroachDB (crAccount: PostgresAccountDB, crDbUri: string): Promise<void> {
let error: boolean = false
do {
try {
await crAccount.init()
error = false
} catch (e) {
console.error('Error while initializing postgres account db', e, crDbUri)
error = true
await new Promise((resolve) => setTimeout(resolve, 1000))
}
} while (error)
}
async function initPostgreSQL (adminClientPGRef: PostgresClientReference, dbUuid: string): Promise<void> {
const adminClientPg = await adminClientPGRef.getClient()
// Clean up any leftover test databases with prefix 'accountdb' for Postgres
const existingPgs = await adminClientPg`SELECT datname FROM pg_database WHERE datname LIKE 'accountdb%'`
for (const row of existingPgs) {
try {
await adminClientPg`DROP DATABASE IF EXISTS ${adminClientPg(row.datname)}`
} catch (err: any) {
// Ignore, Postgress says database is being used by other users
}
}
await adminClientPg`CREATE DATABASE ${adminClientPg(dbUuid)}`
}
async function initCockroachDB (adminClientCRRef: PostgresClientReference, dbUuid: string): Promise<void> {
const adminClient = await adminClientCRRef.getClient()
// Clean up any leftover test databases with prefix 'accountdb'
const existingCrs = await adminClient`SELECT datname FROM pg_database WHERE datname LIKE 'accountdb%'`
for (const row of existingCrs) {
await adminClient`DROP DATABASE IF EXISTS ${adminClient(row.datname)} CASCADE`
}
await adminClient`CREATE DATABASE ${adminClient(dbUuid)}`
}
server/account/src/__tests__/postgres.test.ts
+18 -12
View File
@@ -330,7 +330,7 @@ describe('AccountPostgresDbCollection', () => {
expect(mockClient.unsafe).toHaveBeenCalledWith(
`SELECT * FROM (
SELECT
SELECT
a.uuid,
a.timezone,
a.locale,
@@ -678,7 +678,7 @@ describe('PostgresAccountDB', () => {
expect(
mockClient.unsafe.mock.calls[0][0].replace(/\s+/g, ' ').replace(/\(\s/g, '(').replace(/\s\)/g, ')')
).toEqual(
`SELECT
`SELECT
w.uuid,
w.name,
w.url,
@@ -688,7 +688,7 @@ describe('PostgresAccountDB', () => {
w.region,
w.created_by,
w.created_on,
w.billing_account,
w.billing_account,
json_build_object(
'mode', s.mode,
'processing_progress', s.processing_progress,
@@ -747,7 +747,7 @@ describe('PostgresAccountDB', () => {
expect(
mockClient.unsafe.mock.calls[0][0].replace(/\s+/g, ' ').replace(/\(\s/g, '(').replace(/\s\)/g, ')')
).toEqual(
`SELECT
`SELECT
w.uuid,
w.name,
w.url,
@@ -757,7 +757,7 @@ describe('PostgresAccountDB', () => {
w.region,
w.created_by,
w.created_on,
w.billing_account,
w.billing_account,
json_build_object(
'mode', s.mode,
'processing_progress', s.processing_progress,
@@ -820,7 +820,7 @@ describe('PostgresAccountDB', () => {
expect(
mockClient.unsafe.mock.calls[0][0].replace(/\s+/g, ' ').replace(/\(\s/g, '(').replace(/\s\)/g, ')')
).toEqual(
`SELECT
`SELECT
w.uuid,
w.name,
w.url,
@@ -830,7 +830,7 @@ describe('PostgresAccountDB', () => {
w.region,
w.created_by,
w.created_on,
w.billing_account,
w.billing_account,
json_build_object(
'mode', s.mode,
'processing_progress', s.processing_progress,
@@ -912,7 +912,7 @@ describe('PostgresAccountDB', () => {
expect(
mockClient.unsafe.mock.calls[0][0].replace(/\s+/g, ' ').replace(/\(\s/g, '(').replace(/\s\)/g, ')')
).toEqual(
`SELECT
`SELECT
w.uuid,
w.name,
w.url,
@@ -922,7 +922,7 @@ describe('PostgresAccountDB', () => {
w.region,
w.created_by,
w.created_on,
w.billing_account,
w.billing_account,
json_build_object(
'mode', s.mode,
'processing_progress', s.processing_progress,
@@ -963,8 +963,8 @@ describe('PostgresAccountDB', () => {
expect(
mockClient.unsafe.mock.calls[1][0].replace(/\s+/g, ' ').replace(/\(\s/g, '(').replace(/\s\)/g, ')')
).toEqual(
`UPDATE global_account.workspace_status
SET processing_attempts = processing_attempts + 1, "last_processing_time" = $1
`UPDATE global_account.workspace_status
SET processing_attempts = processing_attempts + 1, "last_processing_time" = $1
WHERE workspace_uuid = $2`
.replace(/\s+/g, ' ')
.replace(/\(\s/g, '(')
@@ -997,7 +997,13 @@ describe('PostgresAccountDB', () => {
expect(mockClient).toHaveBeenCalledWith('global_account.account_passwords')
expect(mockClient).toHaveBeenCalledWith(
['UPSERT INTO ', ' (account_uuid, hash, salt) VALUES (', ', ', '::bytea, ', '::bytea)'],
[
'INSERT INTO ',
' (account_uuid, hash, salt) VALUES (',
', ',
'::bytea, ',
'::bytea) ON CONFLICT (account_uuid) DO UPDATE SET hash = EXCLUDED.hash, salt = EXCLUDED.salt;'
],
expect.anything(),
accountId,
hash.buffer,
server/account/src/collections/postgres/migrations.ts
+342 -182
View File
@@ -13,86 +13,158 @@
// limitations under the License.
//
export function getMigrations (ns: string): [string, string][] {
import { type DBFlavor } from '../../types'
// Type definitions for different database flavors.
// The keys match the DBFlavor type ('postgres' and 'cockroach').
// The SQL syntax used (e.g., BIGSERIAL, JSONB, generated columns) is compatible with
// modern PostgreSQL versions (v13+) and is expected to be forward-compatible with future versions like 18.1.
const dbTypes = {
['cockroach' as DBFlavor]: {
string: 'STRING',
bytes: 'BYTES',
int2: 'INT2',
int4: 'INT4',
int8: 'INT8',
bool: 'BOOL',
// unique_rowid() generates a unique INT8
autoIncrementInt8: (ns: string) => 'INT8 NOT NULL DEFAULT unique_rowid()'
},
['postgres' as DBFlavor]: {
string: 'TEXT',
bytes: 'BYTEA',
int2: 'SMALLINT',
int4: 'INTEGER',
int8: 'BIGINT',
bool: 'BOOLEAN',
// Use special function to generate a cryptographic random bigint
autoIncrementInt8: (ns: string) => `BIGINT NOT NULL DEFAULT ${ns}.gen_random_bigint()`
}
}
export function getMigrations (ns: string, flavor: DBFlavor): [string, string][] {
if (flavor === 'unknown') {
throw new Error('Cannot generate migrations for an unknown database flavor.')
}
const types = dbTypes[flavor]
if (types === undefined) {
// This should not happen if DBFlavor is typed correctly, but it's a good safeguard.
throw new Error(`Unsupported database flavor: ${flavor}`)
}
return [
getV1Migration(ns),
getV2Migration1(ns),
getV2Migration2(ns),
getV2Migration3(ns),
getV3Migration(ns),
getV4Migration(ns),
getV4Migration1(ns),
getV5Migration(ns),
getV6Migration(ns),
getV7Migration(ns),
getV8Migration(ns),
getV9Migration(ns),
getV10Migration1(ns),
getV10Migration2(ns),
getV11Migration(ns),
getV12Migration(ns),
getV13Migration(ns),
getV14Migration(ns),
getV15Migration(ns),
getV16Migration(ns),
getV17Migration(ns),
getV18Migration(ns),
getV19Migration(ns),
getV20Migration(ns),
getV21Migration(ns),
getV22Migration(ns),
getV23Migration(ns),
getV24Migration(ns)
getV1Migration(ns, flavor),
getV2Migration1(ns, flavor),
getV2Migration2(ns, flavor),
getV2Migration3(ns, flavor),
getV3Migration(ns, flavor),
getV4Migration(ns, flavor),
getV4Migration1(ns, flavor),
getV5Migration(ns, flavor),
getV6Migration(ns, flavor),
getV7Migration(ns, flavor),
getV8Migration(ns, flavor),
getV9Migration(ns, flavor),
getV10Migration1(ns, flavor),
getV10Migration2(ns, flavor),
getV11Migration(ns, flavor),
getV12Migration(ns, flavor),
getV13Migration(ns, flavor),
getV14Migration(ns, flavor),
getV15Migration(ns, flavor),
getV16Migration(ns, flavor),
getV17Migration(ns, flavor),
getV18Migration(ns, flavor),
getV19Migration(ns, flavor),
getV20Migration(ns, flavor),
getV21Migration(ns, flavor),
getV22Migration(ns, flavor),
getV23Migration(ns, flavor),
getV24Migration(ns, flavor)
]
}
// NOTE: NEVER MODIFY EXISTING MIGRATIONS. IF YOU NEED TO ADJUST THE SCHEMA, ADD A NEW MIGRATION.
function getV1Migration (ns: string): [string, string] {
function getV1Migration (ns: string, flavor: DBFlavor): [string, string] {
const types = dbTypes[flavor]
// Define the generated 'key' column with flavor-specific syntax.
// For PostgreSQL, we use a custom immutable function to ensure the expression is valid.
const keyColumnDefinition =
flavor === 'postgres'
? `key ${types.string} GENERATED ALWAYS AS (${ns}.social_id_type_to_text(type) || ':' || value) STORED`
: `key ${types.string} AS (CONCAT(type::TEXT, ':', value)) STORED`
return [
'account_db_v1_global_init',
`
/* ======= FUNCTIONS ======= */
CREATE OR REPLACE FUNCTION current_epoch_ms()
RETURNS BIGINT AS $$
SELECT (extract(epoch from current_timestamp) * 1000)::bigint;
CREATE OR REPLACE FUNCTION current_epoch_ms()
RETURNS BIGINT AS $$ SELECT (extract(epoch from current_timestamp) * 1000)::bigint;
$$ LANGUAGE SQL;
/* ======= E X T E N S I O N S ======= */
-- Enable the pgcrypto extension for cryptographic functions, e.g., gen_random_bytes().
-- This is required for secure, non-sequential ID generation in PostgreSQL.
-- This is a no-op for CockroachDB as it doesn't support extensions this way.
${flavor === 'postgres' ? 'CREATE EXTENSION IF NOT EXISTS pgcrypto;' : '-- pgcrypto not needed for CockroachDB'}
/* ======= T Y P E S ======= */
CREATE TYPE IF NOT EXISTS ${ns}.social_id_type AS ENUM ('email', 'github', 'google', 'phone', 'oidc', 'huly', 'telegram');
CREATE TYPE IF NOT EXISTS ${ns}.location AS ENUM ('kv', 'weur', 'eeur', 'wnam', 'enam', 'apac');
CREATE TYPE IF NOT EXISTS ${ns}.workspace_role AS ENUM ('OWNER', 'MAINTAINER', 'USER', 'GUEST', 'DOCGUEST');
CREATE TYPE ${ns}.social_id_type AS ENUM ('email', 'github', 'google', 'phone', 'oidc', 'huly', 'telegram');
CREATE TYPE ${ns}.location AS ENUM ('kv', 'weur', 'eeur', 'wnam', 'enam', 'apac');
CREATE TYPE ${ns}.workspace_role AS ENUM ('OWNER', 'MAINTAINER', 'USER', 'GUEST', 'DOCGUEST');
/* ======= HELPER FUNCTIONS FOR GENERATED COLUMNS ======= */
-- Create an immutable function to cast the social_id_type enum to text.
-- This is required for PostgreSQL to consider the generated expression immutable.
CREATE OR REPLACE FUNCTION ${ns}.social_id_type_to_text(val ${ns}.social_id_type)
RETURNS TEXT AS $$ SELECT val::TEXT;
$$ LANGUAGE SQL IMMUTABLE;
${
flavor === 'postgres'
? `
-- Create a function to generate a random, non-sequential, non-negative BIGINT for secure IDs.
-- This prevents enumeration attacks. We use a standard method of encoding to hex and casting.
CREATE OR REPLACE FUNCTION ${ns}.gen_random_bigint()
RETURNS BIGINT AS $$ SELECT ('x' || encode(gen_random_bytes(8), 'hex'))::bit(64)::bigint & 9223372036854775807::bigint;
$$ LANGUAGE SQL VOLATILE;
`
: ''
}
/* ======= P E R S O N ======= */
CREATE TABLE IF NOT EXISTS ${ns}.person (
uuid UUID NOT NULL DEFAULT gen_random_uuid(),
first_name STRING NOT NULL,
last_name STRING NOT NULL,
country STRING,
city STRING,
first_name ${types.string} NOT NULL,
last_name ${types.string} NOT NULL,
country ${types.string},
city ${types.string},
CONSTRAINT person_pk PRIMARY KEY (uuid)
);
/* ======= A C C O U N T ======= */
CREATE TABLE IF NOT EXISTS ${ns}.account (
uuid UUID NOT NULL,
timezone STRING,
locale STRING,
timezone ${types.string},
locale ${types.string},
CONSTRAINT account_pk PRIMARY KEY (uuid),
CONSTRAINT account_person_fk FOREIGN KEY (uuid) REFERENCES ${ns}.person(uuid)
);
CREATE TABLE IF NOT EXISTS ${ns}.account_passwords (
account_uuid UUID NOT NULL,
hash BYTES NOT NULL,
salt BYTES NOT NULL,
hash ${types.bytes} NOT NULL,
salt ${types.bytes} NOT NULL,
CONSTRAINT account_auth_pk PRIMARY KEY (account_uuid),
CONSTRAINT account_passwords_account_fk FOREIGN KEY (account_uuid) REFERENCES ${ns}.account(uuid)
);
CREATE TABLE IF NOT EXISTS ${ns}.account_events (
account_uuid UUID NOT NULL,
event_type STRING NOT NULL,
event_type ${types.string} NOT NULL,
time BIGINT NOT NULL DEFAULT current_epoch_ms(),
data JSONB,
CONSTRAINT account_events_pk PRIMARY KEY (account_uuid, event_type, time),
@@ -102,26 +174,25 @@ function getV1Migration (ns: string): [string, string] {
/* ======= S O C I A L I D S ======= */
CREATE TABLE IF NOT EXISTS ${ns}.social_id (
type ${ns}.social_id_type NOT NULL,
value STRING NOT NULL,
key STRING AS (CONCAT(type::STRING, ':', value)) STORED,
value ${types.string} NOT NULL,
${keyColumnDefinition},
person_uuid UUID NOT NULL,
created_on BIGINT NOT NULL DEFAULT current_epoch_ms(),
verified_on BIGINT,
CONSTRAINT social_id_pk PRIMARY KEY (type, value),
CONSTRAINT social_id_key_unique UNIQUE (key),
INDEX social_id_account_idx (person_uuid),
CONSTRAINT social_id_person_fk FOREIGN KEY (person_uuid) REFERENCES ${ns}.person(uuid)
);
/* ======= W O R K S P A C E ======= */
CREATE TABLE IF NOT EXISTS ${ns}.workspace (
uuid UUID NOT NULL DEFAULT gen_random_uuid(),
name STRING NOT NULL,
url STRING NOT NULL,
data_id STRING,
branding STRING,
name ${types.string} NOT NULL,
url ${types.string} NOT NULL,
data_id ${types.string},
branding ${types.string},
location ${ns}.location,
region STRING,
region ${types.string},
created_by UUID, -- account uuid
created_on BIGINT NOT NULL DEFAULT current_epoch_ms(),
billing_account UUID,
@@ -133,16 +204,16 @@ function getV1Migration (ns: string): [string, string] {
CREATE TABLE IF NOT EXISTS ${ns}.workspace_status (
workspace_uuid UUID NOT NULL,
mode STRING,
processing_progress INT2 DEFAULT 0,
version_major INT2 NOT NULL DEFAULT 0,
version_minor INT2 NOT NULL DEFAULT 0,
version_patch INT4 NOT NULL DEFAULT 0,
mode ${types.string},
processing_progress ${types.int2} DEFAULT 0,
version_major ${types.int2} NOT NULL DEFAULT 0,
version_minor ${types.int2} NOT NULL DEFAULT 0,
version_patch ${types.int4} NOT NULL DEFAULT 0,
last_processing_time BIGINT DEFAULT 0,
last_visit BIGINT,
is_disabled BOOL DEFAULT FALSE,
processing_attempts INT2 DEFAULT 0,
processing_message STRING,
is_disabled ${types.bool} DEFAULT FALSE,
processing_attempts ${types.int2} DEFAULT 0,
processing_message ${types.string},
backup_info JSONB,
CONSTRAINT workspace_status_pk PRIMARY KEY (workspace_uuid),
CONSTRAINT workspace_status_workspace_fk FOREIGN KEY (workspace_uuid) REFERENCES ${ns}.workspace(uuid)
@@ -163,8 +234,8 @@ function getV1Migration (ns: string): [string, string] {
/* ======= O T P ======= */
CREATE TABLE IF NOT EXISTS ${ns}.otp (
social_id STRING NOT NULL,
code STRING NOT NULL,
social_id ${types.string} NOT NULL,
code ${types.string} NOT NULL,
expires_on BIGINT NOT NULL,
created_on BIGINT NOT NULL DEFAULT current_epoch_ms(),
CONSTRAINT otp_pk PRIMARY KEY (social_id, code),
@@ -173,67 +244,93 @@ function getV1Migration (ns: string): [string, string] {
/* ======= I N V I T E ======= */
CREATE TABLE IF NOT EXISTS ${ns}.invite (
id INT8 NOT NULL DEFAULT unique_rowid(),
id ${types.autoIncrementInt8(ns)} PRIMARY KEY,
workspace_uuid UUID NOT NULL,
expires_on BIGINT NOT NULL,
email_pattern STRING,
remaining_uses INT2,
email_pattern ${types.string},
remaining_uses ${types.int2},
role ${ns}.workspace_role NOT NULL DEFAULT 'USER',
migrated_from STRING,
CONSTRAINT invite_pk PRIMARY KEY (id),
INDEX workspace_invite_idx (workspace_uuid),
INDEX migrated_from_idx (migrated_from),
migrated_from ${types.string},
CONSTRAINT invite_workspace_fk FOREIGN KEY (workspace_uuid) REFERENCES ${ns}.workspace(uuid)
);
/* ======= I N D E X E S ======= */
CREATE INDEX IF NOT EXISTS social_id_account_idx ON ${ns}.social_id (person_uuid);
CREATE INDEX IF NOT EXISTS workspace_invite_idx ON ${ns}.invite (workspace_uuid);
CREATE INDEX IF NOT EXISTS migrated_from_idx ON ${ns}.invite (migrated_from);
`
]
}
function getV2Migration1 (ns: string): [string, string] {
function getV2Migration1 (ns: string, flavor: DBFlavor): [string, string] {
const types = dbTypes[flavor]
return [
'account_db_v2_social_id_id_add',
`
-- Add _id column to social_id table
-- Add the _id column to the social_id table with a random default value.
ALTER TABLE ${ns}.social_id
ADD COLUMN IF NOT EXISTS _id INT8 NOT NULL DEFAULT unique_rowid();
ADD COLUMN IF NOT EXISTS _id ${types.autoIncrementInt8(ns)};
`
]
}
function getV2Migration2 (ns: string): [string, string] {
function getV2Migration2 (ns: string, flavor: DBFlavor): [string, string] {
return [
'account_db_v2_social_id_pk_change',
`
-- Drop existing otp foreign key constraint
-- Drop the existing otp foreign key constraint
ALTER TABLE ${ns}.otp
DROP CONSTRAINT IF EXISTS otp_social_id_fk;
-- Drop existing primary key on social_id
-- Drop the existing primary key on social_id
ALTER TABLE ${ns}.social_id
DROP CONSTRAINT IF EXISTS social_id_pk;
-- Add new primary key on _id
-- Add the new primary key on _id
ALTER TABLE ${ns}.social_id
ADD CONSTRAINT social_id_pk PRIMARY KEY (_id);
`
]
}
function getV2Migration3 (ns: string): [string, string] {
function getV2Migration3 (ns: string, flavor: DBFlavor): [string, string] {
const types = dbTypes[flavor]
// Generate flavor-specific SQL for adding the constraint.
const addConstraintSql =
flavor === 'postgres'
? `
-- Add unique constraint on type, value (PostgreSQL compatible with DO block)
DO $$ BEGIN
IF NOT EXISTS (
SELECT 1 FROM pg_catalog.pg_constraint
WHERE conname = 'social_id_tv_key_unique'
AND connamespace = (SELECT oid FROM pg_namespace WHERE nspname = '${ns}')
AND conrelid = (SELECT oid FROM pg_class WHERE relname = 'social_id' AND relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = '${ns}'))
) THEN
ALTER TABLE ${ns}.social_id
ADD CONSTRAINT social_id_tv_key_unique UNIQUE (type, value);
END IF;
END $$;
`
: `
-- Add unique constraint on type, value (CockroachDB compatible)
ALTER TABLE ${ns}.social_id
ADD CONSTRAINT IF NOT EXISTS social_id_tv_key_unique UNIQUE (type, value);
`
return [
'account_db_v2_social_id_constraints',
`
-- Add unique constraint on type, value
ALTER TABLE ${ns}.social_id
ADD CONSTRAINT social_id_tv_key_unique UNIQUE (type, value);
${addConstraintSql}
-- Drop old table
DROP TABLE ${ns}.otp;
-- Drop the old table
DROP TABLE IF EXISTS ${ns}.otp;
-- Create new OTP table with correct column type
-- Create the new OTP table with the correct column type
CREATE TABLE ${ns}.otp (
social_id INT8 NOT NULL,
code STRING NOT NULL,
social_id ${types.int8} NOT NULL,
code ${types.string} NOT NULL,
expires_on BIGINT NOT NULL,
created_on BIGINT NOT NULL DEFAULT current_epoch_ms(),
CONSTRAINT otp_new_pk PRIMARY KEY (social_id, code),
@@ -243,42 +340,46 @@ function getV2Migration3 (ns: string): [string, string] {
]
}
function getV3Migration (ns: string): [string, string] {
function getV3Migration (ns: string, flavor: DBFlavor): [string, string] {
const types = dbTypes[flavor]
return [
'account_db_v3_add_invite_auto_join_final',
`
ALTER TABLE ${ns}.invite
ADD COLUMN IF NOT EXISTS email STRING,
ADD COLUMN IF NOT EXISTS auto_join BOOL DEFAULT FALSE;
ADD COLUMN IF NOT EXISTS email ${types.string},
ADD COLUMN IF NOT EXISTS auto_join ${types.bool} DEFAULT FALSE;
ALTER TABLE ${ns}.account
ADD COLUMN IF NOT EXISTS automatic BOOL;
ADD COLUMN IF NOT EXISTS automatic ${types.bool};
`
]
}
function getV4Migration (ns: string): [string, string] {
function getV4Migration (ns: string, flavor: DBFlavor): [string, string] {
const types = dbTypes[flavor]
return [
'account_db_v4_mailbox',
`
CREATE TABLE IF NOT EXISTS ${ns}.mailbox (
account_uuid UUID NOT NULL,
mailbox STRING NOT NULL,
mailbox ${types.string} NOT NULL,
CONSTRAINT mailbox_pk PRIMARY KEY (mailbox),
CONSTRAINT mailbox_account_fk FOREIGN KEY (account_uuid) REFERENCES ${ns}.account(uuid)
);
CREATE TABLE IF NOT EXISTS ${ns}.mailbox_secrets (
mailbox STRING NOT NULL,
app STRING,
secret STRING NOT NULL,
mailbox ${types.string} NOT NULL,
app ${types.string},
secret ${types.string} NOT NULL,
CONSTRAINT mailbox_secret_mailbox_fk FOREIGN KEY (mailbox) REFERENCES ${ns}.mailbox(mailbox)
);
`
]
}
function getV4Migration1 (ns: string): [string, string] {
function getV4Migration1 (ns: string, flavor: DBFlavor): [string, string] {
return [
'account_db_v4_remove_mailbox_account_fk',
`
@@ -288,46 +389,54 @@ function getV4Migration1 (ns: string): [string, string] {
]
}
function getV5Migration (ns: string): [string, string] {
function getV5Migration (ns: string, flavor: DBFlavor): [string, string] {
const types = dbTypes[flavor]
return [
'account_db_v5_social_id_is_deleted',
`
ALTER TABLE ${ns}.social_id
ADD COLUMN IF NOT EXISTS is_deleted BOOL NOT NULL DEFAULT FALSE;
ADD COLUMN IF NOT EXISTS is_deleted ${types.bool} NOT NULL DEFAULT FALSE;
`
]
}
function getV6Migration (ns: string): [string, string] {
function getV6Migration (ns: string, flavor: DBFlavor): [string, string] {
const types = dbTypes[flavor]
// Generated column syntax: PostgreSQL uses GENERATED ALWAYS AS (...) STORED
// CockroachDB supports both syntaxes, so we use PostgreSQL-compatible one
return [
'account_db_v6_add_social_id_integrations',
`
CREATE TABLE IF NOT EXISTS ${ns}.integrations (
social_id INT8 NOT NULL,
kind STRING NOT NULL,
social_id ${types.int8} NOT NULL,
kind ${types.string} NOT NULL,
workspace_uuid UUID,
_def_ws_uuid UUID NOT NULL GENERATED ALWAYS AS (COALESCE(workspace_uuid, '00000000-0000-0000-0000-000000000000')) STORED NOT VISIBLE,
_def_ws_uuid UUID NOT NULL GENERATED ALWAYS AS (COALESCE(workspace_uuid, '00000000-0000-0000-0000-000000000000'::UUID)) STORED,
data JSONB,
CONSTRAINT integrations_pk PRIMARY KEY (social_id, kind, _def_ws_uuid),
INDEX integrations_kind_idx (kind),
CONSTRAINT integrations_social_id_fk FOREIGN KEY (social_id) REFERENCES ${ns}.social_id(_id),
CONSTRAINT integrations_workspace_fk FOREIGN KEY (workspace_uuid) REFERENCES ${ns}.workspace(uuid)
);
CREATE TABLE IF NOT EXISTS ${ns}.integration_secrets (
social_id INT8 NOT NULL,
kind STRING NOT NULL,
social_id ${types.int8} NOT NULL,
kind ${types.string} NOT NULL,
workspace_uuid UUID,
_def_ws_uuid UUID NOT NULL GENERATED ALWAYS AS (COALESCE(workspace_uuid, '00000000-0000-0000-0000-000000000000')) STORED NOT VISIBLE,
key STRING,
secret STRING NOT NULL,
_def_ws_uuid UUID NOT NULL GENERATED ALWAYS AS (COALESCE(workspace_uuid, '00000000-0000-0000-0000-000000000000'::UUID)) STORED,
key ${types.string},
secret ${types.string} NOT NULL,
CONSTRAINT integration_secrets_pk PRIMARY KEY (social_id, kind, _def_ws_uuid, key)
);
/* ======= I N D E X E S ======= */
CREATE INDEX IF NOT EXISTS integrations_kind_idx ON ${ns}.integrations (kind);
`
]
}
function getV7Migration (ns: string): [string, string] {
function getV7Migration (ns: string, flavor: DBFlavor): [string, string] {
return [
'account_db_v7_add_display_value',
`
@@ -337,7 +446,7 @@ function getV7Migration (ns: string): [string, string] {
]
}
function getV8Migration (ns: string): [string, string] {
function getV8Migration (ns: string, flavor: DBFlavor): [string, string] {
return [
'account_db_v8_add_account_max_workspaces',
`
@@ -347,7 +456,7 @@ function getV8Migration (ns: string): [string, string] {
]
}
function getV9Migration (ns: string): [string, string] {
function getV9Migration (ns: string, flavor: DBFlavor): [string, string] {
return [
'account_db_v9_add_migrated_to_person',
`
@@ -358,130 +467,172 @@ function getV9Migration (ns: string): [string, string] {
]
}
function getV10Migration1 (ns: string): [string, string] {
return [
'account_db_v10_add_readonly_role',
function getV10Migration1 (ns: string, flavor: DBFlavor): [string, string] {
// For PostgreSQL, we need to check if the value exists before adding it
const addValueSql =
flavor === 'postgres'
? `
-- Add READONLYGUEST value to workspace_role enum (PostgreSQL)
DO $$ BEGIN
IF NOT EXISTS (
SELECT 1 FROM pg_enum
WHERE enumlabel = 'READONLYGUEST'
AND enumtypid = (SELECT oid FROM pg_type WHERE typname = 'workspace_role' AND typnamespace = (SELECT oid FROM pg_namespace WHERE nspname = '${ns}'))
) THEN
ALTER TYPE ${ns}.workspace_role ADD VALUE 'READONLYGUEST';
END IF;
END $$;
`
ALTER TYPE ${ns}.workspace_role ADD VALUE 'READONLYGUEST' AFTER 'DOCGUEST';
: `
-- Add READONLYGUEST value to workspace_role enum (CockroachDB)
ALTER TYPE ${ns}.workspace_role ADD VALUE IF NOT EXISTS 'READONLYGUEST';
`
]
return ['account_db_v10_add_readonly_role', addValueSql]
}
function getV10Migration2 (ns: string): [string, string] {
function getV10Migration2 (ns: string, flavor: DBFlavor): [string, string] {
const types = dbTypes[flavor]
return [
'account_db_v10_add_allow_guests_flag_to_workspace',
`
ALTER TABLE ${ns}.workspace
ADD COLUMN IF NOT EXISTS allow_read_only_guest BOOL NOT NULL DEFAULT FALSE;
ADD COLUMN IF NOT EXISTS allow_read_only_guest ${types.bool} NOT NULL DEFAULT FALSE;
`
]
}
function getV11Migration (ns: string): [string, string] {
function getV11Migration (ns: string, flavor: DBFlavor): [string, string] {
const types = dbTypes[flavor]
return [
'account_db_v10_add_migrated_to_person',
'account_db_v11_add_pending_workspace_lock',
`
CREATE TABLE IF NOT EXISTS ${ns}._pending_workspace_lock (
id INT8 DEFAULT 1 PRIMARY KEY,
id ${types.int8} DEFAULT 1 PRIMARY KEY,
CONSTRAINT single_row CHECK (id = 1)
);
INSERT INTO ${ns}._pending_workspace_lock (id) VALUES (1)
ON CONFLICT (id) DO NOTHING;
`
]
}
function getV12Migration (ns: string): [string, string] {
function getV12Migration (ns: string, flavor: DBFlavor): [string, string] {
return [
'account_db_v12_update_account_events_fk',
`
-- Drop existing foreign key constraint
-- Drop the existing foreign key constraint
ALTER TABLE ${ns}.account_events
DROP CONSTRAINT IF EXISTS account_events_account_fk;
-- Add new foreign key constraint referencing person table
-- Add the new foreign key constraint referencing the person table
ALTER TABLE ${ns}.account_events
ADD CONSTRAINT account_events_person_fk FOREIGN KEY (account_uuid) REFERENCES ${ns}.person(uuid);
`
]
}
function getV13Migration (ns: string): [string, string] {
function getV13Migration (ns: string, flavor: DBFlavor): [string, string] {
return [
'account_db_v13_update_workspace_fk_to_person',
`
-- Drop existing foreign key constraints
-- Drop the existing foreign key constraints
ALTER TABLE ${ns}.workspace
DROP CONSTRAINT IF EXISTS workspace_created_by_fk,
DROP CONSTRAINT IF EXISTS workspace_billing_account_fk;
-- Add new foreign key constraints referencing person table
-- Add the new foreign key constraints referencing the person table
ALTER TABLE ${ns}.workspace
ADD CONSTRAINT workspace_created_by_person_fk
ADD CONSTRAINT workspace_created_by_person_fk
FOREIGN KEY (created_by) REFERENCES ${ns}.person(uuid),
ADD CONSTRAINT workspace_billing_account_person_fk
ADD CONSTRAINT workspace_billing_account_person_fk
FOREIGN KEY (billing_account) REFERENCES ${ns}.person(uuid);
`
]
}
function getV14Migration (ns: string): [string, string] {
function getV14Migration (ns: string, flavor: DBFlavor): [string, string] {
const types = dbTypes[flavor]
return [
'account_db_v14_add_allow_guest_signup_flag_to_workspace',
`
ALTER TABLE ${ns}.workspace
ADD COLUMN IF NOT EXISTS allow_guest_sign_up BOOL NOT NULL DEFAULT FALSE;
ADD COLUMN IF NOT EXISTS allow_guest_sign_up ${types.bool} NOT NULL DEFAULT FALSE;
`
]
}
function getV15Migration (ns: string): [string, string] {
function getV15Migration (ns: string, flavor: DBFlavor): [string, string] {
const types = dbTypes[flavor]
return [
'account_db_v15_add_target_region_to_workspace_status',
`
ALTER TABLE ${ns}.workspace_status
ADD COLUMN IF NOT EXISTS target_region STRING;
ADD COLUMN IF NOT EXISTS target_region ${types.string};
`
]
}
function getV16Migration (ns: string): [string, string] {
return [
'account_db_v16_add_huly_assistant_social_id_type',
function getV16Migration (ns: string, flavor: DBFlavor): [string, string] {
// For PostgreSQL, we need to check if the value exists before adding it
const addValueSql =
flavor === 'postgres'
? `
-- Add huly-assistant value to social_id_type enum (PostgreSQL)
DO $$ BEGIN
IF NOT EXISTS (
SELECT 1 FROM pg_enum
WHERE enumlabel = 'huly-assistant'
AND enumtypid = (SELECT oid FROM pg_type WHERE typname = 'social_id_type' AND typnamespace = (SELECT oid FROM pg_namespace WHERE nspname = '${ns}'))
) THEN
ALTER TYPE ${ns}.social_id_type ADD VALUE 'huly-assistant';
END IF;
END $$;
`
ALTER TYPE ${ns}.social_id_type ADD VALUE 'huly-assistant' AFTER 'telegram';
: `
-- Add huly-assistant value to social_id_type enum (CockroachDB)
ALTER TYPE ${ns}.social_id_type ADD VALUE IF NOT EXISTS 'huly-assistant';
`
]
return ['account_db_v16_add_huly_assistant_social_id_type', addValueSql]
}
function getV17Migration (ns: string): [string, string] {
function getV17Migration (ns: string, flavor: DBFlavor): [string, string] {
const types = dbTypes[flavor]
return [
'account_db_v17_create_user_profile_table',
`
/* ======= U S E R P R O F I L E ======= */
CREATE TABLE IF NOT EXISTS ${ns}.user_profile (
person_uuid UUID NOT NULL,
bio STRING,
city STRING,
country STRING,
website STRING,
bio ${types.string},
city ${types.string},
country ${types.string},
website ${types.string},
social_links JSONB,
is_public BOOL NOT NULL DEFAULT FALSE,
is_public ${types.bool} NOT NULL DEFAULT FALSE,
CONSTRAINT user_profile_pk PRIMARY KEY (person_uuid),
CONSTRAINT user_profile_person_fk FOREIGN KEY (person_uuid) REFERENCES ${ns}.person(uuid) ON DELETE CASCADE,
INDEX user_profile_is_public_idx (is_public) WHERE is_public = TRUE
CONSTRAINT user_profile_person_fk FOREIGN KEY (person_uuid) REFERENCES ${ns}.person(uuid) ON DELETE CASCADE
);
/* Remove city and country from person table */
/* Remove city and country from the person table */
ALTER TABLE ${ns}.person
DROP COLUMN IF EXISTS city,
DROP COLUMN IF EXISTS country;
/* ======= I N D E X E S ======= */
CREATE INDEX IF NOT EXISTS user_profile_is_public_idx ON ${ns}.user_profile (is_public) WHERE is_public = TRUE;
`
]
}
function getV18Migration (ns: string): [string, string] {
function getV18Migration (ns: string, flavor: DBFlavor): [string, string] {
return [
'account_db_v18_populate_user_profiles',
`
@@ -497,17 +648,19 @@ function getV18Migration (ns: string): [string, string] {
]
}
function getV19Migration (ns: string): [string, string] {
function getV19Migration (ns: string, flavor: DBFlavor): [string, string] {
const types = dbTypes[flavor]
return [
'account_db_v19_subscription_table',
`
/* ======= S U B S C R I P T I O N ======= */
/* Provider-agnostic subscription information for workspaces */
/* Managed by billing service via payment provider webhooks (e.g. Polar.sh, Stripe) */
/* Managed by a billing service via payment provider webhooks (e.g. Polar.sh, Stripe) */
/* Multiple active subscriptions allowed per workspace (tier + addons + support) */
/* Historical subscriptions preserved with status: canceled/expired */
CREATE TYPE IF NOT EXISTS ${ns}.subscription_status AS ENUM (
CREATE TYPE ${ns}.subscription_status AS ENUM (
'active',
'trialing',
'past_due',
@@ -515,55 +668,57 @@ function getV19Migration (ns: string): [string, string] {
'paused',
'expired'
);
CREATE TABLE IF NOT EXISTS ${ns}.subscription (
id STRING NOT NULL DEFAULT gen_random_uuid()::STRING,
id ${types.string} NOT NULL DEFAULT gen_random_uuid()::TEXT,
workspace_uuid UUID NOT NULL,
account_uuid UUID NOT NULL, -- Account that paid for the subscription
-- Provider details
provider STRING NOT NULL, -- Payment provider identifier (e.g. 'polar', 'stripe', 'manual')
provider_subscription_id STRING NOT NULL, -- External subscription ID from the provider
provider_checkout_id STRING, -- External checkout/session ID that created this subscription
provider ${types.string} NOT NULL, -- Payment provider identifier (e.g. 'polar', 'stripe', 'manual')
provider_subscription_id ${types.string} NOT NULL, -- External subscription ID from the provider
provider_checkout_id ${types.string}, -- External checkout/session ID that created this subscription
-- Subscription classification
type STRING NOT NULL DEFAULT 'tier', -- tier, support, etc.
type ${types.string} NOT NULL DEFAULT 'tier', -- tier, support, etc.
status ${ns}.subscription_status NOT NULL DEFAULT 'active',
plan STRING NOT NULL, -- Plan identifier (e.g. 'rare', 'epic', 'legendary', 'custom')
plan ${types.string} NOT NULL, -- Plan identifier (e.g. 'rare', 'epic', 'legendary', 'custom')
-- Amount paid (in cents, e.g. 9999 = $99.99)
-- Used primarily for pay-what-you-want/donation subscriptions to track actual payment
amount INT8,
amount ${types.int8},
-- Billing period (optional)
period_start BIGINT,
period_end BIGINT,
-- Trial information (optional)
trial_end BIGINT,
-- Cancellation info (optional)
canceled_at BIGINT,
will_cancel_at BIGINT, -- Scheduled cancellation date (cancel at period end)
-- Provider-specific data (stored as JSONB for flexibility)
-- e.g. customerExternalId, metadata, etc.
provider_data JSONB,
created_on BIGINT NOT NULL DEFAULT current_epoch_ms(),
updated_on BIGINT NOT NULL DEFAULT current_epoch_ms(),
CONSTRAINT subscription_pk PRIMARY KEY (id),
CONSTRAINT subscription_provider_subscription_id_unique UNIQUE (provider, provider_subscription_id),
CONSTRAINT subscription_workspace_fk FOREIGN KEY (workspace_uuid) REFERENCES ${ns}.workspace(uuid),
CONSTRAINT subscription_account_fk FOREIGN KEY (account_uuid) REFERENCES ${ns}.account(uuid),
INDEX subscription_workspace_status_idx (workspace_uuid, status)
CONSTRAINT subscription_account_fk FOREIGN KEY (account_uuid) REFERENCES ${ns}.account(uuid)
);
/* ======= I N D E X E S ======= */
CREATE INDEX IF NOT EXISTS subscription_workspace_status_idx ON ${ns}.subscription (workspace_uuid, status);
`
]
}
function getV20Migration (ns: string): [string, string] {
function getV20Migration (ns: string, flavor: DBFlavor): [string, string] {
return [
'account_db_v20_usage_info',
`
@@ -573,7 +728,7 @@ function getV20Migration (ns: string): [string, string] {
]
}
function getV21Migration (ns: string): [string, string] {
function getV21Migration (ns: string, flavor: DBFlavor): [string, string] {
return [
'account_db_v21_add_failed_login_attempts',
`
@@ -583,7 +738,7 @@ function getV21Migration (ns: string): [string, string] {
]
}
function getV22Migration (ns: string): [string, string] {
function getV22Migration (ns: string, flavor: DBFlavor): [string, string] {
return [
'account_db_v22_add_password_change_event_index',
`
@@ -593,7 +748,7 @@ function getV22Migration (ns: string): [string, string] {
]
}
function getV23Migration (ns: string): [string, string] {
function getV23Migration (ns: string, flavor: DBFlavor): [string, string] {
return [
'account_db_v23_add_password_aging_rule_to_workspace',
`
@@ -603,7 +758,8 @@ function getV23Migration (ns: string): [string, string] {
]
}
function getV24Migration (ns: string): [string, string] {
function getV24Migration (ns: string, flavor: DBFlavor): [string, string] {
const types = dbTypes[flavor]
return [
'account_db_v24_add_workspace_permissions_table',
`
@@ -611,14 +767,18 @@ function getV24Migration (ns: string): [string, string] {
CREATE TABLE IF NOT EXISTS ${ns}.workspace_permissions (
workspace_uuid UUID NOT NULL,
account_uuid UUID NOT NULL,
permission STRING NOT NULL,
created_on BIGINT NOT NULL DEFAULT current_epoch_ms(),
permission ${types.string} NOT NULL,
created_on ${types.int8} NOT NULL DEFAULT current_epoch_ms(),
CONSTRAINT workspace_permissions_pk PRIMARY KEY (workspace_uuid, account_uuid, permission),
CONSTRAINT workspace_permissions_workspace_fk FOREIGN KEY (workspace_uuid) REFERENCES ${ns}.workspace(uuid),
CONSTRAINT workspace_permissions_account_fk FOREIGN KEY (account_uuid) REFERENCES ${ns}.account(uuid),
INDEX workspace_permissions_account_idx (account_uuid),
INDEX workspace_permissions_permission_idx (permission)
CONSTRAINT workspace_permissions_account_fk FOREIGN KEY (account_uuid) REFERENCES ${ns}.account(uuid)
);
CREATE INDEX IF NOT EXISTS workspace_permissions_account_idx
ON ${ns}.workspace_permissions (account_uuid);
CREATE INDEX IF NOT EXISTS workspace_permissions_permission_idx
ON ${ns}.workspace_permissions (permission);
`
]
}
server/account/src/collections/postgres/postgres.ts
+59 -39
View File
@@ -49,7 +49,8 @@ import type {
AccountAggregatedInfo,
UserProfile,
Subscription,
WorkspacePermission
WorkspacePermission,
DBFlavor
} from '../../types'
function toSnakeCase (str: string): string {
@@ -117,7 +118,8 @@ implements DbCollection<T> {
constructor (
readonly name: string,
readonly client: Sql,
readonly options: PostgresDbCollectionOptions<T, K> = {}
readonly options: PostgresDbCollectionOptions<T, K> = {},
readonly filterFields: string[] = []
) {}
get ns (): string {
@@ -249,6 +251,14 @@ implements DbCollection<T> {
for (const field of this.timestampFields) {
res[field] = convertTimestamp(res[field])
}
if (this.filterFields.length > 0) {
for (const key of Object.keys(res)) {
if (this.filterFields.includes(key.toLowerCase())) {
// eslint-disable-next-line @typescript-eslint/no-dynamic-delete
delete res[key]
}
}
}
return res as T
}
@@ -334,7 +344,7 @@ implements DbCollection<T> {
.join(', ')
const sql = `
INSERT INTO ${this.getTableName()}
INSERT INTO ${this.getTableName()}
(${columnsList.map((k) => `"${k}"`).join(', ')})
VALUES ${placeholders}
RETURNING *
@@ -432,7 +442,7 @@ export class AccountPostgresDbCollection
protected buildSelectClause (): string {
return `SELECT * FROM (
SELECT
SELECT
a.uuid,
a.timezone,
a.locale,
@@ -528,7 +538,8 @@ export class PostgresAccountDB implements AccountDB {
constructor (
readonly client: Sql,
readonly ns: string = 'global_account'
readonly ns: string = 'global_account',
readonly dbFlavor: DBFlavor = 'cockroach'
) {
const withRetryClient = this.withRetry
this.person = new PostgresDbCollection<Person, 'uuid'>('person', client, { ns, idKey: 'uuid', withRetryClient })
@@ -564,11 +575,19 @@ export class PostgresAccountDB implements AccountDB {
})
this.mailbox = new PostgresDbCollection<Mailbox, 'mailbox'>('mailbox', client, { ns, withRetryClient })
this.mailboxSecret = new PostgresDbCollection<MailboxSecret>('mailbox_secrets', client, { ns, withRetryClient })
this.integration = new PostgresDbCollection<Integration>('integrations', client, { ns, withRetryClient })
this.integrationSecret = new PostgresDbCollection<IntegrationSecret>('integration_secrets', client, {
ns,
withRetryClient
})
this.integration = new PostgresDbCollection<Integration>('integrations', client, { ns, withRetryClient }, [
'_def_ws_uuid',
'_defwsuuid'
])
this.integrationSecret = new PostgresDbCollection<IntegrationSecret>(
'integration_secrets',
client,
{
ns,
withRetryClient
},
['_def_ws_uuid', '_defwsuuid']
)
this.userProfile = new PostgresDbCollection<UserProfile, 'personUuid'>('user_profile', client, {
ns,
idKey: 'personUuid',
@@ -614,8 +633,8 @@ export class PostgresAccountDB implements AccountDB {
const executeMigration = async (client: Sql): Promise<void> => {
updateInterval = setInterval(() => {
this.client`
UPDATE ${this.client(this.ns)}._account_applied_migrations
SET last_processed_at = NOW()
UPDATE ${this.client(this.ns)}._account_applied_migrations
SET last_processed_at = NOW()
WHERE identifier = ${name} AND applied_at IS NULL
`.catch((err) => {
console.error(`Failed to update last_processed_at for migration ${name}:`, err)
@@ -634,7 +653,7 @@ export class PostgresAccountDB implements AccountDB {
// Only locks if row exists and is not already locked
const existing = await client`
SELECT identifier, applied_at, last_processed_at
FROM ${this.client(this.ns)}._account_applied_migrations
FROM ${this.client(this.ns)}._account_applied_migrations
WHERE identifier = ${name}
FOR UPDATE NOWAIT
`
@@ -649,7 +668,7 @@ export class PostgresAccountDB implements AccountDB {
) {
// Take over the stale migration
await client`
UPDATE ${this.client(this.ns)}._account_applied_migrations
UPDATE ${this.client(this.ns)}._account_applied_migrations
SET last_processed_at = NOW()
WHERE identifier = ${name}
`
@@ -658,8 +677,8 @@ export class PostgresAccountDB implements AccountDB {
}
} else {
const res = await client`
INSERT INTO ${this.client(this.ns)}._account_applied_migrations
(identifier, ddl, last_processed_at)
INSERT INTO ${this.client(this.ns)}._account_applied_migrations
(identifier, ddl, last_processed_at)
VALUES (${name}, ${ddl}, NOW())
ON CONFLICT (identifier) DO NOTHING
`
@@ -674,8 +693,8 @@ export class PostgresAccountDB implements AccountDB {
if (executed) {
await this.client`
UPDATE ${this.client(this.ns)}._account_applied_migrations
SET applied_at = NOW()
UPDATE ${this.client(this.ns)}._account_applied_migrations
SET applied_at = NOW()
WHERE identifier = ${name}
`
migrationComplete = true
@@ -715,14 +734,14 @@ export class PostgresAccountDB implements AccountDB {
, last_processed_at TIMESTAMP WITH TIME ZONE
);
ALTER TABLE ${this.ns}._account_applied_migrations
ALTER TABLE ${this.ns}._account_applied_migrations
ADD COLUMN IF NOT EXISTS last_processed_at TIMESTAMP WITH TIME ZONE;
`
)
const constraintsExist = await this.client`
SELECT 1
FROM information_schema.columns
SELECT 1
FROM information_schema.columns
WHERE table_schema = ${this.ns}
AND table_name = '_account_applied_migrations'
AND column_name = 'applied_at'
@@ -733,10 +752,10 @@ export class PostgresAccountDB implements AccountDB {
try {
await this.client.unsafe(
`
ALTER TABLE ${this.ns}._account_applied_migrations
ALTER TABLE ${this.ns}._account_applied_migrations
ALTER COLUMN applied_at DROP DEFAULT;
ALTER TABLE ${this.ns}._account_applied_migrations
ALTER TABLE ${this.ns}._account_applied_migrations
ALTER COLUMN applied_at DROP NOT NULL;
`
)
@@ -815,7 +834,7 @@ export class PostgresAccountDB implements AccountDB {
const values = data.flat()
const sql = `
INSERT INTO ${this.getWsMembersTableName()}
INSERT INTO ${this.getWsMembersTableName()}
(account_uuid, workspace_uuid, role)
VALUES ${placeholders}
`
@@ -868,7 +887,7 @@ export class PostgresAccountDB implements AccountDB {
}
async getAccountWorkspaces (accountUuid: AccountUuid): Promise<WorkspaceInfoWithStatus[]> {
const sql = `SELECT
const sql = `SELECT
w.uuid,
w.name,
w.url,
@@ -892,8 +911,8 @@ export class PostgresAccountDB implements AccountDB {
'processing_message', s.processing_message,
'backup_info', s.backup_info,
'usage_info', s.usage_info
) status
FROM ${this.getWsMembersTableName()} as m
) status
FROM ${this.getWsMembersTableName()} as m
INNER JOIN ${this.workspace.getTableName()} as w ON m.workspace_uuid = w.uuid
INNER JOIN ${this.workspaceStatus.getTableName()} as s ON s.workspace_uuid = w.uuid
WHERE m.account_uuid = $1
@@ -922,7 +941,7 @@ export class PostgresAccountDB implements AccountDB {
wsLivenessMs?: number
): Promise<WorkspaceInfoWithStatus | undefined> {
const sqlChunks: string[] = [
`SELECT
`SELECT
w.uuid,
w.name,
w.url,
@@ -932,7 +951,7 @@ export class PostgresAccountDB implements AccountDB {
w.region,
w.created_by,
w.created_on,
w.billing_account,
w.billing_account,
json_build_object(
'mode', s.mode,
'processing_progress', s.processing_progress,
@@ -1028,7 +1047,7 @@ export class PostgresAccountDB implements AccountDB {
async setPassword (accountUuid: AccountUuid, hash: Buffer, salt: Buffer): Promise<void> {
await this.withRetry(
async (rTx) =>
await rTx`UPSERT INTO ${this.client(this.account.getPasswordsTableName())} (account_uuid, hash, salt) VALUES (${accountUuid}, ${hash.buffer as any}::bytea, ${salt.buffer as any}::bytea)`
await rTx`INSERT INTO ${this.client(this.account.getPasswordsTableName())} (account_uuid, hash, salt) VALUES (${accountUuid}, ${hash.buffer as any}::bytea, ${salt.buffer as any}::bytea) ON CONFLICT (account_uuid) DO UPDATE SET hash = EXCLUDED.hash, salt = EXCLUDED.salt;`
)
}
@@ -1070,7 +1089,7 @@ export class PostgresAccountDB implements AccountDB {
const sqlChunks: string[] = [
`
WITH account_data AS (
SELECT
SELECT
a.uuid,
a.timezone,
a.locale,
@@ -1078,16 +1097,16 @@ export class PostgresAccountDB implements AccountDB {
a.max_workspaces,
p.first_name,
p.last_name,
p.country,
p.city,
up.country,
up.city,
p.migrated_to,
(
SELECT jsonb_agg(jsonb_build_object(
'socialId', i.social_id,
'kind', i.kind,
'workspaceUuid', i.workspace_uuid
))
FROM ${this.integration.getTableName()} i
))
FROM ${this.integration.getTableName()} i
WHERE i.social_id IN (SELECT _id FROM ${this.socialId.getTableName()} s WHERE s.person_uuid = a.uuid)
) as integrations,
(
@@ -1121,6 +1140,7 @@ export class PostgresAccountDB implements AccountDB {
) as workspaces
FROM ${this.account.getTableName()} a
INNER JOIN ${this.ns}.person p ON p.uuid = a.uuid
LEFT JOIN ${this.userProfile.getTableName()} up ON up.person_uuid = p.uuid
`
]
@@ -1129,11 +1149,11 @@ export class PostgresAccountDB implements AccountDB {
if (search !== undefined && search !== '') {
sqlChunks.push(`
WHERE
p.first_name ILIKE $${paramIndex} OR
WHERE
p.first_name ILIKE $${paramIndex} OR
p.last_name ILIKE $${paramIndex} OR
EXISTS (
SELECT 1 FROM ${this.socialId.getTableName()} s
SELECT 1 FROM ${this.socialId.getTableName()} s
WHERE s.person_uuid = a.uuid AND s.value ILIKE $${paramIndex}
)
`)
@@ -1193,7 +1213,7 @@ export class PostgresAccountDB implements AccountDB {
}
protected getMigrations (): [string, string][] {
return getMigrations(this.ns)
return getMigrations(this.ns, this.dbFlavor)
}
async batchAssignWorkspacePermission (
server/account/src/types.ts
+2
View File
@@ -304,6 +304,8 @@ export type WorkspaceStatusData = Omit<WorkspaceStatus, 'workspaceUuid'>
export type WorkspaceInviteData = Omit<WorkspaceInvite, 'id'>
export type DBFlavor = 'postgres' | 'cockroach' | 'unknown'
/* ========= D A T A B A S E C O L L E C T I O N S ========= */
export interface AccountDB {
person: DbCollection<Person>
server/account/src/utils.ts
+36 -2
View File
@@ -64,12 +64,31 @@ import {
type WorkspaceInvite,
type WorkspaceJoinInfo,
type WorkspaceLoginInfo,
type WorkspaceStatus
type WorkspaceStatus,
type DBFlavor
} from './types'
import { isAdminEmail } from './admin'
import { type Sql } from 'postgres'
export const GUEST_ACCOUNT = 'b6996120-416f-49cd-841e-e4a5d2e49c9b' as PersonUuid
export async function getDbFlavor (pgClient: Sql<any>): Promise<DBFlavor> {
// Run the version query
const [{ version }] = await pgClient`SELECT version()`
// CockroachDB’s string contains “Cockroach” (case‑insensitive)
if (/cockroach/i.test(version)) {
return 'cockroach'
}
// Anything else that looks like a PostgreSQL version string
if (/postgresql/i.test(version)) {
return 'postgres'
}
// Fallback – could be a custom build or something unexpected
return 'unknown'
}
export async function getAccountDB (
uri: string,
dbNs?: string,
@@ -98,10 +117,25 @@ export async function getAccountDB (
})
const client = getDBClient(uri)
const pgClient = await client.getClient()
const pgAccount = new PostgresAccountDB(pgClient, dbNs ?? 'global_account')
let flavor: DBFlavor = 'unknown'
let error = false
do {
try {
flavor = await getDbFlavor(pgClient)
error = false
} catch (err: any) {
error = true
console.error('Error while initializing postgres account db', err.message)
await new Promise((resolve) => setTimeout(resolve, 1000))
}
} while (error)
error = false
const pgAccount = new PostgresAccountDB(pgClient, dbNs ?? 'global_account', flavor)
do {
try {
await pgAccount.init()
services/datalake/pod-datalake/src/datalake/db.ts
+52 -27
View File
@@ -90,12 +90,15 @@ export async function createDb (ctx: MeasureContext, connectionString: string):
fetch_types: false,
prepare: false,
types: {
// https://jdbc.postgresql.org/documentation/publicapi/constant-values.html
int8: {
to: 0,
// Corrected OID for BIGINT is 20.
// We parse BIGINT as Number to maintain compatibility with parseInt() used in the codebase.
// WARNING: This can lead to precision loss for values larger than Number.MAX_SAFE_INTEGER.
// If you expect to handle sizes larger than ~9 PB, consider migrating the codebase to use BigInt.
to: 20,
from: [20],
serialize: (value: string) => value.toString(),
parse: (value: number) => Number(value)
serialize: (value: number | string) => value.toString(),
parse: (value: string) => Number(value)
}
}
})
@@ -143,7 +146,7 @@ export class PostgresDB implements BlobDB {
)`)
const appliedMigrations = (await this.execute<Row[]>('SELECT name FROM blob.migrations')).map((row) => row.name)
ctx.info('applied migrations', { migrations: appliedMigrations })
ctx.info('Applied migrations', { migrations: appliedMigrations })
for (const [name, sql] of getMigrations()) {
if (appliedMigrations.includes(name)) {
@@ -151,11 +154,11 @@ export class PostgresDB implements BlobDB {
}
try {
ctx.warn('applying migration', { migration: name })
ctx.warn('Applying migration', { migration: name })
await this.execute(sql)
await this.execute('INSERT INTO blob.migrations (name) VALUES ($1)', [name])
} catch (err: any) {
ctx.error('failed to apply migration', { migration: name, error: err })
ctx.error('Failed to apply migration', { migration: name, error: err })
throw err
}
}
@@ -237,8 +240,13 @@ export class PostgresDB implements BlobDB {
await this.execute(
`
UPSERT INTO blob.blob (workspace, name, hash, location, parent, deleted_at)
INSERT INTO blob.blob (workspace, name, hash, location, parent, deleted_at)
VALUES ($1, $2, $3, $4, $5, NULL)
ON CONFLICT (workspace, name) DO UPDATE SET
hash = EXCLUDED.hash,
location = EXCLUDED.location,
parent = EXCLUDED.parent,
deleted_at = EXCLUDED.deleted_at
`,
[workspace, name, hash, location, parent]
)
@@ -249,8 +257,12 @@ export class PostgresDB implements BlobDB {
await this.execute(
`
UPSERT INTO blob.data (hash, location, filename, size, type)
INSERT INTO blob.data (hash, location, filename, size, type)
VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (hash, location) DO UPDATE SET
filename = EXCLUDED.filename,
size = EXCLUDED.size,
type = EXCLUDED.type
`,
[hash, location, filename, size, type]
)
@@ -259,18 +271,29 @@ export class PostgresDB implements BlobDB {
async createBlobData (ctx: MeasureContext, data: BlobWithDataRecord): Promise<void> {
const { workspace, name, hash, location, parent, filename, size, type } = data
// First upsert into data table
await this.execute(
`
UPSERT INTO blob.data (hash, location, filename, size, type)
INSERT INTO blob.data (hash, location, filename, size, type)
VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (hash, location) DO UPDATE SET
filename = EXCLUDED.filename,
size = EXCLUDED.size,
type = EXCLUDED.type
`,
[hash, location, filename, size, type]
)
// Then upsert into blob table
await this.execute(
`
UPSERT INTO blob.blob (workspace, name, hash, location, parent, deleted_at)
INSERT INTO blob.blob (workspace, name, hash, location, parent, deleted_at)
VALUES ($1, $2, $3, $4, $5, NULL)
ON CONFLICT (workspace, name) DO UPDATE SET
hash = EXCLUDED.hash,
location = EXCLUDED.location,
parent = EXCLUDED.parent,
deleted_at = EXCLUDED.deleted_at
`,
[workspace, name, hash, location, parent]
)
@@ -336,10 +359,12 @@ export class PostgresDB implements BlobDB {
await this.execute(
`
UPSERT INTO blob.meta (workspace, name, meta)
VALUES ($1, $2, $3)
INSERT INTO blob.meta (workspace, name, meta)
VALUES ($1, $2, $3::jsonb)
ON CONFLICT (workspace, name) DO UPDATE SET
meta = EXCLUDED.meta
`,
[workspace, name, meta]
[workspace, name, JSON.stringify(meta)]
)
}
@@ -564,7 +589,7 @@ export function escape (value: any): string {
if (value instanceof Date) {
return `'${value.toISOString()}'`
} else {
return `'${JSON.stringify(value)}'`
return `'${JSON.stringify(value).replace(/'/g, "''")}'`
}
default:
throw new Error(`Unsupported value type: ${typeof value}`)
@@ -577,33 +602,33 @@ function getMigrations (): [string, string][] {
function migrationV1 (): [string, string] {
const sql = `
CREATE TYPE IF NOT EXISTS blob.location AS ENUM ('eu', 'weur', 'eeur', 'wnam', 'enam', 'apac');
CREATE TYPE blob.location AS ENUM ('eu', 'weur', 'eeur', 'wnam', 'enam', 'apac');
CREATE TABLE IF NOT EXISTS blob.data (
hash UUID NOT NULL,
location blob.location NOT NULL,
size INT8 NOT NULL,
filename STRING(255) NOT NULL,
type STRING(255) NOT NULL,
size BIGINT NOT NULL,
filename VARCHAR(255) NOT NULL,
type VARCHAR(255) NOT NULL,
CONSTRAINT pk_data PRIMARY KEY (hash, location)
);
CREATE TABLE IF NOT EXISTS blob.blob (
workspace STRING(255) NOT NULL,
name STRING(255) NOT NULL,
workspace VARCHAR(255) NOT NULL,
name VARCHAR(255) NOT NULL,
hash UUID NOT NULL,
location blob.location NOT NULL,
parent STRING(255) DEFAULT NULL,
parent VARCHAR(255) DEFAULT NULL,
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
deleted_at TIMESTAMP DEFAULT NULL,
CONSTRAINT pk_blob PRIMARY KEY (workspace, name),
CONSTRAINT fk_data FOREIGN KEY (hash, location) REFERENCES blob.data (hash, location),
CONSTRAINT fk_parent FOREIGN KEY (workspace, parent) REFERENCES blob.blob (workspace, name) ON DELETE CASCADE
CONSTRAINT fk_parent FOREIGN KEY (workspace, parent) REFERENCES blob.blob (workspace, name) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS blob.meta (
workspace STRING(255) NOT NULL,
name STRING(255) NOT NULL,
workspace VARCHAR(255) NOT NULL,
name VARCHAR(255) NOT NULL,
meta JSONB NOT NULL,
CONSTRAINT pk_meta PRIMARY KEY (workspace, name),
CONSTRAINT fk_blob FOREIGN KEY (workspace, name) REFERENCES blob.blob (workspace, name)
@@ -623,7 +648,7 @@ function migrationV2 (): [string, string] {
AND NOT EXISTS (
SELECT 1
FROM blob.blob existing
WHERE existing.workspace = w.uuid::string
WHERE existing.workspace = w.uuid::text
AND existing.name = blob.blob.name
);
@@ -634,7 +659,7 @@ function migrationV2 (): [string, string] {
AND NOT EXISTS (
SELECT 1
FROM blob.meta existing
WHERE existing.workspace = w.uuid::string
WHERE existing.workspace = w.uuid::text
AND existing.name = blob.meta.name
);
tests/.env
+1
View File
@@ -6,4 +6,5 @@ BACKUP_BUCKET_NAME=dev-backups
DB_URL=mongodb://mongodb:27018
MONGO_URL=mongodb://mongodb:27018
DB_PG_URL=postgresql://root@cockroach:26257/defaultdb?sslmode=disable
DB_PURE_PG_URL=postgresql://postgres:postgres@postgres:5433/postgres
QUEUE_CONFIG='redpanda:9093;-staging'
tests/docker-compose.purepg.yaml
+52
View File
@@ -0,0 +1,52 @@
services:
account:
environment:
- DB_URL=${DB_PURE_PG_URL}
- STORAGE_CONFIG=${DATALAKE_STORAGE_CONFIG}
- PROCEED_V7_MONGO=false
transactor:
environment:
- DB_URL=${DB_PURE_PG_URL}
- STORAGE_CONFIG=${DATALAKE_STORAGE_CONFIG}
workspace:
environment:
- DB_URL=${DB_PURE_PG_URL}
- STORAGE_CONFIG=${DATALAKE_STORAGE_CONFIG}
- ACCOUNTS_DB_URL=${DB_PURE_PG_URL}
fulltext:
environment:
- DB_URL=${DB_PURE_PG_URL}
- STORAGE_CONFIG=${DATALAKE_STORAGE_CONFIG}
front:
environment:
- STORAGE_CONFIG=${DATALAKE_STORAGE_CONFIG}
- FILES_URL=http://localhost:4031/blob/:workspace/:blobId/:filename
- UPLOAD_URL=http://localhost:4031/upload/form-data/:workspace
- DATALAKE_URL=http://localhost:4031
- PREVIEW_URL=http://localhost:4031
collaborator:
environment:
- STORAGE_CONFIG=${DATALAKE_STORAGE_CONFIG}
datalake:
image: hardcoreeng/datalake
depends_on:
minio:
condition: service_healthy
cockroach:
condition: service_started
stats:
condition: service_started
account:
condition: service_started
ports:
- 4031:4031
environment:
- PORT=4031
- SECRET=secret
- ACCOUNTS_URL=http://account:3003
- STATS_URL=http://stats:4901
- STREAM_URL=http://localhost:1081/recording
- DB_URL=${DB_PURE_PG_URL}
- BUCKETS=blobs,eu|http://minio:9000?accessKey=minioadmin&secretKey=minioadmin
- QUEUE_CONFIG=${QUEUE_CONFIG}
restart: unless-stopped
tests/docker-compose.yaml
+21
View File
@@ -34,6 +34,21 @@ services:
- '18089:8080'
command: start-single-node --insecure
restart: unless-stopped
postgres:
image: postgres:18.1
command: ["postgres", "-p", "5433"]
healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres -p 5433"]
interval: 5s
timeout: 5s
retries: 10
ports:
- 5433:5433
environment:
- POSTGRES_USER=postgres
- POSTGRES_PASSWORD=postgres
- POSTGRES_DB=postgres
restart: unless-stopped
redpanda:
image: docker.redpanda.com/redpandadata/redpanda:v24.3.6
command:
@@ -110,10 +125,14 @@ services:
account:
image: hardcoreeng/account
pull_policy: never
depends_on:
postgres:
condition: service_healthy
links:
- mongodb
- minio
- cockroach
- postgres
ports:
- 3003:3003
volumes:
@@ -136,6 +155,7 @@ services:
links:
- mongodb
- cockroach
- postgres
- minio
- redpanda
depends_on:
@@ -198,6 +218,7 @@ services:
- minio
- rekoni
- cockroach
- postgres
- account
- stats
- redpanda
tests/prepare-cockroach.sh
Executable
+30
View File
@@ -0,0 +1,30 @@
#!/usr/bin/env bash
docker compose -p sanity kill
docker compose -p sanity down --volumes
docker compose -p sanity up -d --force-recreate --renew-anon-volumes
docker_exit=$?
if [ ${docker_exit} -eq 0 ]; then
echo "Container started successfully"
else
echo "Container started with errors"
exit ${docker_exit}
fi
if [ "x$DO_CLEAN" == 'xtrue' ]; then
echo 'Do docker Clean'
docker system prune -a -f
fi
./wait-elastic.sh 9201
# Create user record in accounts
./tool-cockroach.sh create-account user1 -f John -l Appleseed -p 1234
./tool-cockroach.sh create-account user2 -f Kainin -l Dirak -p 1234
./tool-cockroach.sh create-account admin -f Super -l User -p 1234
# Create workspace record in accounts
./tool-cockroach.sh create-workspace sanity-ws email:user1
./restore-cockroach.sh
rm -rf ./sanity/.auth
tests/prepare-pg.sh
+1 -1
View File
@@ -2,7 +2,7 @@
docker compose -p sanity kill
docker compose -p sanity down --volumes
docker compose -p sanity up -d --force-recreate --renew-anon-volumes
docker compose -f docker-compose.yaml -f docker-compose.purepg.yaml -p sanity up -d --force-recreate --renew-anon-volumes
docker_exit=$?
if [ ${docker_exit} -eq 0 ]; then
echo "Container started successfully"
tests/prepare-tests.sh
+2 -2
View File
@@ -2,7 +2,7 @@
docker compose -p sanity kill
docker compose -p sanity down --volumes
docker compose -f docker-compose.yaml -p sanity up elastic mongodb cockroach redpanda redpanda_console -d --force-recreate --renew-anon-volumes
docker compose -f docker-compose.yaml -p sanity up elastic mongodb cockroach postgres redpanda -d --force-recreate --renew-anon-volumes
docker_exit=$?
if [ ${docker_exit} -eq 0 ]; then
echo "Container started successfully"
@@ -11,4 +11,4 @@ else
exit ${docker_exit}
fi
./wait-elastic.sh 9201
./wait-elastic.sh 9201
tests/restore-cockroach.sh
Executable
+18
View File
@@ -0,0 +1,18 @@
#!/usr/bin/env bash
# Restore workspace contents in mongo/elastic
./tool-cockroach.sh backup-restore ./sanity-ws sanity-ws --upgrade
# ./tool-cockroach.sh upgrade-workspace sanity-ws
# Re-assign user to workspace.
./tool-cockroach.sh assign-workspace user1 sanity-ws
./tool-cockroach.sh assign-workspace user2 sanity-ws
./tool-cockroach.sh set-user-role user1 sanity-ws OWNER
./tool-cockroach.sh set-user-role user2 sanity-ws OWNER
./tool-cockroach.sh configure sanity-ws --enable=*
./tool-cockroach.sh configure sanity-ws --list
# setup issue createdOn for yesterday
./tool-cockroach.sh change-field sanity-ws --objectId 65e47f1f1b875b51e3b4b983 --objectClass tracker:class:Issue --attribute createdOn --value $(($(date +%s)*1000 - 86400000)) --type number
tests/tool-cockroach.sh
Executable
+14
View File
@@ -0,0 +1,14 @@
#!/usr/bin/env bash
export MODEL_VERSION=$(node ../common/scripts/show_version.js)
export STORAGE_CONFIG="datalake|http://localhost:4031"
export ACCOUNTS_URL=http://localhost:3003
export TRANSACTOR_URL=ws://localhost:3334
export ACCOUNT_DB_URL=postgresql://root@localhost:26258/defaultdb?sslmode=disable
export MONGO_URL=mongodb://localhost:27018
export ELASTIC_URL=http://localhost:9201
export SERVER_SECRET=secret
export DB_URL=postgresql://root@localhost:26258/defaultdb?sslmode=disable
export QUEUE_CONFIG='localhost:19093;-staging'
node ${TOOL_OPTIONS} ../dev/tool/bundle/bundle.js $@
tests/tool-pg.sh
+3 -3
View File
@@ -4,11 +4,11 @@ export MODEL_VERSION=$(node ../common/scripts/show_version.js)
export STORAGE_CONFIG="datalake|http://localhost:4031"
export ACCOUNTS_URL=http://localhost:3003
export TRANSACTOR_URL=ws://localhost:3334
export ACCOUNT_DB_URL=postgresql://root@localhost:26258/defaultdb?sslmode=disable
export ACCOUNT_DB_URL=postgresql://postgres:postgres@localhost:5433/postgres
export MONGO_URL=mongodb://localhost:27018
export ELASTIC_URL=http://localhost:9201
export SERVER_SECRET=secret
export DB_URL=postgresql://root@localhost:26258/defaultdb?sslmode=disable
export DB_URL=postgresql://postgres:postgres@localhost:5433/postgres
export QUEUE_CONFIG='localhost:19093;-staging'
node ${TOOL_OPTIONS} ../dev/tool/bundle/bundle.js $@
node ${TOOL_OPTIONS} ../dev/tool/bundle/bundle.js $@
ws-tests/docker-compose.yaml
+3
View File
@@ -147,6 +147,9 @@ services:
pull_policy: never
extra_hosts:
- 'huly.local:host-gateway'
depends_on:
cockroach:
condition: service_started
links:
- mongodb
- minio