From 49e416075b73305c1ab3da6804d8dabfccd86262 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Wed, 12 Aug 2026 09:47:13 +0200 Subject: [PATCH] =?UTF-8?q?=E2=99=BB=EF=B8=8F(yhub)=20maintain=20database?= =?UTF-8?q?=20schema=20using=20npm=20run=20init-db?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The yhub database schema have new update and will probably be modified in the future. We don't want to maintain this sql schema in the Docs repo, we want to reuse what is directly made in the yhub project. For this we reuse the existing bin/init-db.js script --- .github/workflows/impress.yml | 15 ++++++++------- CHANGELOG.md | 18 ++++++++++++++++++ Makefile | 15 +++++++++++++++ compose.yml | 6 ++++-- docker/files/yhub/initdb/01-yhub.sql | 14 -------------- src/yhub-server/README.md | 19 +++++++++++++++++++ src/yhub-server/package.json | 3 ++- 7 files changed, 66 insertions(+), 24 deletions(-) delete mode 100644 docker/files/yhub/initdb/01-yhub.sql diff --git a/.github/workflows/impress.yml b/.github/workflows/impress.yml index 272f523eb..f9c13f2a0 100644 --- a/.github/workflows/impress.yml +++ b/.github/workflows/impress.yml @@ -226,13 +226,6 @@ jobs: working-directory: . run: bin/generate-jwt-private-key.sh - - name: Create the collaboration server database - run: | - PGPASSWORD=pass psql -h localhost -U dinum -d impress \ - -c 'CREATE DATABASE yhub' - PGPASSWORD=pass psql -h localhost -U dinum -d yhub \ - -f ../../docker/files/yhub/initdb/01-yhub.sql - - name: Generate a MO file from strings extracted from the project run: uv run python manage.py compilemessages @@ -245,6 +238,14 @@ jobs: working-directory: src/yhub-server run: npm ci --omit=dev + # yhub ships its own DDL and creates the database as well, so this needs + # the dependencies installed above — hence its place after them + - name: Create the collaboration server database + working-directory: src/yhub-server + env: + POSTGRES: postgres://dinum:pass@localhost:5432/yhub + run: npm run init-db + - name: Start the backend for the collaboration server to authenticate against env: DJANGO_ALLOWED_HOSTS: "*" diff --git a/CHANGELOG.md b/CHANGELOG.md index a792d50e6..66421ef6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,24 @@ and this project adheres to ### Added +- ⬆️(collaboration) upgrade yhub to 0.6.0, which needs a schema change: a + `yhub_ydoc_tombstones_v1` table (it adds document deletion) and four + `*_is_reference` markers on `yhub_ydoc_v1`. Neither is optional — every + document read joins the tombstone table, so without them yhub answers + `relation "yhub_ydoc_tombstones_v1" does not exist`. Existing rows read as + "may be a reference", exactly as before the markers existed, so there is no + backfill and no downtime beyond applying the DDL +- 🔧(collaboration) let yhub own its schema: `npm run init-db` in + `src/yhub-server` runs the DDL script yhub ships (`bin/init-db.js`), which + creates the database when missing and every table the installed version + needs. It replaces the copy of the schema we kept in + `docker/files/yhub/initdb/`, which only replayed on a fresh postgres volume — + so an upgrade that added a table silently skipped an existing database, and + the copy had to be updated by hand on every upgrade. `make migrate-yhub` runs + it against the dev stack, the counterpart of `make migrate` for the Django + database, and is part of `make bootstrap`; CI runs the same script instead of + applying the SQL by hand. It is the only thing that runs DDL — the server and + the worker never do — and it is idempotent, so re-running it is always safe - ✅(collaboration) add integration tests covering both legacy migrations, running against a real yhub: CI now starts the collaboration server and a valkey alongside the backend test job. They skip themselves when nothing diff --git a/Makefile b/Makefile index 6f0a033b2..82729249a 100644 --- a/Makefile +++ b/Makefile @@ -101,6 +101,7 @@ pre-bootstrap: \ post-bootstrap: \ migrate \ + migrate-yhub \ demo \ back-i18n-compile \ mails-install \ @@ -332,6 +333,20 @@ migrate: ## run django migrations for the impress project. @$(MANAGE) migrate .PHONY: migrate +# Runs the DDL script yhub ships (`bin/init-db.js`, wrapped as `npm run +# init-db`): it creates the yhub database when missing, then every table the +# installed @y/hub version needs. yhub never runs DDL from the server or the +# worker, so this is what applies a schema change after an upgrade. +# Both stores are started because the script also creates the valkey worker +# stream and connects to it whenever REDIS is set. Re-running is safe and +# expected; on an existing stream it logs a harmless `BUSYGROUP` error and +# still exits 0, since the server creates that stream at startup anyway. +migrate-yhub: ## create or upgrade the collaboration server (yhub) schema. + @echo "$(BOLD)Running yhub migrations$(RESET)" + @$(COMPOSE) up -d yhub-postgres yhub-valkey + @$(COMPOSE_RUN) --no-deps yhub npm run init-db +.PHONY: migrate-yhub + superuser: ## Create an admin superuser with password "admin" @echo "$(BOLD)Creating a Django superuser$(RESET)" @$(MANAGE) createsuperuser --email admin@example.com --password admin diff --git a/compose.yml b/compose.yml index 02bf221bc..e4d307f78 100644 --- a/compose.yml +++ b/compose.yml @@ -221,14 +221,16 @@ services: POSTGRES_DB: yhub volumes: - yhub-pgdata:/var/lib/postgresql/data - # NOTE: initdb.d only runs on a FRESH volume; schema changes need `podman volume rm` - - ./docker/files/yhub/initdb:/docker-entrypoint-initdb.d:ro healthcheck: test: ["CMD-SHELL", "pg_isready -U yhub"] interval: 1s timeout: 2s retries: 60 # no published port (Django's postgres already publishes) + # the schema is not seeded here: initdb.d would only replay on a fresh + # volume, so an upgrade that adds a table would silently skip an existing + # one. `make migrate-yhub` runs yhub's own DDL script instead, the same way + # `make migrate` runs Django's migrations. yhub: user: ${DOCKER_USER:-1000} diff --git a/docker/files/yhub/initdb/01-yhub.sql b/docker/files/yhub/initdb/01-yhub.sql deleted file mode 100644 index 0a28fb3c0..000000000 --- a/docker/files/yhub/initdb/01-yhub.sql +++ /dev/null @@ -1,14 +0,0 @@ --- Column-for-column from yhub bin/init-db.js (unquoted identifiers so --- case-folding matches yhub's persistence.js queries). -CREATE TABLE IF NOT EXISTS yhub_ydoc_v1 ( - org text, - docid text, - branch text, - t text, - created INT8, - gcDoc bytea, - nongcDoc bytea, - contentmap bytea, - contentids bytea, - PRIMARY KEY (org,docid,branch,t) -); diff --git a/src/yhub-server/README.md b/src/yhub-server/README.md index 4be057c33..7c6e0544e 100644 --- a/src/yhub-server/README.md +++ b/src/yhub-server/README.md @@ -52,6 +52,25 @@ backend-internal and should not be routed through the public ingress. The `Dockerfile` builds the container image used by the `yhub` service in `compose.yml`. +## Database schema (`npm run init-db`) + +yhub never runs DDL from the server or the worker, so the schema is created by +the script it ships (`node_modules/@y/hub/bin/init-db.js`), wrapped here as +`npm run init-db`. It reads `POSTGRES` from the environment, creates the +database when it does not exist, then every table and index the **installed** +yhub version needs. It is idempotent, so re-running it is always safe. + +Run it whenever `@y/hub` is upgraded — releases that add a table or a column +say so in their changelog, and the server fails on every document read until +the DDL is applied (`relation "yhub_ydoc_tombstones_v1" does not exist`, for +instance). Nothing in this repository copies the schema, so an upgrade is +`package.json` plus this script and nothing else. + +From the repository root, `make migrate-yhub` runs it against the dev stack — +the counterpart of `make migrate` for the Django database. `make bootstrap` +already includes it, so a fresh checkout needs nothing extra; an upgrade is +`make migrate-yhub` and restart the service. + ## Soft migration (`SOFT_MIGRATION=true`) Documents were historically stored by the Django backend in the S3 media diff --git a/src/yhub-server/package.json b/src/yhub-server/package.json index 17beae20f..998924faf 100644 --- a/src/yhub-server/package.json +++ b/src/yhub-server/package.json @@ -3,7 +3,8 @@ "private": true, "type": "module", "scripts": { - "start": "node server.js" + "start": "node server.js", + "init-db": "node node_modules/@y/hub/bin/init-db.js" }, "dependencies": { "@y/hub": "0.6.0",