From 82d43d01feb1ff67e7bfca078be603ea923e7392 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Fri, 16 Oct 2026 07:49:47 +0200 Subject: [PATCH] =?UTF-8?q?=E2=AC=86=EF=B8=8F(yhub)=20upgrade=20@y/hub=20t?= =?UTF-8?q?o=200.10.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Last version of @y/hub contains a fix we need, it fixes the websocket reconnection loop for anonymous user on a read only document sending a keepalive every 20 seconds. In both version 0.10.0 and 0.10.1 breaking changes are introduced, one is impacting us by removing the computePool.patchYdoc function used by the create-ydoc endpoint declated in our yhub server. To fix it, we are doing what the version 0.9.0 was doing and we keep the same behavior. A new environment variable YHUB_AUTH_TIMEOUT_MS is added to configure the server.authTimeout parameters in the createYhub options --- CHANGELOG.md | 1 + UPGRADE.md | 15 + documentation/env.md | 1 + src/helm/impress/README.md | 1 + src/helm/impress/values.yaml | 1 + src/yhub-server/README.md | 1 + src/yhub-server/__tests__/api.spec.ts | 266 ++++++++++++++++++ src/yhub-server/__tests__/config.spec.ts | 23 ++ src/yhub-server/__tests__/permissions.spec.ts | 42 +++ src/yhub-server/__tests__/server.spec.ts | 32 +++ src/yhub-server/package.json | 4 +- src/yhub-server/src/api.ts | 74 +++-- src/yhub-server/src/config.ts | 15 +- src/yhub-server/src/server.ts | 5 + src/yhub-server/yarn.lock | 33 ++- 15 files changed, 465 insertions(+), 49 deletions(-) create mode 100644 src/yhub-server/__tests__/api.spec.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index b978fcf96..745a456f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to - ♻️(frontend) replace outdated AGPL mentions of BlockNote XL packages #2775 - πŸ’„(frontend) update upload loader and error ui #2778 +- ⬆️(yhub) upgrade @y/hub to 0.10.1 #2853 ## [v6.0.1] - 2026-10-15 diff --git a/UPGRADE.md b/UPGRADE.md index bf20282c1..109be3c42 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -16,6 +16,21 @@ the following command inside your docker container: ## [Unreleased] +- ⚠️ **The collaboration server moves to `@y/hub` 0.10.1: run its schema + script again before starting it.** It adds the `yhub_ydoc_versions_v1` + table, without which every hard deletion of a document fails, and two + indexes. The helm chart's `init-db` job does it on sync (`yhub.initDb`); with + docker compose, `make migrate-yhub`. Each index build holds a SHARE lock on + its table, which pauses compaction for a few seconds per million rows: run it + in a quiet moment. +- Redis 6.2 or later (or Valkey) is now required by the collaboration server. +- Roll its workers before its servers when they are deployed apart + (`yhub.worker.enabled`). The other order is harmless: a stale entry in a + Redis index until the document's next write. +- yhub now gives up on an authentication that takes longer than + `YHUB_AUTH_TIMEOUT_MS` (default 30000) and asks the caller to retry. Raise it + along with `YHUB_BACKEND_REQUEST_TIMEOUT_MS`. + ## [6.0.0] - 2026-10-05 ⚠️ This release replaces the collaboration server. The content of a document diff --git a/documentation/env.md b/documentation/env.md index 41ed7d818..8a7d82abd 100644 --- a/documentation/env.md +++ b/documentation/env.md @@ -203,6 +203,7 @@ documents what each of them changes. | YHUB_TASK_DEBOUNCE_MS | How long an update waits on the redis stream before a worker persists it | 10000 | | YHUB_MIN_MESSAGE_LIFETIME_MS | How long persisted updates stay replayable from redis rather than read back from postgres | 60000 | | YHUB_BACKEND_REQUEST_TIMEOUT_MS | How long a call to the backend (`users/me`, `documents/{id}`, `accesses/me`) may take before it is given up on and the caller told to retry, in ms | 5000 | +| YHUB_AUTH_TIMEOUT_MS | How long yhub waits for the whole authentication of a connection (backend calls, legacy seed) before the caller is told to retry, in ms | 30000 | | YHUB_IDENTITY_CACHE_TTL_MS | How long a resolved identity is reused for the same session cookie, in ms. 0 disables this layer (default) | 0 | | YHUB_ABILITIES_CACHE_TTL_MS | How long a document abilities verdict is served from cache, in ms, before it is re-read; deleted cluster-wide by the reset-connections kick. 0 disables it | 0 | | YHUB_ACCESS_CACHE_TTL_MS | How long an access date (the history bound) is served from cache, in ms. 0 disables it (default) | 0 | diff --git a/src/helm/impress/README.md b/src/helm/impress/README.md index b4331bebd..e984297e8 100644 --- a/src/helm/impress/README.md +++ b/src/helm/impress/README.md @@ -442,6 +442,7 @@ Allow: / | `yhub.envVars.YHUB_TASK_DEBOUNCE_MS` | How long an update waits on the redis stream before a worker persists it, in ms (default: 10000) | | | `yhub.envVars.YHUB_MIN_MESSAGE_LIFETIME_MS` | How long persisted updates stay replayable from redis, in ms (default: 60000) | | | `yhub.envVars.YHUB_BACKEND_REQUEST_TIMEOUT_MS` | How long a call to the backend may take before yhub gives up on it and tells the caller to retry, in ms (default: 5000) | | +| `yhub.envVars.YHUB_AUTH_TIMEOUT_MS` | How long yhub waits for the whole authentication of a connection or request (backend calls and legacy seed) before it tells the caller to retry, in ms (default: 30000) | | | `yhub.envVars.SOFT_MIGRATION` | Set to "true" to seed rooms from the legacy Django/S3 document store on first access | | | `yhub.envVars.LEGACY_S3_ENDPOINT_URL` | Required by SOFT_MIGRATION, endpoint of the legacy Django media bucket, without a path (e.g. https://s3.example.com) | | | `yhub.envVars.LEGACY_S3_ACCESS_KEY_ID` | Required by SOFT_MIGRATION, read access to the legacy bucket (or LEGACY_S3_ACCESS_KEY_ID_FILE) | | diff --git a/src/helm/impress/values.yaml b/src/helm/impress/values.yaml index 6645675d0..e4bc30e52 100644 --- a/src/helm/impress/values.yaml +++ b/src/helm/impress/values.yaml @@ -1189,6 +1189,7 @@ yhub: ## @extra yhub.envVars.YHUB_TASK_DEBOUNCE_MS How long an update waits on the redis stream before a worker persists it, in ms (default: 10000) ## @extra yhub.envVars.YHUB_MIN_MESSAGE_LIFETIME_MS How long persisted updates stay replayable from redis, in ms (default: 60000) ## @extra yhub.envVars.YHUB_BACKEND_REQUEST_TIMEOUT_MS How long a call to the backend may take before yhub gives up on it and tells the caller to retry, in ms (default: 5000) + ## @extra yhub.envVars.YHUB_AUTH_TIMEOUT_MS How long yhub waits for the whole authentication of a connection or request (backend calls and legacy seed) before it tells the caller to retry, in ms (default: 30000) ## @extra yhub.envVars.YHUB_IDENTITY_CACHE_TTL_MS How long a resolved identity is reused for the same session cookie, in ms; 0 disables this layer (default) ## @extra yhub.envVars.YHUB_ABILITIES_CACHE_TTL_MS How long a document abilities verdict is served from cache, in ms, before it is re-read; deleted cluster-wide by the reset-connections kick; 0 disables it ## @extra yhub.envVars.YHUB_ACCESS_CACHE_TTL_MS How long an access date (the history bound) is served from cache, in ms; 0 disables it (default) diff --git a/src/yhub-server/README.md b/src/yhub-server/README.md index 57ada590a..8c98b589d 100644 --- a/src/yhub-server/README.md +++ b/src/yhub-server/README.md @@ -274,6 +274,7 @@ were configurable: | `YHUB_TASK_DEBOUNCE_MS` | `10000` | How long an update waits on the stream before a worker persists it | | `YHUB_MIN_MESSAGE_LIFETIME_MS` | `60000` | How long persisted updates stay replayable from redis | | `YHUB_BACKEND_REQUEST_TIMEOUT_MS` | `5000` | How long a call to the Docs backend may take before it is given up on | +| `YHUB_AUTH_TIMEOUT_MS` | `30000` | How long yhub waits for `authenticate` or `authorize` as a whole (backend calls, legacy seed) before it answers 503; raise it with `YHUB_BACKEND_REQUEST_TIMEOUT_MS` | | `YHUB_IDENTITY_CACHE_TTL_MS` | `0` | How long a resolved `users/me` identity is reused for the same Cookie header (authz cache, off by default) | | `YHUB_ABILITIES_CACHE_TTL_MS` | `0` | How long a `documents/{id}` verdict is served from cache before it is re-read (deleted by `reset-connections`, see the authz-cache section) | | `YHUB_ACCESS_CACHE_TTL_MS` | `0` | How long an `accesses/me` date is served from cache before it is re-read | diff --git a/src/yhub-server/__tests__/api.spec.ts b/src/yhub-server/__tests__/api.spec.ts new file mode 100644 index 000000000..4cbf4ecb2 --- /dev/null +++ b/src/yhub-server/__tests__/api.spec.ts @@ -0,0 +1,266 @@ +// src/api.ts β€” the custom REST endpoints. What is covered here is +// `create-ydoc`, the one handler that built its stream message out of a yhub +// internal: it diffed the posted update in the compute pool's `patchYdoc`, +// which yhub 0.9.1 removed. The handler wrapped that call in a catch-all that +// answers "Invalid Yjs update", so the removal turned every document creation +// into a 400 without a type error at runtime, and with no test failing. +// +// `@y/hub` is faked down to what the module imports; `@y/y` is real, so the +// updates are genuine and the contentmap is decoded the way yhub decodes it. +// The fake instance has the compute pool of yhub 0.10 β€” no `patchYdoc` β€” and +// these tests would answer 400 again if the handler went back to it. + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import * as Y from '@y/y'; + +import { makeUpdate } from './_helpers.js'; + +vi.mock('@y/hub', () => { + const logger = { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + child: () => logger, + }; + return { + logger, + checkPermissions: vi.fn(), + createApiEndpoint: (name: string, opts: unknown) => ({ name, opts }), + createDocumentPermissions: (x: unknown) => x, + }; +}); + +vi.mock('../src/backend.js', () => ({ backendPublicJwk: null })); +vi.mock('../src/authz-cache.js', () => ({ invalidateDocumentAuthz: vi.fn() })); +vi.mock('../src/migration.js', () => ({ + SOFT_MIGRATION: false, + fullMigrate: vi.fn(), +})); + +const DOCID = '00000000-0000-4000-8000-000000000000'; + +type Handler = (req: unknown) => Promise; + +const createYdoc = async (): Promise => { + const { api } = await import('../src/api.js'); + const endpoint = ( + api as unknown as Array<{ + name: string; + opts: { post: { handler: Handler } }; + }> + ).find((e) => e.name === 'create-ydoc'); + if (endpoint == null) throw new Error('create-ydoc is not mounted'); + return endpoint.opts.post.handler; +}; + +// The surface of yhub 0.10 the handler reaches: the room read, the stream +// write, and a compute pool without the `patchYdoc` 0.9.1 removed. +const makeYhub = (gcDoc: Uint8Array | null = null) => ({ + getDoc: vi.fn(async () => ({ gcDoc })), + stream: { addMessage: vi.fn(async () => {}) }, + computePool: { mergeUpdates: vi.fn() }, +}); + +const makeReq = ( + body: Uint8Array, + { + yhub = makeYhub(), + authInfo = { userid: 'system', admin: true } as Record< + string, + unknown + > | null, + headers = { 'x-user-id': 'user-1' } as Record, + } = {}, +) => ({ + org: 'docs', + docid: DOCID, + branch: 'main', + docRef: { org: 'docs', docid: DOCID, branch: 'main' }, + // uws hands the body over as a Node Buffer, which the handler re-views + bytes: async () => Buffer.from(body), + headers, + authInfo, + permissions: {}, + query: {}, + yhub, +}); + +// Every attribute the contentmap puts on the inserted ids, by name. +const insertAttributes = (contentmap: Uint8Array) => { + const attrs = new Map(); + Y.decodeContentMap(contentmap).inserts.clients.forEach((ranges) => + ranges + .getIds() + .forEach((range) => + range.attrs.forEach((attr) => attrs.set(attr.name, attr.val)), + ), + ); + return attrs; +}; + +beforeEach(() => { + vi.resetModules(); +}); + +describe('create-ydoc', () => { + it('creates the document without the compute pool’s removed patchYdoc', async () => { + const handler = await createYdoc(); + const yhub = makeYhub(); + const update = makeUpdate('hello world'); + + const res = await handler(makeReq(update, { yhub })); + + expect(res.status).toBe(201); + expect(await res.json()).toEqual({ message: 'Document created' }); + expect(yhub.computePool.mergeUpdates).not.toHaveBeenCalled(); + expect(yhub.stream.addMessage).toHaveBeenCalledTimes(1); + }); + + it('appends the update as posted, as yhub’s own PATCH does since 0.9.1', async () => { + const handler = await createYdoc(); + const yhub = makeYhub(); + const update = makeUpdate('hello world'); + + await handler(makeReq(update, { yhub })); + + const [docRef, message] = yhub.stream.addMessage.mock + .calls[0] as unknown as [ + unknown, + { type: string; update: Uint8Array; contentmap: Uint8Array }, + ]; + expect(docRef).toEqual({ org: 'docs', docid: DOCID, branch: 'main' }); + expect(message.type).toBe('ydoc:update:v1'); + // an exact Uint8Array, not the Buffer `req.bytes()` resolved to: the + // stream-message schema compares constructors + expect(message.update.constructor).toBe(Uint8Array); + expect(Array.from(message.update)).toEqual(Array.from(update)); + // and it is a document yhub can read back + const doc = new Y.Doc(); + Y.applyUpdate(doc, message.update); + expect(doc.get('t', 'text').toString()).toBe('hello world'); + }); + + it('attributes every inserted id to the user the admin token names, now', async () => { + const handler = await createYdoc(); + const yhub = makeYhub(); + const update = makeUpdate('hello world'); + const before = Date.now(); + + await handler(makeReq(update, { yhub })); + + const [, { contentmap }] = yhub.stream.addMessage.mock + .calls[0] as unknown as [unknown, { contentmap: Uint8Array }]; + // the contentmap covers exactly the ids the update inserts + const covered = Y.createContentIdsFromContentMap( + Y.decodeContentMap(contentmap), + ); + const posted = Y.createContentIdsFromUpdate(update); + expect(Y.excludeContentIds(posted, covered).inserts.clients.size).toBe(0); + expect(Y.excludeContentIds(covered, posted).inserts.clients.size).toBe(0); + // with the author and the timestamp the activity timeline reads + const attrs = insertAttributes(contentmap); + expect(attrs.get('insert')).toBe('user-1'); + expect(attrs.get('insertAt')).toBeGreaterThanOrEqual(before); + expect(attrs.get('insertAt')).toBeLessThanOrEqual(Date.now()); + }); + + it('attributes a user’s own create to them, whatever X-User-Id says', async () => { + const handler = await createYdoc(); + const yhub = makeYhub(); + + await handler( + makeReq(makeUpdate(), { + yhub, + authInfo: { userid: 'user-2' }, + headers: { 'x-user-id': 'someone-else' }, + }), + ); + + const [, { contentmap }] = yhub.stream.addMessage.mock + .calls[0] as unknown as [unknown, { contentmap: Uint8Array }]; + expect(insertAttributes(contentmap).get('insert')).toBe('user-2'); + }); + + it('falls back to the admin identity when no X-User-Id is sent', async () => { + const handler = await createYdoc(); + const yhub = makeYhub(); + + await handler(makeReq(makeUpdate(), { yhub, headers: {} })); + + const [, { contentmap }] = yhub.stream.addMessage.mock + .calls[0] as unknown as [unknown, { contentmap: Uint8Array }]; + expect(insertAttributes(contentmap).get('insert')).toBe('system'); + }); + + it.each([ + ['truncated', makeUpdate('hello world').slice(0, -5)], + ['garbage', new Uint8Array([7, 7, 7, 7, 7, 7, 7, 7, 7, 7])], + [ + 'corrupted', + (() => { + const update = makeUpdate('hello world'); + update[5] ^= 0xff; + update[10] ^= 0xff; + return update; + })(), + ], + ])('refuses a %s update with 400 and writes nothing', async (_, update) => { + const handler = await createYdoc(); + const yhub = makeYhub(); + + const res = await handler(makeReq(update, { yhub })); + + expect(res.status).toBe(400); + expect(await res.json()).toEqual({ error: 'Invalid Yjs update' }); + expect(yhub.stream.addMessage).not.toHaveBeenCalled(); + }); + + it('refuses an update that inserts nothing, which would create no room', async () => { + // a delete set alone is a valid update of more than three bytes + const doc = new Y.Doc(); + doc.get('t', 'text').insert(0, 'abc'); + const stateVector = Y.encodeStateVector(doc); + doc.get('t', 'text').delete(0, 3); + const deleteOnly = Y.encodeStateAsUpdate(doc, stateVector); + expect(deleteOnly.byteLength).toBeGreaterThan(3); + const handler = await createYdoc(); + const yhub = makeYhub(); + + const res = await handler(makeReq(deleteOnly, { yhub })); + + expect(res.status).toBe(400); + expect(await res.json()).toEqual({ error: 'Empty update' }); + expect(yhub.stream.addMessage).not.toHaveBeenCalled(); + }); + + it('refuses an empty body before reading the room', async () => { + const handler = await createYdoc(); + const yhub = makeYhub(); + + const res = await handler(makeReq(new Uint8Array([0, 0]), { yhub })); + + expect(res.status).toBe(400); + expect(yhub.getDoc).not.toHaveBeenCalled(); + }); + + it('answers 409 on a room that already has content, and writes nothing', async () => { + const handler = await createYdoc(); + const yhub = makeYhub(makeUpdate('already here')); + + const res = await handler(makeReq(makeUpdate(), { yhub })); + + expect(res.status).toBe(409); + expect(yhub.stream.addMessage).not.toHaveBeenCalled(); + }); + + it('refuses an anonymous caller', async () => { + const handler = await createYdoc(); + const yhub = makeYhub(); + + const res = await handler(makeReq(makeUpdate(), { yhub, authInfo: null })); + + expect(res.status).toBe(401); + expect(yhub.stream.addMessage).not.toHaveBeenCalled(); + }); +}); diff --git a/src/yhub-server/__tests__/config.spec.ts b/src/yhub-server/__tests__/config.spec.ts index 8b60146f4..5b630f04b 100644 --- a/src/yhub-server/__tests__/config.spec.ts +++ b/src/yhub-server/__tests__/config.spec.ts @@ -25,6 +25,7 @@ beforeEach(() => { 'YHUB_TASK_DEBOUNCE_MS', 'YHUB_MIN_MESSAGE_LIFETIME_MS', 'YHUB_BACKEND_REQUEST_TIMEOUT_MS', + 'YHUB_AUTH_TIMEOUT_MS', 'YHUB_IDENTITY_CACHE_TTL_MS', 'YHUB_ABILITIES_CACHE_TTL_MS', 'YHUB_ACCESS_CACHE_TTL_MS', @@ -117,6 +118,28 @@ describe('the numeric tuning knobs', () => { vi.stubEnv('YHUB_BACKEND_REQUEST_TIMEOUT_MS', '5s'); await expect(load()).rejects.toThrow(/YHUB_BACKEND_REQUEST_TIMEOUT_MS/); }); + + it('gives the auth hooks 30s, above the 10s yhub 0.10 defaults to', async () => { + // `authorize` chains two backend calls and the 10s S3 fetch of a legacy + // seed: yhub's own default would cut a slow seed short + const { AUTH_TIMEOUT_MS } = await load(); + expect(AUTH_TIMEOUT_MS).toBe(30000); + + vi.resetModules(); + vi.stubEnv('YHUB_AUTH_TIMEOUT_MS', '45000'); + expect((await load()).AUTH_TIMEOUT_MS).toBe(45000); + }); + + it('refuses an auth timeout that is not a positive whole number of ms', async () => { + vi.stubEnv('YHUB_AUTH_TIMEOUT_MS', '0'); + await expect(load()).rejects.toThrow( + /YHUB_AUTH_TIMEOUT_MS must be an integer >= 1/, + ); + + vi.resetModules(); + vi.stubEnv('YHUB_AUTH_TIMEOUT_MS', '30s'); + await expect(load()).rejects.toThrow(/YHUB_AUTH_TIMEOUT_MS/); + }); }); describe('the origin allowlist', () => { diff --git a/src/yhub-server/__tests__/permissions.spec.ts b/src/yhub-server/__tests__/permissions.spec.ts index c83e52914..46ef5b267 100644 --- a/src/yhub-server/__tests__/permissions.spec.ts +++ b/src/yhub-server/__tests__/permissions.spec.ts @@ -173,11 +173,43 @@ describe('everything the browser must not reach', () => { 'reset-connections', 'restore-ydoc', 'reset-ydoc', + // the built-ins yhub 0.9.1 and 0.10.1 added: named versions, and the + // listing of the documents an org changed + 'version', + 'updated', 'some-endpoint-added-later', ])('refuses %s to an editor β€” there is no "*" fallback', (name) => { expect(grants(editor, { endpoint: { [name]: '-r--' } })).toBe(false); expect(grants(editor, { endpoint: { [name]: 'c---' } })).toBe(false); }); + + // yhub 0.9.1 added `version` (crud over named versions) and `publish` to the + // history facet. Docs builds its version panel from the activity alone, so + // neither is granted: a named version would outlive the access that wrote it. + it('grants no named versions to anyone, an editor included', () => { + for (const who of [reader, editor]) { + expect( + grants(who, { history: { from: ACCESS_SINCE, version: '-r--' } }), + ).toBe(false); + expect( + grants(who, { history: { from: ACCESS_SINCE, version: 'c---' } }), + ).toBe(false); + expect( + grants(who, { history: { from: ACCESS_SINCE, publish: true } }), + ).toBe(false); + } + }); + + it('normalizes the two new history keys to nothing granted', () => { + // what yhub hands a handler as `req.permissions.history` since 0.9.1 + expect(editor.history).toEqual({ + from: ACCESS_SINCE, + rollback: true, + prune: false, + version: '----', + publish: false, + }); + }); }); describe('the admin token', () => { @@ -197,6 +229,16 @@ describe('the admin token', () => { expect(grants(admin, { delete: ['hard'] })).toBe(false); expect(grants(admin, { history: { from: 0, prune: true } })).toBe(false); }); + + it('reaches the named-version route but is granted no version in it', () => { + // the `'*'` endpoint fallback covers yhub 0.9.1's `version` route, and the + // history facet is what still refuses every verb on it + expect(grants(admin, { endpoint: { version: 'crud' } })).toBe(true); + expect(grants(admin, { history: { from: 0, version: '-r--' } })).toBe( + false, + ); + expect(grants(admin, { history: { from: 0, publish: true } })).toBe(false); + }); }); describe('the public global routes', () => { diff --git a/src/yhub-server/__tests__/server.spec.ts b/src/yhub-server/__tests__/server.spec.ts index 6bd73eb9e..2911dbd75 100644 --- a/src/yhub-server/__tests__/server.spec.ts +++ b/src/yhub-server/__tests__/server.spec.ts @@ -107,6 +107,7 @@ beforeEach(() => { 'YHUB_TASK_DEBOUNCE_MS', 'YHUB_MIN_MESSAGE_LIFETIME_MS', 'YHUB_BACKEND_REQUEST_TIMEOUT_MS', + 'YHUB_AUTH_TIMEOUT_MS', 'YHUB_S3_PERSISTENCE', 'YHUB_S3_ENDPOINT_URL', 'YHUB_S3_ACCESS_KEY_ID', @@ -144,6 +145,19 @@ describe('a valid, minimal environment', () => { expect(redis.minMessageLifetime).toBe(60000); expect(worker.taskConcurrency).toBe(5); }); + + it('hands yhub our auth timeout rather than leaving its 10s default', async () => { + // yhub 0.10 abandons an auth hook after `server.authTimeout`; left unset, + // a legacy seed slower than 10s would refuse the connection it serves + await boot(); + expect(createYHub.lastConfig.server.authTimeout).toBe(30000); + }); + + it('threads YHUB_AUTH_TIMEOUT_MS into the yhub config', async () => { + vi.stubEnv('YHUB_AUTH_TIMEOUT_MS', '45000'); + await boot(); + expect(createYHub.lastConfig.server.authTimeout).toBe(45000); + }); }); describe('the process role', () => { @@ -342,4 +356,22 @@ describe('the authorize authz-cache wiring', () => { ).resolves.toBeNull(); expect(fetchMock).toHaveBeenCalledTimes(1); }); + + it('denies the org scope, which closes yhub 0.10’s updated-documents listing', async () => { + // `GET /updated/v1/{org}` is authorized against the org scope: a plugin + // that answered it would list every changed document id of the org + const { authorize, fetchMock } = await bootAuthorize(); + // the org scope's resource is the org name, not a docRef + const authorizeOrg = authorize as unknown as ( + scope: 'org', + org: string, + user: unknown, + ) => Promise; + + await expect(authorizeOrg('org', 'docs', user('u1'))).resolves.toBeNull(); + await expect( + authorizeOrg('org', 'docs', { userid: 'system', admin: true }), + ).resolves.toBeNull(); + expect(fetchMock).not.toHaveBeenCalled(); + }); }); diff --git a/src/yhub-server/package.json b/src/yhub-server/package.json index b30370c8d..002672ca9 100644 --- a/src/yhub-server/package.json +++ b/src/yhub-server/package.json @@ -17,8 +17,8 @@ "@aws-sdk/client-s3": "3.1146.0", "@sentry/node": "10.75.0", "@sentry/profiling-node": "10.75.0", - "@y/hub": "0.9.0", - "@y/y": "14.0.0-rc.25", + "@y/hub": "0.10.1", + "@y/y": "14.0.0-rc.29", "jose": "6.2.12", "lru-cache": "11.5.3", "prom-client": "15.1.3" diff --git a/src/yhub-server/src/api.ts b/src/yhub-server/src/api.ts index a72a8bbca..85757e8db 100644 --- a/src/yhub-server/src/api.ts +++ b/src/yhub-server/src/api.ts @@ -15,12 +15,12 @@ import { createDocumentPermissions, logger, } from '@y/hub'; +import * as Y from '@y/y'; import { backendPublicJwk } from './backend.js'; import { invalidateDocumentAuthz } from './authz-cache.js'; import { EMPTY_UPDATE_MAX_BYTES, - EMPTY_YDOC, MAX_CREATE_BYTES, ORG, READINESS_TIMEOUT_MS, @@ -71,6 +71,30 @@ const eraseContent = async ( await yhubApi.persistence.deleteTombstone(docRef); }; +// The contentmap yhub's own `PATCH ydoc` stamps on an update (its +// `createContentMap`, which the package does not export): every id the update +// inserts or deletes, attributed to `userid` at this moment. The insertAt is +// what gives the seeded content its entry in the activity timeline. +const attributeTo = ( + contentids: Y.ContentIds, + userid: string, +): Uint8Array => { + const now = Date.now(); + return Y.encodeContentMap( + Y.createContentMapFromContentIds( + contentids, + [ + Y.createContentAttribute('insert', userid), + Y.createContentAttribute('insertAt', now), + ], + [ + Y.createContentAttribute('delete', userid), + Y.createContentAttribute('deleteAt', now), + ], + ), + ); +}; + const readyLog = logger.child({ module: 'readiness' }); const resetLog = logger.child({ module: 'reset-ydoc' }); @@ -272,7 +296,7 @@ export const api = [ return jsonResponse(400, { error: 'Unknown branch' }); } const body = await req.bytes(); - // req.bytes() resolves to a Node Buffer, but the compute-task schema + // req.bytes() resolves to a Node Buffer, but the stream-message schema // requires an exact Uint8Array (lib0 $constructedBy compares the // constructor) β€” re-view the same bytes without copying const update = new Uint8Array( @@ -286,8 +310,8 @@ export const api = [ // ("HTTP/1.1 413 ") β€” legal, and callers switch on the code return jsonResponse(413, { error: 'Update too large' }); } - // yhub's "no effective content" convention β€” reject before it reaches - // a worker + // yhub's "no effective content" convention β€” reject before reading the + // room if (update.byteLength <= EMPTY_UPDATE_MAX_BYTES) { return jsonResponse(400, { error: 'Empty update' }); } @@ -316,38 +340,32 @@ export const api = [ const userid = (authInfo.admin === true && req.headers['x-user-id']) || authInfo.userid; - let result; + let contentids: Y.ContentIds; try { - // diffs the posted update against the (empty) current doc and - // stamps the attribution contentmap - result = await req.yhub.computePool.patchYdoc( - { - update, - currentDoc: gcDoc ?? EMPTY_YDOC, - userid, - customAttributions: [], - }, - { docRef: req.docRef }, - ); + // The structural scan yhub runs on an update before appending it, on + // the socket and in its own PATCH: a malformed or truncated update + // throws here. It used to be the compute worker's `patchYdoc` that + // threw, until yhub 0.9.1 removed it and started appending updates + // as they come. The update is the only untrusted input here, so a + // throw maps to 400; getDoc / addMessage failures stay generic 500s. + contentids = Y.createContentIdsFromUpdate(update); } catch { - // a malformed update makes the compute worker throw (yhub logs - // 'worker failed' and replaces the thread). The update is the only - // untrusted input here, so a rejection maps to 400; getDoc / - // addMessage failures stay generic 500s. return jsonResponse(400, { error: 'Invalid Yjs update' }); } - if (result == null) { - // structurally valid but no effective content (e.g. delete-set - // only). A "successful" create that leaves the room nonexistent - // would lie to the caller β€” a later create would not 409. + if (contentids.inserts.clients.size === 0) { + // structurally valid but no content (e.g. delete-set only). A + // "successful" create that leaves the room nonexistent would lie to + // the caller β€” a later create would not 409. return jsonResponse(400, { error: 'Empty update' }); } - // on a fresh room this creates the stream, schedules compaction, and - // fans out to any live subscribers β€” nothing else to do + // Appended as posted, which is what yhub's PATCH does since 0.9.1: the + // room was checked empty above, so there is nothing to diff it against. + // On a fresh room this creates the stream, schedules compaction, and + // fans out to any live subscribers β€” nothing else to do. await req.yhub.stream.addMessage(req.docRef, { type: 'ydoc:update:v1', - contentmap: result.contentmap, - update: result.update, + contentmap: attributeTo(contentids, userid), + update, }); return jsonResponse(201, { message: 'Document created' }); }, diff --git a/src/yhub-server/src/config.ts b/src/yhub-server/src/config.ts index dbbf4272d..d7119c979 100644 --- a/src/yhub-server/src/config.ts +++ b/src/yhub-server/src/config.ts @@ -152,15 +152,12 @@ export const YHUB_AUDIENCE = 'yhub'; // miss its S3 object and fork the document's lineage) export const UUID4 = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; -// an empty Yjs update (what `Y.encodeStateAsUpdate(new Y.Doc())` encodes to) β€” -// hardcoded so we don't import @y/y for two bytes -export const EMPTY_YDOC = new Uint8Array([0, 0]); // yhub's "no effective content" convention: an empty update encodes to 2 bytes, // and anything up to 3 is read as an empty document export const EMPTY_UPDATE_MAX_BYTES = 3; // uws buffers the whole body before the handler sees it, so this cap does not -// bound upload memory β€” it bounds what a single create hands to a compute -// worker and writes to the valkey stream as one message. Creates carry one +// bound upload memory β€” it bounds what a single create scans on the main +// thread and writes to the valkey stream as one message. Creates carry one // freshly-converted snapshot (typically KBs); anything bigger belongs on the // websocket path. export const MAX_CREATE_BYTES = 10 * 1024 * 1024; @@ -178,6 +175,14 @@ export const BACKEND_REQUEST_TIMEOUT_MS = intEnv( 'YHUB_BACKEND_REQUEST_TIMEOUT_MS', 5000, ); +// How long yhub waits for `authenticate` or `authorize` before it gives up on +// the call and answers 503 (a websocket recheck closes with 1013); yhub +// defaults to 10s since 0.10.0. Our `authorize` chains two backend calls and, +// on the first open of a legacy document, its seed from S3 (10s on its own), +// so 10s would turn every slow seed into a refused connection even though each +// of those steps is bounded already. This is the backstop above them, and it +// has to be raised along with YHUB_BACKEND_REQUEST_TIMEOUT_MS. +export const AUTH_TIMEOUT_MS = intEnv('YHUB_AUTH_TIMEOUT_MS', 30000); // The authz cache (see authz-cache.ts): how long each layer may serve what a // previous connection read from the backend. All three default to 0 β€” caching // off, the behavior yhub has always had β€” and a deployment turns them up once diff --git a/src/yhub-server/src/server.ts b/src/yhub-server/src/server.ts index b63b6ea4a..21369e691 100644 --- a/src/yhub-server/src/server.ts +++ b/src/yhub-server/src/server.ts @@ -23,6 +23,7 @@ import { import { ANONYMOUS_USERID, API_PREFIX, + AUTH_TIMEOUT_MS, BACKEND_REQUEST_TIMEOUT_MS, MIN_MESSAGE_LIFETIME_MS, ORG, @@ -468,6 +469,9 @@ const yhub: YHub = (await createYHub({ // on a cross-origin `fetch`; it is also why the list has to be concrete, browsers // refusing "*" together with Access-Control-Allow-Credentials. cors: { origin: allowedOrigins, credentials: true }, + // see AUTH_TIMEOUT_MS: yhub's own default is shorter than the legacy + // seed `authorize` may legitimately wait on + authTimeout: AUTH_TIMEOUT_MS, } : null, worker: RUNS_WORKER @@ -502,6 +506,7 @@ logger.info( taskDebounceMs: yhub.stream.taskDebounce, minMessageLifetimeMs: yhub.stream.minMessageLifetime, backendRequestTimeoutMs: BACKEND_REQUEST_TIMEOUT_MS, + authTimeoutMs: RUNS_SERVER ? AUTH_TIMEOUT_MS : null, }, 'yhub configuration', ); diff --git a/src/yhub-server/yarn.lock b/src/yhub-server/yarn.lock index 940c22ca9..618411bc4 100644 --- a/src/yhub-server/yarn.lock +++ b/src/yhub-server/yarn.lock @@ -823,16 +823,16 @@ resolved "https://registry.yarnpkg.com/@y-crdt/yn/-/yn-0.1.4.tgz#8cd8329bcf0b30a7614a10fbd1018dde5ab54f45" integrity sha512-BrRpTE4tvONSx+hYXbpZERu7Cx3/xyHFNXKNnBd9/maTJNsCGaVwfkfH3PN9fx/IrIXxa3GTjX0t8zJam273BQ== -"@y/hub@0.9.0": - version "0.9.0" - resolved "https://registry.yarnpkg.com/@y/hub/-/hub-0.9.0.tgz#9a323fd43cb2e728920b9d9d51358f347fc08656" - integrity sha512-xGzLoPpJ1lrZo++ULu0IAs3bnLpjCNhsEgHcRiIf+XaTO5iu7GjwnbO6HGqI4LiP45Dn952kkh+WrmhK8qkfaQ== +"@y/hub@0.10.1": + version "0.10.1" + resolved "https://registry.yarnpkg.com/@y/hub/-/hub-0.10.1.tgz#6e0145c1bdcab3ac26283a673ad850556b9e841f" + integrity sha512-etcSmaZs4ItqArc0v+bwIixs03quzDBMGJWixpNHA27eZj+AutHMlykXWuSz6H41J/wGYPd6mCLdiYbSdiEGJQ== dependencies: "@y-crdt/yn" "^0.1.4" "@y/protocols" "^1.0.6-rc.1" - "@y/y" "^14.0.0-rc.24" - lib0 "^1.0.0-rc.27" - minio "^8.0.6" + "@y/y" "^14.0.0-rc.29" + lib0 "^1.0.0-rc.36" + minio "^8.0.7" pino "^10.3.1" postgres "^3.4.3" redis "^5.10.0" @@ -845,12 +845,12 @@ dependencies: lib0 "^1.0.0-rc.1" -"@y/y@14.0.0-rc.25", "@y/y@^14.0.0-rc.24": - version "14.0.0-rc.25" - resolved "https://registry.yarnpkg.com/@y/y/-/y-14.0.0-rc.25.tgz#e5b2d825406984dc71fe65a71977ffe005ffd273" - integrity sha512-kG61I9GLF85aNtBelJKcf6D6PhLoWUBWQ2kyTAKwOWRUnuCkzGRCp3gUdwcTB9TINtgC9w5kPAKzstNDIpwSXw== +"@y/y@14.0.0-rc.29", "@y/y@^14.0.0-rc.29": + version "14.0.0-rc.29" + resolved "https://registry.yarnpkg.com/@y/y/-/y-14.0.0-rc.29.tgz#0a0a4edea1e8f64308ab5e0ee62dd4bde8bf3dd0" + integrity sha512-A4W9Bnrfj7N7IgampoaMSygJlzShLkWZ2uqKjx92tsEqGnbjpfr7jFP1i3zQ37h3L1cXIeipr9QDneUrZHuDjA== dependencies: - lib0 "^1.0.0-rc.28" + lib0 "^1.0.0-rc.35" acorn-jsx@^5.3.2: version "5.3.2" @@ -1374,11 +1374,16 @@ levn@^0.4.1: prelude-ls "^1.2.1" type-check "~0.4.0" -lib0@^1.0.0-rc.1, lib0@^1.0.0-rc.27, lib0@^1.0.0-rc.28: +lib0@^1.0.0-rc.1: version "1.0.0-rc.30" resolved "https://registry.yarnpkg.com/lib0/-/lib0-1.0.0-rc.30.tgz#2c2aec2c91a99230891918e0a551472a6cc8052e" integrity sha512-WhIbHSbBMttOT3Ke/HDA5+kIf9KRDSd5L+aJMBkvJSxwqL7w7004/0X5fjXj8loBIm5PkdgSMRmlPjC2mIzuCw== +lib0@^1.0.0-rc.35, lib0@^1.0.0-rc.36: + version "1.0.0-rc.36" + resolved "https://registry.yarnpkg.com/lib0/-/lib0-1.0.0-rc.36.tgz#aa0b3cdfacb275c2cbce401cae3c8a99a3a49cc2" + integrity sha512-7t/I+PCQqakal22rhXq5giOUtDP/RfecQtdeKvzCtnAvPkxSLAE+/Zvzi/kJDopg+hBAYKo46qmxmlkhX0tYlQ== + lightningcss-android-arm64@1.33.0: version "1.33.0" resolved "https://registry.yarnpkg.com/lightningcss-android-arm64/-/lightningcss-android-arm64-1.33.0.tgz#9a6841f88ae50fc83502903892b41af41bc2b907" @@ -1496,7 +1501,7 @@ minimatch@^10.2.1, minimatch@^10.2.2, minimatch@^10.2.4, minimatch@^10.2.5: dependencies: brace-expansion "^5.0.8" -minio@^8.0.6: +minio@^8.0.7: version "8.0.7" resolved "https://registry.yarnpkg.com/minio/-/minio-8.0.7.tgz#6a651151f3d2df1077ca894e46efc47f4a1ec25c" integrity sha512-E737MgufW8CeQAsTAtnEMrxZ9scMSf29kkhZoXzDTKj/Jszzo2SfeZUH9wbDQH2Rsq6TCtl/yQL0+XdVKZansQ==