diff --git a/CHANGELOG.md b/CHANGELOG.md index e78088fe8..e5ee1017e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ and this project adheres to - ✨(collaboration) add a get-ydoc endpoint on yhub - ✨(backend) duplicate a document through the collaboration server - ✨(backend) serve `documents/{id}/formatted-content/` from yhub +- ✨(collaboration) notify the backend when the worker persists new content ### Changed diff --git a/Makefile b/Makefile index 82729249a..5753f4475 100644 --- a/Makefile +++ b/Makefile @@ -69,11 +69,16 @@ data/media: data/static: @mkdir -p data/static -# RSA key signing the JWT tokens the backend issues. Generated locally, never -# committed: "data/" is gitignored. Regenerate it by deleting the file. +# RSA keys signing the JWT tokens the services issue: one for the backend, one +# for the collaboration server. Generated locally, never committed: "data/" is +# gitignored. Regenerate one by deleting the file. Both are listed, so a stack +# set up before the collaboration server had a key of its own gets it too. data/jwt/private.pem: @bin/generate-jwt-private-key.sh +data/jwt/yhub-private.pem: + @bin/generate-jwt-private-key.sh + # -- Project create-env-local-files: ## create env.local files in env.d/development @@ -87,7 +92,7 @@ create-env-local-files: generate-secret-keys: generate-secret-keys: ## generate the secret keys needed by the dev stack -generate-secret-keys: data/jwt/private.pem +generate-secret-keys: data/jwt/private.pem data/jwt/yhub-private.pem @bin/generate-oidc-store-refresh-token-key.sh .PHONY: generate-secret-keys diff --git a/UPGRADE.md b/UPGRADE.md index 06629eb5d..b7d3f5ac0 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -31,6 +31,28 @@ the following command inside your docker container: tokens Docs issues to call external services. If you enabled the resource server (`OIDC_RESOURCE_SERVER_ENABLED`), update the JWKS URI declared to your OIDC provider accordingly. +- ⚠️ The collaboration server now calls the backend on its own, to declare that + a document was edited, and signs those calls: **it needs an RSA private key + of its own**, which it had not before. Generate one and give it to the + collaboration server in `YHUB_JWT_PRIVATE_KEY`, or in a file + `YHUB_JWT_PRIVATE_KEY_FILE` points at: + + ```bash + openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out yhub-private.pem + ``` + + There is nothing to configure on the backend side: it reads the public half + from the JWKS the collaboration server publishes on `/collaboration/jwks/v1`, + which it fetches over `YHUB_API_BASE_URL` — so the two only need to reach + each other, and this key can be rolled without the backend being touched. + Do not share the backend key (`JWT_PRIVATE_KEY`) with it: each service signs + with a key of its own. + + Without this key the collaboration server keeps serving documents, and warns + at startup that it will not notify the backend: the `updated_at` of a document + then stops following the edits made in the editor, and the lists ordered by it + drift out of date. In a development environment, + `make generate-secret-keys` creates the key in `data/jwt/`. ### [5.0.0] - 2026-04-30 diff --git a/bin/generate-jwt-private-key.sh b/bin/generate-jwt-private-key.sh index 3d2963bd0..76d62eb48 100755 --- a/bin/generate-jwt-private-key.sh +++ b/bin/generate-jwt-private-key.sh @@ -1,23 +1,39 @@ #!/usr/bin/env bash -# Generate the RSA private key signing the JWT tokens issued by the backend. +# Generate the RSA keys signing the JWT tokens exchanged between the services. # -# Development only. The key is generated locally and never committed: it lands -# in "data/", which is gitignored. The dev stack mounts it in the backend -# containers, where JWT_PRIVATE_KEY_FILE points at it. +# Two directions, hence two keys: +# - "private.pem" signs the tokens the backend issues to call the converter and +# the collaboration server. +# - "yhub-private.pem" signs the calls the collaboration server makes to the +# backend. # -# Idempotent: an existing key is kept. Delete the file to roll the key. +# Only the private halves exist as files: each service publishes the public half +# of its own key on its JWKS endpoint, where the other one reads it. +# +# Development only. The keys are generated locally and never committed: they +# land in "data/", which is gitignored. The dev stack mounts them in the +# containers, where the *_FILE settings point at them. +# +# Idempotent: existing keys are kept. Delete a file to roll it. set -eo pipefail REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -KEY_PATH="${REPO_DIR}/data/jwt/private.pem" +KEY_DIR="${REPO_DIR}/data/jwt" -if [ -f "${KEY_PATH}" ]; then - exit 0 +mkdir -p "${KEY_DIR}" + +if [ ! -f "${KEY_DIR}/private.pem" ]; then + openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 \ + -out "${KEY_DIR}/private.pem" 2>/dev/null + chmod 600 "${KEY_DIR}/private.pem" + echo "✓ backend JWT private key generated in ${KEY_DIR}/private.pem" fi -mkdir -p "$(dirname "${KEY_PATH}")" -openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${KEY_PATH}" 2>/dev/null -chmod 600 "${KEY_PATH}" -echo "✓ JWT private key generated in ${KEY_PATH}" +if [ ! -f "${KEY_DIR}/yhub-private.pem" ]; then + openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 \ + -out "${KEY_DIR}/yhub-private.pem" 2>/dev/null + chmod 600 "${KEY_DIR}/yhub-private.pem" + echo "✓ collaboration JWT private key generated in ${KEY_DIR}/yhub-private.pem" +fi diff --git a/compose.yml b/compose.yml index 087b048fe..dd7b5940f 100644 --- a/compose.yml +++ b/compose.yml @@ -246,9 +246,13 @@ services: # seed rooms from the legacy Django/S3 document store on first access — # S3 endpoint/credentials come from env.d/development/common SOFT_MIGRATION: "true" + # signs the calls made to the backend, which holds the public half + YHUB_JWT_PRIVATE_KEY_FILE: /data/jwt/yhub-private.pem env_file: - env.d/development/common - env.d/development/common.local + volumes: + - ./data/jwt:/data/jwt:ro restart: unless-stopped ports: - "3002:3002" diff --git a/src/backend/core/api/viewsets.py b/src/backend/core/api/viewsets.py index 0bb1e29f2..330de4ab7 100644 --- a/src/backend/core/api/viewsets.py +++ b/src/backend/core/api/viewsets.py @@ -934,6 +934,37 @@ class DocumentViewSet( {"id": str(document.id)}, status=status.HTTP_201_CREATED ) + @drf.decorators.action( + authentication_classes=[authentication.CollaborationServerAuthentication], + detail=True, + methods=["post"], + permission_classes=[], + url_path="content-updated", + ) + def content_updated(self, request, *args, **kwargs): + """ + Record that the collaboration server saved a new content for a document. + + The content of a document does not go through Django anymore, so nothing + would refresh its "updated_at" as it is edited and the lists ordered by + it would freeze. The collaboration server calls this once it persisted + the changes of a document, at most once per debounce window. + + The update is written without going through the model, saving it would + trigger a re-indexing of a content Django did not see change. + """ + try: + document_id = uuid.UUID(kwargs["pk"]) + except ValueError as err: + raise Http404 from err + + if not models.Document.objects.filter(pk=document_id).update( + updated_at=timezone.now() + ): + raise Http404 + + return drf_response.Response(status=status.HTTP_204_NO_CONTENT) + @drf.decorators.action(detail=True, methods=["post"]) @transaction.atomic def move(self, request, *args, **kwargs): diff --git a/src/backend/core/authentication/__init__.py b/src/backend/core/authentication/__init__.py index c5fa0c711..6c615dacb 100644 --- a/src/backend/core/authentication/__init__.py +++ b/src/backend/core/authentication/__init__.py @@ -2,9 +2,13 @@ from django.conf import settings +import jwt from rest_framework.authentication import BaseAuthentication from rest_framework.exceptions import AuthenticationFailed +from core.services.jwt_services import JWTError +from core.services.yhub_services import YHubError, YHubService + class ServerToServerAuthentication(BaseAuthentication): """ @@ -50,3 +54,64 @@ class ServerToServerAuthentication(BaseAuthentication): def authenticate_header(self, request): """Return the WWW-Authenticate header value.""" return f"{self.TOKEN_TYPE} realm='Create document server to server'" + + +class CollaborationServerAuthentication(BaseAuthentication): + """ + Authenticate the collaboration server on the JWT it signs. + + The mirror of the token the backend signs to call it: the collaboration + server holds the private key and publishes the public half on its JWKS, + which is where we read it, and the tokens it mints live for a few minutes. + Nothing long-lived is shared between them, and either side can roll its key + without the other being reconfigured. + """ + + AUTH_HEADER = "Authorization" + TOKEN_TYPE = "Bearer" # noqa S105 + ALGORITHM = "RS256" + # The collaboration server mints its tokens for us, and only us: a token it + # signed for another service is refused here. + AUDIENCE = "docs-backend" + + def authenticate(self, request): + """ + Authenticate the request on the signature, the audience and the expiry + of the token it carries. The key validating the signature is the one + the collaboration server publishes for the "kid" the token names. + + Returns: + None: If authentication is successful, no user acts behind a call + of the collaboration server. + + Raises: + AuthenticationFailed: If the Authorization header is missing, + malformed, or carries a token we cannot validate. + """ + auth_header = request.headers.get(self.AUTH_HEADER) + if not auth_header: + raise AuthenticationFailed("Authorization header is missing.") + + auth_parts = auth_header.split(" ") + if len(auth_parts) != 2 or auth_parts[0] != self.TOKEN_TYPE: + raise AuthenticationFailed("Invalid authorization header.") + + token = auth_parts[1] + try: + signing_key = YHubService().jwks.get_signing_key(token) + jwt.decode( + token, + signing_key.key, + algorithms=[self.ALGORITHM], + audience=self.AUDIENCE, + ) + # a collaboration server we cannot reach, or one that publishes nothing + # we can verify a token with, authenticates nobody either + except (jwt.PyJWTError, JWTError, YHubError) as err: + raise AuthenticationFailed("Invalid collaboration server token.") from err + + # Authentication is successful, but no user is authenticated + + def authenticate_header(self, request): + """Return the WWW-Authenticate header value.""" + return f"{self.TOKEN_TYPE} realm='Collaboration server'" diff --git a/src/backend/core/services/jwt_services.py b/src/backend/core/services/jwt_services.py index 7ac666fba..6edafb5d7 100644 --- a/src/backend/core/services/jwt_services.py +++ b/src/backend/core/services/jwt_services.py @@ -12,12 +12,24 @@ from django.core.cache import cache from django.utils import timezone import jwt +import requests from joserfc.jwk import KeySet, RSAKey logger = logging.getLogger(__name__) ALGORITHM = "RS256" CACHE_KEY_PREFIX = "jwt_token" +JWKS_CACHE_KEY_PREFIX = "jwks" +# How long a fetched JWKS is served from the cache before being fetched again. +JWKS_CACHE_TIMEOUT = 300 +# Minimum delay, in seconds, between two fetches of the same JWKS. A token +# names the key that signed it and anybody can name one that does not exist, so +# the refresh a rotation needs is rate limited: an unknown key costs at most one +# fetch per window, not one per request. +JWKS_REFRESH_COOLDOWN = 30 +# Timeout, in seconds, of the fetch of a JWKS. Short: it happens while +# authenticating a request. +JWKS_FETCH_TIMEOUT = 10 class Audiences(StrEnum): @@ -39,6 +51,10 @@ class TokenGenerationError(JWTError): """Raised when a token cannot be signed.""" +class JWKSError(JWTError): + """Raised when the keys validating the tokens of a service cannot be used.""" + + @functools.cache def import_private_key(private_key): """ @@ -66,6 +82,98 @@ def import_private_key(private_key): raise ConfigurationError("The JWT private key cannot be imported.") from err +@functools.cache +def import_jwks(jwks): + """ + Import a JSON Web Key Set, as published by the service issuing the tokens. + + Importing keys is expensive, hence the cache. It is keyed on the document + itself, so a service publishing a new key gets it imported instead of the + previous set being served forever. + """ + try: + return jwt.PyJWKSet.from_json(jwks) + # a JWKS is fetched from another service: anything malformed in it, from + # the JSON to the key material, must surface as a JWKS error + except (jwt.PyJWTError, AttributeError, TypeError, ValueError) as err: + raise JWKSError("The JWKS cannot be imported.") from err + + +class JWKSClient: + """ + Client of the JSON Web Key Set a service publishes to let us verify the + tokens it signs. + + Fetching and importing the keys on every token would be wasteful, so the + document is cached — in the Django cache, hence shared by our processes — + and its import memoized. The service can still roll its key without + anything changing here: a token signed by a key we do not know refreshes + the set. + """ + + def __init__(self, url, timeout=JWKS_FETCH_TIMEOUT): + """Bind the client to the url a service publishes its keys at.""" + self.url = url + self.timeout = timeout + + @property + def cache_key(self): + """Build the cache key holding the document published at our url.""" + digest = hashlib.sha256(self.url.encode("utf-8")).hexdigest() + return f"{JWKS_CACHE_KEY_PREFIX}:{digest}" + + def fetch(self): + """Fetch the published document and cache it, as published.""" + try: + response = requests.get(self.url, timeout=self.timeout) + response.raise_for_status() + except requests.RequestException as err: + logger.exception("Unable to fetch the JWKS at %s", self.url) + raise JWKSError(f"Unable to fetch the JWKS at {self.url}") from err + + cache.set(self.cache_key, response.text, JWKS_CACHE_TIMEOUT) + + return response.text + + def get_keys(self, refresh=False): + """Return the published keys, from the cache unless a refresh is asked.""" + jwks = None if refresh else cache.get(self.cache_key) + if jwks is None: + jwks = self.fetch() + + return import_jwks(jwks) + + def get_signing_key(self, token): + """ + Return the key a token was signed with, among the published ones. + + The header of the token names it, which is what makes a rotation + transparent: a key we do not know yet is looked for again in a freshly + fetched set. That name is not authenticated though, so the refresh is + rate limited, and a token naming a key nobody published is refused. + """ + try: + kid = jwt.get_unverified_header(token)["kid"] + except (jwt.PyJWTError, KeyError) as err: + raise JWKSError("The token does not name the key that signed it.") from err + + try: + return self.get_keys()[kid] + except KeyError: + pass + + # `add` only succeeds for the first caller of the cooldown window, + # whichever process it runs in + if not cache.add(f"{self.cache_key}:refresh", True, JWKS_REFRESH_COOLDOWN): + raise JWKSError(f'The JWKS at {self.url} has no key "{kid}".') + + logger.info('Unknown key "%s", refreshing the JWKS at %s', kid, self.url) + try: + return self.get_keys(refresh=True)[kid] + except KeyError as err: + raise JWKSError(f'The JWKS at {self.url} has no key "{kid}".') from err + + class JWTService: """ Service class issuing RS256 signed JSON Web Tokens. diff --git a/src/backend/core/services/yhub_services.py b/src/backend/core/services/yhub_services.py index c0f74c208..fed8c3ce5 100644 --- a/src/backend/core/services/yhub_services.py +++ b/src/backend/core/services/yhub_services.py @@ -13,6 +13,10 @@ update), `rollback`, `prune`, `changeset` and `activity`, all at `v1`. yhub also accepts a `branch` query parameter, but our auth plugin only ever grants access to the `main` branch, so this service never sends it. +A few routes are about the server itself rather than about a document, and +carry no room: `/{prefix}/jwks/{version}` publishes the public keys validating +the tokens yhub signs to call us back. + This service only owns the transport for now, the endpoints are added as we need them. """ @@ -23,7 +27,7 @@ from django.conf import settings import requests -from core.services.jwt_services import Audiences, JWTService +from core.services.jwt_services import Audiences, JWKSClient, JWTService logger = logging.getLogger(__name__) @@ -134,6 +138,22 @@ class YHubService: ) return f"Bearer {token}" + @property + def jwks_url(self): + """Return the url yhub publishes its public keys at.""" + return f"{self.base_url}/{self.api_prefix}/jwks/{self.api_version}" + + @property + def jwks(self): + """ + Return the client of the keys validating the tokens yhub signs. + + The mirror of the JWKS we publish for the tokens we sign to call it: + neither side holds a copy of the key of the other, so either can roll + its own without the other being reconfigured. + """ + return JWKSClient(self.jwks_url) + def build_url(self, endpoint, document): """Build the url of a document scoped endpoint of the yhub API.""" return ( diff --git a/src/backend/core/tests/documents/test_api_documents_content_updated.py b/src/backend/core/tests/documents/test_api_documents_content_updated.py new file mode 100644 index 000000000..c77ea75ab --- /dev/null +++ b/src/backend/core/tests/documents/test_api_documents_content_updated.py @@ -0,0 +1,273 @@ +""" +Tests for Documents API endpoint in impress's core app: content updated +""" + +from datetime import datetime, timedelta +from datetime import timezone as tz +from uuid import uuid4 + +import jwt +import pytest +import responses +from freezegun import freeze_time +from rest_framework.test import APIClient + +from core import factories +from core.authentication import CollaborationServerAuthentication +from core.models import Document +from core.tests.utils.jwt_helper import build_jwks, generate_key_pair, key_id + +pytestmark = pytest.mark.django_db + +# Generating an RSA key is expensive, do it once for the whole module +PRIVATE_KEY, PUBLIC_KEY = generate_key_pair() +JWKS_URL = "http://yhub:3002/collaboration/jwks/v1" + + +@pytest.fixture(name="yhub_jwks", autouse=True) +def yhub_jwks_fixture(settings): + """ + Publish the collaboration server keys where the backend reads them. + + It only ever holds the public half of that key, and not even in its + configuration: it fetches it from the collaboration server itself. + """ + settings.YHUB_API_BASE_URL = "http://yhub:3002" + + with responses.RequestsMock(assert_all_requests_are_fired=False) as mock: + mock.get(JWKS_URL, json=build_jwks(PUBLIC_KEY)) + yield mock + + +def collaboration_token(private_key=PRIVATE_KEY, public_key=PUBLIC_KEY, **claims): + """Sign a token the way the collaboration server does.""" + issued_at = datetime.now(tz=tz.utc) + + return jwt.encode( + { + "iss": "yhub", + "aud": CollaborationServerAuthentication.AUDIENCE, + "iat": issued_at, + "exp": issued_at + timedelta(seconds=60), + **claims, + }, + private_key, + algorithm="RS256", + headers={"kid": key_id(public_key)}, + ) + + +def test_api_documents_content_updated_anonymous(): + """Anonymous users should not be allowed to declare a content update.""" + document = factories.DocumentFactory() + + response = APIClient().post(f"/api/v1.0/documents/{document.id!s}/content-updated/") + + assert response.status_code == 401 + + +def test_api_documents_content_updated_authenticated(): + """A logged-in user is not the collaboration server, their session is no credential.""" + user = factories.UserFactory() + client = APIClient() + client.force_login(user) + document = factories.DocumentFactory(users=[(user, "owner")]) + + response = client.post(f"/api/v1.0/documents/{document.id!s}/content-updated/") + + assert response.status_code == 401 + + +def test_api_documents_content_updated_token_signed_by_another_key(): + """🔒 A token signed by another key should not be allowed, published "kid" or not.""" + document = factories.DocumentFactory() + other_private_key, _other_public_key = generate_key_pair() + + response = APIClient().post( + f"/api/v1.0/documents/{document.id!s}/content-updated/", + # the key that signs it is not the one it names + HTTP_AUTHORIZATION=f"Bearer {collaboration_token(other_private_key)}", + ) + + assert response.status_code == 401 + + +def test_api_documents_content_updated_token_naming_an_unpublished_key(): + """A token naming a key the collaboration server does not publish is refused.""" + document = factories.DocumentFactory() + token = jwt.encode( + { + "aud": CollaborationServerAuthentication.AUDIENCE, + "exp": datetime.now(tz=tz.utc) + timedelta(seconds=60), + }, + PRIVATE_KEY, + algorithm="RS256", + headers={"kid": "a-key-nobody-published"}, + ) + + response = APIClient().post( + f"/api/v1.0/documents/{document.id!s}/content-updated/", + HTTP_AUTHORIZATION=f"Bearer {token}", + ) + + assert response.status_code == 401 + + +def test_api_documents_content_updated_token_naming_no_key(): + """A token that does not name the key it was signed with is refused.""" + document = factories.DocumentFactory() + token = jwt.encode( + { + "aud": CollaborationServerAuthentication.AUDIENCE, + "exp": datetime.now(tz=tz.utc) + timedelta(seconds=60), + }, + PRIVATE_KEY, + algorithm="RS256", + ) + + response = APIClient().post( + f"/api/v1.0/documents/{document.id!s}/content-updated/", + HTTP_AUTHORIZATION=f"Bearer {token}", + ) + + assert response.status_code == 401 + + +def test_api_documents_content_updated_token_for_another_audience(): + """A token the collaboration server minted for another service should be refused.""" + document = factories.DocumentFactory() + + response = APIClient().post( + f"/api/v1.0/documents/{document.id!s}/content-updated/", + HTTP_AUTHORIZATION=f"Bearer {collaboration_token(aud='somewhere-else')}", + ) + + assert response.status_code == 401 + + +def test_api_documents_content_updated_expired_token(): + """An expired token should be refused, they are short-lived on purpose.""" + document = factories.DocumentFactory() + expired = datetime.now(tz=tz.utc) - timedelta(seconds=60) + + response = APIClient().post( + f"/api/v1.0/documents/{document.id!s}/content-updated/", + HTTP_AUTHORIZATION=f"Bearer {collaboration_token(exp=expired)}", + ) + + assert response.status_code == 401 + + +def test_api_documents_content_updated_collaboration_server_not_configured(settings): + """Without a collaboration server to read the keys from, nothing is authenticated.""" + settings.YHUB_API_BASE_URL = None + document = factories.DocumentFactory() + + response = APIClient().post( + f"/api/v1.0/documents/{document.id!s}/content-updated/", + HTTP_AUTHORIZATION=f"Bearer {collaboration_token()}", + ) + + assert response.status_code == 401 + + +def test_api_documents_content_updated_jwks_unavailable(yhub_jwks): + """A collaboration server that publishes no key authenticates nobody.""" + yhub_jwks.reset() + yhub_jwks.get(JWKS_URL, status=500) + document = factories.DocumentFactory() + + response = APIClient().post( + f"/api/v1.0/documents/{document.id!s}/content-updated/", + HTTP_AUTHORIZATION=f"Bearer {collaboration_token()}", + ) + + assert response.status_code == 401 + + +def test_api_documents_content_updated_rolled_key(yhub_jwks): + """ + A key rolled on the collaboration server should be picked up on its own. + + This is what publishing a JWKS buys over a key pinned in our settings: + the tokens signed with the new key name a key we do not know, and looking + it up fetches the set again. + """ + document = factories.DocumentFactory() + url = f"/api/v1.0/documents/{document.id!s}/content-updated/" + + # a first call caches the keys published so far + response = APIClient().post( + url, HTTP_AUTHORIZATION=f"Bearer {collaboration_token()}" + ) + assert response.status_code == 204 + + new_private_key, new_public_key = generate_key_pair() + yhub_jwks.reset() + yhub_jwks.get(JWKS_URL, json=build_jwks(new_public_key)) + + response = APIClient().post( + url, + HTTP_AUTHORIZATION=( + f"Bearer {collaboration_token(new_private_key, new_public_key)}" + ), + ) + + assert response.status_code == 204 + + +def test_api_documents_content_updated(): + """The collaboration server should be able to refresh the date of a document.""" + with freeze_time("2026-08-01 12:00:00"): + # no content: writing one to S3 under a frozen clock breaks its signature + document = factories.DocumentFactory(title="my document", content="") + + with freeze_time("2026-08-06 12:00:00"): + response = APIClient().post( + f"/api/v1.0/documents/{document.id!s}/content-updated/", + HTTP_AUTHORIZATION=f"Bearer {collaboration_token()}", + ) + + assert response.status_code == 204 + + document.refresh_from_db() + assert document.updated_at == datetime(2026, 8, 6, 12, 0, 0, tzinfo=tz.utc) + # the document itself is left alone + assert document.created_at == datetime(2026, 8, 1, 12, 0, 0, tzinfo=tz.utc) + assert document.title == "my document" + + +def test_api_documents_content_updated_restricted_document(): + """ + The collaboration server acts for whoever is editing, the access of a + document is not its business. + """ + document = factories.DocumentFactory(link_reach="restricted") + + response = APIClient().post( + f"/api/v1.0/documents/{document.id!s}/content-updated/", + HTTP_AUTHORIZATION=f"Bearer {collaboration_token()}", + ) + + assert response.status_code == 204 + + +def test_api_documents_content_updated_unknown_document(): + """A document deleted in the meantime should answer a 404.""" + response = APIClient().post( + f"/api/v1.0/documents/{uuid4()!s}/content-updated/", + HTTP_AUTHORIZATION=f"Bearer {collaboration_token()}", + ) + + assert response.status_code == 404 + assert not Document.objects.exists() + + +def test_api_documents_content_updated_invalid_document_id(): + """A room name that is no document id should answer a 404, not a 500.""" + response = APIClient().post( + "/api/v1.0/documents/not-an-uuid/content-updated/", + HTTP_AUTHORIZATION=f"Bearer {collaboration_token()}", + ) + + assert response.status_code == 404 diff --git a/src/backend/core/tests/test_services_jwks_client.py b/src/backend/core/tests/test_services_jwks_client.py new file mode 100644 index 000000000..a96566426 --- /dev/null +++ b/src/backend/core/tests/test_services_jwks_client.py @@ -0,0 +1,165 @@ +""" +This module contains tests for the JWKSClient class in the +core.services.jwt_services module. +""" + +from datetime import datetime, timedelta, timezone + +import jwt +import pytest +import responses + +from core.services.jwt_services import JWKSClient, JWKSError +from core.tests.utils.jwt_helper import build_jwks, generate_key_pair, key_id + +# Generating RSA keys is expensive, do it once for the whole module +PRIVATE_KEY, PUBLIC_KEY = generate_key_pair() +OTHER_PRIVATE_KEY, OTHER_PUBLIC_KEY = generate_key_pair() + +JWKS_URL = "http://service.example.com/jwks" + + +def signed_token(private_key=PRIVATE_KEY, public_key=PUBLIC_KEY, headers=None): + """ + Sign a token the way a service publishing a JWKS does. + + Unless the caller wants something else in there, the header names the key + the token can be validated with. + """ + return jwt.encode( + {"exp": datetime.now(tz=timezone.utc) + timedelta(seconds=60)}, + private_key, + algorithm="RS256", + headers={"kid": key_id(public_key)} if headers is None else headers, + ) + + +@pytest.fixture(name="jwks") +def jwks_fixture(): + """Serve the JWKS of the key this module signs its tokens with.""" + with responses.RequestsMock(assert_all_requests_are_fired=False) as mock: + mock.get(JWKS_URL, json=build_jwks(PUBLIC_KEY)) + yield mock + + +@pytest.mark.usefixtures("jwks") +def test_get_signing_key_returns_the_published_key(): + """The key a token names should validate its signature.""" + token = signed_token() + + key = JWKSClient(JWKS_URL).get_signing_key(token) + + assert jwt.decode(token, key.key, algorithms=["RS256"]) + + +def test_the_document_is_fetched_once(jwks): + """Validating tokens should not call the publisher every time.""" + client = JWKSClient(JWKS_URL) + + client.get_signing_key(signed_token()) + client.get_signing_key(signed_token()) + # the document is cached for the url, not for the client instance + JWKSClient(JWKS_URL).get_signing_key(signed_token()) + + assert len(jwks.calls) == 1 + + +def test_an_unknown_key_is_looked_for_in_a_fresh_document(jwks): + """A key published after the document was cached should still be found.""" + client = JWKSClient(JWKS_URL) + client.get_signing_key(signed_token()) + + # the service rolls its key and publishes the new one + jwks.reset() + jwks.get(JWKS_URL, json=build_jwks(OTHER_PUBLIC_KEY)) + token = signed_token(OTHER_PRIVATE_KEY, OTHER_PUBLIC_KEY) + + key = client.get_signing_key(token) + + assert jwt.decode(token, key.key, algorithms=["RS256"]) + assert len(jwks.calls) == 1 # the fetch of the refreshed document + + +@pytest.mark.usefixtures("jwks") +def test_a_key_nobody_published_is_refused(): + """A token can name any key, only a published one validates it.""" + with pytest.raises(JWKSError, match="has no key"): + JWKSClient(JWKS_URL).get_signing_key( + signed_token(headers={"kid": "a-key-nobody-published"}) + ) + + +def test_an_unknown_key_only_refreshes_once_per_cooldown(jwks): + """ + 🔒 A forged "kid" must not turn every request into a call to the publisher. + + Nothing authenticates the key a token names, so without a cooldown an + unauthenticated caller would have us fetch the document as often as it asks. + """ + client = JWKSClient(JWKS_URL) + + for _ in range(5): + with pytest.raises(JWKSError): + client.get_signing_key( + signed_token(headers={"kid": "a-key-nobody-published"}) + ) + + # the first call fills the cache, the second is the refresh of the window + assert len(jwks.calls) == 2 + + +@pytest.mark.usefixtures("jwks") +def test_a_token_naming_no_key_is_refused(): + """A token that does not name its key cannot be matched to one.""" + with pytest.raises(JWKSError, match="does not name"): + JWKSClient(JWKS_URL).get_signing_key(signed_token(headers={})) + + +@pytest.mark.usefixtures("jwks") +def test_a_token_that_is_not_a_token_is_refused(): + """A header we cannot even read is reported like any unusable token.""" + with pytest.raises(JWKSError, match="does not name"): + JWKSClient(JWKS_URL).get_signing_key("not-a-token") + + +@responses.activate +def test_an_unreachable_publisher_is_reported(): + """A service we cannot fetch the keys from validates no token.""" + responses.get(JWKS_URL, status=500) + + with pytest.raises(JWKSError, match="Unable to fetch"): + JWKSClient(JWKS_URL).get_signing_key(signed_token()) + + +@responses.activate +def test_a_malformed_document_is_reported(): + """A published document we cannot import validates no token either.""" + responses.get(JWKS_URL, body="not a jwks") + + with pytest.raises(JWKSError, match="cannot be imported"): + JWKSClient(JWKS_URL).get_signing_key(signed_token()) + + +@responses.activate +def test_a_document_without_any_usable_key_is_reported(): + """A publisher answering an empty set is not a publisher we can use.""" + responses.get(JWKS_URL, json={"keys": []}) + + with pytest.raises(JWKSError, match="cannot be imported"): + JWKSClient(JWKS_URL).get_signing_key(signed_token()) + + +@responses.activate +def test_the_documents_of_two_services_do_not_share_a_cache_entry(): + """Two publishers are two documents, whatever each of them holds.""" + other_url = "http://other-service.example.com/jwks" + responses.get(JWKS_URL, json=build_jwks(PUBLIC_KEY)) + responses.get(other_url, json=build_jwks(OTHER_PUBLIC_KEY)) + + JWKSClient(JWKS_URL).get_signing_key(signed_token()) + key = JWKSClient(other_url).get_signing_key( + signed_token(OTHER_PRIVATE_KEY, OTHER_PUBLIC_KEY) + ) + + assert key.key_id == key_id(OTHER_PUBLIC_KEY) + assert len(responses.calls) == 2 diff --git a/src/backend/core/tests/test_services_yhub_services.py b/src/backend/core/tests/test_services_yhub_services.py index a9dfaf7c4..08d08bf57 100644 --- a/src/backend/core/tests/test_services_yhub_services.py +++ b/src/backend/core/tests/test_services_yhub_services.py @@ -60,6 +60,14 @@ def test_build_url(): assert url == f"http://yhub:3002/collaboration/ydoc/v1/docs/{DOCUMENT.id!s}" +def test_jwks_url(): + """The keys validating what yhub signs should be read from yhub itself.""" + service = YHubService() + + assert service.jwks_url == "http://yhub:3002/collaboration/jwks/v1" + assert service.jwks.url == service.jwks_url + + def test_auth_header(): """The auth header should carry an admin JWT signed with the configured key.""" scheme, token = YHubService().auth_header.split(" ") diff --git a/src/backend/core/tests/utils/jwt_helper.py b/src/backend/core/tests/utils/jwt_helper.py index f97bb18ea..139e6e0c1 100644 --- a/src/backend/core/tests/utils/jwt_helper.py +++ b/src/backend/core/tests/utils/jwt_helper.py @@ -2,6 +2,7 @@ from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import rsa +from joserfc.jwk import KeySet, RSAKey def generate_key_pair(): @@ -21,3 +22,24 @@ def generate_key_pair(): .decode("utf-8") ) return private_pem, public_pem + + +def key_id(public_pem): + """ + Return the "kid" naming a key, the way every service here names its own. + + It is the RFC 7638 thumbprint of the key, computed from its public + components: the signer stamps it in the header of its tokens and publishes + it in its JWKS, which is how the two are matched. + """ + return RSAKey.import_key(public_pem).thumbprint() + + +def build_jwks(public_pem): + """Build the JWKS a service publishes for a PEM encoded RSA public key.""" + key = RSAKey.import_key( + public_pem, + parameters={"alg": "RS256", "use": "sig", "kid": key_id(public_pem)}, + ) + + return KeySet([key]).as_dict(private=False) diff --git a/src/yhub-server/README.md b/src/yhub-server/README.md index c87929235..0d517869f 100644 --- a/src/yhub-server/README.md +++ b/src/yhub-server/README.md @@ -39,14 +39,30 @@ It is not a fork of yhub — it is a thin wrapper: - exposes `POST /collaboration/migrate/v1/{org}/{docid}`, which replays a document's **full** legacy version history out of the S3 media bucket (see "Full migration" below) — admin JWT only, like `reset-connections`, +- exposes `GET /collaboration/get-ydoc/v1/{org}/{docid}`, the read counterpart + of `create-ydoc`: the current state of a document as a raw binary update + (204 when it has no content), which the Django backend reads to export or + duplicate a document. Guarded by standard document read access, +- notifies the Django backend on + `POST /api/v1.0/documents/{id}/content-updated/` whenever the worker + persists new content for a document, so that lists ordered by `updated_at` + follow the edits made here. Signed with an RS256 JWT of our own + (`YHUB_JWT_PRIVATE_KEY`, `aud: "docs-backend"`, one minute), best effort: a + notification the backend refuses or never receives is logged and dropped, +- publishes the public half of that key on `GET /collaboration/jwks/v1`, where + the backend reads it. The exact mirror of the JWKS the backend publishes for + its own tokens: neither side is configured with a copy of the key of the + other, so either can roll its key on its own. Served unauthenticated, as any + JWKS is, - mirrors the environment conventions used elsewhere in this repository (`*_FILE` secret indirection, `COLLABORATION_SERVER_ORIGIN` allowlist, …). Public exposure: route the whole `/collaboration/` prefix to this server — the websocket and the built-in document APIs (`ydoc`, `rollback`, `prune`, `changeset`, `activity`) are all guarded by the same cookie-based document -authorization and are meant to be reachable by browsers. The one exception -is `/collaboration/reset-connections/` and `/collaboration/migrate/`, which are +authorization and are meant to be reachable by browsers, as is +`/collaboration/jwks/v1`, which carries public keys and nothing else. The one +exception is `/collaboration/reset-connections/` and `/collaboration/migrate/`, which are backend-internal and should not be routed through the public ingress. ## Container image diff --git a/src/yhub-server/server.js b/src/yhub-server/server.js index f2d3e293a..dbafe6da4 100644 --- a/src/yhub-server/server.js +++ b/src/yhub-server/server.js @@ -1,12 +1,22 @@ -import { createHash } from 'node:crypto'; +import { createHash, createPublicKey, randomUUID } from 'node:crypto'; +import { readFileSync } from 'node:fs'; import { apiError, createApiEndpoint, createAuthPlugin, createYHub, + logger, } from '@y/hub'; -import { createRemoteJWKSet, jwtVerify } from 'jose'; +import { + calculateJwkThumbprint, + createRemoteJWKSet, + exportJWK, + importPKCS8, + jwtVerify, + SignJWT, +} from 'jose'; +import { Client as S3Client } from 'minio'; import { secret } from './env.js'; // legacy Django/S3 document store — see migration.js and README.md @@ -29,6 +39,13 @@ const allowedOrigins = ( ).split(','); const Y_PROVIDER_API_KEY = secret('Y_PROVIDER_API_KEY', 'yprovider-api-key'); const ORG = process.env.YHUB_ORG || 'docs'; +// Segment every route is mounted under (`server.apiPrefix` below), matching the +// URL scheme Docs already routes to the collaboration server. Hardcoded like +// the audiences: the backend builds its urls with the same prefix. +const API_PREFIX = 'collaboration'; +// Path the JWKS endpoint declared in `api` is mounted at. readAuthInfo reads +// the raw request, without any route context, hence the duplication. +const JWKS_PATH = `/${API_PREFIX}/jwks/v1`; // Requiring this audience stops a valid admin JWT that Django issued for // another service (today: the y-converter token in converter_services.py, // which is handed to the converter process) from being replayed against yhub. @@ -51,6 +68,77 @@ const EMPTY_YDOC = new Uint8Array([0, 0]); // websocket path. const MAX_CREATE_BYTES = 10 * 1024 * 1024; +const touchLog = logger.child({ module: 'updated-at-notifier' }); + +const BACKEND_NOTIFY_TIMEOUT_MS = 5000; +// Audience of the tokens the backend accepts from us. It must match the one +// its CollaborationServerAuthentication requires, a token minted for anything +// else is refused there. +const BACKEND_AUDIENCE = 'docs-backend'; +const BACKEND_TOKEN_LIFETIME_S = 60; +// Renew this long before expiry so a token never dies in flight. +const BACKEND_TOKEN_MARGIN_MS = 10000; + +// We sign the calls we make to the backend, the mirror of the admin JWT it +// signs to call us: no long-lived shared secret, only our private key here and +// its public half published on the JWKS endpoint below. +const YHUB_JWT_PRIVATE_KEY = secret('YHUB_JWT_PRIVATE_KEY', ''); +const backendSigningKey = YHUB_JWT_PRIVATE_KEY + ? await importPKCS8(YHUB_JWT_PRIVATE_KEY, 'RS256') + : null; + +if (backendSigningKey == null) { + // not fatal, documents keep being served — only their `updated_at` freezes + touchLog.warn( + 'YHUB_JWT_PRIVATE_KEY is empty, the backend will not be notified of content updates', + ); +} + +// The public half of the signing key, as published on the JWKS endpoint. Its +// "kid" is the RFC 7638 thumbprint of the key: computed from the public +// components only, it is stable across restarts and changes on its own when +// the key is rolled. Every token we sign carries it, which is how the backend +// picks the matching key — and how it knows to fetch the set again when it +// does not know the key yet, so rolling this key needs no change on its side. +const backendPublicJwk = + backendSigningKey == null + ? null + : await (async () => { + // derived from the PEM rather than exported from `backendSigningKey`: + // exporting a private key as a JWK would carry its private components + const jwk = await exportJWK(createPublicKey(YHUB_JWT_PRIVATE_KEY)); + return { + ...jwk, + alg: 'RS256', + use: 'sig', + kid: await calculateJwkThumbprint(jwk), + }; + })(); + +/** + * @type {{ token: string, expiresAt: number } | null} + */ +let backendToken = null; + +// The token carries no per-document claim, so one is reused until it is about +// to expire rather than signing on every notification. +const getBackendToken = async () => { + const now = Date.now(); + if (backendToken != null && backendToken.expiresAt - BACKEND_TOKEN_MARGIN_MS > now) { + return backendToken.token; + } + const token = await new SignJWT({}) + // the "kid" names the key in our JWKS the backend must verify it with + .setProtectedHeader({ alg: 'RS256', kid: backendPublicJwk.kid }) + .setIssuer('yhub') + .setAudience(BACKEND_AUDIENCE) + .setIssuedAt() + .setExpirationTime(`${BACKEND_TOKEN_LIFETIME_S}s`) + .sign(backendSigningKey); + backendToken = { token, expiresAt: now + BACKEND_TOKEN_LIFETIME_S * 1000 }; + return token; +}; + // Public keys verifying the RS256 admin tokens Django issues (JWTService). // Lazily fetched on first use; jose caches the keys and refetches on unknown // "kid", so Django can rotate the signing key without a yhub restart. @@ -110,10 +198,19 @@ const seedFromLegacyStore = async (room) => { const auth = createAuthPlugin({ // uws req is only valid synchronously — read headers AND query before first await. async readAuthInfo(req) { + const url = req.getUrl(); const authorization = req.getHeader('authorization'); const cookie = req.getHeader('cookie'); const origin = req.getHeader('origin'); const gcOff = req.getQuery('gc') === 'false'; + // The JWKS holds public keys and nothing else, and the backend must be + // able to fetch it before it can authenticate anything we send it — so it + // is served to anyone, as the backend serves its own. This identity is + // granted the 'jwks' purpose and nothing else (getGlobalAccessType), and + // the check is on the exact path of that one route. + if (url === JWKS_PATH) { + return { userid: 'anonymous' }; + } if (authorization !== '') { // backend-to-server call: RS256 JWT signed by Django, verified against // its JWKS. A browser cannot attach an Authorization header to a ws @@ -183,6 +280,10 @@ const auth = createAuthPlugin({ return { userid: `anon:${anon}`, cookie, origin }; } }, + // Authorizes the global-scoped endpoints, of which the JWKS is the only one. + async getGlobalAccessType(authInfo, purpose) { + return purpose === 'jwks' ? 'r' : null; + }, async getAccessType(authInfo, { org, docid, branch }, purpose) { if (authInfo.admin === true) { // Django's admin token: full access. It still goes through the legacy @@ -254,6 +355,24 @@ const jsonResponse = (status, body) => }); const api = [ + // GET /collaboration/jwks/v1 — the public keys verifying the tokens we sign + // to call the backend, in the JSON Web Key Set format (RFC 7517). Global + // scope: it is about this server, not about a document, so the route carries + // no org and no docid. The counterpart of the backend's own /api/v1.0/jwks, + // which we read above to verify its tokens: neither side stores a copy of + // the other's key, so either can be rolled without the other being changed. + createApiEndpoint('jwks', { + scope: 'global', + accessPurpose: 'jwks', + get: { + // an empty set when no key is configured: honest, and the backend + // refuses our (equally absent) tokens rather than trusting anything + handler: () => + jsonResponse(200, { + keys: backendPublicJwk == null ? [] : [backendPublicJwk], + }), + }, + }), // POST /collaboration/reset-connections/v1/{org}/{docid} — replaces // y-provider's /collaboration/api/reset-connections/?room=. Doc-scoped, so // the room comes from the path; access is gated to the admin token via the @@ -475,8 +594,46 @@ const api = [ }), ]; -// referenced by getAccessType above — safe: auth callbacks only fire once the -// server is up, i.e. after this assignment +// Django orders the document lists by `updated_at` and no edit goes through it +// anymore, so it is told here that a document moved on. +const touchDocument = async (docid) => { + if (backendSigningKey == null) return; + try { + const res = await fetch( + `${COLLABORATION_BACKEND_BASE_URL}/api/v1.0/documents/${docid}/content-updated/`, + { + method: 'POST', + headers: { authorization: `Bearer ${await getBackendToken()}` }, + signal: AbortSignal.timeout(BACKEND_NOTIFY_TIMEOUT_MS), + }, + ); + if (!res.ok) { + touchLog.warn({ docid, status: res.status }, 'backend refused the notification'); + } + } catch (err) { + // best effort: a lost notification only leaves `updated_at` behind until + // the document is edited again, it must never fail a compaction + touchLog.warn({ err, docid }, 'could not notify the backend'); + } +}; + +// `docUpdate` is the worker event for "this compaction found new content": the +// task returns before it when it has nothing to persist, so the awareness-only +// traffic of someone merely opening a document never reaches it. Since yhub +// 0.5.0 it is handed the room of the task alongside the merged document. +const workerEvents = { + docUpdate: ({ room }) => { + // Django knows the documents of this org, on the main branch, by their uuid + if (room.org !== ORG || room.branch !== 'main' || !UUID4.test(room.docid)) { + return; + } + // deliberately not awaited: a slow backend must not hold the worker + touchDocument(room.docid); + }, +}; + +// the instance is referenced by the soft-migration helpers above — safe: auth +// callbacks only fire once the server is up, i.e. after this assignment const yhub = await createYHub({ redis: { url: REDIS, @@ -486,12 +643,8 @@ const yhub = await createYHub({ }, postgres: POSTGRES, persistence: [], // blobs live in yhub's postgres - // apiPrefix mounts every route — built-ins, reset-connections, and the - // websocket (/collaboration/ws/v1/{org}/{docid}) — under /collaboration/, - // matching the URL scheme Docs already routes to the collaboration server. - server: { port: PORT, auth, api, apiPrefix: 'collaboration' }, - worker: { taskConcurrency: 5 }, - // TODO(yhub): worker.events.docUpdate could push snapshots to Django and replace the - // client useSaveDoc PATCH flow. No longer blocked upstream — yhub 0.5.0 adds `room` - // to the event payload, which was the missing piece. + // apiPrefix mounts every route — built-ins, our custom endpoints, and the + // websocket (/collaboration/ws/v1/{org}/{docid}) — under /collaboration/. + server: { port: PORT, auth, api, apiPrefix: API_PREFIX }, + worker: { taskConcurrency: 5, events: workerEvents }, });