From 935143501e9331e6c15d18e6083ef09dfbbf6c07 Mon Sep 17 00:00:00 2001 From: Julien Maupetit Date: Tue, 8 Sep 2026 18:36:47 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=A7=91=E2=80=8D=F0=9F=92=BB(project)=20im?= =?UTF-8?q?prove=20e2e=20tests=20environment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Running end-to-end tests locally can be a burden as there are many services to start and configure. It can be improved by avoiding to restart the whole stack at every test run. We've also added documentation and fix permission issues regarding volumes and locally created files and directories. --- .github/workflows/drive-frontend.yml | 50 +++++-------- Makefile | 102 +++++++++++++++++++++------ README.md | 85 +++++++++++++++------- compose.yaml | 39 ++++++++-- 4 files changed, 192 insertions(+), 84 deletions(-) diff --git a/.github/workflows/drive-frontend.yml b/.github/workflows/drive-frontend.yml index 79d846fa..e0f0e950 100644 --- a/.github/workflows/drive-frontend.yml +++ b/.github/workflows/drive-frontend.yml @@ -85,12 +85,12 @@ jobs: fail-on-cache-miss: true - name: Build frontend (static export, .env.development) - run: | - cd src/frontend/apps/drive - set -a - . ./.env.development - set +a - yarn build + env: + NEXT_PUBLIC_S3_DOMAIN_REPLACE: http://localhost:9000 + NEXT_PUBLIC_API_ORIGIN: http://localhost:8071 + NEXT_PUBLIC_POSTHOG_TEST_MODE: true + working-directory: src/frontend/apps/drive + run: yarn build - name: Upload frontend bundle uses: actions/upload-artifact@v4 @@ -151,24 +151,21 @@ jobs: - name: Install Playwright Browsers (cache miss) if: steps.playwright-cache.outputs.cache-hit != 'true' - run: | - cd src/frontend/apps/e2e - npx playwright install --with-deps ${{ matrix.browser }} + working-directory: src/frontend/apps/e2e + run: npx playwright install --with-deps ${{ matrix.browser }} - name: Install Playwright system deps (cache hit) if: steps.playwright-cache.outputs.cache-hit == 'true' - run: | - cd src/frontend/apps/e2e - npx playwright install-deps ${{ matrix.browser }} + working-directory: src/frontend/apps/e2e + run: npx playwright install-deps ${{ matrix.browser }} - name: Build the drive SDK - run: | - cd src/frontend/packages/sdk - yarn build + working-directory: src/frontend/packages/sdk + run: yarn build - name: Start sdk-consumer dev server + working-directory: src/frontend/apps/sdk-consumer run: | - cd src/frontend/apps/sdk-consumer nohup yarn dev > /tmp/sdk-consumer.log 2>&1 & echo "sdk-consumer PID $!" @@ -180,27 +177,14 @@ jobs: - name: Start Docker services run: | - make bootstrap-e2e + make bootstrap-e2e run-backend-e2e - name: Serve frontend (stock nginx + prod vhost) - run: | - docker run -d --rm --name drive-static -p 3000:3000 \ - -v "$PWD/src/frontend/apps/drive/out:/usr/share/nginx/html:ro" \ - -v "$PWD/src/frontend/apps/drive/conf/default.conf:/etc/nginx/conf.d/default.conf:ro" \ - nginx:1.25 - - - name: Wait for Keycloak to be ready - run: | - timeout 30 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' http://localhost:8083/)" != "302" ]]; do echo "Waiting for Keycloak..." && sleep 2; done' && echo "Keycloak is ready!" - - - name: Wait for frontend to be ready - run: | - timeout 30 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' http://localhost:3000/)" != "200" ]]; do echo "Waiting for frontend..." && sleep 1; done' && echo "Frontend is ready!" + run: make ci-serve-frontend-build - name: Run e2e tests - run: | - cd src/frontend/apps/e2e - yarn test --project=${{ matrix.browser }} --shard=${{ matrix.shard }} + working-directory: src/frontend/apps/e2e + run: yarn test --project=${{ matrix.browser }} --shard=${{ matrix.shard }} - uses: actions/upload-artifact@v4 if: always() diff --git a/Makefile b/Makefile index da58f48a..8d143986 100644 --- a/Makefile +++ b/Makefile @@ -49,8 +49,10 @@ MAIL_YARN = $(COMPOSE_RUN) -w /app/src/mail node yarn PSQL = ./bin/psql # -- Frontend -PATH_FRONT = ./src/frontend -PATH_FRONT_DRIVE = $(PATH_FRONT)/apps/drive +FRONTEND_PATH = ./src/frontend +DRIVE_APP_FRONTEND_PATH = $(FRONTEND_PATH)/apps/drive +CONSUMER_APP_FRONTEND_PATH = $(FRONTEND_PATH)/apps/sdk-consumer +DRIVE_SDK_FRONTEND_PATH = $(FRONTEND_PATH)/packages/sdk # ============================================================================== # RULES @@ -63,15 +65,47 @@ data/media: data/static: @mkdir -p data/static +data/postgresql.local: + @mkdir -p data/postgresql.local + +data/postgresql.e2e: + @mkdir -p data/postgresql.e2e + +env.d/development/crowdin.local: + @touch env.d/development/crowdin.local + +env.d/development/common.local: + @touch env.d/development/common.local + +env.d/development/postgresql.local: + @touch env.d/development/postgresql.local + +env.d/development/kc_postgresql.local: + @touch env.d/development/kc_postgresql.local + +src/frontend/node_modules: + @mkdir -p src/frontend/node_modules + +src/frontend/apps/drive/node_modules: + @mkdir -p src/frontend/apps/drive/node_modules + +src/frontend/apps/drive/out/index.html: + @mkdir -p src/frontend/apps/drive/out/ + @touch src/frontend/apps/drive/out/index.html + # -- Project -create-env-local-files: ## create env.local files in env.d/development -create-env-local-files: - @touch env.d/development/crowdin.local - @touch env.d/development/common.local - @touch env.d/development/postgresql.local - @touch env.d/development/kc_postgresql.local -.PHONY: create-env-local-files +create-dev-local-files: ## create local files and directories for development +create-dev-local-files: \ + data/postgresql.local \ + data/postgresql.e2e \ + src/frontend/node_modules \ + src/frontend/apps/drive/node_modules \ + env.d/development/crowdin.local \ + env.d/development/common.local \ + env.d/development/postgresql.local \ + env.d/development/kc_postgresql.local +.PHONY: create-dev-local-files create-docker-network: ## create the docker network if it doesn't exist @docker network create lasuite-network || true @@ -81,7 +115,7 @@ bootstrap: ## Prepare Docker images for the project bootstrap: \ data/media \ data/static \ - create-env-local-files \ + create-dev-local-files \ build \ create-docker-network \ migrate \ @@ -127,26 +161,37 @@ bootstrap-e2e: ## bootstrap the backend container for e2e tests, without fronten bootstrap-e2e: \ data/media \ data/static \ - create-env-local-files \ + create-dev-local-files \ build-backend \ create-docker-network \ back-i18n-compile \ - run-backend-e2e + migrate-e2e \ + frontend-development-install .PHONY: bootstrap-e2e clear-db-e2e: ## quickly clears the database for e2e tests, used in the e2e tests POSTGRES_DB=drive_e2e $(PSQL) < bin/clear_records.sql .PHONY: clear-db-e2e +is-e2e-backend-running: ## check if the backend is running (with configured e2e database) + @CONTAINER_ID=$$($(COMPOSE) ps app-dev --filter status=running -q | grep -v "🐳"); \ + docker inspect $$CONTAINER_ID --format "{{ range .Config.Env }}{{ println . }}{{ end }}" | \ + grep DB_NAME=drive_e2e || \ + (echo -e "e2e backend is not running. You should run the following command(s) first:\nmake bootstrap-e2e && make run-backend-e2e" && false) +.PHONY: is-e2e-backend-running + +migrate-e2e: ## run backend migrations for the e2e database + $(COMPOSE) stop postgresql app-dev + ENV_OVERRIDE=e2e $(MAKE) migrate +.PHONY: migrate-e2e + run-backend-e2e: ## start the backend container for e2e tests, always remove the postgresql.e2e volume first - @$(MAKE) stop - rm -rf data/postgresql.e2e - @ENV_OVERRIDE=e2e $(MAKE) run-backend - @ENV_OVERRIDE=e2e $(MAKE) migrate + $(COMPOSE) stop postgresql app-dev + ENV_OVERRIDE=e2e $(MAKE) run-backend .PHONY: run-backend-e2e run-tests-e2e: ## run the e2e tests, example: make run-tests-e2e -- --project chromium --headed - @$(MAKE) run-backend-e2e +run-tests-e2e: is-e2e-backend-running @args="$(filter-out $@,$(MAKECMDGOALS))" && \ cd src/frontend/apps/e2e && yarn test $${args:-${1}} .PHONY: run-tests-e2e @@ -348,20 +393,28 @@ help: # Front frontend-development-install: ## install the frontend locally - cd $(PATH_FRONT_DRIVE) && yarn + cd $(DRIVE_APP_FRONTEND_PATH) && yarn .PHONY: frontend-development-install frontend-lint: ## run the frontend linter - cd $(PATH_FRONT) && yarn lint + cd $(FRONTEND_PATH) && yarn lint .PHONY: frontend-lint run-frontend-development: ## Run the frontend in development mode @$(COMPOSE) stop frontend-dev - cd $(PATH_FRONT_DRIVE) && yarn dev + cd $(DRIVE_APP_FRONTEND_PATH) && yarn dev .PHONY: run-frontend-development +run-frontend-sdk-development: ## Run the frontend SDK consumer in development mode + cd $(CONSUMER_APP_FRONTEND_PATH) && yarn dev +.PHONY: run-frontend-development + +build-frontend-sdk: ## Build drive SDK package + cd $(DRIVE_SDK_FRONTEND_PATH) && yarn build +.PHONY: build-frontend-sdk + frontend-i18n-extract: ## Extract the frontend translation inside a json to be used for crowdin - cd $(PATH_FRONT) && yarn i18n:extract + cd $(FRONTEND_PATH) && yarn i18n:extract .PHONY: frontend-i18n-extract frontend-i18n-generate: ## Generate the frontend json files used for crowdin @@ -371,9 +424,14 @@ frontend-i18n-generate: \ .PHONY: frontend-i18n-generate frontend-i18n-compile: ## Format the crowin json files used deploy to the apps - cd $(PATH_FRONT) && yarn i18n:deploy + cd $(FRONTEND_PATH) && yarn i18n:deploy .PHONY: frontend-i18n-compile +ci-serve-frontend-build: ## service static build (used in the CI) +ci-serve-frontend-build: src/frontend/apps/drive/out/index.html + $(COMPOSE) up -d --wait static +.PHONY: ci-serve-frontend-build + # -- K8S build-k8s-cluster: ## build the kubernetes cluster using kind ./bin/start-kind.sh diff --git a/README.md b/README.md index d39c3eb2..ecbf0dbd 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@

# La Suite Drive: Collaborative File Sharing + **LaSuite Drive, where your files become collaborative assets through seamless teamwork.** @@ -31,22 +32,27 @@ LaSuite Drive is an open-source tool focused on file storage, editing and sharing with granular access control. ## Why use Drive ❓ + LaSuite Drive empowers teams to securely store, share, and collaborate on files while maintaining full control over their data through a user-friendly, open-source platform. ### Store + - Store your files securely in a centralized location - Access your files from anywhere with our web-based interface ### Find + - Powerful search capabilities to quickly locate files and folders - Organized file structure with intuitive navigation and filtering ### Collaborate -- Share files and folders with your team members + +- Share files and folders with your team members - Granular access control to ensure your information is secure and only shared with the right people - Create workspaces to organize team collaboration and manage shared resources #### Features + - File & folder upload with drag & drop support - File preview for PDF, images, audio, and video directly in the browser - Online document editing via WOPI protocol (compatible with Collabora, OnlyOffice) @@ -62,9 +68,11 @@ LaSuite Drive empowers teams to securely store, share, and collaborate on files ## Getting started 🔧 ### Self-host -* 🚀 Easy to install, scalable and secure file storage solution + +- 🚀 Easy to install, scalable and secure file storage solution #### LaSuite Drive is easy to install on your own servers + We use Kubernetes for our [production instance](https://fichiers.numerique.gouv.fr/). Check out the [docs](https://github.com/suitenumerique/drive/blob/main/docs/installation/kubernetes.md) to get detailed instructions and examples. For now we only have a documentation to install it on Kubernetes. We will be more than happy to improve this documentation with other methods. Feel free to make a PR ! @@ -72,14 +80,15 @@ For now we only have a documentation to install it on Kubernetes. We will be mor **Questions?** Open an issue on [GitHub](https://github.com/suitenumerique/drive/issues/new?template=Bug_report.md) or join our [Matrix community](https://matrix.to/#/#drive-official:matrix.org). #### Known instances + We hope to see many more, here is an incomplete list of public LaSuite Drive instances. Feel free to make a PR to add ones that are not listed below🙏 -| URL | Organisation | AccĂšs | -|--------------------------------------|--------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------| -| [fichiers.numerique.gouv.fr](/) | DINUM | RĂ©servĂ© aux agents publics français travaillant pour l'administration centrale et la sphĂšre publique Ă©largie. Connexion via ProConnect requise. | -| [fichiers.suite.anct.gouv.fr](/) | ANCT | RĂ©servĂ© aux agents publics français travaillant pour l'administration territoriale et la sphĂšre publique Ă©largie. Connexion via ProConnect requise. | -| [fichiers.lasuite.coop](/) | lasuite.coop | DĂ©monstration gratuite et ouverte Ă  tous. Les contenus et comptes sont rĂ©initialisĂ©s aprĂšs un mois. | -| [mosacloud.cloud](/) | mosa.cloud | Instance de dĂ©monstration de mosa.cloud, une entreprise nĂ©erlandaise proposant des services autour des applications La Suite. | +| URL | Organisation | AccĂšs | +| -------------------------------- | ------------ | --------------------------------------------------------------------------------------------------------------------------------------------------- | +| [fichiers.numerique.gouv.fr](/) | DINUM | RĂ©servĂ© aux agents publics français travaillant pour l'administration centrale et la sphĂšre publique Ă©largie. Connexion via ProConnect requise. | +| [fichiers.suite.anct.gouv.fr](/) | ANCT | RĂ©servĂ© aux agents publics français travaillant pour l'administration territoriale et la sphĂšre publique Ă©largie. Connexion via ProConnect requise. | +| [fichiers.lasuite.coop](/) | lasuite.coop | DĂ©monstration gratuite et ouverte Ă  tous. Les contenus et comptes sont rĂ©initialisĂ©s aprĂšs un mois. | +| [mosacloud.cloud](/) | mosa.cloud | Instance de dĂ©monstration de mosa.cloud, une entreprise nĂ©erlandaise proposant des services autour des applications La Suite. | ### Local Development (for contributors) @@ -108,6 +117,7 @@ The easiest way to start working on the project is to use GNU Make: ```bash $ make bootstrap ``` + This command builds the `app-dev` and `frontend-dev` containers, installs dependencies, performs database migrations and compile translations. It's a good idea to use this command each time you are pulling code from the project repository to avoid @@ -123,35 +133,65 @@ You will be prompted to log in. The default credentials are: username: drive password: drive ``` + Note that if you need to run them afterward, you can use the eponym Make rule: ```bash $ make run ``` + You can check all available Make rules using: ```bash $ make help ``` +#### Running end-to-end tests + +Prepare end-to-end test environment database and dependencies using: + +```bash +$ make bootstrap-e2e +``` + +Run the backend configured for end-to-end testing along with the frontend: + +```bash +$ make run-backend-e2e run-frontend-development +``` + +One also need to run the SDK-consumer that needs to be build first: + +```bash +$ make build-frontend-sdk run-frontend-sdk-development +``` + +Once the backend and its dependencies are running, start testing using playwright _via_: + +```bash +$ make run-tests-e2e +``` + #### Frontend development mode ⚠ For frontend work, it is often better to run the frontend in development mode locally. To do so, install the frontend dependencies with the following command: -```shellscript +```bash $ make frontend-development-install ``` + And run the frontend locally in development mode with the following command: -```shellscript +```bash $ make run-frontend-development ``` + #### Backend only To start all the services, except the frontend container, you can use the following command: -```shellscript +```bash $ make run-backend ``` @@ -165,6 +205,7 @@ You first need to create a superuser account: ```bash $ make superuser ``` + You can then login with sub `admin@example.com` and password `admin`. ### Feedback @@ -175,18 +216,19 @@ We'd love to hear your thoughts and hear about your experiments, so come and say This project is community-driven and PRs are welcome. Do not hesitate to get in touch if you have any question related to our implementation or design decisions. We <3 contributions of any kind, big and small : -* [Contribution guide](https://github.com/suitenumerique/drive/blob/main/CONTRIBUTING.md) -* [Translations](https://crowdin.com/project/lasuite-drive) -* [Chat with us!](https://matrix.to/#/#drive-official:matrix.org) -* Open a PR (see our instructions on [developing La Suite Drive locally](https://github.com/suitenumerique/drive/blob/main/docs/installation/README.md)) -* Submit a [feature request](https://github.com/suitenumerique/drive/issues/new?assignees=&labels=enhancement&template=Feature_request.md) or [bug report](https://github.com/suitenumerique/drive/issues/new?assignees=&labels=bug&template=Bug_report.md) +- [Contribution guide](https://github.com/suitenumerique/drive/blob/main/CONTRIBUTING.md) +- [Translations](https://crowdin.com/project/lasuite-drive) +- [Chat with us!](https://matrix.to/#/#drive-official:matrix.org) +- Open a PR (see our instructions on [developing La Suite Drive locally](https://github.com/suitenumerique/drive/blob/main/docs/installation/README.md)) +- Submit a [feature request](https://github.com/suitenumerique/drive/issues/new?assignees=&labels=enhancement&template=Feature_request.md) or [bug report](https://github.com/suitenumerique/drive/issues/new?assignees=&labels=bug&template=Bug_report.md) #### Gov ❀ open source -All features we develop will always remain open-source. + +All features we develop will always remain open-source. Come help us make LaSuite Drive even better. We're growing fast and would love some help. We are always looking for new partners, feel free to [contact us](mailto:fichiers@numerique.gouv.fr) if you are interested in using or contributing to LaSuite Drive.  - ## Roadmap + Curious where LaSuite Drive is headed? Explore upcoming features, priorities and long-term direction on our [public roadmap](https://docs.numerique.gouv.fr/docs/eacaabdb-d92b-465d-bedf-75d28b397221/). @@ -194,13 +236,8 @@ Explore upcoming features, priorities and long-term direction on our [public roa This work is released under the MIT License (see [LICENSE](./LICENSE)). -While LaSuite Drive is a public driven initiative our licence choice is an invitation for private sector actors to use, sell and contribute to the project. +While LaSuite Drive is a public driven initiative our licence choice is an invitation for private sector actors to use, sell and contribute to the project. ## Credits LaSuite Drive is built on top of [Django Rest Framework](https://www.django-rest-framework.org/), [Next.js](https://nextjs.org/). We thank the contributors of all these projects for their awesome work! - - - - - diff --git a/compose.yaml b/compose.yaml index 1122fbcf..29f941df 100644 --- a/compose.yaml +++ b/compose.yaml @@ -2,6 +2,7 @@ name: drive services: postgresql: + user: ${DOCKER_USER:-1000} image: postgres:16 ports: - "6434:5432" @@ -163,8 +164,26 @@ services: volumes: - ./docker/files/development/etc/nginx/conf.d:/etc/nginx/conf.d:ro depends_on: - - keycloak - - app-dev + keycloak: + condition: service_healthy + restart: true + app-dev: + condition: service_started + + # The static service is only used in the CI to serve the build + static: + image: nginx:1.25 + ports: + - "3000:3000" + healthcheck: + test: curl -sf http://127.0.0.1:3000/ || exit 1 + interval: 2s + timeout: 10s + retries: 30 + start_period: 2s + volumes: + - ./src/frontend/apps/drive/out:/usr/share/nginx/html:ro + - ./src/frontend/apps/drive/conf/default.conf:/etc/nginx/conf.d/default.conf:ro frontend-dev: user: "${DOCKER_USER:-1000}" @@ -177,8 +196,8 @@ services: image: drive:frontend-development volumes: - ./src/frontend/:/home/frontend/ - - /home/frontend/node_modules - - /home/frontend/apps/drive/node_modules + - ./src/frontend/node_modules:/home/frontend/node_modules + - ./src/frontend/apps/drive/node_modules:/home/frontend/apps/drive/node_modules ports: - "3000:3000" @@ -224,6 +243,14 @@ services: - --proxy=edge - --hostname=http://localhost:8083 - --hostname-strict=false + - --health-enabled=true + - --metrics-enabled=true + healthcheck: + test: ['CMD-SHELL', 'exec 3<>/dev/tcp/localhost/9000; echo -e "GET /health/live HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n" >&3; grep "HTTP/1.1 200 OK" <&3'] + start_period: 5s + interval: 1s + timeout: 2s + retries: 300 environment: KEYCLOAK_ADMIN: admin KEYCLOAK_ADMIN_PASSWORD: admin @@ -237,7 +264,9 @@ services: ports: - "8080:8080" depends_on: - - kc_postgresql + kc_postgresql: + condition: service_healthy + restart: true networks: - lasuite - default