From 0d40256363ad84d3cc3947d99d54c2edd2845d6d Mon Sep 17 00:00:00 2001
From: Nicolas Clerc <kernicpanel@nclerc.fr>
Date: Wed, 8 Jul 2026 10:20:48 +0200
Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B(nginx)=20let=20search=20engines=20?=
 =?UTF-8?q?see=20the=20noindex=20directive?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

robots.txt blocked all crawling, so crawlers could never fetch the
pages and never saw the noindex meta tag: URLs discovered through
external links could still end up listed in search results.

Allow crawling in robots.txt and add a X-Robots-Tag noindex header on
frontend pages and media files so every response carries the directive.
---
 CHANGELOG.md                                  |  1 +
 .../development/etc/nginx/conf.d/default.conf |  2 ++
 .../production/etc/nginx/conf.d/default.conf  |  2 ++
 src/frontend/apps/drive/conf/default.conf     |  1 +
 src/frontend/apps/drive/public/robots.txt     |  5 +++-
 .../e2e/__tests__/app-drive/noindex.spec.ts   | 25 ++++++++++++++++---
 6 files changed, 31 insertions(+), 5 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e0a3c298..1b84f21c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,6 +16,7 @@ and this project adheres to
 
 ### Fixed
 
+- 🐛(nginx) let search engines see the noindex directive to drop public URLs
 - 🐛(backend) find deleted root items when searching the trashbin
 - 🐛(backend) exclude folders from file type search results
 - 🐛(frontend) keep uploaded items usable while malware analysis runs
diff --git a/docker/files/development/etc/nginx/conf.d/default.conf b/docker/files/development/etc/nginx/conf.d/default.conf
index 18af0dbf..1b7682f1 100644
--- a/docker/files/development/etc/nginx/conf.d/default.conf
+++ b/docker/files/development/etc/nginx/conf.d/default.conf
@@ -24,6 +24,7 @@ server {
         # proxy_pass http://ds-proxy:4444/upstream/drive-media-storage/;
         # proxy_set_header Host ds-proxy:4444;
         add_header Content-Disposition "attachment";
+        add_header X-Robots-Tag noindex always;
     }
 
     # Proxy auth for media
@@ -45,6 +46,7 @@ server {
         # To use with ds_proxy
         # proxy_pass http://ds-proxy:4444/upstream/drive-media-storage/;
         # proxy_set_header Host ds-proxy:4444;
+        add_header X-Robots-Tag noindex always;
     }
 
     location /media-auth {
diff --git a/docker/files/production/etc/nginx/conf.d/default.conf b/docker/files/production/etc/nginx/conf.d/default.conf
index 64c7b7f6..87ca696e 100644
--- a/docker/files/production/etc/nginx/conf.d/default.conf
+++ b/docker/files/production/etc/nginx/conf.d/default.conf
@@ -62,6 +62,8 @@ server {
         # Get resource from Minio
         proxy_pass http://minio:9000/docs-media-storage/;
         proxy_set_header Host minio:9000;
+
+        add_header X-Robots-Tag noindex always;
     }
 
     location /media-auth {
diff --git a/src/frontend/apps/drive/conf/default.conf b/src/frontend/apps/drive/conf/default.conf
index 92c81305..8b8eb5f7 100644
--- a/src/frontend/apps/drive/conf/default.conf
+++ b/src/frontend/apps/drive/conf/default.conf
@@ -8,6 +8,7 @@ server {
   root /usr/share/nginx/html;
 
   add_header X-Frame-Options DENY always;
+  add_header X-Robots-Tag noindex always;
 
   location / {
     try_files $uri index.html $uri/ =404;
diff --git a/src/frontend/apps/drive/public/robots.txt b/src/frontend/apps/drive/public/robots.txt
index 1f53798b..e806af66 100644
--- a/src/frontend/apps/drive/public/robots.txt
+++ b/src/frontend/apps/drive/public/robots.txt
@@ -1,2 +1,5 @@
+# Crawling is allowed on purpose: crawlers can only honor the noindex
+# directive (meta tag and X-Robots-Tag header) on pages they can fetch.
+# A "Disallow: /" would leave already discovered URLs indexed forever.
 User-agent: *
-Disallow: /
+Disallow:
diff --git a/src/frontend/apps/e2e/__tests__/app-drive/noindex.spec.ts b/src/frontend/apps/e2e/__tests__/app-drive/noindex.spec.ts
index 4921aae0..738e8bb8 100644
--- a/src/frontend/apps/e2e/__tests__/app-drive/noindex.spec.ts
+++ b/src/frontend/apps/e2e/__tests__/app-drive/noindex.spec.ts
@@ -13,14 +13,31 @@ test.describe("Search engine indexing prevention", () => {
     await expect(robotsMeta).toBeAttached();
   });
 
-  test("should serve robots.txt that disallows all crawlers", async ({
-    page,
-  }) => {
+  test("should serve robots.txt that allows crawling", async ({ page }) => {
+    // Crawling must stay allowed: crawlers can only honor the noindex
+    // directive on pages they are able to fetch.
     const response = await page.request.get("/robots.txt");
     expect(response.status()).toBe(200);
 
     const content = await response.text();
     expect(content).toContain("User-agent: *");
-    expect(content).toContain("Disallow: /");
+
+    const rules = content
+      .split("\n")
+      .filter((line) => !line.startsWith("#"))
+      .join("\n");
+    expect(rules).not.toMatch(/Disallow: \//);
+  });
+
+  test("should serve pages with a X-Robots-Tag noindex header", async ({
+    page,
+  }) => {
+    test.skip(
+      !process.env.CI,
+      "the header is added by nginx, not by the local dev server",
+    );
+
+    const response = await page.request.get("/");
+    expect(response.headers()["x-robots-tag"]).toBe("noindex");
   });
 });