🐛(nginx) let search engines see the noindex directive

robots.txt blocked all crawling, so crawlers could never fetch the
pages and never saw the noindex meta tag: URLs discovered through
external links could still end up listed in search results.

Allow crawling in robots.txt and add a X-Robots-Tag noindex header on
frontend pages and media files so every response carries the directive.

CHANGELOG.md

@@ -16,6 +16,7 @@ and this project adheres to
 
 ### Fixed
 
+- 🐛(nginx) let search engines see the noindex directive to drop public URLs
 - 🐛(backend) find deleted root items when searching the trashbin
 - 🐛(backend) exclude folders from file type search results
 - 🐛(frontend) keep uploaded items usable while malware analysis runs

docker/files/development/etc/nginx/conf.d/default.conf

@@ -24,6 +24,7 @@ server {
         # proxy_pass http://ds-proxy:4444/upstream/drive-media-storage/;
         # proxy_set_header Host ds-proxy:4444;
         add_header Content-Disposition "attachment";
+        add_header X-Robots-Tag noindex always;
     }
 
     # Proxy auth for media
@@ -45,6 +46,7 @@ server {
         # To use with ds_proxy
         # proxy_pass http://ds-proxy:4444/upstream/drive-media-storage/;
         # proxy_set_header Host ds-proxy:4444;
+        add_header X-Robots-Tag noindex always;
     }
 
     location /media-auth {

docker/files/production/etc/nginx/conf.d/default.conf

@@ -62,6 +62,8 @@ server {
         # Get resource from Minio
         proxy_pass http://minio:9000/docs-media-storage/;
         proxy_set_header Host minio:9000;
+
+        add_header X-Robots-Tag noindex always;
     }
 
     location /media-auth {

src/frontend/apps/drive/conf/default.conf

@@ -8,6 +8,7 @@ server {
   root /usr/share/nginx/html;
 
   add_header X-Frame-Options DENY always;
+  add_header X-Robots-Tag noindex always;
 
   location / {
     try_files $uri index.html $uri/ =404;

src/frontend/apps/drive/public/robots.txt

@@ -1,2 +1,5 @@
+# Crawling is allowed on purpose: crawlers can only honor the noindex
+# directive (meta tag and X-Robots-Tag header) on pages they can fetch.
+# A "Disallow: /" would leave already discovered URLs indexed forever.
 User-agent: *
-Disallow: /
+Disallow:

src/frontend/apps/e2e/__tests__/app-drive/noindex.spec.ts

@@ -13,14 +13,31 @@ test.describe("Search engine indexing prevention", () => {
     await expect(robotsMeta).toBeAttached();
   });
 
-  test("should serve robots.txt that disallows all crawlers", async ({
-    page,
-  }) => {
+  test("should serve robots.txt that allows crawling", async ({ page }) => {
+    // Crawling must stay allowed: crawlers can only honor the noindex
+    // directive on pages they are able to fetch.
     const response = await page.request.get("/robots.txt");
     expect(response.status()).toBe(200);
 
     const content = await response.text();
     expect(content).toContain("User-agent: *");
-    expect(content).toContain("Disallow: /");
+
+    const rules = content
+      .split("\n")
+      .filter((line) => !line.startsWith("#"))
+      .join("\n");
+    expect(rules).not.toMatch(/Disallow: \//);
+  });
+
+  test("should serve pages with a X-Robots-Tag noindex header", async ({
+    page,
+  }) => {
+    test.skip(
+      !process.env.CI,
+      "the header is added by nginx, not by the local dev server",
+    );
+
+    const response = await page.request.get("/");
+    expect(response.headers()["x-robots-tag"]).toBe("noindex");
   });
 });