diff --git a/public/robots.txt b/public/robots.txt index c62aca60..22a08769 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -10,8 +10,6 @@ Sitemap: https://www.bentopdf.com/sitemap.xml # Disallow internal/system paths Disallow: /api/ -Disallow: /_next/ -Disallow: /assets/ Disallow: /src/ Disallow: /*.wasm$ Disallow: /*.wasm.gz$ @@ -23,13 +21,3 @@ Disallow: /tmp/ Disallow: /temp/ Disallow: /uploads/ -# Disallow docs internal files (but allow /docs/ pages) -Disallow: /docs/assets/ - -# Google-specific -User-agent: Googlebot -Allow: / - -# Bing-specific -User-agent: Bingbot -Allow: / \ No newline at end of file