remove disallow rules from robots.txt

public/robots.txt

@@ -10,8 +10,6 @@ Sitemap: https://www.bentopdf.com/sitemap.xml
 
 # Disallow internal/system paths
 Disallow: /api/
-Disallow: /_next/
-Disallow: /assets/
 Disallow: /src/
 Disallow: /*.wasm$
 Disallow: /*.wasm.gz$
@@ -23,13 +21,3 @@ Disallow: /tmp/
 Disallow: /temp/
 Disallow: /uploads/
 
-# Disallow docs internal files (but allow /docs/ pages)
-Disallow: /docs/assets/
-
-# Google-specific
-User-agent: Googlebot
-Allow: /
-
-# Bing-specific
-User-agent: Bingbot
-Allow: /