From 8c2de4d24262aaa05245b055257381927c304526 Mon Sep 17 00:00:00 2001 From: s0lray Date: Sat, 21 Mar 2026 23:00:15 -0400 Subject: [PATCH] Phase 1: CI/CD pipeline, dead link cleanup, and URL normalization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebased cleanly onto current master (resolving conflicts from community PRs #240-#471 merged while this branch was in flight). Changes: - Add GitHub Actions CI workflow (JSON lint, link check, smoke tests) - Remove 176 confirmed dead links from arf.json - Normalize 393 URLs from http:// to https:// - Fix invalid JSON (missing commas/braces from old format) - Remove Bower dependency; lock D3 to npm - Add link audit tooling (tools/) - Cherry-pick alive tools from community PR #391: FaceCheckID, Cisco Talos, VerifyEmail, YouTube Geofind, Whitepages, Hiya, Berla Vehicle Lookup, OpenAIP, Valkyrie File Analysis - Fix OpenInfrastructureMap URL (missing https://) Community PR #359 (Risk Toolbox / fraudfightersalliance.com): DNS does not resolve — all 27 proposed additions are dead, not merged. Co-Authored-By: Paperclip --- .github/workflows/ci.yml | 90 + .gitignore | 6 + bower.json | 13 - package-lock.json | 844 ++- package.json | 6 +- public/arf.json | 12886 ++++++++++++++++----------------- tools/link_audit_report.json | 2734 +++++++ tools/link_checker.py | 177 + tools/remove_dead_links.py | 89 + 9 files changed, 10061 insertions(+), 6784 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 bower.json create mode 100644 tools/link_audit_report.json create mode 100644 tools/link_checker.py create mode 100644 tools/remove_dead_links.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..742c2fa --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,90 @@ +name: CI + +on: + push: + branches: [master] + pull_request: + branches: [master] + schedule: + # Run link check weekly on Sunday at 02:00 UTC + - cron: '0 2 * * 0' + +jobs: + json-lint: + name: JSON Lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Validate arf.json is well-formed + run: python3 -m json.tool public/arf.json > /dev/null + + smoke-test: + name: Smoke Test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Check index.html local asset references exist + run: | + check_asset() { + local path="public/$1" + if [ ! -f "$path" ]; then + echo "MISSING: $path" + exit 1 + else + echo "OK: $path" + fi + } + check_asset "css/arf.css" + check_asset "js/d3.v3.min.js" + check_asset "js/arf.js" + check_asset "arf.json" + + link-check: + name: Link Check + runs-on: ubuntu-latest + # Only run on schedule and direct pushes to master, not on every PR + # (avoids hammering external sites during code review) + if: github.event_name == 'schedule' || (github.event_name == 'push' && github.ref == 'refs/heads/master') + steps: + - uses: actions/checkout@v4 + + - name: Extract URLs from arf.json + run: | + python3 -c " + import json, re, sys + + with open('public/arf.json') as f: + data = json.load(f) + + def extract_urls(node): + urls = [] + url = node.get('url', '') + if url and url.startswith('http'): + urls.append(url) + for child in node.get('children', []): + urls.extend(extract_urls(child)) + return urls + + urls = extract_urls(data) + print(f'Found {len(urls)} URLs', file=sys.stderr) + with open('urls.txt', 'w') as f: + f.write('\n'.join(urls)) + " + + - name: Run lychee link checker + uses: lycheeverse/lychee-action@v2 + with: + args: | + --verbose + --no-progress + --timeout 20 + --max-retries 2 + --retry-wait-time 5 + --exclude-mail + --accept 200,201,204,206,301,302,307,308,403,429 + urls.txt + fail: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 8b9e4f1..6aadf92 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,13 @@ +# agent / org files # +###################### +agents/ + # vendor packages # ###################### public/js/vendor node_modules +bower_components +.bowerrc # OS generated files # ###################### diff --git a/bower.json b/bower.json deleted file mode 100644 index f42d5b1..0000000 --- a/bower.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "name": "ARF", - "main": "index.html", - "homepage": "https://github.com/lockfale/ARF", - "authors": [ - "Justin Nordine (s0lray)" - ], - "description": "Advanced Reconnaissance Framework", - "license": "", - "dependencies": { - "d3": "3.5.16" - } -} diff --git a/package-lock.json b/package-lock.json index 8d6de02..5ef8ce2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "hasInstallScript": true, "devDependencies": { "copyfiles": "^1.0.0", - "d3": "^3.5.16" + "d3": "^7.9.0" } }, "node_modules/balanced-match": { @@ -29,6 +29,16 @@ "concat-map": "0.0.1" } }, + "node_modules/commander": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", + "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -60,10 +70,447 @@ "dev": true }, "node_modules/d3": { - "version": "3.5.17", - "resolved": "https://registry.npmjs.org/d3/-/d3-3.5.17.tgz", - "integrity": "sha512-yFk/2idb8OHPKkbAL8QaOaqENNoMhIaSHZerk3oQsECwkObkCpJyjYwCe+OHiq6UEdhe1m8ZGARRRO3ljFjlKg==", - "dev": true + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/d3/-/d3-7.9.0.tgz", + "integrity": "sha512-e1U46jVP+w7Iut8Jt8ri1YsPOvFpg46k+K8TpCb0P+zjCkjkPnV7WzfDJzMHy1LnA+wj5pLT1wjO901gLXeEhA==", + "dev": true, + "license": "ISC", + "dependencies": { + "d3-array": "3", + "d3-axis": "3", + "d3-brush": "3", + "d3-chord": "3", + "d3-color": "3", + "d3-contour": "4", + "d3-delaunay": "6", + "d3-dispatch": "3", + "d3-drag": "3", + "d3-dsv": "3", + "d3-ease": "3", + "d3-fetch": "3", + "d3-force": "3", + "d3-format": "3", + "d3-geo": "3", + "d3-hierarchy": "3", + "d3-interpolate": "3", + "d3-path": "3", + "d3-polygon": "3", + "d3-quadtree": "3", + "d3-random": "3", + "d3-scale": "4", + "d3-scale-chromatic": "3", + "d3-selection": "3", + "d3-shape": "3", + "d3-time": "3", + "d3-time-format": "4", + "d3-timer": "3", + "d3-transition": "3", + "d3-zoom": "3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-array": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz", + "integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==", + "dev": true, + "license": "ISC", + "dependencies": { + "internmap": "1 - 2" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-axis": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-axis/-/d3-axis-3.0.0.tgz", + "integrity": "sha512-IH5tgjV4jE/GhHkRV0HiVYPDtvfjHQlQfJHs0usq7M30XcSBvOotpmH1IgkcXsO/5gEQZD43B//fc7SRT5S+xw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-brush": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-brush/-/d3-brush-3.0.0.tgz", + "integrity": "sha512-ALnjWlVYkXsVIGlOsuWH1+3udkYFI48Ljihfnh8FZPF2QS9o+PzGLBslO0PjzVoHLZ2KCVgAM8NVkXPJB2aNnQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-drag": "2 - 3", + "d3-interpolate": "1 - 3", + "d3-selection": "3", + "d3-transition": "3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-chord": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-chord/-/d3-chord-3.0.1.tgz", + "integrity": "sha512-VE5S6TNa+j8msksl7HwjxMHDM2yNK3XCkusIlpX5kwauBfXuyLAtNg9jCp/iHH61tgI4sb6R/EIMWCqEIdjT/g==", + "dev": true, + "license": "ISC", + "dependencies": { + "d3-path": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-color": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz", + "integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-contour": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/d3-contour/-/d3-contour-4.0.2.tgz", + "integrity": "sha512-4EzFTRIikzs47RGmdxbeUvLWtGedDUNkTcmzoeyg4sP/dvCexO47AaQL7VKy/gul85TOxw+IBgA8US2xwbToNA==", + "dev": true, + "license": "ISC", + "dependencies": { + "d3-array": "^3.2.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-delaunay": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/d3-delaunay/-/d3-delaunay-6.0.4.tgz", + "integrity": "sha512-mdjtIZ1XLAM8bm/hx3WwjfHt6Sggek7qH043O8KEjDXN40xi3vx/6pYSVTwLjEgiXQTbvaouWKynLBiUZ6SK6A==", + "dev": true, + "license": "ISC", + "dependencies": { + "delaunator": "5" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-dispatch": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-dispatch/-/d3-dispatch-3.0.1.tgz", + "integrity": "sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-drag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-drag/-/d3-drag-3.0.0.tgz", + "integrity": "sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg==", + "dev": true, + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-selection": "3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-dsv": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-dsv/-/d3-dsv-3.0.1.tgz", + "integrity": "sha512-UG6OvdI5afDIFP9w4G0mNq50dSOsXHJaRE8arAS5o9ApWnIElp8GZw1Dun8vP8OyHOZ/QJUKUJwxiiCCnUwm+Q==", + "dev": true, + "license": "ISC", + "dependencies": { + "commander": "7", + "iconv-lite": "0.6", + "rw": "1" + }, + "bin": { + "csv2json": "bin/dsv2json.js", + "csv2tsv": "bin/dsv2dsv.js", + "dsv2dsv": "bin/dsv2dsv.js", + "dsv2json": "bin/dsv2json.js", + "json2csv": "bin/json2dsv.js", + "json2dsv": "bin/json2dsv.js", + "json2tsv": "bin/json2dsv.js", + "tsv2csv": "bin/dsv2dsv.js", + "tsv2json": "bin/dsv2json.js" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-ease": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz", + "integrity": "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-fetch": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-fetch/-/d3-fetch-3.0.1.tgz", + "integrity": "sha512-kpkQIM20n3oLVBKGg6oHrUchHM3xODkTzjMoj7aWQFq5QEM+R6E4WkzT5+tojDY7yjez8KgCBRoj4aEr99Fdqw==", + "dev": true, + "license": "ISC", + "dependencies": { + "d3-dsv": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-force": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-force/-/d3-force-3.0.0.tgz", + "integrity": "sha512-zxV/SsA+U4yte8051P4ECydjD/S+qeYtnaIyAs9tgHCqfguma/aAQDjo85A9Z6EKhBirHRJHXIgJUlffT4wdLg==", + "dev": true, + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-quadtree": "1 - 3", + "d3-timer": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-format": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/d3-format/-/d3-format-3.1.2.tgz", + "integrity": "sha512-AJDdYOdnyRDV5b6ArilzCPPwc1ejkHcoyFarqlPqT7zRYjhavcT3uSrqcMvsgh2CgoPbK3RCwyHaVyxYcP2Arg==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-geo": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/d3-geo/-/d3-geo-3.1.1.tgz", + "integrity": "sha512-637ln3gXKXOwhalDzinUgY83KzNWZRKbYubaG+fGVuc/dxO64RRljtCTnf5ecMyE1RIdtqpkVcq0IbtU2S8j2Q==", + "dev": true, + "license": "ISC", + "dependencies": { + "d3-array": "2.5.0 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-hierarchy": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/d3-hierarchy/-/d3-hierarchy-3.1.2.tgz", + "integrity": "sha512-FX/9frcub54beBdugHjDCdikxThEqjnR93Qt7PvQTOHxyiNCAlvMrHhclk3cD5VeAaq9fxmfRp+CnWw9rEMBuA==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-interpolate": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz", + "integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==", + "dev": true, + "license": "ISC", + "dependencies": { + "d3-color": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-path": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz", + "integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-polygon": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-polygon/-/d3-polygon-3.0.1.tgz", + "integrity": "sha512-3vbA7vXYwfe1SYhED++fPUQlWSYTTGmFmQiany/gdbiWgU/iEyQzyymwL9SkJjFFuCS4902BSzewVGsHHmHtXg==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-quadtree": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-quadtree/-/d3-quadtree-3.0.1.tgz", + "integrity": "sha512-04xDrxQTDTCFwP5H6hRhsRcb9xxv2RzkcsygFzmkSIOJy3PeRJP7sNk3VRIbKXcog561P9oU0/rVH6vDROAgUw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-random": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-random/-/d3-random-3.0.1.tgz", + "integrity": "sha512-FXMe9GfxTxqd5D6jFsQ+DJ8BJS4E/fT5mqqdjovykEB2oFbTMDVdg1MGFxfQW+FBOGoB++k8swBrgwSHT1cUXQ==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-scale": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz", + "integrity": "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "d3-array": "2.10.0 - 3", + "d3-format": "1 - 3", + "d3-interpolate": "1.2.0 - 3", + "d3-time": "2.1.1 - 3", + "d3-time-format": "2 - 4" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-scale-chromatic": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-scale-chromatic/-/d3-scale-chromatic-3.1.0.tgz", + "integrity": "sha512-A3s5PWiZ9YCXFye1o246KoscMWqf8BsD9eRiJ3He7C9OBaxKhAd5TFCdEx/7VbKtxxTsu//1mMJFrEt572cEyQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "d3-color": "1 - 3", + "d3-interpolate": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-selection": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", + "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", + "dev": true, + "license": "ISC", + "peer": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-shape": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz", + "integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==", + "dev": true, + "license": "ISC", + "dependencies": { + "d3-path": "^3.1.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-time": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-time/-/d3-time-3.1.0.tgz", + "integrity": "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==", + "dev": true, + "license": "ISC", + "dependencies": { + "d3-array": "2 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-time-format": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-4.1.0.tgz", + "integrity": "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==", + "dev": true, + "license": "ISC", + "dependencies": { + "d3-time": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-timer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz", + "integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-transition": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-transition/-/d3-transition-3.0.1.tgz", + "integrity": "sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w==", + "dev": true, + "license": "ISC", + "dependencies": { + "d3-color": "1 - 3", + "d3-dispatch": "1 - 3", + "d3-ease": "1 - 3", + "d3-interpolate": "1 - 3", + "d3-timer": "1 - 3" + }, + "engines": { + "node": ">=12" + }, + "peerDependencies": { + "d3-selection": "2 - 3" + } + }, + "node_modules/d3-zoom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-zoom/-/d3-zoom-3.0.0.tgz", + "integrity": "sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw==", + "dev": true, + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-drag": "2 - 3", + "d3-interpolate": "1 - 3", + "d3-selection": "2 - 3", + "d3-transition": "2 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/delaunator": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/delaunator/-/delaunator-5.0.1.tgz", + "integrity": "sha512-8nvh+XBe96aCESrGOqMp/84b13H9cdKbG5P2ejQCh4d4sK9RL4371qou9drQjMhvnPmhWl5hnmqbEE0fXr9Xnw==", + "dev": true, + "license": "ISC", + "dependencies": { + "robust-predicates": "^3.0.2" + } }, "node_modules/fs.realpath": { "version": "1.0.0", @@ -92,6 +539,19 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -109,6 +569,16 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "dev": true }, + "node_modules/internmap": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz", + "integrity": "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, "node_modules/isarray": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", @@ -203,12 +673,33 @@ "string_decoder": "~0.10.x" } }, + "node_modules/robust-predicates": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/robust-predicates/-/robust-predicates-3.0.2.tgz", + "integrity": "sha512-IXgzBWvWQwE6PrDI05OvmXUIruQTcoMDzRsOd5CDvHCVLcLHMTSYvOK5Cm46kWqlV3yAbuSpBZdJ5oP5OUoStg==", + "dev": true, + "license": "Unlicense" + }, + "node_modules/rw": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/rw/-/rw-1.3.3.tgz", + "integrity": "sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==", + "dev": true, + "license": "BSD-3-Clause" + }, "node_modules/safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", "dev": true }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true, + "license": "MIT" + }, "node_modules/string_decoder": { "version": "0.10.31", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", @@ -294,6 +785,12 @@ "concat-map": "0.0.1" } }, + "commander": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", + "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", + "dev": true + }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -321,11 +818,309 @@ "dev": true }, "d3": { - "version": "3.5.17", - "resolved": "https://registry.npmjs.org/d3/-/d3-3.5.17.tgz", - "integrity": "sha512-yFk/2idb8OHPKkbAL8QaOaqENNoMhIaSHZerk3oQsECwkObkCpJyjYwCe+OHiq6UEdhe1m8ZGARRRO3ljFjlKg==", + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/d3/-/d3-7.9.0.tgz", + "integrity": "sha512-e1U46jVP+w7Iut8Jt8ri1YsPOvFpg46k+K8TpCb0P+zjCkjkPnV7WzfDJzMHy1LnA+wj5pLT1wjO901gLXeEhA==", + "dev": true, + "requires": { + "d3-array": "3", + "d3-axis": "3", + "d3-brush": "3", + "d3-chord": "3", + "d3-color": "3", + "d3-contour": "4", + "d3-delaunay": "6", + "d3-dispatch": "3", + "d3-drag": "3", + "d3-dsv": "3", + "d3-ease": "3", + "d3-fetch": "3", + "d3-force": "3", + "d3-format": "3", + "d3-geo": "3", + "d3-hierarchy": "3", + "d3-interpolate": "3", + "d3-path": "3", + "d3-polygon": "3", + "d3-quadtree": "3", + "d3-random": "3", + "d3-scale": "4", + "d3-scale-chromatic": "3", + "d3-selection": "3", + "d3-shape": "3", + "d3-time": "3", + "d3-time-format": "4", + "d3-timer": "3", + "d3-transition": "3", + "d3-zoom": "3" + } + }, + "d3-array": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz", + "integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==", + "dev": true, + "requires": { + "internmap": "1 - 2" + } + }, + "d3-axis": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-axis/-/d3-axis-3.0.0.tgz", + "integrity": "sha512-IH5tgjV4jE/GhHkRV0HiVYPDtvfjHQlQfJHs0usq7M30XcSBvOotpmH1IgkcXsO/5gEQZD43B//fc7SRT5S+xw==", "dev": true }, + "d3-brush": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-brush/-/d3-brush-3.0.0.tgz", + "integrity": "sha512-ALnjWlVYkXsVIGlOsuWH1+3udkYFI48Ljihfnh8FZPF2QS9o+PzGLBslO0PjzVoHLZ2KCVgAM8NVkXPJB2aNnQ==", + "dev": true, + "requires": { + "d3-dispatch": "1 - 3", + "d3-drag": "2 - 3", + "d3-interpolate": "1 - 3", + "d3-selection": "3", + "d3-transition": "3" + } + }, + "d3-chord": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-chord/-/d3-chord-3.0.1.tgz", + "integrity": "sha512-VE5S6TNa+j8msksl7HwjxMHDM2yNK3XCkusIlpX5kwauBfXuyLAtNg9jCp/iHH61tgI4sb6R/EIMWCqEIdjT/g==", + "dev": true, + "requires": { + "d3-path": "1 - 3" + } + }, + "d3-color": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz", + "integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==", + "dev": true + }, + "d3-contour": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/d3-contour/-/d3-contour-4.0.2.tgz", + "integrity": "sha512-4EzFTRIikzs47RGmdxbeUvLWtGedDUNkTcmzoeyg4sP/dvCexO47AaQL7VKy/gul85TOxw+IBgA8US2xwbToNA==", + "dev": true, + "requires": { + "d3-array": "^3.2.0" + } + }, + "d3-delaunay": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/d3-delaunay/-/d3-delaunay-6.0.4.tgz", + "integrity": "sha512-mdjtIZ1XLAM8bm/hx3WwjfHt6Sggek7qH043O8KEjDXN40xi3vx/6pYSVTwLjEgiXQTbvaouWKynLBiUZ6SK6A==", + "dev": true, + "requires": { + "delaunator": "5" + } + }, + "d3-dispatch": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-dispatch/-/d3-dispatch-3.0.1.tgz", + "integrity": "sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==", + "dev": true + }, + "d3-drag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-drag/-/d3-drag-3.0.0.tgz", + "integrity": "sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg==", + "dev": true, + "requires": { + "d3-dispatch": "1 - 3", + "d3-selection": "3" + } + }, + "d3-dsv": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-dsv/-/d3-dsv-3.0.1.tgz", + "integrity": "sha512-UG6OvdI5afDIFP9w4G0mNq50dSOsXHJaRE8arAS5o9ApWnIElp8GZw1Dun8vP8OyHOZ/QJUKUJwxiiCCnUwm+Q==", + "dev": true, + "requires": { + "commander": "7", + "iconv-lite": "0.6", + "rw": "1" + } + }, + "d3-ease": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz", + "integrity": "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==", + "dev": true + }, + "d3-fetch": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-fetch/-/d3-fetch-3.0.1.tgz", + "integrity": "sha512-kpkQIM20n3oLVBKGg6oHrUchHM3xODkTzjMoj7aWQFq5QEM+R6E4WkzT5+tojDY7yjez8KgCBRoj4aEr99Fdqw==", + "dev": true, + "requires": { + "d3-dsv": "1 - 3" + } + }, + "d3-force": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-force/-/d3-force-3.0.0.tgz", + "integrity": "sha512-zxV/SsA+U4yte8051P4ECydjD/S+qeYtnaIyAs9tgHCqfguma/aAQDjo85A9Z6EKhBirHRJHXIgJUlffT4wdLg==", + "dev": true, + "requires": { + "d3-dispatch": "1 - 3", + "d3-quadtree": "1 - 3", + "d3-timer": "1 - 3" + } + }, + "d3-format": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/d3-format/-/d3-format-3.1.2.tgz", + "integrity": "sha512-AJDdYOdnyRDV5b6ArilzCPPwc1ejkHcoyFarqlPqT7zRYjhavcT3uSrqcMvsgh2CgoPbK3RCwyHaVyxYcP2Arg==", + "dev": true + }, + "d3-geo": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/d3-geo/-/d3-geo-3.1.1.tgz", + "integrity": "sha512-637ln3gXKXOwhalDzinUgY83KzNWZRKbYubaG+fGVuc/dxO64RRljtCTnf5ecMyE1RIdtqpkVcq0IbtU2S8j2Q==", + "dev": true, + "requires": { + "d3-array": "2.5.0 - 3" + } + }, + "d3-hierarchy": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/d3-hierarchy/-/d3-hierarchy-3.1.2.tgz", + "integrity": "sha512-FX/9frcub54beBdugHjDCdikxThEqjnR93Qt7PvQTOHxyiNCAlvMrHhclk3cD5VeAaq9fxmfRp+CnWw9rEMBuA==", + "dev": true + }, + "d3-interpolate": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz", + "integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==", + "dev": true, + "requires": { + "d3-color": "1 - 3" + } + }, + "d3-path": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz", + "integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==", + "dev": true + }, + "d3-polygon": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-polygon/-/d3-polygon-3.0.1.tgz", + "integrity": "sha512-3vbA7vXYwfe1SYhED++fPUQlWSYTTGmFmQiany/gdbiWgU/iEyQzyymwL9SkJjFFuCS4902BSzewVGsHHmHtXg==", + "dev": true + }, + "d3-quadtree": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-quadtree/-/d3-quadtree-3.0.1.tgz", + "integrity": "sha512-04xDrxQTDTCFwP5H6hRhsRcb9xxv2RzkcsygFzmkSIOJy3PeRJP7sNk3VRIbKXcog561P9oU0/rVH6vDROAgUw==", + "dev": true + }, + "d3-random": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-random/-/d3-random-3.0.1.tgz", + "integrity": "sha512-FXMe9GfxTxqd5D6jFsQ+DJ8BJS4E/fT5mqqdjovykEB2oFbTMDVdg1MGFxfQW+FBOGoB++k8swBrgwSHT1cUXQ==", + "dev": true + }, + "d3-scale": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz", + "integrity": "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==", + "dev": true, + "requires": { + "d3-array": "2.10.0 - 3", + "d3-format": "1 - 3", + "d3-interpolate": "1.2.0 - 3", + "d3-time": "2.1.1 - 3", + "d3-time-format": "2 - 4" + } + }, + "d3-scale-chromatic": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-scale-chromatic/-/d3-scale-chromatic-3.1.0.tgz", + "integrity": "sha512-A3s5PWiZ9YCXFye1o246KoscMWqf8BsD9eRiJ3He7C9OBaxKhAd5TFCdEx/7VbKtxxTsu//1mMJFrEt572cEyQ==", + "dev": true, + "requires": { + "d3-color": "1 - 3", + "d3-interpolate": "1 - 3" + } + }, + "d3-selection": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", + "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", + "dev": true, + "peer": true + }, + "d3-shape": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz", + "integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==", + "dev": true, + "requires": { + "d3-path": "^3.1.0" + } + }, + "d3-time": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-time/-/d3-time-3.1.0.tgz", + "integrity": "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==", + "dev": true, + "requires": { + "d3-array": "2 - 3" + } + }, + "d3-time-format": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-4.1.0.tgz", + "integrity": "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==", + "dev": true, + "requires": { + "d3-time": "1 - 3" + } + }, + "d3-timer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz", + "integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==", + "dev": true + }, + "d3-transition": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-transition/-/d3-transition-3.0.1.tgz", + "integrity": "sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w==", + "dev": true, + "requires": { + "d3-color": "1 - 3", + "d3-dispatch": "1 - 3", + "d3-ease": "1 - 3", + "d3-interpolate": "1 - 3", + "d3-timer": "1 - 3" + } + }, + "d3-zoom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-zoom/-/d3-zoom-3.0.0.tgz", + "integrity": "sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw==", + "dev": true, + "requires": { + "d3-dispatch": "1 - 3", + "d3-drag": "2 - 3", + "d3-interpolate": "1 - 3", + "d3-selection": "2 - 3", + "d3-transition": "2 - 3" + } + }, + "delaunator": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/delaunator/-/delaunator-5.0.1.tgz", + "integrity": "sha512-8nvh+XBe96aCESrGOqMp/84b13H9cdKbG5P2ejQCh4d4sK9RL4371qou9drQjMhvnPmhWl5hnmqbEE0fXr9Xnw==", + "dev": true, + "requires": { + "robust-predicates": "^3.0.2" + } + }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -346,6 +1141,15 @@ "path-is-absolute": "^1.0.0" } }, + "iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dev": true, + "requires": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + } + }, "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -362,6 +1166,12 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "dev": true }, + "internmap": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz", + "integrity": "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==", + "dev": true + }, "isarray": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", @@ -441,12 +1251,30 @@ "string_decoder": "~0.10.x" } }, + "robust-predicates": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/robust-predicates/-/robust-predicates-3.0.2.tgz", + "integrity": "sha512-IXgzBWvWQwE6PrDI05OvmXUIruQTcoMDzRsOd5CDvHCVLcLHMTSYvOK5Cm46kWqlV3yAbuSpBZdJ5oP5OUoStg==", + "dev": true + }, + "rw": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/rw/-/rw-1.3.3.tgz", + "integrity": "sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==", + "dev": true + }, "safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", "dev": true }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true + }, "string_decoder": { "version": "0.10.31", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", diff --git a/package.json b/package.json index f685fd6..8588aa0 100644 --- a/package.json +++ b/package.json @@ -4,8 +4,8 @@ "description": "Advanced Reconnaissance Framework", "main": "public/index.html", "scripts": { - "postinstall": "copyfiles -f ./node_modules/d3/d3.min.js ./public/js/vendor/d3", - "start": "cd ./public; python -m SimpleHTTPServer 8000" + "postinstall": "copyfiles -f ./node_modules/d3/dist/d3.min.js ./public/js/", + "start": "cd ./public; python3 -m http.server 8000" }, "repository": { "type": "git", @@ -19,6 +19,6 @@ "homepage": "https://github.com/lockfale/ARF#readme", "devDependencies": { "copyfiles": "^1.0.0", - "d3": "^3.5.16" + "d3": "^7.9.0" } } diff --git a/public/arf.json b/public/arf.json index d789b0c..1777475 100644 --- a/public/arf.json +++ b/public/arf.json @@ -2,6916 +2,6282 @@ "name": "OSINT Framework", "type": "folder", "children": [ - { - "name": "Username", - "type": "folder", - "children": [ { - "name": "Username Search Engines", - "type": "folder", - "children": [ - { - "name": "Sylva Identity Discovery (T)", - "type": "url", - "url": "https://sylva.pfeister.dev/" - }, - { - "name": "Sherlock (T)", - "type": "url", - "url": "https://github.com/sherlock-project/sherlock" - }, - { - "name": "WhatsMyName", - "type": "url", - "url": "https://whatsmyname.app/" - }, - { - "name": "Namechk", - "type": "url", - "url": "https://namechk.com/" - }, - { - "name": "UserSearch.org", - "type": "url", - "url": "https://usersearch.org/" - }, - { - "name": "WhatsMyName (T)", - "type": "url", - "url": "https://github.com/WebBreacher/WhatsMyName" - }, - { - "name": "WhatsMyName", - "type": "url", - "url": "https://whatsmyname.app/" - }, - { - "name": "Thats Them", - "type": "url", - "url": "https://thatsthem.com/" - }, - { - "name": "NameCheckup", - "type": "url", - "url": "https://namecheckup.com/" - }, - { - "name": "Instant Username Search", - "type": "url", - "url": "https://instantusername.com/" - }, - { - "name": "Sherlock", - "type": "url", - "url": "https://github.com/sherlock-project/sherlock" - }, - { - "name": "IDCrawl Username Search", - "type": "url", - "url": "https://www.idcrawl.com/username" - }, - { - "name": "Names Directory", - "type": "url", - "url": "https://namesdir.com/" - }] - }, - { - "name": "Specific Sites", - "type": "folder", - "children": [ - { - "name": "Amazon Usernames (M)", - "type": "url", - "url": "https://www.google.com/search?q=site:amazon.com+%3Cusername%3E" - }, - { - "name": "Github User (M)", - "type": "url", - "url": "https://api.github.com/users/%3Cusername%3E/events/public" - }, - { - "name": "Tinder Usernames (M)", - "type": "url", - "url": "https://www.gotinder.com/@%3Cusername%3E" - }, - { - "name": "Keybase", - "type": "url", - "url": "https://keybase.io/" - }, - { - "name": "MIT PGP Key Server", - "type": "url", - "url": "http://pgp.mit.edu/" - } - { - "name": "ProtonMail users (M)", - "type": "url", - "url": "https://api.protonmail.ch/pks/lookup?op=index&search=@protonmail.com" - }, - { - "name": "ProtonMail Domains (M)", - "type": "url", - "url": "https://api.protonmail.ch/pks/lookup?op=index&search=" - }] - }] - }, - { - "name": "Email Address", - "type": "folder", - "children": [ - { - "name": "Email Search", - "type": "folder", - "children": [ - { - "name": "Sylva Identity Discovery (T)", - "type": "url", - "url": "https://sylva.pfeister.dev/" - }, - { - "name": "ThatsThem", - "type": "url", - "url": "https://thatsthem.com/reverse-email-lookup" - }, - { - "name": "Hunter", - "type": "url", - "url": "https://hunter.io/" - }, - { - "name": "Email to Address (R)", - "type": "url", - "url": "http://www.melissadata.com/lookups/emails.asp" - }, - { - "name": "VoilaNorbert", - "type": "url", - "url": "https://www.voilanorbert.com/" - }, - { - "name": "GHunt (T)", - "type": "url", - "url": "https://github.com/mxrch/GHunt" - }, - { - "name": "OSINT Industries", - "type": "url", - "url": "https://osint.industries/" - }, - { - "name": "theHarvester (T)", - "type": "url", - "url": "https://github.com/laramies/theHarvester" - }, - { - "name": "Infoga (T)", - "type": "url", - "url": "https://github.com/m4ll0k/infoga" - }, - { - "name": "Skymem", - "type": "url", - "url": "http://www.skymem.info/" - }, - { - "name": "Epieos Email Tool", - "type": "url", - "url": "https://tools.epieos.com/email.php" - }, - { - "name": "Holehe (T)", - "type": "url", - "url": "https://github.com/megadose/holehe" - }] - }, - { - "name": "Common Email Formats", - "type": "folder", - "children": [ - { - "name": "Email Format", - "type": "url", - "url": "https://www.email-format.com/" - }, - { - "name": "Email Permutator", - "type": "url", - "url": "http://metricsparrow.com/toolkit/email-permutator/" - }, - { - "name": "OneLook Reverse Dictionary and Thesaurus", - "type": "url", - "url": "http://www.onelook.com/reverse-dictionary.shtml" - }] - }, - { - "name": "Email Verification", - "type": "folder", - "children": [ - { - "name": "Reacher Github (T)", - "type": "url", - "url": "https://github.com/reacherhq/check-if-email-exists" - }, - { - "name": "Reacher Demo", - "type": "url", - "url": "https://reacher.email" - }, - { - "name": "MailScrap", - "type": "url", - "url": "https://mailscrap.com/" - }, - { - "name": "Read Notify", - "type": "url", - "url": "http://www.readnotify.com/" - }, - { - "name": "Email Reputation", - "type": "url", - "url": "https://emailrep.io/" - }, - { - "name": "MailboxValidator", - "type": "url", - "url": "https://www.mailboxvalidator.com/demo" - }] - }, - { - "name": "Breach Data", - "type": "folder", - "children": [ - { - "name": "Have I been pwned?", - "type": "url", - "url": "https://haveibeenpwned.com/" - }, - { - "name": "Hudson Rock", - "type": "url", - "url": "https://www.hudsonrock.com/threat-intelligence-cybercrime-tools" - }, - { - "name": "DeHashed (R)", - "type": "url", - "url": "https://dehashed.com/" - }, - { - "name": "Vigilante.pw", - "type": "url", - "url": "https://www.vigilante.pw/" - }, - { - "name": "Ashley Madison Emails", - "type": "url", - "url": "https://ashley.cynic.al/" - }] - }, - { - "name": "Mail Blacklists", - "type": "folder", - "children": [ - { - "name": "MxToolbox", - "type": "url", - "url": "http://mxtoolbox.com/" - }] - }] - }, - { - "name": "Domain Name", - "type": "folder", - "children": [ - { - "name": "Whois Records", - "type": "folder", - "children": [ - { - "name": "Domain Dossier", - "type": "url", - "url": "http://centralops.net/co/DomainDossier.aspx" - }, - { - "name": "domainIQ", - "type": "url", - "url": "https://www.domainiq.com/" - }, - { - "name": "DomainTools Whois", - "type": "url", - "url": "http://whois.domaintools.com/" - }, - { - "name": "SWITCH Internet Domains Whois (.ch)", - "type": "url", - "url": "https://www.nic.ch/whois/" - }, - { - "name": "Domain Big Data", - "type": "url", - "url": "http://domainbigdata.com/" - }, - { - "name": "Whoisology", - "type": "url", - "url": "https://whoisology.com/#advanced" - }, - { - "name": "Whois ARIN", - "type": "url", - "url": "https://whois.arin.net/ui/advanced.jsp" - }, - { - "name": "DNSstuff", - "type": "url", - "url": "https://tools.dnsstuff.com/" - }, - { - "name": "Robtex (R)", - "type": "url", - "url": "https://www.robtex.com/" - }, - { - "name": "Domaincrawler.com", - "type": "url", - "url": "http://www.domaincrawler.com/" - }, - { - "name": "MarkMonitor Whois Search", - "type": "url", - "url": "https://domains.markmonitor.com/whois/" - }, - { - "name": "easyWhois", - "type": "url", - "url": "https://www.easywhois.com/" - }, - { - "name": "Website Informer", - "type": "url", - "url": "http://website.informer.com/" - }, - { - "name": "Who.is", - "type": "url", - "url": "https://who.is/" - }, - { - "name": "Whois AMPed", - "type": "url", - "url": "https://whoisamped.com/" - }, - { - "name": "ViewDNS.info", - "type": "url", - "url": "http://viewdns.info/" - }, - { - "name": "Daily DNS Changes", - "type": "url", - "url": "http://www.dailychanges.com/" - }, - { - "name": "IP2WHOIS", - "type": "url", - "url": "https://www.ip2whois.com" - }, - { - "name": "Netlas.io", - "type": "url", - "url": "https://app.netlas.io/whois_domains/" - }] - }, - { - "name": "Subdomains", + "name": "Username", "type": "folder", "children": [ { - "name": "SynapsInt", - "type": "url", - "url": "https://synapsint.com" - }, - { - "name": "Aquatone (T)", - "type": "url", - "url": "https://github.com/michenriksen/aquatone" - }, - { - "name": "FindSubDomains", - "type": "url", - "url": "https://findsubdomains.com/" - }, - { - "name": "Google Subdomains (D)", - "type": "url", - "url": "https://www.google.com/?gws_rd=ssl#q=site:%3Cdomain.com%3E" - }, - { - "name": "Recon-ng (T)", - "type": "url", - "url": "https://github.com/lanmaster53/recon-ng" - }, - { - "name": "XRay", - "type": "url", - "url": "https://github.com/evilsocket/xray" - }, - { - "name": "DNS Recon (T)", - "type": "url", - "url": "https://github.com/darkoperator/dnsrecon" - }, - { - "name": "Gobuster (T)", - "type": "url", - "url": "https://github.com/OJ/gobuster" - }, - { - "name": "Fierce Domain Scanner (T)", - "type": "url", - "url": "https://github.com/davidpepper/fierce-domain-scanner" - }, - { - "name": "Bluto (T)", - "type": "url", - "url": "https://github.com/darryllane/Bluto" - }, - { - "name": "OWASP Maryam (T)", - "type": "url", - "url": "https://github.com/saeeddhqan/Maryam" - }, - { - "name": "theHarvester (T)", - "type": "url", - "url": "https://github.com/laramies/theHarvester" - }, - { - "name": "Pentest-tools.com Subdomains", - "type": "url", - "url": "https://pentest-tools.com/information-gathering/find-subdomains-of-domain" - }, - { - "name": "SecLists DNS Subdomains (T)", - "type": "url", - "url": "https://github.com/danielmiessler/SecLists/tree/master/Discovery/DNS" - }, - { - "name": "dnspop (T)", - "type": "url", - "url": "https://github.com/bitquark/dnspop" - }, - { - "name": "gdns (T)", - "type": "url", - "url": "https://github.com/hrbrmstr/gdns" - }, - { - "name": "Sublist3r", - "type": "url", - "url": "https://github.com/aboul3la/Sublist3r" - }, - { - "name": "AltDNS (T)", - "type": "url", - "url": "https://github.com/infosec-au/altdns" - }, - { - "name": "Netlas.io", - "type": "url", - "url": "https://app.netlas.io/domains/" - }] - }, - { - "name": "Discovery", - "type": "folder", - "children": [ - { - "name": "Shodan", - "type": "url", - "url": "https://www.shodan.io/" - }, - { - "name": "Netlas.io", - "type": "url", - "url": "https://app.netlas.io/" - }, - { - "name": "ODIN (R)", - "type": "url", - "url": "https://www.getodin.com/" - }, - { - "name": "Kraken (T)", - "type": "url", - "url": "https://github.com/Sw4mpf0x/Kraken" - }, - { - "name": "SecurityTrails (R)", - "type": "url", - "url": "https://securitytrails.com/" - }, - { - "name": "Online Nikto scanner", - "type": "url", - "url": "https://nikto.online/" - }, - { - "name": "urlscan.io", - "type": "url", - "url": "https://urlscan.io/search/#*" - }, - { - "name": "Daily DNS Changes", - "type": "url", - "url": "http://www.dailychanges.com/" - }, - { - "name": "Redirect Detective", - "type": "url", - "url": "http://redirectdetective.com/" - }, - { - "name": "Sitediff (T)", - "type": "url", - "url": "https://github.com/digininja/sitediff" - }, - { - "name": "BuiltWith", - "type": "url", - "url": "https://builtwith.com/" - }, - { - "name": "Wappalyzer", - "type": "url", - "url": "https://www.wappalyzer.com/" - }, - { - "name": "AnalyzeID", - "type": "url", - "url": "http://analyzeid.com/" - }, - { - "name": "Criminal IP Search", - "type": "url", - "url": "https://www.criminalip.io/" - }, - { - "name": "urlDNA", - "type": "url", - "url": "https://urldna.io" - }] - }, - { - "name": "Certificate Search", - "type": "folder", - "children": [ - { - "name": "Shodan", - "type": "url", - "url": "https://www.shodan.io/" - }, - { - "name": "Google's Certificate Transparency", - "type": "url", - "url": "https://www.certificate-transparency.org/known-logs" - }, - { - "name": "Spyse", - "type": "url", - "url": "https://spyse.com/search/certificate" - }, - { - "name": "Censys", - "type": "url", - "url": "https://censys.io/" - }, - { - "name": "crt.sh - Certificate Search", - "type": "url", - "url": "https://crt.sh/?" - }, - { - "name": "certgraph (T)", - "type": "url", - "url": "https://github.com/lanrat/certgraph" - }, - { - "name": "ODIN (R)", - "type": "url", - "url": "https://www.getodin.com/" - }, - { - "name": "Netlas.io", - "type": "url", - "url": "https://app.netlas.io/certs/" - }] - }, - { - "name": "PassiveDNS", - "type": "folder", - "children": [ - { - "name": "Security Trails (R)", - "type": "url", - "url": "https://securitytrails.com/" - }, - { - "name": "Mnemonic", - "type": "url", - "url": "http://passivedns.mnemonic.no/" - }, - { - "name": "DNS History", - "type": "url", - "url": "http://dnshistory.org/" - }, - { - "name": "PTRarchive.com", - "type": "url", - "url": "http://ptrarchive.com/" - }, - { - "name": "DNS Dumpster", - "type": "url", - "url": "https://dnsdumpster.com/" - }, - { - "name": "Deteque (R)", - "type": "url", - "url": "https://www.deteque.com/" - }] - }, - { - "name": "Reputation", - "type": "folder", - "children": [ - { - "name": "AbuseIPDB (Webpage and API)", - "type": "url", - "url": "https://www.abuseipdb.com/" - }, - { - "name": "UrlQuery.net", - "type": "url", - "url": "http://urlquery.net/" - }, - { - "name": "PassiveTotal", - "type": "url", - "url": "https://www.passivetotal.org/" - }, - { - "name": "URL Void", - "type": "url", - "url": "http://www.urlvoid.com/" - }, - { - "name": "Threat Crowd", - "type": "url", - "url": "https://www.threatcrowd.org/" - }, - { - "name": "FortiGuard Reputation Service", - "type": "url", - "url": "http://fortiguard.com/iprep" - }, - { - "name": "McAfee TrustedSource", - "type": "url", - "url": "http://www.trustedsource.org/" - }, - { - "name": "Trend Micro Site Safety Center", - "type": "url", - "url": "https://global.sitesafety.trendmicro.com/" - }, - { - "name": "WatchGuard ReputationAuthority", - "type": "url", - "url": "http://www.reputationauthority.org/" - }, - { - "name": "Sucuri SiteCheck", - "type": "url", - "url": "https://sitecheck.sucuri.net/" - }, - { - "name": "ThreatMiner.org", - "type": "url", - "url": "https://www.threatminer.org/" - }, - { - "name": "BlueCoat WebPulse", - "type": "url", - "url": "https://sitereview.bluecoat.com/sitereview.jsp" - }, - { - "name": "Zscaler Zulu URL Risk Analyzer", - "type": "url", - "url": "http://zulu.zscaler.com/" - }, - { - "name": "Joe Sandbox Url Analyzer", - "type": "url", - "url": "https://www.url-analyzer.net/" - }, - { - "name": "Deepviz Domain Search", - "type": "url", - "url": "https://search.deepviz.com/" - }, - { - "name": "Cisco SenderBase", - "type": "url", - "url": "http://www.senderbase.org/" - }, - { - "name": "AVG Threat Labs", - "type": "url", - "url": "http://www.avgthreatlabs.com/ww-en/website-safety-reports/" - }, - { - "name": "Webroot BrightCloud URL/IP Lookup", - "type": "url", - "url": "http://www.brightcloud.com/tools/url-ip-lookup.php" - }, - { - "name": "vURL Online", - "type": "url", - "url": "https://vurldissect.co.uk/" - }, - { - "name": "AlienVault Open Threat Exchange", - "type": "url", - "url": "https://otx.alienvault.com/browse/pulses/" - }, - { - "name": "Malware Domain List", - "type": "url", - "url": "http://www.malwaredomainlist.com/mdl.php" - }, - { - "name": "Web Inspector Online Scan", - "type": "url", - "url": "http://app.webinspector.com/" - }, - { - "name": "Google Safe Browsing API", - "type": "url", - "url": "https://developers.google.com/safe-browsing/?csw=1" - }, - { - "name": "hpHosts Online", - "type": "url", - "url": "http://hosts-file.net/" - }] - }, - { - "name": "Domain Blacklists", - "type": "folder", - "children": [ - { - "name": "Ransomware Tracker Abuse.ch", - "type": "url", - "url": "http://ransomwaretracker.abuse.ch/downloads/RW_DOMBL.txt" - }, - { - "name": "Threatexpert.com Malicious URLs", - "type": "url", - "url": "http://www.networksec.org/grabbho/block.txt" - }, - { - "name": "Zeus C2 Tracker", - "type": "url", - "url": "https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist" - }, - { - "name": "Malware Domains Blacklist", - "type": "url", - "url": "http://mirror1.malwaredomains.com/files/domains.txt" - }, - { - "name": "Blackweb", - "type": "url", - "url": "https://github.com/maravento/blackweb" - }, - { - "name": "Critical Stack Intel (R)", - "type": "url", - "url": "https://intel.criticalstack.com/" - }, - { - "name": "DNS Sinkhole", - "type": "url", - "url": "http://malc0de.com/bl/" - }, - { - "name": "DNS-BH Malware Domain Blocklist", - "type": "url", - "url": "http://www.malwaredomains.com/wordpress/?page_id=66" - }, - { - "name": "Malware Domain List", - "type": "url", - "url": "http://www.malwaredomainlist.com/hostslist/hosts.txt" - }, - { - "name": "Malware Patrol (R)", - "type": "url", - "url": "http://www.malware.com.br/open-source.shtml" - }, - { - "name": "MalwareURL (R)", - "type": "url", - "url": "http://www.malwareurl.com/index.php" - }, - { - "name": "scumware.org", - "type": "url", - "url": "https://www.scumware.org/" - }, - { - "name": "ZeuS Tracker", - "type": "url", - "url": "https://zeustracker.abuse.ch/blocklist.php" - }, - { - "name": "Shadowserver Foundation", - "type": "url", - "url": "http://www.shadowserver.org/wiki/pmwiki.php?n=Services/Reports" - }, - { - "name": "Email Domain Validation", - "type": "url", - "url": "https://www.mailboxvalidator.com/domain" - }] - }, - { - "name": "Typosquatting", - "type": "folder", - "children": [ - { - "name": "DNS Twist (T)", - "type": "url", - "url": "https://github.com/elceef/dnstwist" - }, - { - "name": "URLCrazy (T)", - "type": "url", - "url": "http://www.morningstarsecurity.com/research/urlcrazy" - }, - { - "name": "dnstwister", - "type": "url", - "url": "https://dnstwister.report/" - }, - { - "name": "Catphish (T)", - "type": "url", - "url": "https://github.com/ring0lab/catphish" - }] - }, - { - "name": "Analytics", - "type": "folder", - "children": [ - { - "name": "BuiltWith", - "type": "url", - "url": "http://builtwith.com/" - }, - { - "name": "SiteSleuth", - "type": "url", - "url": "https://www.sitesleuth.io/" - }, - { - "name": "Wappalyzer (T)", - "type": "url", - "url": "https://wappalyzer.com/" - }, - { - "name": "SEMrush", - "type": "url", - "url": "https://www.semrush.com/" - }, - { - "name": "TopGainers", - "type": "url", - "url": "https://topgainers.net/" - }, - { - "name": "Moonsearch", - "type": "url", - "url": "http://moonsearch.com/" - }, - { - "name": "StackShare", - "type": "url", - "url": "https://stackshare.io/" - }, - { - "name": "Ewhois", - "type": "url", - "url": "https://ewhois.com/" - }, - { - "name": "Netcraft", - "type": "url", - "url": "http://toolbar.netcraft.com/site_report?url=undefined#last_reboot" - }, - { - "name": "StatsCrop", - "type": "url", - "url": "http://www.statscrop.com/" - }, - { - "name": "Open Site Explorer", - "type": "url", - "url": "https://moz.com/researchtools/ose/" - }, - { - "name": "SpyOnWeb", - "type": "url", - "url": "http://www.spyonweb.com/" - }, - { - "name": "SecurityHeaders.io", - "type": "url", - "url": "https://securityheaders.io/" - }, - { - "name": "Keyword Density", - "type": "url", - "url": "http://tools.seobook.com/general/keyword-density/" - }, - { - "name": "Alexa Site Statistics", - "type": "url", - "url": "http://www.alexa.com/siteinfo" - }, - { - "name": "Cisco Umbrella Popularity List", - "type": "url", - "url": "http://s3-us-west-1.amazonaws.com/umbrella-static/index.html" - }, - { - "name": "Alexa Top 500 Global Sites", - "type": "url", - "url": "http://www.alexa.com/topsites" - }, - { - "name": "W3bin.com", - "type": "url", - "url": "https://w3bin.com/" - }, - { - "name": "Sitedossier", - "type": "url", - "url": "http://www.sitedossier.com/" - }, - { - "name": "Visual Site Mapper", - "type": "url", - "url": "http://www.visualsitemapper.com/" - }, - { - "name": "ClearWebStats.com", - "type": "url", - "url": "https://www.clearwebstats.com/" - }, - { - "name": "PubDB", - "type": "url", - "url": "http://pub-db.com/" - }, - { - "name": "WWW Domain Tools", - "type": "url", - "url": "https://w3dt.net/" - }, - { - "name": "SimilarWeb", - "type": "url", - "url": "https://www.similarweb.com/" - }, - { - "name": "Website Outlook", - "type": "url", - "url": "http://websiteoutlook.com/" - }, - { - "name": "Siteliner", - "type": "url", - "url": "http://siteliner.com/" - }, - { - "name": "WebPagetest", - "type": "url", - "url": "https://www.webpagetest.org/" - }, - { - "name": "WhatWeb", - "type": "url", - "url": "https://github.com/urbanadventurer/WhatWeb" - }] - }, - { - "name": "URL Expanders", - "type": "folder", - "children": [ - { - "name": "Link Expander", - "type": "url", - "url": "http://www.linkexpander.com/" - }, - { - "name": "GetLinkInfo", - "type": "url", - "url": "http://www.getlinkinfo.com/" - }, - { - "name": "CheckShortURL", - "type": "url", - "url": "http://checkshorturl.com/" - }, - { - "name": "Lengthen Me", - "type": "url", - "url": "https://lengthen.me/" - }, - { - "name": "URL Expander", - "type": "url", - "url": "http://urlex.org/" - }, - { - "name": "URL Unshortener", - "type": "url", - "url": "http://www.urlunshortener.com/" - }, - { - "name": "Where Does This Link Go?", - "type": "url", - "url": "http://wheredoesthislinkgo.com/" - }, - { - "name": "KnowURL", - "type": "url", - "url": "http://www.knowurl.com/" - }] - }, - { - "name": "Change Detection", - "type": "folder", - "children": [ - { - "name": "VisualPing", - "type": "url", - "url": "http://visualping.io/" - }, - { - "name": "Change Detection", - "type": "url", - "url": "http://www.changedetection.com/" - }, - { - "name": "Check site availability Online", - "type": "url", - "url": "https://upcheck.online/" - }, - { - "name": "Follow That Page", - "type": "url", - "url": "https://www.followthatpage.com/" - }, - { - "name": "Urlwatch", - "type": "url", - "url": "https://github.com/thp/urlwatch" - }, - { - "name": "WatchThatPage", - "type": "url", - "url": "http://watchthatpage.com/" - }, - { - "name": "ChangeDetect", - "type": "url", - "url": "http://www.changedetect.com/" - }] - }, - { - "name": "Social Analysis", - "type": "folder", - "children": [ - { - "name": "Google Trends", - "type": "url", - "url": "https://www.google.com/trends/" - }, - { - "name": "Reddit (M)", - "type": "url", - "url": "https://www.reddit.com/domain/%3CURLhere%3E" - }] - }, - { - "name": "DNSSEC", - "type": "folder", - "children": [ - { - "name": "DNSSEC Analyzer", - "type": "url", - "url": "http://dnssec-debugger.verisignlabs.com/" - }, - { - "name": "DNSViz", - "type": "url", - "url": "http://dnsviz.net/" - }] - }, - { - "name": "Cloud Resources", - "type": "folder", - "children": [ - { - "name": "Public Buckets", - "type": "url", - "url": "https://buckets.grayhatwarfare.com/" - }, - { - "name": "Online Port scanner", - "type": "url", - "url": "https://portscanner.online/" - }, - { - "name": "CloudScraper (T)", - "type": "url", - "url": "https://github.com/jordanpotti/cloudscraper" - }, - { - "name": "Check site availability Online", - "type": "url", - "url": "https://upcheck.online/" - }] - }, - { - "name": "Vulnerabilities", - "type": "folder", - "children": [ - { - "name": "Scanners", - "type": "folder", - "children": [ - { - "name": "Sn1per (T)", - "type": "url", - "url": "https://github.com/1N3/Sn1per" + "name": "Username Search Engines", + "type": "folder", + "children": [ + { + "name": "Sylva Identity Discovery (T)", + "type": "url", + "url": "https://sylva.pfeister.dev/" + }, + { + "name": "Sherlock (T)", + "type": "url", + "url": "https://github.com/sherlock-project/sherlock" + }, + { + "name": "Namechk", + "type": "url", + "url": "https://namechk.com/" + }, + { + "name": "WhatsMyName (T)", + "type": "url", + "url": "https://github.com/WebBreacher/WhatsMyName" + }, + { + "name": "Thats Them", + "type": "url", + "url": "https://thatsthem.com/" + }, + { + "name": "NameCheckup", + "type": "url", + "url": "https://namecheckup.com/" + }, + { + "name": "Sherlock", + "type": "url", + "url": "https://github.com/sherlock-project/sherlock" + }, + { + "name": "Names Directory", + "type": "url", + "url": "https://namesdir.com/" + } + ] }, { - "name": "Online Nikto scanner", - "type": "url", - "url": "https://nikto.online/" - }] - }, - { - "name": "Disclosure Sites", - "type": "folder", - "children": [ - { - "name": "XSSposed.org", - "type": "url", - "url": "https://www.xssposed.org/" - }, - { - "name": "Zone-H.org", - "type": "url", - "url": "http://zone-h.org/archive" - }] - }, - { - "name": "RobotsDisallowed", - "type": "url", - "url": "https://github.com/danielmiessler/RobotsDisallowed" - }] + "name": "Specific Sites", + "type": "folder", + "children": [ + { + "name": "Amazon Usernames (M)", + "type": "url", + "url": "https://www.google.com/search?q=site:amazon.com+%3Cusername%3E" + }, + { + "name": "Github User (M)", + "type": "url", + "url": "https://api.github.com/users/%3Cusername%3E/events/public" + }, + { + "name": "Tinder Usernames (M)", + "type": "url", + "url": "https://www.gotinder.com/@%3Cusername%3E" + }, + { + "name": "Keybase", + "type": "url", + "url": "https://keybase.io/" + }, + { + "name": "MIT PGP Key Server", + "type": "url", + "url": "https://pgp.mit.edu/" + }, + { + "name": "ProtonMail users (M)", + "type": "url", + "url": "https://api.protonmail.ch/pks/lookup?op=index&search=@protonmail.com" + }, + { + "name": "ProtonMail Domains (M)", + "type": "url", + "url": "https://api.protonmail.ch/pks/lookup?op=index&search=" + } + ] + } + ] }, { - "name": "Tools", + "name": "Email Address", "type": "folder", "children": [ - { - "name": "Belati (T)", - "type": "url", - "url": "https://github.com/aancw/Belati" - }, - { - "name": "BlackWidow (T)", - "type": "url", - "url": "https://softbytelabs.com/wp/blackwidow/" - }, - { - "name": "Burp Suite (T)", - "type": "url", - "url": "https://portswigger.net/burp/" - }, - { - "name": "EyeWitness (T)", - "type": "url", - "url": "https://github.com/ChrisTruncer/EyeWitness" - }, - { - "name": "IntelliTamper (T)", - "type": "url", - "url": "http://www.softpedia.com/get/Internet/Other-Internet-Related/IntelliTamper.shtml" - }, - { - "name": "Hunting-New-Registered-Domains (T)", - "type": "url", - "url": "https://github.com/gfek/Hunting-New-Registered-Domains" - }, - { - "name": "International Domain Name Conversion Tool", - "type": "url", - "url": "http://mct.verisign-grs.com/" - }, - { - "name": "Online Nikto scanner", - "type": "url", - "url": "https://nikto.online/" - }] - }, - { - "name": "Report Malicious Sites", - "type": "folder", - "children": [ - { - "name": "Microsoft", - "type": "url", - "url": "https://www.microsoft.com/en-us/wdsi/support/report-unsafe-site-guest" - }, - { - "name": "Google", - "type": "url", - "url": "https://safebrowsing.google.com/safebrowsing/report_phish/?hl=en" - }] - }] - }, - { - "name": "IP & MAC Address", - "type": "folder", - "children": [ - { - "name": "Geolocation", - "type": "folder", - "children": [ - { - "name": "MaxMind Demo", - "type": "url", - "url": "https://www.maxmind.com/en/home" - }, - { - "name": "IPv4/IPv6 lists by country code", - "type": "url", - "url": "http://ipverse.net/" - }, - { - "name": "IP2Location.com", - "type": "url", - "url": "https://www.ip2location.com/demo" - }, - { - "name": "IP Fingerprints", - "type": "url", - "url": "http://www.ipfingerprints.com/" - }, - { - "name": "DB-IP", - "type": "url", - "url": "https://db-ip.com/" - }, - { - "name": "IP Location Finder", - "type": "url", - "url": "https://www.iplocation.net/" - }, - { - "name": "Info Sniper", - "type": "url", - "url": "http://www.infosniper.net/" - }, - { - "name": "utrace", - "type": "url", - "url": "http://en.utrace.de/" - }, - { - "name": "InfobyIP.com", - "type": "url", - "url": "https://www.infobyip.com/" - }, - { - "name": "ipTRACKERonline", - "type": "url", - "url": "https://www.iptrackeronline.com/" - }, - { - "name": "My IP Address", - "type": "url", - "url": "https://www.ipaddress.my/" - }] + { + "name": "Email Search", + "type": "folder", + "children": [ + { + "name": "Sylva Identity Discovery (T)", + "type": "url", + "url": "https://sylva.pfeister.dev/" + }, + { + "name": "ThatsThem", + "type": "url", + "url": "https://thatsthem.com/reverse-email-lookup" + }, + { + "name": "Hunter", + "type": "url", + "url": "https://hunter.io/" + }, + { + "name": "Email to Address (R)", + "type": "url", + "url": "https://www.melissa.com/" + }, + { + "name": "VoilaNorbert", + "type": "url", + "url": "https://www.voilanorbert.com/" + }, + { + "name": "GHunt (T)", + "type": "url", + "url": "https://github.com/mxrch/GHunt" + }, + { + "name": "OSINT Industries", + "type": "url", + "url": "https://osint.industries/" + }, + { + "name": "theHarvester (T)", + "type": "url", + "url": "https://github.com/laramies/theHarvester" + }, + { + "name": "Infoga (T)", + "type": "url", + "url": "https://github.com/m4ll0k/infoga" + }, + { + "name": "Skymem", + "type": "url", + "url": "https://www.skymem.info/" + }, + { + "name": "Epieos Email Tool", + "type": "url", + "url": "https://tools.epieos.com/email.php" + }, + { + "name": "Holehe (T)", + "type": "url", + "url": "https://github.com/megadose/holehe" + } + ] + }, + { + "name": "Common Email Formats", + "type": "folder", + "children": [ + { + "name": "Email Format", + "type": "url", + "url": "https://www.email-format.com/" + }, + { + "name": "Email Permutator", + "type": "url", + "url": "https://metricsparrow.com/toolkit/email-permutator/" + } + ] + }, + { + "name": "Email Verification", + "type": "folder", + "children": [ + { + "name": "Reacher Github (T)", + "type": "url", + "url": "https://github.com/reacherhq/check-if-email-exists" + }, + { + "name": "Reacher Demo", + "type": "url", + "url": "https://reacher.email" + }, + { + "name": "MailScrap", + "type": "url", + "url": "https://mailscrap.com/" + }, + { + "name": "Read Notify", + "type": "url", + "url": "https://www.readnotify.com/" + }, + { + "name": "Email Reputation", + "type": "url", + "url": "https://emailrep.io/" + }, + { + "name": "MailboxValidator", + "type": "url", + "url": "https://www.mailboxvalidator.com/demo" + }, + { + "name": "VerifyEmail (R$)", + "type": "url", + "url": "https://emailable.com/" + } + ] + }, + { + "name": "Breach Data", + "type": "folder", + "children": [ + { + "name": "Have I been pwned?", + "type": "url", + "url": "https://haveibeenpwned.com/" + }, + { + "name": "Hudson Rock", + "type": "url", + "url": "https://www.hudsonrock.com/threat-intelligence-cybercrime-tools" + }, + { + "name": "DeHashed (R)", + "type": "url", + "url": "https://dehashed.com/" + }, + { + "name": "Vigilante.pw", + "type": "url", + "url": "https://www.vigilante.pw/" + } + ] + }, + { + "name": "Mail Blacklists", + "type": "folder", + "children": [ + { + "name": "MxToolbox", + "type": "url", + "url": "https://mxtoolbox.com/" + } + ] + } + ] }, { - "name": "Host / Port Discovery", + "name": "Domain Name", "type": "folder", "children": [ - { - "name": "urlscan.io", - "type": "url", - "url": "https://urlscan.io/search/#*" - }, - { - "name": "Spyse", - "type": "url", - "url": "https://spyse.com/search/ip" - }, - { - "name": "Shodan", - "type": "url", - "url": "https://www.shodan.io/" - }, - { - "name": "Netlas.io", - "type": "url", - "url": "https://netlas.io/" - }, - { - "name": "Portmap", - "type": "url", - "url": "https://portmap.com/" - }, - { - "name": "Scans.io", - "type": "url", - "url": "https://scans.io/" - }, - { - "name": "ZoomEye", - "type": "url", - "url": "https://www.zoomeye.org/" - }, - { - "name": "Nmap (T)", - "type": "url", - "url": "https://nmap.org/download.html" - }, - { - "name": "Online Port scanner", - "type": "url", - "url": "https://portscanner.online/" - }, - { - "name": "Internet Census Search", - "type": "url", - "url": "http://www.exfiltrated.com/querystart.php" - }, - { - "name": "Criminal IP Search", - "type": "url", - "url": "https://www.criminalip.io/" - }, - { - "name": "Scanless (T)", - "type": "url", - "url": "https://github.com/vesche/scanless" - }, - { - "name": "BinaryEdge (R)", - "type": "url", - "url": "https://www.binaryedge.io/" - }, - { - "name": "Masscan (T)", - "type": "url", - "url": "https://github.com/robertdavidgraham/masscan" - }, - { - "name": "ODIN (R)", - "type": "url", - "url": "https://www.getodin.com/" - }] + { + "name": "Whois Records", + "type": "folder", + "children": [ + { + "name": "Domain Dossier", + "type": "url", + "url": "https://centralops.net/co/DomainDossier.aspx" + }, + { + "name": "domainIQ", + "type": "url", + "url": "https://www.domainiq.com/" + }, + { + "name": "DomainTools Whois", + "type": "url", + "url": "https://whois.domaintools.com/" + }, + { + "name": "SWITCH Internet Domains Whois (.ch)", + "type": "url", + "url": "https://www.nic.ch/whois/" + }, + { + "name": "Whoisology", + "type": "url", + "url": "https://whoisology.com/#advanced" + }, + { + "name": "Whois ARIN", + "type": "url", + "url": "https://whois.arin.net/ui/advanced.jsp" + }, + { + "name": "DNSstuff", + "type": "url", + "url": "https://tools.dnsstuff.com/" + }, + { + "name": "Robtex (R)", + "type": "url", + "url": "https://www.robtex.com/" + }, + { + "name": "Domaincrawler.com", + "type": "url", + "url": "https://domaincrawler.com/" + }, + { + "name": "MarkMonitor Whois Search", + "type": "url", + "url": "https://domains.markmonitor.com/whois/" + }, + { + "name": "easyWhois", + "type": "url", + "url": "https://www.easywhois.com/" + }, + { + "name": "Website Informer", + "type": "url", + "url": "https://website.informer.com/" + }, + { + "name": "Who.is", + "type": "url", + "url": "https://who.is/" + }, + { + "name": "Whois AMPed", + "type": "url", + "url": "https://whoisamped.com/" + }, + { + "name": "ViewDNS.info", + "type": "url", + "url": "https://viewdns.info/" + }, + { + "name": "Daily DNS Changes", + "type": "url", + "url": "https://dailychanges.domaintools.com/" + }, + { + "name": "IP2WHOIS", + "type": "url", + "url": "https://www.ip2whois.com" + }, + { + "name": "Netlas.io", + "type": "url", + "url": "https://app.netlas.io/whois_domains/" + } + ] + }, + { + "name": "Subdomains", + "type": "folder", + "children": [ + { + "name": "SynapsInt", + "type": "url", + "url": "https://synapsint.com" + }, + { + "name": "Aquatone (T)", + "type": "url", + "url": "https://github.com/michenriksen/aquatone" + }, + { + "name": "FindSubDomains", + "type": "url", + "url": "https://findsubdomains.com/" + }, + { + "name": "Google Subdomains (D)", + "type": "url", + "url": "https://www.google.com/?gws_rd=ssl#q=site:%3Cdomain.com%3E" + }, + { + "name": "Recon-ng (T)", + "type": "url", + "url": "https://github.com/lanmaster53/recon-ng" + }, + { + "name": "XRay", + "type": "url", + "url": "https://github.com/evilsocket/xray" + }, + { + "name": "DNS Recon (T)", + "type": "url", + "url": "https://github.com/darkoperator/dnsrecon" + }, + { + "name": "Gobuster (T)", + "type": "url", + "url": "https://github.com/OJ/gobuster" + }, + { + "name": "Fierce Domain Scanner (T)", + "type": "url", + "url": "https://github.com/davidpepper/fierce-domain-scanner" + }, + { + "name": "Bluto (T)", + "type": "url", + "url": "https://github.com/darryllane/Bluto" + }, + { + "name": "OWASP Maryam (T)", + "type": "url", + "url": "https://github.com/saeeddhqan/Maryam" + }, + { + "name": "theHarvester (T)", + "type": "url", + "url": "https://github.com/laramies/theHarvester" + }, + { + "name": "Pentest-tools.com Subdomains", + "type": "url", + "url": "https://pentest-tools.com/information-gathering/find-subdomains-of-domain" + }, + { + "name": "SecLists DNS Subdomains (T)", + "type": "url", + "url": "https://github.com/danielmiessler/SecLists/tree/master/Discovery/DNS" + }, + { + "name": "dnspop (T)", + "type": "url", + "url": "https://github.com/bitquark/dnspop" + }, + { + "name": "gdns (T)", + "type": "url", + "url": "https://github.com/hrbrmstr/gdns" + }, + { + "name": "Sublist3r", + "type": "url", + "url": "https://github.com/aboul3la/Sublist3r" + }, + { + "name": "AltDNS (T)", + "type": "url", + "url": "https://github.com/infosec-au/altdns" + }, + { + "name": "Netlas.io", + "type": "url", + "url": "https://app.netlas.io/domains/" + } + ] + }, + { + "name": "Discovery", + "type": "folder", + "children": [ + { + "name": "Shodan", + "type": "url", + "url": "https://www.shodan.io/" + }, + { + "name": "Netlas.io", + "type": "url", + "url": "https://app.netlas.io/" + }, + { + "name": "Kraken (T)", + "type": "url", + "url": "https://github.com/Sw4mpf0x/Kraken" + }, + { + "name": "Online Nikto scanner", + "type": "url", + "url": "https://nikto.online/" + }, + { + "name": "urlscan.io", + "type": "url", + "url": "https://urlscan.io/search/#*" + }, + { + "name": "Daily DNS Changes", + "type": "url", + "url": "https://dailychanges.domaintools.com/" + }, + { + "name": "Redirect Detective", + "type": "url", + "url": "https://redirectdetective.com/" + }, + { + "name": "Sitediff (T)", + "type": "url", + "url": "https://github.com/digininja/sitediff" + }, + { + "name": "BuiltWith", + "type": "url", + "url": "https://builtwith.com/" + }, + { + "name": "Wappalyzer", + "type": "url", + "url": "https://www.wappalyzer.com/" + }, + { + "name": "AnalyzeID", + "type": "url", + "url": "https://analyzeid.com/" + }, + { + "name": "Criminal IP Search", + "type": "url", + "url": "https://www.criminalip.io/" + }, + { + "name": "urlDNA", + "type": "url", + "url": "https://urldna.io" + } + ] + }, + { + "name": "Certificate Search", + "type": "folder", + "children": [ + { + "name": "Shodan", + "type": "url", + "url": "https://www.shodan.io/" + }, + { + "name": "Google's Certificate Transparency", + "type": "url", + "url": "https://www.certificate-transparency.org/known-logs" + }, + { + "name": "Spyse", + "type": "url", + "url": "https://spyse.com/search/certificate" + }, + { + "name": "Censys", + "type": "url", + "url": "https://censys.io/" + }, + { + "name": "crt.sh - Certificate Search", + "type": "url", + "url": "https://crt.sh/?" + }, + { + "name": "certgraph (T)", + "type": "url", + "url": "https://github.com/lanrat/certgraph" + }, + { + "name": "Netlas.io", + "type": "url", + "url": "https://app.netlas.io/certs/" + } + ] + }, + { + "name": "PassiveDNS", + "type": "folder", + "children": [ + { + "name": "Mnemonic", + "type": "url", + "url": "https://passivedns.mnemonic.no/" + }, + { + "name": "DNS Dumpster", + "type": "url", + "url": "https://dnsdumpster.com/" + }, + { + "name": "Deteque (R)", + "type": "url", + "url": "https://www.deteque.com/" + } + ] + }, + { + "name": "Reputation", + "type": "folder", + "children": [ + { + "name": "UrlQuery.net", + "type": "url", + "url": "https://urlquery.net/" + }, + { + "name": "URL Void", + "type": "url", + "url": "https://www.urlvoid.com/" + }, + { + "name": "FortiGuard Reputation Service", + "type": "url", + "url": "https://fortiguard.com/iprep" + }, + { + "name": "Trend Micro Site Safety Center", + "type": "url", + "url": "https://global.sitesafety.trendmicro.com/" + }, + { + "name": "WatchGuard ReputationAuthority", + "type": "url", + "url": "https://www.reputationauthority.org/" + }, + { + "name": "Sucuri SiteCheck", + "type": "url", + "url": "https://sitecheck.sucuri.net/" + }, + { + "name": "ThreatMiner.org", + "type": "url", + "url": "https://www.threatminer.org/" + }, + { + "name": "BlueCoat WebPulse", + "type": "url", + "url": "https://sitereview.bluecoat.com/sitereview.jsp" + }, + { + "name": "Zscaler Zulu URL Risk Analyzer", + "type": "url", + "url": "https://zulu.zscaler.com/" + }, + { + "name": "Joe Sandbox Url Analyzer", + "type": "url", + "url": "https://www.url-analyzer.net/" + }, + { + "name": "Deepviz Domain Search", + "type": "url", + "url": "https://search.deepviz.com/" + }, + { + "name": "AVG Threat Labs", + "type": "url", + "url": "https://www.avg.com/en/signal/website-safety" + }, + { + "name": "Webroot BrightCloud URL/IP Lookup", + "type": "url", + "url": "https://www.brightcloud.com/tools/url-ip-lookup.php" + }, + { + "name": "vURL Online", + "type": "url", + "url": "https://vurldissect.co.uk/" + }, + { + "name": "AlienVault Open Threat Exchange", + "type": "url", + "url": "https://otx.alienvault.com/browse/pulses/" + }, + { + "name": "Malware Domain List", + "type": "url", + "url": "https://www.malwaredomainlist.com/mdl.php" + }, + { + "name": "Web Inspector Online Scan", + "type": "url", + "url": "https://www.webinspector.com/website-malware-scanner/" + }, + { + "name": "Google Safe Browsing API", + "type": "url", + "url": "https://developers.google.com/safe-browsing/?csw=1" + }, + { + "name": "Cisco Talos", + "type": "url", + "url": "https://talosintelligence.com/" + } + ] + }, + { + "name": "Domain Blacklists", + "type": "folder", + "children": [ + { + "name": "Threatexpert.com Malicious URLs", + "type": "url", + "url": "https://www.networksec.org/grabbho/block.txt" + }, + { + "name": "Zeus C2 Tracker", + "type": "url", + "url": "https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist" + }, + { + "name": "Malware Domains Blacklist", + "type": "url", + "url": "https://mirror1.malwaredomains.com/files/domains.txt" + }, + { + "name": "Blackweb", + "type": "url", + "url": "https://github.com/maravento/blackweb" + }, + { + "name": "Critical Stack Intel (R)", + "type": "url", + "url": "https://intel.criticalstack.com/" + }, + { + "name": "DNS Sinkhole", + "type": "url", + "url": "https://malc0de.com/bl/" + }, + { + "name": "DNS-BH Malware Domain Blocklist", + "type": "url", + "url": "https://www.malwaredomains.com/wordpress/?page_id=66" + }, + { + "name": "Malware Domain List", + "type": "url", + "url": "https://www.malwaredomainlist.com/hostslist/hosts.txt" + }, + { + "name": "MalwareURL (R)", + "type": "url", + "url": "https://www.malwareurl.com/index.php" + }, + { + "name": "scumware.org", + "type": "url", + "url": "https://www.scumware.org/" + }, + { + "name": "ZeuS Tracker", + "type": "url", + "url": "https://zeustracker.abuse.ch/blocklist.php" + }, + { + "name": "Shadowserver Foundation", + "type": "url", + "url": "https://www.shadowserver.org/" + }, + { + "name": "Email Domain Validation", + "type": "url", + "url": "https://www.mailboxvalidator.com/domain" + } + ] + }, + { + "name": "Typosquatting", + "type": "folder", + "children": [ + { + "name": "DNS Twist (T)", + "type": "url", + "url": "https://github.com/elceef/dnstwist" + }, + { + "name": "URLCrazy (T)", + "type": "url", + "url": "https://www.morningstarsecurity.com/research/urlcrazy" + }, + { + "name": "dnstwister", + "type": "url", + "url": "https://dnstwister.report/" + }, + { + "name": "Catphish (T)", + "type": "url", + "url": "https://github.com/ring0lab/catphish" + } + ] + }, + { + "name": "Analytics", + "type": "folder", + "children": [ + { + "name": "BuiltWith", + "type": "url", + "url": "https://builtwith.com/" + }, + { + "name": "SiteSleuth", + "type": "url", + "url": "https://www.sitesleuth.io/" + }, + { + "name": "Wappalyzer (T)", + "type": "url", + "url": "https://wappalyzer.com/" + }, + { + "name": "SEMrush", + "type": "url", + "url": "https://www.semrush.com/" + }, + { + "name": "Moonsearch", + "type": "url", + "url": "https://moonsearch.com/" + }, + { + "name": "Ewhois", + "type": "url", + "url": "https://ewhois.com/" + }, + { + "name": "StatsCrop", + "type": "url", + "url": "https://www.statscrop.com/" + }, + { + "name": "Open Site Explorer", + "type": "url", + "url": "https://moz.com/researchtools/ose/" + }, + { + "name": "SpyOnWeb", + "type": "url", + "url": "https://www.spyonweb.com/" + }, + { + "name": "Keyword Density", + "type": "url", + "url": "https://tools.seobook.com/general/keyword-density/" + }, + { + "name": "Alexa Site Statistics", + "type": "url", + "url": "https://alexa.amazon.com/about" + }, + { + "name": "Cisco Umbrella Popularity List", + "type": "url", + "url": "https://s3-us-west-1.amazonaws.com/umbrella-static/index.html" + }, + { + "name": "Alexa Top 500 Global Sites", + "type": "url", + "url": "https://alexa.amazon.com/about" + }, + { + "name": "Sitedossier", + "type": "url", + "url": "https://www.sitedossier.com/" + }, + { + "name": "Visual Site Mapper", + "type": "url", + "url": "https://github.com/alentum/sitemapper-nodejs" + }, + { + "name": "ClearWebStats.com", + "type": "url", + "url": "https://www.clearwebstats.com/" + }, + { + "name": "PubDB", + "type": "url", + "url": "https://pub-db.com/" + }, + { + "name": "SimilarWeb", + "type": "url", + "url": "https://www.similarweb.com/" + }, + { + "name": "Siteliner", + "type": "url", + "url": "https://www.siteliner.com/" + }, + { + "name": "WhatWeb", + "type": "url", + "url": "https://github.com/urbanadventurer/WhatWeb" + } + ] + }, + { + "name": "URL Expanders", + "type": "folder", + "children": [ + { + "name": "Link Expander", + "type": "url", + "url": "https://www.linkexpander.com/" + }, + { + "name": "CheckShortURL", + "type": "url", + "url": "https://checkshorturl.com/" + }, + { + "name": "URL Expander", + "type": "url", + "url": "https://urlex.org/" + }, + { + "name": "Where Does This Link Go?", + "type": "url", + "url": "https://wheregoes.com/" + }, + { + "name": "KnowURL", + "type": "url", + "url": "https://www.knowurl.com/" + } + ] + }, + { + "name": "Change Detection", + "type": "folder", + "children": [ + { + "name": "VisualPing", + "type": "url", + "url": "https://visualping.io:443/" + }, + { + "name": "Change Detection", + "type": "url", + "url": "https://visualping.io/" + }, + { + "name": "Check site availability Online", + "type": "url", + "url": "https://upcheck.online/" + }, + { + "name": "Follow That Page", + "type": "url", + "url": "https://www.followthatpage.com/" + }, + { + "name": "Urlwatch", + "type": "url", + "url": "https://github.com/thp/urlwatch" + }, + { + "name": "WatchThatPage", + "type": "url", + "url": "https://watchthatpage.com/" + }, + { + "name": "ChangeDetect", + "type": "url", + "url": "https://www.eatgrainmaker.com/home/" + } + ] + }, + { + "name": "Social Analysis", + "type": "folder", + "children": [ + { + "name": "Google Trends", + "type": "url", + "url": "https://www.google.com/trends/" + } + ] + }, + { + "name": "DNSSEC", + "type": "folder", + "children": [ + { + "name": "DNSSEC Analyzer", + "type": "url", + "url": "https://dnssec-analyzer.verisignlabs.com/" + }, + { + "name": "DNSViz", + "type": "url", + "url": "https://dnsviz.net/" + } + ] + }, + { + "name": "Cloud Resources", + "type": "folder", + "children": [ + { + "name": "Public Buckets", + "type": "url", + "url": "https://buckets.grayhatwarfare.com/" + }, + { + "name": "Online Port scanner", + "type": "url", + "url": "https://portscanner.online/" + }, + { + "name": "CloudScraper (T)", + "type": "url", + "url": "https://github.com/jordanpotti/cloudscraper" + }, + { + "name": "Check site availability Online", + "type": "url", + "url": "https://upcheck.online/" + } + ] + }, + { + "name": "Vulnerabilities", + "type": "folder", + "children": [ + { + "name": "Scanners", + "type": "folder", + "children": [ + { + "name": "Sn1per (T)", + "type": "url", + "url": "https://github.com/1N3/Sn1per" + }, + { + "name": "Online Nikto scanner", + "type": "url", + "url": "https://nikto.online/" + } + ] + }, + { + "name": "Disclosure Sites", + "type": "folder", + "children": [ + { + "name": "Zone-H.org", + "type": "url", + "url": "https://zone-h.org/archive" + } + ] + }, + { + "name": "RobotsDisallowed", + "type": "url", + "url": "https://github.com/danielmiessler/RobotsDisallowed" + } + ] + }, + { + "name": "Tools", + "type": "folder", + "children": [ + { + "name": "Belati (T)", + "type": "url", + "url": "https://github.com/aancw/Belati" + }, + { + "name": "Burp Suite (T)", + "type": "url", + "url": "https://portswigger.net/burp/" + }, + { + "name": "EyeWitness (T)", + "type": "url", + "url": "https://github.com/ChrisTruncer/EyeWitness" + }, + { + "name": "Hunting-New-Registered-Domains (T)", + "type": "url", + "url": "https://github.com/gfek/Hunting-New-Registered-Domains" + }, + { + "name": "International Domain Name Conversion Tool", + "type": "url", + "url": "https://mct.verisign-grs.com/" + }, + { + "name": "Online Nikto scanner", + "type": "url", + "url": "https://nikto.online/" + } + ] + }, + { + "name": "Report Malicious Sites", + "type": "folder", + "children": [ + { + "name": "Microsoft", + "type": "url", + "url": "https://www.microsoft.com/en-us/wdsi/support/report-unsafe-site-guest" + }, + { + "name": "Google", + "type": "url", + "url": "https://safebrowsing.google.com/safebrowsing/report_phish/?hl=en" + } + ] + } + ] }, { - "name": "IPv4", + "name": "IP & MAC Address", "type": "folder", "children": [ - { - "name": "ASlookup.com", - "type": "url", - "url": "https://aslookup.com/" - }, - { - "name": "Port scanner Online", - "type": "url", - "url": "https://portscanner.online/" - }, - { - "name": "Onyphe", - "type": "url", - "url": "https://www.onyphe.io/" - }, - { - "name": "ODIN (R)", - "type": "url", - "url": "https://www.getodin.com/" - }, - { - "name": "IPv4 CIDR Report", - "type": "url", - "url": "http://www.cidr-report.org/as2.0/" - }, - { - "name": "Reverse.report", - "type": "url", - "url": "https://reverse.report/" - }, - { - "name": "Team Cymru IP to ASN", - "type": "url", - "url": "https://asn.cymru.com/" - }, - { - "name": "IP to ASN DB", - "type": "url", - "url": "https://iptoasn.com/" - }, - { - "name": "Hacker Target - Reverse DNS", - "type": "url", - "url": "https://hackertarget.com/reverse-dns-lookup/" - }] + { + "name": "Geolocation", + "type": "folder", + "children": [ + { + "name": "MaxMind Demo", + "type": "url", + "url": "https://www.maxmind.com/en/home" + }, + { + "name": "IPv4/IPv6 lists by country code", + "type": "url", + "url": "https://github.com/ipverse/" + }, + { + "name": "IP2Location.com", + "type": "url", + "url": "https://www.ip2location.com/demo" + }, + { + "name": "IP Fingerprints", + "type": "url", + "url": "https://ipfingerprints.com/" + }, + { + "name": "DB-IP", + "type": "url", + "url": "https://db-ip.com/" + }, + { + "name": "IP Location Finder", + "type": "url", + "url": "https://www.iplocation.net/" + }, + { + "name": "Info Sniper", + "type": "url", + "url": "https://www.infosniper.net/" + }, + { + "name": "utrace", + "type": "url", + "url": "https://en.utrace.de/" + } + ] + }, + { + "name": "Host / Port Discovery", + "type": "folder", + "children": [ + { + "name": "urlscan.io", + "type": "url", + "url": "https://urlscan.io/search/#*" + }, + { + "name": "Spyse", + "type": "url", + "url": "https://spyse.com/search/ip" + }, + { + "name": "Shodan", + "type": "url", + "url": "https://www.shodan.io/" + }, + { + "name": "Netlas.io", + "type": "url", + "url": "https://netlas.io/" + }, + { + "name": "Portmap", + "type": "url", + "url": "https://portmap.com/" + }, + { + "name": "Scans.io", + "type": "url", + "url": "https://scans.io/" + }, + { + "name": "Nmap (T)", + "type": "url", + "url": "https://nmap.org/download.html" + }, + { + "name": "Online Port scanner", + "type": "url", + "url": "https://portscanner.online/" + }, + { + "name": "Internet Census Search", + "type": "url", + "url": "https://www.exfiltrated.com/querystart.php" + }, + { + "name": "Criminal IP Search", + "type": "url", + "url": "https://www.criminalip.io/" + }, + { + "name": "Scanless (T)", + "type": "url", + "url": "https://github.com/vesche/scanless" + }, + { + "name": "BinaryEdge (R)", + "type": "url", + "url": "https://www.binaryedge.io/" + }, + { + "name": "Masscan (T)", + "type": "url", + "url": "https://github.com/robertdavidgraham/masscan" + } + ] + }, + { + "name": "IPv4", + "type": "folder", + "children": [ + { + "name": "ASlookup.com", + "type": "url", + "url": "https://aslookup.com/" + }, + { + "name": "Port scanner Online", + "type": "url", + "url": "https://portscanner.online/" + }, + { + "name": "Onyphe", + "type": "url", + "url": "https://www.onyphe.io/" + }, + { + "name": "IPv4 CIDR Report", + "type": "url", + "url": "https://www.cidr-report.org/as2.0/" + }, + { + "name": "Reverse.report", + "type": "url", + "url": "https://reverse.report/" + }, + { + "name": "Team Cymru IP to ASN", + "type": "url", + "url": "https://asn.cymru.com/" + }, + { + "name": "IP to ASN DB", + "type": "url", + "url": "https://iptoasn.com/" + }, + { + "name": "Hacker Target - Reverse DNS", + "type": "url", + "url": "https://hackertarget.com/reverse-dns-lookup/" + } + ] + }, + { + "name": "IPv6", + "type": "folder", + "children": [ + { + "name": "IPv6 CIDR Report", + "type": "url", + "url": "https://www.cidr-report.org/v6/as2.0/" + } + ] + }, + { + "name": "BGP", + "type": "folder", + "children": [ + { + "name": "Hurricane Electric BGP Toolkit", + "type": "url", + "url": "https://bgp.he.net/" + }, + { + "name": "BGP Malicious Content Ranking", + "type": "url", + "url": "https://bgpranking.circl.lu/" + }, + { + "name": "PeeringDB", + "type": "url", + "url": "https://www.peeringdb.com/advanced_search" + }, + { + "name": "BGP Tools", + "type": "url", + "url": "https://www.bgp4.as/tools" + } + ] + }, + { + "name": "Reputation", + "type": "folder", + "children": [ + { + "name": "IP Void", + "type": "url", + "url": "https://www.ipvoid.com/" + }, + { + "name": "ExoneraTor", + "type": "url", + "url": "https://exonerator.torproject.org/" + }, + { + "name": "Grey Noise", + "type": "url", + "url": "https://viz.greynoise.io/" + } + ] + }, + { + "name": "Blacklists", + "type": "folder", + "children": [ + { + "name": "Blocklist.de", + "type": "url", + "url": "https://www.blocklist.de/en/index.html" + }, + { + "name": "DShield API", + "type": "url", + "url": "https://isc.sans.edu/api/" + }, + { + "name": "FireHOL IP Lists ", + "type": "url", + "url": "https://iplists.firehol.org/" + }, + { + "name": "Project Honey Pot", + "type": "url", + "url": "https://www.projecthoneypot.org/list_of_ips.php" + } + ] + }, + { + "name": "Neighbor Domains", + "type": "folder", + "children": [ + { + "name": "IP Fingerprints - Reverse IP Lookup", + "type": "url", + "url": "https://ipfingerprints.com/reverseip.php" + }, + { + "name": "Bing IP Search (D)", + "type": "url", + "url": "https://www.bing.com/search?q=ip%3A8.8.8.8" + }, + { + "name": "TCP/IP Utils - Domain Neighbors", + "type": "url", + "url": "https://dnslytics.com/" + }, + { + "name": "MyIPNeighbors", + "type": "url", + "url": "https://www.my-ip-neighbors.com/" + } + ] + }, + { + "name": "Protected by Cloud Services", + "type": "folder", + "children": [ + { + "name": "CloudFlare Watch", + "type": "url", + "url": "https://www.crimeflare.com/" + }, + { + "name": "CloudFail (T)", + "type": "url", + "url": "https://github.com/m0rtem/CloudFail" + } + ] + }, + { + "name": "Wireless Network Info", + "type": "folder", + "children": [ + { + "name": "WiGLE: Wireless Network Mapping", + "type": "url", + "url": "https://wigle.net/" + }, + { + "name": "OpenCellid: Database of Cell Towers", + "type": "url", + "url": "https://opencellid.org/" + } + ] + }, + { + "name": "Network Analysis Tools", + "type": "folder", + "children": [ + { + "name": "Wireshark", + "type": "url", + "url": "https://www.wireshark.org/download.html" + }, + { + "name": "NetworkMiner", + "type": "url", + "url": "https://www.netresec.com/?page=Networkminer" + }, + { + "name": "Packet Total", + "type": "url", + "url": "https://www.packettotal.com/" + } + ] + }, + { + "name": "IP Loggers", + "type": "folder", + "children": [ + { + "name": "Ki.tc", + "type": "url", + "url": "https://ki.tc" + }, + { + "name": "Grabify", + "type": "url", + "url": "https://grabify.link" + }, + { + "name": "IP Logger", + "type": "url", + "url": "https://iplogger.com/" + } + ] + } + ] }, { - "name": "IPv6", + "name": "Images / Videos / Docs", "type": "folder", "children": [ - { - "name": "IPv6 CIDR Report", - "type": "url", - "url": "http://www.cidr-report.org/v6/as2.0/" - }] + { + "name": "Images", + "type": "folder", + "children": [ + { + "name": "Search", + "type": "folder", + "children": [ + { + "name": "Google Images", + "type": "url", + "url": "https://images.google.com/?gws_rd=ssl" + }, + { + "name": "Bing Images", + "type": "url", + "url": "https://www.bing.com/images" + }, + { + "name": "FaceCheck Facial Recognition Search", + "type": "url", + "url": "https://facecheck.id/" + }, + { + "name": "TinEye Reverse Image Search", + "type": "url", + "url": "https://tineye.com/" + }, + { + "name": "PimEyes Face Search Engine", + "type": "url", + "url": "https://pimeyes.com/en" + }, + { + "name": "Yandex Images", + "type": "url", + "url": "https://www.yandex.com/images/" + }, + { + "name": "Baidu Images", + "type": "url", + "url": "https://graph.baidu.com/pcpage/index?tpl_from=pc" + }, + { + "name": "Twitter Image Search (M)", + "type": "url", + "url": "https://twitter.com/search?q=%3Csearchterm%3E&src=typd&vertical=default&f=images" + }, + { + "name": "Imgur Search", + "type": "url", + "url": "https://imgur.com/search" + }, + { + "name": "Photobucket", + "type": "url", + "url": "https://photobucket.com/" + }, + { + "name": "7Photos.net", + "type": "url", + "url": "https://7photos.net/" + }, + { + "name": "Panoramio", + "type": "url", + "url": "https://www.panoramio.com/" + }, + { + "name": "Current Location", + "type": "url", + "url": "https://current-location.com/" + }, + { + "name": "Lakako Photo Search", + "type": "url", + "url": "https://www.lakako.com/" + }, + { + "name": "CC Search", + "type": "url", + "url": "https://ccsearch.creativecommons.org/" + }, + { + "name": "CamFind App", + "type": "url", + "url": "https://camfindapp.com/" + }, + { + "name": "RevEye Reverse Image Search (T)", + "type": "url", + "url": "https://chrome.google.com/webstore/detail/reveye-reverse-image-sear/keaaclcjhehbbapnphnmpiklalfhelgf?hl=en" + }, + { + "name": "SmugMug Search", + "type": "url", + "url": "https://www.smugmug.com/" + }, + { + "name": "ImageNet", + "type": "url", + "url": "https://image-net.org/" + }, + { + "name": "Places2", + "type": "url", + "url": "https://places2.csail.mit.edu/explore.html" + }, + { + "name": "Image Identification Project", + "type": "url", + "url": "https://www.imageidentify.com/" + }, + { + "name": "SauceNAO", + "type": "url", + "url": "https://saucenao.com/" + }, + { + "name": "Picarta", + "type": "url", + "url": "https://picarta.ai/" + }, + { + "name": "FaceSeek Face Search Engine", + "type": "url", + "url": "https://faceseek.online" + } + ] + }, + { + "name": "Instagram", + "type": "folder", + "children": [ + { + "name": "Webstigram", + "type": "url", + "url": "https://websta.me/search-engine-optimization/" + }, + { + "name": "Instagram", + "type": "url", + "url": "https://www.instagram.com/" + }, + { + "name": "Mini Instagram", + "type": "url", + "url": "https://mininsta.net/" + }, + { + "name": "Imgrab", + "type": "url", + "url": "https://imgrab.com/" + }, + { + "name": "Tofo.me", + "type": "url", + "url": "https://tofo.me/" + } + ] + }, + { + "name": "Flickr", + "type": "folder", + "children": [ + { + "name": "Flickr", + "type": "url", + "url": "https://www.flickr.com/" + }, + { + "name": "Flickr Map", + "type": "url", + "url": "https://www.flickr.com/map/" + }, + { + "name": "My Pics Map", + "type": "url", + "url": "https://www.mypicsmap.com/" + }, + { + "name": "idGettr", + "type": "url", + "url": "https://www.webfx.com/tools/idgettr/" + }, + { + "name": "Flickr Hive Mind", + "type": "url", + "url": "https://flickrhivemind.net/" + } + ] + }, + { + "name": "Metadata", + "type": "folder", + "children": [ + { + "name": "ExifTool (T)", + "type": "url", + "url": "https://exiftool.org/" + }, + { + "name": "ExifEditor", + "type": "url", + "url": "https://exifeditor.io" + }, + { + "name": "Jeffrey's Exif Viewer", + "type": "url", + "url": "https://exif.regex.info/" + }, + { + "name": "ExifViewer", + "type": "url", + "url": "https://www.exifviewer.org/" + }, + { + "name": "Search by Exif", + "type": "url", + "url": "https://www.exif-search.com/" + }, + { + "name": "ImgOps", + "type": "url", + "url": "https://imgops.com/" + }, + { + "name": "Metapicz", + "type": "url", + "url": "https://metapicz.com/#landing" + }, + { + "name": "JPEGsnoop (T)", + "type": "url", + "url": "https://www.impulseadventure.com/photo/jpeg-snoop.html" + }, + { + "name": "GeoSetter", + "type": "url", + "url": "https://geosetter.de/en/main-en/" + }, + { + "name": "xeuledoc - Fetch metadata about any public Google document", + "type": "url", + "url": "https://github.com/Malfrats/xeuledoc" + } + ] + }, + { + "name": "Forensics", + "type": "folder", + "children": [ + { + "name": "FotoForensics", + "type": "url", + "url": "https://fotoforensics.com/" + }, + { + "name": "Ghiro (T)", + "type": "url", + "url": "https://github.com/ghirensics/ghiro" + }, + { + "name": "Camera Trace", + "type": "url", + "url": "https://www.cameratrace.com/trace" + } + ] + }, + { + "name": "OCR", + "type": "folder", + "children": [ + { + "name": "Online OCR", + "type": "url", + "url": "https://www.sodapdf.com/pdf-tools/ocr-pdf/" + }, + { + "name": "i2OCR", + "type": "url", + "url": "https://www.i2ocr.com/" + }, + { + "name": "New OCR", + "type": "url", + "url": "https://www.newocr.com/" + }, + { + "name": "Online OCR", + "type": "url", + "url": "https://www.onlineocr.net/" + } + ] + }, + { + "name": "Tools", + "type": "folder", + "children": [ + { + "name": "Creepy (T)", + "type": "url", + "url": "https://www.geocreepy.com/" + } + ] + } + ] + }, + { + "name": "Videos", + "type": "folder", + "children": [ + { + "name": "Search", + "type": "folder", + "children": [ + { + "name": "Google Videos", + "type": "url", + "url": "https://www.google.com/videohp?gws_rd=ssl" + }, + { + "name": "Bing Videos", + "type": "url", + "url": "https://www.bing.com/videos" + }, + { + "name": "Internet Archive Videos", + "type": "url", + "url": "https://archive.org/details/opensource_movies" + }, + { + "name": "Vines (D)", + "type": "url", + "url": "https://www.google.com/search?q=site:vine.co+%3Csearchterm%3E" + }, + { + "name": "Dogpile Web Search", + "type": "url", + "url": "https://www.dogpile.com/" + }, + { + "name": "Geo Search Tool", + "type": "url", + "url": "https://www.geosearchtool.com/" + }, + { + "name": "blinkx Video Search", + "type": "url", + "url": "https://blinkx.com/" + }, + { + "name": "Facebook Live Map", + "type": "url", + "url": "https://www.facebook.com/livemap#" + }, + { + "name": "LiveLeak", + "type": "url", + "url": "https://www.liveleak.com/" + }, + { + "name": "Metatube", + "type": "url", + "url": "https://www.metatube.com/" + }, + { + "name": "Yahoo Video Search", + "type": "url", + "url": "https://video.search.yahoo.com/" + }, + { + "name": "Search YouTube by Location", + "type": "url", + "url": "https://mattw.io/youtube-geofind/location" + } + ] + }, + { + "name": "Analyze / Record", + "type": "folder", + "children": [ + { + "name": "DetURL", + "type": "url", + "url": "https://deturl.com/" + }, + { + "name": "Print YouTube StoryBoard Instructions", + "type": "url", + "url": "https://www.labnol.org/internet/print-youtube-video/28217" + }, + { + "name": "Print Storyboard from Youtube", + "type": "url", + "url": "javascript:(function(){a=ytplayer.config.args.storyboard_spec;if(!a){alert(\"Sorry we cannot process this YouTube video. Could you please try another one\");exit();}b=a.split(\"|\");base=b[0].split(\"$\")[0]+\"2/M\";c=b[3].split(\"%23\");sigh=c[c.length-1];var imgs=\"\";t=ytplayer.config.args.length_seconds;n=Math.ceil(c[2]/(c[3]*c[4]));for(i=0;i
\";}var title=ytplayer.config.args.title;msg=\"
\";}var title=ytplayer.config.args.title;msg=\"" + }, + { + "url": "http://answerthepublic.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://backtweets.com/", + "status": null, + "error": "timed out" + }, + { + "url": "http://browserspy.dk/browser.php", + "status": 500, + "error": "HTTP Error 500: Internal Server Error" + }, + { + "url": "http://chroniclingamerica.loc.gov/", + "status": 308, + "error": "HTTP Error 308: Permanent Redirect" + }, + { + "url": "http://chroniclingamerica.loc.gov/search/titles/", + "status": 308, + "error": "HTTP Error 308: Permanent Redirect" + }, + { + "url": "http://claz.org/", + "status": 429, + "error": "HTTP Error 429: 429" + }, + { + "url": "http://corporateinformation.com/", + "status": 308, + "error": "HTTP Error 308: Permanent Redirect" + }, + { + "url": "http://darkfailllnkf4vf.onion/", + "status": null, + "error": "[Errno 8] nodename nor servname provided, or not known" + }, + { + "url": "http://dnshistory.org/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://domainbigdata.com/", + "status": 530, + "error": "HTTP Error 530: " + }, + { + "url": "http://eqt5g4fuenphqinx.onion/", + "status": null, + "error": "[Errno 8] nodename nor servname provided, or not known" + }, + { + "url": "http://flightaware.com/live/", + "status": 308, + "error": "HTTP Error 308: Permanent Redirect" + }, + { + "url": "http://freecarrierlookup.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://gjobqjj7wyczbqie.onion/", + "status": null, + "error": "[Errno 8] nodename nor servname provided, or not known" + }, + { + "url": "http://hosts-file.net/", + "status": null, + "error": "timed out" + }, + { + "url": "http://hss3uro2hsxfogfq.onion/", + "status": null, + "error": "[Errno 8] nodename nor servname provided, or not known" + }, + { + "url": "http://images.yahoo.com/", + "status": 500, + "error": "HTTP Error 500: Service not available" + }, + { + "url": "http://infospace.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://investing.businessweek.com/research/common/symbollookup/symbollookup.asp", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://karmadecay.com/", + "status": null, + "error": "timed out" + }, + { + "url": "http://labs.teehanlax.com/project/hyperlapse", + "status": 308, + "error": "HTTP Error 308: Permanent Redirect" + }, + { + "url": "http://map.daum.net/", + "status": 308, + "error": "HTTP Error 308: Permanent Redirect" + }, + { + "url": "http://mediabugs.org/", + "status": 500, + "error": "HTTP Error 500: Internal Server Error" + }, + { + "url": "http://pqdtopen.proquest.com/search.html", + "status": 521, + "error": "HTTP Error 521: " + }, + { + "url": "http://ptrarchive.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://radaris.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://ransomwaretracker.abuse.ch/downloads/RW_DOMBL.txt", + "status": 503, + "error": "HTTP Error 503: certificate has expired" + }, + { + "url": "http://search-id.com/", + "status": 522, + "error": "HTTP Error 522: " + }, + { + "url": "http://socialblade.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://sortedbyname.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://ssrn.com/en/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://toolbar.netcraft.com/site_report?url=undefined#last_reboot", + "status": 400, + "error": "HTTP Error 400: Bad Request" + }, + { + "url": "http://tor66sezptuu2nta.onion/", + "status": null, + "error": "[Errno 8] nodename nor servname provided, or not known" + }, + { + "url": "http://tracker.geops.ch/", + "status": 308, + "error": "HTTP Error 308: Permanent Redirect" + }, + { + "url": "http://vin.place/", + "status": null, + "error": "[Errno 61] Connection refused" + }, + { + "url": "http://wamba.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://websiteoutlook.com/", + "status": null, + "error": "[Errno 8] nodename nor servname provided, or not known" + }, + { + "url": "http://www.amnestyusa.org/citizenevidence/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://www.andyroid.net/", + "status": 503, + "error": "HTTP Error 503: Service Temporarily Unavailable" + }, + { + "url": "http://www.blackpeoplemeet.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://www.browserscope.org/", + "status": 503, + "error": "HTTP Error 503: Service Temporarily Unavailable" + }, + { + "url": "http://www.calleridservice.com/", + "status": null, + "error": "timed out" + }, + { + "url": "http://www.canlii.org/en/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://www.codetwo.com/freeware/outlook-export/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://www.corporationwiki.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://www.criminalsearches.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://www.docjax.com/", + "status": 308, + "error": "HTTP Error 308: Permanent Redirect" + }, + { + "url": "http://www.edge-security.com/metagoofil.php", + "status": 308, + "error": "HTTP Error 308: Permanent Redirect" + }, + { + "url": "http://www.familytreenow.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://www.felonspy.com/search.html", + "status": null, + "error": "Remote end closed connection without response" + }, + { + "url": "http://www.flippity.com/", + "status": 522, + "error": "HTTP Error 522: " + }, + { + "url": "http://www.followthemoney.org/", + "status": 400, + "error": "HTTP Error 400: Bad Request" + }, + { + "url": "http://www.genealogy.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://www.getlinkinfo.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://www.hoovers.com/", + "status": null, + "error": "The read operation timed out" + }, + { + "url": "http://www.hunch.ly/", + "status": 308, + "error": "HTTP Error 308: Permanent Redirect" + }, + { + "url": "http://www.iboogie.com/", + "status": 503, + "error": "HTTP Error 503: Service Unavailable" + }, + { + "url": "http://www.indeed.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://www.infobel.com/en/world/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://www.intelius.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://www.kompass.com/selectcountry/", + "status": 308, + "error": "HTTP Error 308: Permanent Redirect" + }, + { + "url": "http://www.malware.com.br/open-source.shtml", + "status": 522, + "error": "HTTP Error 522: " + }, + { + "url": "http://www.marinetraffic.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://www.match.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://www.onelook.com/reverse-dictionary.shtml", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://www.onion.link/", + "status": null, + "error": "timed out" + }, + { + "url": "http://www.oodle.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://www.opensecrets.org/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://www.ripoffreport.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://www.senderbase.org/", + "status": 503, + "error": "HTTP Error 503: Service Unavailable" + }, + { + "url": "http://www.softpedia.com/get/Internet/Other-Internet-Related/IntelliTamper.shtml", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://www.spiderfoot.net/", + "status": 308, + "error": "HTTP Error 308: Permanent Redirect" + }, + { + "url": "http://www.talkwalker.com/alerts", + "status": 308, + "error": "HTTP Error 308: Permanent Redirect" + }, + { + "url": "http://www.thepaperboy.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://www.trustedsource.org/", + "status": null, + "error": "timed out" + }, + { + "url": "http://www.urlunshortener.com/", + "status": 308, + "error": "HTTP Error 308: Permanent Redirect" + }, + { + "url": "http://www.ussearch.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://www.vindecoderz.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://www.worldlingo.com/en/products_services/worldlingo_translator.html", + "status": 302, + "error": "HTTP Error 302: The HTTP server returned a redirect error that would lead to an infinite loop.\nThe last 30x error message was:\nFound" + }, + { + "url": "http://www.yournavigation.org/", + "status": null, + "error": "timed out" + }, + { + "url": "http://www.zabasearch.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://www.zillow.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://www.zoominfo.com/company-directory/us", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "http://www.zoominfo.com/people_directory/professional_profile/A-0-1", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://africacheck.org/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://ashley.cynic.al/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://batchgeo.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://bgpstream.com/", + "status": null, + "error": "timed out" + }, + { + "url": "https://blockchain.info/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://blockexplorer.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://chainradar.com/xmr/blocks", + "status": null, + "error": "[SSL: TLSV1_UNRECOGNIZED_NAME] tlsv1 unrecognized name (_ssl.c:1129)" + }, + { + "url": "https://chatgpt.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://community.blueliv.com/#!/discover", + "status": null, + "error": "timed out" + }, + { + "url": "https://community.blueliv.com/#!/sandbox", + "status": null, + "error": "timed out" + }, + { + "url": "https://connect.data.com/", + "status": null, + "error": "timed out" + }, + { + "url": "https://cubib.com/", + "status": 522, + "error": "HTTP Error 522: " + }, + { + "url": "https://elevenlabs.io/dubbing", + "status": 308, + "error": "HTTP Error 308: Permanent Redirect" + }, + { + "url": "https://epieos.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://etherchain.org/accounts/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://flowgpt.com/p/wormgpt-6", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://graphsense.info/", + "status": 500, + "error": "HTTP Error 500: Internal Server Error" + }, + { + "url": "https://instantusername.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://intrigue.io/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://justpaste.it/", + "status": null, + "error": "[SSL: SSLV3_ALERT_HANDSHAKE_FAILURE] sslv3 alert handshake failure (_ssl.c:1129)" + }, + { + "url": "https://labs.inquest.net/dfi", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://landsatlook.usgs.gov/sentinel2/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://lengthen.me/", + "status": null, + "error": "timed out" + }, + { + "url": "https://live.ether.camp/transactions", + "status": 530, + "error": "HTTP Error 530: " + }, + { + "url": "https://millionshort.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://mobility.portal.geops.io/", + "status": 308, + "error": "HTTP Error 308: Permanent Redirect" + }, + { + "url": "https://nuwber.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://oxt.me", + "status": null, + "error": "[Errno 51] Network is unreachable" + }, + { + "url": "https://proxycheck.haschek.at/", + "status": null, + "error": "[SSL: TLSV1_ALERT_INTERNAL_ERROR] tlsv1 alert internal error (_ssl.c:1129)" + }, + { + "url": "https://securityheaders.io/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://securitytrails.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://shiptracker.shodan.io/", + "status": null, + "error": "timed out" + }, + { + "url": "https://softbytelabs.com/wp/blackwidow/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://stackshare.io/", + "status": 429, + "error": "HTTP Error 429: Too Many Requests" + }, + { + "url": "https://topgainers.net/", + "status": null, + "error": "timed out" + }, + { + "url": "https://torguard.net/vpn-dns-leak-test.php", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://urlebird.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://usersearch.org/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://vimeo.com/search?", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://voterrecords.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://w3bin.com/", + "status": null, + "error": "Remote end closed connection without response" + }, + { + "url": "https://w3dt.net/", + "status": null, + "error": "Remote end closed connection without response" + }, + { + "url": "https://web-beta.archive.org/#/", + "status": 403, + "error": "HTTP Error 403: FORBIDDEN" + }, + { + "url": "https://whatsmyname.app/", + "status": 464, + "error": "HTTP Error 464: " + }, + { + "url": "https://www.abuseipdb.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://www.advancedbackgroundchecks.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://www.ashleymadison.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://www.companieslist.co.uk/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://www.corporationwiki.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://www.fastpeoplesearch.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://www.fontspring.com/matcherator", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://www.getodin.com/", + "status": null, + "error": "[SSL: TLSV1_ALERT_PROTOCOL_VERSION] tlsv1 alert protocol version (_ssl.c:1129)" + }, + { + "url": "https://www.glassdoor.com/Reviews/index.htm", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://www.hathitrust.org/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://www.heavy.ai/demos/tweetmap", + "status": null, + "error": "[SSL: TLSV1_ALERT_INTERNAL_ERROR] tlsv1 alert internal error (_ssl.c:1129)" + }, + { + "url": "https://www.homefacts.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://www.idcrawl.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://www.idcrawl.com/username", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://www.imageraider.com/", + "status": 500, + "error": "HTTP Error 500: Internal Server Error" + }, + { + "url": "https://www.infobyip.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://www.ipaddress.my/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://www.iptrackeronline.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://www.liveipmap.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://www.manta.com/business-directory", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://www.mend.io/vulnerability-database/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://www.mylife.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://www.networktotal.com/", + "status": null, + "error": "timed out" + }, + { + "url": "https://www.nicb.org/theft_and_fraud_awareness/vincheck", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://www.okcupid.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://www.opencnam.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://www.passivetotal.org/", + "status": null, + "error": "timed out" + }, + { + "url": "https://www.pexels.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://www.reddit.com/domain/%3CURLhere%3E", + "status": 404, + "error": "HTTP Error 404: Not Found" + }, + { + "url": "https://www.safeshepherd.com/handbook", + "status": 308, + "error": "HTTP Error 308: Permanent Redirect" + }, + { + "url": "https://www.sans.org/sec487", + "status": 308, + "error": "HTTP Error 308: Permanent Redirect" + }, + { + "url": "https://www.threatcrowd.org/", + "status": 503, + "error": "HTTP Error 503: Service Temporarily Unavailable" + }, + { + "url": "https://www.threatpipes.com", + "status": null, + "error": "timed out" + }, + { + "url": "https://www.truepeoplesearch.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://www.usphonebook.com", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://www.wdl.org/en/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://www.webpagetest.org/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + }, + { + "url": "https://www.xssposed.org/", + "status": null, + "error": "[SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:1129)" + }, + { + "url": "https://www.zoomeye.org/", + "status": 521, + "error": "HTTP Error 521: " + }, + { + "url": "https://www.zoosk.com/", + "status": 403, + "error": "HTTP Error 403: Forbidden" + } + ], + "redirected": [ + { + "url": "http://0day.today/", + "status": 200, + "final_url": "https://0day.today/" + }, + { + "url": "http://7photos.net/", + "status": 200, + "final_url": "http://ww1.7photos.net/" + }, + { + "url": "http://adultfriendfinder.com/", + "status": 200, + "final_url": "https://adultfriendfinder.com/" + }, + { + "url": "http://analyzeid.com/", + "status": 200, + "final_url": "https://analyzeid.com/" + }, + { + "url": "http://app.webinspector.com/", + "status": 200, + "final_url": "https://www.webinspector.com/website-malware-scanner/" + }, + { + "url": "http://archive.is/", + "status": 200, + "final_url": "https://archive.is/" + }, + { + "url": "http://bgp.he.net/", + "status": 200, + "final_url": "https://bgp.he.net/" + }, + { + "url": "http://bgpranking.circl.lu/", + "status": 200, + "final_url": "https://bgpranking.circl.lu/" + }, + { + "url": "http://bitcoinwhoswho.com/", + "status": 200, + "final_url": "https://www.bitcoinwhoswho.com/" + }, + { + "url": "http://blog.didierstevens.com/programs/pdf-tools/", + "status": 200, + "final_url": "https://blog.didierstevens.com/programs/pdf-tools/" + }, + { + "url": "http://blog.didierstevens.com/programs/xorsearch/", + "status": 200, + "final_url": "https://blog.didierstevens.com/programs/xorsearch/" + }, + { + "url": "http://boardreader.com/", + "status": 200, + "final_url": "https://boardreader.com/" + }, + { + "url": "http://botscout.com/", + "status": 200, + "final_url": "https://botscout.com/" + }, + { + "url": "http://browserspy.dk/", + "status": 200, + "final_url": "https://browserspy.dk/" + }, + { + "url": "http://builtwith.com/", + "status": 200, + "final_url": "https://builtwith.com/" + }, + { + "url": "http://cdrex.com/", + "status": 200, + "final_url": "https://www.cdrex.com/" + }, + { + "url": "http://checkshorturl.com/", + "status": 200, + "final_url": "https://checkshorturl.com/" + }, + { + "url": "http://commoncrawl.org/", + "status": 200, + "final_url": "https://commoncrawl.org/" + }, + { + "url": "http://copyscape.com/", + "status": 200, + "final_url": "https://www.copyscape.com/" + }, + { + "url": "http://corona.cast.uark.edu/", + "status": 200, + "final_url": "https://corona.cast.uark.edu/" + }, + { + "url": "http://craigslist.org/", + "status": 200, + "final_url": "https://charlotte.craigslist.org/" + }, + { + "url": "http://cryptome.org/", + "status": 200, + "final_url": "https://cryptome.org/" + }, + { + "url": "http://current-location.com/", + "status": 200, + "final_url": "https://current-location.com/" + }, + { + "url": "http://cve.mitre.org/", + "status": 200, + "final_url": "https://www.cve.org/" + }, + { + "url": "http://data.deutschebahn.com/", + "status": 200, + "final_url": "https://data.deutschebahn.com/opendata" + }, + { + "url": "http://data.mashedworld.com/dualmaps/map.htm", + "status": 200, + "final_url": "https://data.mashedworld.com/dualmaps/map.htm" + }, + { + "url": "http://datacatalog.worldbank.org/", + "status": 200, + "final_url": "https://datacatalog.worldbank.org/" + }, + { + "url": "http://discovery.nationalarchives.gov.uk/", + "status": 200, + "final_url": "https://discovery.nationalarchives.gov.uk/" + }, + { + "url": "http://dnsleak.com/", + "status": 200, + "final_url": "https://dnsleak.com/" + }, + { + "url": "http://dnssec-debugger.verisignlabs.com/", + "status": 200, + "final_url": "https://dnssec-analyzer.verisignlabs.com/" + }, + { + "url": "http://dnsviz.net/", + "status": 200, + "final_url": "https://dnsviz.net/" + }, + { + "url": "http://ec.europa.eu/taxation_customs/vies/?locale=en", + "status": 200, + "final_url": "https://ec.europa.eu/taxation_customs/vies/?locale=en" + }, + { + "url": "http://emailipleak.com/", + "status": 200, + "final_url": "https://emailipleak.com/" + }, + { + "url": "http://emm.newsbrief.eu/NewsBrief/clusteredition/en/latest.html", + "status": 200, + "final_url": "https://emm.newsbrief.eu/NewsBrief/clusteredition/en/latest.html" + }, + { + "url": "http://eternal-todo.com/var/scripts/xorbruteforcer", + "status": 200, + "final_url": "https://eternal-todo.com/var/scripts/xorbruteforcer" + }, + { + "url": "http://ether.gtisc.gatech.edu/web_unpack", + "status": 200, + "final_url": "https://ether.gtisc.gatech.edu/web_unpack/" + }, + { + "url": "http://everypolitician.org/", + "status": 200, + "final_url": "https://everypolitician.org:443/" + }, + { + "url": "http://explorer.naco.org/index.html", + "status": 200, + "final_url": "https://explorer.naco.org/index.html" + }, + { + "url": "http://explorer.opensyllabusproject.org/", + "status": 200, + "final_url": "https://www.opensyllabus.org/" + }, + { + "url": "http://fotoforensics.com/", + "status": 200, + "final_url": "https://fotoforensics.com/" + }, + { + "url": "http://francais-a-londres.org/", + "status": 200, + "final_url": "https://francaisalondres.com/" + }, + { + "url": "http://globaledge.msu.edu/global-resources", + "status": 200, + "final_url": "https://globaledge.msu.edu/global-resources" + }, + { + "url": "http://go.microsoft.com/fwlink/?LinkID=158791", + "status": 200, + "final_url": "https://download.microsoft.com/download/1/2/7/127ba59a-4fe1-4acd-ba47-513ceef85a85/OffVis.zip" + }, + { + "url": "http://googleguide.com/advanced_operators_reference.html", + "status": 200, + "final_url": "https://www.googleguide.com/advanced_operators_reference.html" + }, + { + "url": "http://googleguide.com/help/calculator.html", + "status": 200, + "final_url": "https://www.googleguide.com/help/calculator.html" + }, + { + "url": "http://guides.library.harvard.edu/hks/think_tank_search", + "status": 200, + "final_url": "https://guides.library.harvard.edu/hks/think_tank_search" + }, + { + "url": "http://guides.uflib.ufl.edu/az.php", + "status": 200, + "final_url": "https://guides.uflib.ufl.edu/az.php" + }, + { + "url": "http://idgettr.com/", + "status": 200, + "final_url": "https://www.webfx.com/tools/idgettr/" + }, + { + "url": "http://image-net.org/", + "status": 200, + "final_url": "https://image-net.org/" + }, + { + "url": "http://ip-check.info/?lang=en", + "status": 200, + "final_url": "https://ip-check.info/?lang=en" + }, + { + "url": "http://ipv6leak.com/", + "status": 200, + "final_url": "https://ipv6leak.com/" + }, + { + "url": "http://ipverse.net/", + "status": 200, + "final_url": "https://github.com/ipverse/" + }, + { + "url": "http://irc.netsplit.de/channels/search.php", + "status": 200, + "final_url": "https://netsplit.de/channels/search.php" + }, + { + "url": "http://jsbeautifier.org/", + "status": 200, + "final_url": "https://beautifier.io/" + }, + { + "url": "http://jurn.org/#gsc.tab=0", + "status": 200, + "final_url": "https://www.jurn.org/" + }, + { + "url": "http://justdelete.me/fake-identity-generator/", + "status": 200, + "final_url": "https://backgroundchecks.org/justdeleteme/fake-identity-generator/" + }, + { + "url": "http://labnol.org/?p=28217", + "status": 200, + "final_url": "https://www.labnol.org/?p=28217" + }, + { + "url": "http://littlesis.org/", + "status": 200, + "final_url": "https://littlesis.org/" + }, + { + "url": "http://ljseek.com/", + "status": 200, + "final_url": "http://ww1.ljseek.com/" + }, + { + "url": "http://malwareanalysis.tools/", + "status": 200, + "final_url": "https://malwareanalysis.tools/" + }, + { + "url": "http://map.baidu.com/", + "status": 200, + "final_url": "https://map.baidu.com/" + }, + { + "url": "http://map.naver.com/", + "status": 200, + "final_url": "https://map.naver.com/p/" + }, + { + "url": "http://moneroblocks.info/", + "status": 200, + "final_url": "https://localmonero.co/blocks/" + }, + { + "url": "http://mostwantedhf.info/index.php", + "status": 200, + "final_url": "https://mostwantedhf.info/index.php" + }, + { + "url": "http://mrnumber.com/1-888-742-0000", + "status": 200, + "final_url": "https://www.hiya.com/products/apps/hiya-spam-blocker" + }, + { + "url": "http://mugshots.com/", + "status": 200, + "final_url": "https://mugshots.com/" + }, + { + "url": "http://mxtoolbox.com/", + "status": 200, + "final_url": "https://mxtoolbox.com/" + }, + { + "url": "http://newspaperarchive.com/", + "status": 200, + "final_url": "https://newspaperarchive.com/" + }, + { + "url": "http://newspapermap.com/", + "status": 200, + "final_url": "https://newspapermap.com/" + }, + { + "url": "http://ok.ru/", + "status": 200, + "final_url": "https://ok.ru/" + }, + { + "url": "http://omgili.com/", + "status": 200, + "final_url": "https://webz.io/" + }, + { + "url": "http://onemilliontweetmap.com/", + "status": 200, + "final_url": "https://onemilliontweetmap.com/" + }, + { + "url": "http://online-barcode-reader.inliteresearch.com/", + "status": 200, + "final_url": "https://online-barcode-reader.inliteresearch.com/" + }, + { + "url": "http://online.wsj.com/public/resources/documents/TweetMetadata.pdf", + "status": 200, + "final_url": "https://www.wsj.com/public/resources/documents/TweetMetadata.pdf" + }, + { + "url": "http://open-sez.me/", + "status": 200, + "final_url": "https://open-sez.me/" + }, + { + "url": "http://opengrey.eu/", + "status": 200, + "final_url": "https://opengrey.eu/" + }, + { + "url": "http://openstreetcam.org/", + "status": 200, + "final_url": "https://kartaview.org/" + }, + { + "url": "http://passivedns.mnemonic.no/", + "status": 200, + "final_url": "https://passivedns.mnemonic.no/" + }, + { + "url": "http://pdfmyurl.com/", + "status": 200, + "final_url": "https://pdfmyurl.com/" + }, + { + "url": "http://photobucket.com/", + "status": 200, + "final_url": "https://photobucket.com/" + }, + { + "url": "http://pingroupie.com/", + "status": 200, + "final_url": "https://pingroupie.com/" + }, + { + "url": "http://pub-db.com/", + "status": 200, + "final_url": "https://pub-db.com/" + }, + { + "url": "http://publicrecords.netronline.com/", + "status": 200, + "final_url": "https://publicrecords.netronline.com/" + }, + { + "url": "http://publicrecords.searchsystems.net/", + "status": 200, + "final_url": "https://publicrecords.searchsystems.net/" + }, + { + "url": "http://redirectdetective.com/", + "status": 200, + "final_url": "https://redirectdetective.com/" + }, + { + "url": "http://routerpasswords.com/", + "status": 200, + "final_url": "https://www.routerpasswords.com/" + }, + { + "url": "http://sandbox.pikker.ee/", + "status": 200, + "final_url": "https://sandbox.pikker.ee/" + }, + { + "url": "http://scribblemaps.com/", + "status": 200, + "final_url": "https://www.scribblemaps.com/" + }, + { + "url": "http://search.ancestry.com/", + "status": 200, + "final_url": "https://www.ancestry.com/search/" + }, + { + "url": "http://search.findmypast.com/search-world-records", + "status": 200, + "final_url": "https://www.findmypast.com/discover" + }, + { + "url": "http://search.mibbit.com/", + "status": 200, + "final_url": "https://search.mibbit.com/" + }, + { + "url": "http://shitu.baidu.com/", + "status": 200, + "final_url": "https://graph.baidu.com/pcpage/index?tpl_from=pc" + }, + { + "url": "http://siteliner.com/", + "status": 200, + "final_url": "https://www.siteliner.com/" + }, + { + "url": "http://suncalc.net/", + "status": 200, + "final_url": "https://suncalc.net/" + }, + { + "url": "http://thehiddenwiki.org/", + "status": 200, + "final_url": "https://thehiddenwiki.org/" + }, + { + "url": "http://tineye.com/", + "status": 200, + "final_url": "https://tineye.com/" + }, + { + "url": "http://tinfoleak.com/", + "status": 200, + "final_url": "https://tinfoleak.com/" + }, + { + "url": "http://tools.tracemyip.org/user-agent-string-decoder/", + "status": 200, + "final_url": "https://tools.tracemyip.org/user-agent-string-decoder/" + }, + { + "url": "http://travelbydrone.com/", + "status": 200, + "final_url": "https://travelbydrone.com/" + }, + { + "url": "http://tweepsect.com/", + "status": 200, + "final_url": "http://ww1.tweepsect.com/" + }, + { + "url": "http://ukdata.com/", + "status": 200, + "final_url": "https://ukdata.com/" + }, + { + "url": "http://urlquery.net/", + "status": 200, + "final_url": "https://urlquery.net/" + }, + { + "url": "http://video.search.yahoo.com/", + "status": 200, + "final_url": "https://video.search.yahoo.com/" + }, + { + "url": "http://viewdns.info/", + "status": 200, + "final_url": "https://viewdns.info/" + }, + { + "url": "http://visualgenome.org/", + "status": 200, + "final_url": "https://visualgenome.org/" + }, + { + "url": "http://visualping.io/", + "status": 200, + "final_url": "https://visualping.io:443/" + }, + { + "url": "http://webcitation.org/query", + "status": 200, + "final_url": "https://www.webcitation.org:443/query" + }, + { + "url": "http://webmii.com/", + "status": 200, + "final_url": "https://webmii.com/" + }, + { + "url": "http://website.informer.com/", + "status": 200, + "final_url": "https://website.informer.com/" + }, + { + "url": "http://websta.me/search", + "status": 200, + "final_url": "https://websta.me/search-engine-optimization/" + }, + { + "url": "http://wheredoesthislinkgo.com/", + "status": 200, + "final_url": "https://wheregoes.com/" + }, + { + "url": "http://whois.domaintools.com/", + "status": 200, + "final_url": "https://whois.domaintools.com/" + }, + { + "url": "http://www.411.com/reverse_phone", + "status": 200, + "final_url": "https://www.411.com/reverse-phone-lookup" + }, + { + "url": "http://www.addresses.com/", + "status": 200, + "final_url": "https://www.addresses.com/" + }, + { + "url": "http://www.alexa.com/siteinfo", + "status": 200, + "final_url": "https://alexa.amazon.com/about" + }, + { + "url": "http://www.alexa.com/topsites", + "status": 200, + "final_url": "https://alexa.amazon.com/about" + }, + { + "url": "http://www.allyoucanread.com/", + "status": 200, + "final_url": "https://www.allyoucanread.com/" + }, + { + "url": "http://www.annualreports.com/", + "status": 200, + "final_url": "https://www.annualreports.com/" + }, + { + "url": "http://www.antennasearch.com/", + "status": 200, + "final_url": "https://www.antennasearch.com/" + }, + { + "url": "http://www.asianave.com/user_search/index.html", + "status": 200, + "final_url": "https://blackplanet.com/" + }, + { + "url": "http://www.avgthreatlabs.com/ww-en/website-safety-reports/", + "status": 200, + "final_url": "https://www.avg.com/en/signal/website-safety" + }, + { + "url": "http://www.bing.com/maps/", + "status": 200, + "final_url": "https://www.bing.com/maps" + }, + { + "url": "http://www.bishopfox.com/download/405/", + "status": 200, + "final_url": "https://bishopfox.com/resources" + }, + { + "url": "http://www.blackbookonline.info/USA-County-Court-Records.aspx", + "status": 200, + "final_url": "https://www.blackbookonline.info/USA-County-Court-Records.aspx" + }, + { + "url": "http://www.blackbookonline.info/USA-Voter-Records.aspx", + "status": 200, + "final_url": "https://www.blackbookonline.info/USA-Voter-Records.aspx" + }, + { + "url": "http://www.blackbookonline.info/criminalsearch.aspx", + "status": 200, + "final_url": "https://www.blackbookonline.info/criminalsearch.aspx" + }, + { + "url": "http://www.blinkx.com/", + "status": 200, + "final_url": "https://blinkx.com/" + }, + { + "url": "http://www.blocklist.de/en/index.html", + "status": 200, + "final_url": "https://www.blocklist.de/en/index.html" + }, + { + "url": "http://www.bluestacks.com/", + "status": 200, + "final_url": "https://www.bluestacks.com/" + }, + { + "url": "http://www.brbpub.com/", + "status": 200, + "final_url": "https://www.brbpub.com/" + }, + { + "url": "http://www.brightcloud.com/tools/url-ip-lookup.php", + "status": 200, + "final_url": "https://www.brightcloud.com/tools/url-ip-lookup.php" + }, + { + "url": "http://www.buzzfile.com/Home/Basic", + "status": 200, + "final_url": "https://www.buzzfile.com/Home/Basic" + }, + { + "url": "http://www.calleridtest.com/", + "status": 200, + "final_url": "https://calleridtest.com/" + }, + { + "url": "http://www.changedetect.com/", + "status": 200, + "final_url": "https://www.eatgrainmaker.com/home/" + }, + { + "url": "http://www.changedetection.com/", + "status": 200, + "final_url": "https://visualping.io/" + }, + { + "url": "http://www.cidr-report.org/as2.0/", + "status": 200, + "final_url": "https://www.cidr-report.org/as2.0/" + }, + { + "url": "http://www.cidr-report.org/v6/as2.0/", + "status": 200, + "final_url": "https://www.cidr-report.org/v6/as2.0/" + }, + { + "url": "http://www.commercial-register.sg.ch/home/worldwide.html", + "status": 200, + "final_url": "https://www.sg.ch/recht/handelsregister-notariate.html" + }, + { + "url": "http://www.cvedetails.com/", + "status": 200, + "final_url": "https://www.cvedetails.com/" + }, + { + "url": "http://www.cvgadget.com/", + "status": 200, + "final_url": "https://cvgadget.com/" + }, + { + "url": "http://www.dailychanges.com/", + "status": 200, + "final_url": "https://dailychanges.domaintools.com/" + }, + { + "url": "http://www.delphiforums.com/", + "status": 200, + "final_url": "https://www.delphiforums.com/" + }, + { + "url": "http://www.document-analyzer.net/", + "status": 200, + "final_url": "https://www.joesandbox.com/" + }, + { + "url": "http://www.dogpile.com/", + "status": 200, + "final_url": "https://www.dogpile.com/" + }, + { + "url": "http://www.domaincrawler.com/", + "status": 200, + "final_url": "https://domaincrawler.com/" + }, + { + "url": "http://www.earthcam.com/", + "status": 200, + "final_url": "https://www.earthcam.com/" + }, + { + "url": "http://www.ebay.com/", + "status": 200, + "final_url": "https://www.ebay.com/" + }, + { + "url": "http://www.eharmony.com/", + "status": 200, + "final_url": "https://www.eharmony.com/" + }, + { + "url": "http://www.europages.co.uk/", + "status": 200, + "final_url": "https://www.europages.co.uk:443/" + }, + { + "url": "http://www.exifviewer.org/", + "status": 200, + "final_url": "http://ww1.exifviewer.org/" + }, + { + "url": "http://www.factcheck.org/scicheck/", + "status": 200, + "final_url": "https://www.factcheck.org:443/scicheck/" + }, + { + "url": "http://www.fakenamegenerator.com/", + "status": 200, + "final_url": "https://www.fakenamegenerator.com/" + }, + { + "url": "http://www.familywatchdog.us/", + "status": 200, + "final_url": "https://www.familywatchdog.us/" + }, + { + "url": "http://www.farmersonly.com/", + "status": 200, + "final_url": "https://www.farmersonly.com:443/" + }, + { + "url": "http://www.fec.gov/finance/disclosure/norindsea.shtml", + "status": 200, + "final_url": "https://www.fec.gov/data/" + }, + { + "url": "http://www.findagrave.com/index.html", + "status": 200, + "final_url": "https://www.findagrave.com/" + }, + { + "url": "http://www.flashearth.com/", + "status": 200, + "final_url": "https://zoom.earth/" + }, + { + "url": "http://www.fonefinder.net/", + "status": 200, + "final_url": "https://www.fonefinder.net/" + }, + { + "url": "http://www.fortypoundhead.com/tools_dpw.asp", + "status": 200, + "final_url": "https://fortypoundhead.com/tools_dpw.asp" + }, + { + "url": "http://www.fraps.com/", + "status": 200, + "final_url": "https://fraps.com/" + }, + { + "url": "http://www.free-ocr.com/", + "status": 200, + "final_url": "https://www.sodapdf.com/pdf-tools/ocr-pdf/" + }, + { + "url": "http://www.geocreepy.com/", + "status": 200, + "final_url": "https://www.geocreepy.com/" + }, + { + "url": "http://www.geosetter.de/en/", + "status": 200, + "final_url": "https://geosetter.de/en/main-en/" + }, + { + "url": "http://www.gpsvisualizer.com/geocode", + "status": 200, + "final_url": "https://www.gpsvisualizer.com/geocode" + }, + { + "url": "http://www.historicaerials.com/?javascript=&", + "status": 200, + "final_url": "https://www.historicaerials.com/?javascript=&" + }, + { + "url": "http://www.i2ocr.com/", + "status": 200, + "final_url": "https://www.i2ocr.com/" + }, + { + "url": "http://www.impulseadventure.com/photo/jpeg-snoop.html", + "status": 200, + "final_url": "https://www.impulseadventure.com/photo/jpeg-snoop.html" + }, + { + "url": "http://www.infosniper.net/", + "status": 200, + "final_url": "https://www.infosniper.net/" + }, + { + "url": "http://www.instantstreetview.com/", + "status": 200, + "final_url": "https://www.instantstreetview.com/" + }, + { + "url": "http://www.instya.com/#/web/", + "status": 200, + "final_url": "https://www.instya.com/" + }, + { + "url": "http://www.ipfingerprints.com/", + "status": 200, + "final_url": "https://ipfingerprints.com/" + }, + { + "url": "http://www.ipfingerprints.com/reverseip.php", + "status": 200, + "final_url": "https://ipfingerprints.com/reverseip.php" + }, + { + "url": "http://www.ipvoid.com/", + "status": 200, + "final_url": "https://www.ipvoid.com/" + }, + { + "url": "http://www.izito.com/", + "status": 200, + "final_url": "https://www.izito.com/" + }, + { + "url": "http://www.jobster.com/", + "status": 200, + "final_url": "https://jobster.com/" + }, + { + "url": "http://www.kijiji.ca/", + "status": 200, + "final_url": "https://www.kijiji.ca:443/" + }, + { + "url": "http://www.labnol.org/internet/print-youtube-video/28217/", + "status": 200, + "final_url": "https://www.labnol.org/internet/print-youtube-video/28217" + }, + { + "url": "http://www.lazyscholar.org/", + "status": 200, + "final_url": "https://lazyscholar.org/" + }, + { + "url": "http://www.locabrowser.com", + "status": 200, + "final_url": "https://www.locabrowser.com/" + }, + { + "url": "http://www.malware-traffic-analysis.net/index.html", + "status": 200, + "final_url": "https://www.malware-traffic-analysis.net/index.html" + }, + { + "url": "http://www.malwareurl.com/index.php", + "status": 200, + "final_url": "https://www.malwareurl.com/index.php" + }, + { + "url": "http://www.meetup.com/", + "status": 200, + "final_url": "https://www.meetup.com/" + }, + { + "url": "http://www.melissadata.com/lookups/deathcheck.asp", + "status": 200, + "final_url": "https://www.melissa.com/" + }, + { + "url": "http://www.melissadata.com/lookups/emails.asp", + "status": 200, + "final_url": "https://www.melissa.com/" + }, + { + "url": "http://www.melissadata.com/lookups/fec.asp", + "status": 200, + "final_url": "https://www.melissa.com/" + }, + { + "url": "http://www.melissadata.com/lookups/peoplefinder.asp", + "status": 200, + "final_url": "https://www.melissa.com/" + }, + { + "url": "http://www.melissadata.com/lookups/propertyviewer.asp", + "status": 200, + "final_url": "https://www.melissa.com/" + }, + { + "url": "http://www.metatube.com/", + "status": 200, + "final_url": "https://www.metatube.com/" + }, + { + "url": "http://www.mgmaps.com/kml/#view", + "status": 200, + "final_url": "https://www.mgmaps.com/kml/" + }, + { + "url": "http://www.migente.com/user_search/index.html", + "status": 200, + "final_url": "https://migente.com/wp-login.php?redirect_to=https%3A%2F%2Fmigente.com%2Fuser_search%2Findex.html&bp-auth=1&action=bpnoaccess" + }, + { + "url": "http://www.misp-project.org/", + "status": 200, + "final_url": "https://www.misp-project.org/" + }, + { + "url": "http://www.n2yo.com/", + "status": 200, + "final_url": "https://www.n2yo.com/" + }, + { + "url": "http://www.ncbi.nlm.nih.gov/pubmed/", + "status": 200, + "final_url": "https://pubmed.ncbi.nlm.nih.gov/" + }, + { + "url": "http://www.networksec.org/grabbho/block.txt", + "status": 200, + "final_url": "https://www.networksec.org/grabbho/block.txt" + }, + { + "url": "http://www.newsnow.co.uk/h/", + "status": 200, + "final_url": "https://www.newsnow.co.uk/h/" + }, + { + "url": "http://www.newsobserver.com/news/databases/state-pay/", + "status": 200, + "final_url": "https://www.newsobserver.com/news/databases/state-pay/" + }, + { + "url": "http://www.noslang.com/", + "status": 200, + "final_url": "https://www.noslang.com/" + }, + { + "url": "http://www.onlineocr.net/", + "status": 200, + "final_url": "https://www.onlineocr.net/" + }, + { + "url": "http://www.openrailwaymap.org/", + "status": 200, + "final_url": "https://www.openrailwaymap.org/" + }, + { + "url": "http://www.openseamap.org", + "status": 200, + "final_url": "http://www.openseamap.org/index.php?id=openseamap&no_cache=1" + }, + { + "url": "http://www.openstreetmap.org/#map=5/40.614/-100.679", + "status": 200, + "final_url": "https://www.openstreetmap.org/" + }, + { + "url": "http://www.openstreetmap.org/#map=5/51.500/-0.100", + "status": 200, + "final_url": "https://www.openstreetmap.org/" + }, + { + "url": "http://www.page2images.com/URL-Live-Website-Screenshot-Generator", + "status": 200, + "final_url": "https://www.page2images.com/URL-Live-Website-Screenshot-Generator" + }, + { + "url": "http://www.pagepress.org/", + "status": 200, + "final_url": "https://www.pagepress.org/" + }, + { + "url": "http://www.peekyou.com/", + "status": 200, + "final_url": "https://www.peekyou.com/" + }, + { + "url": "http://www.phonevalidator.com/", + "status": 200, + "final_url": "https://www.phonevalidator.com/index.aspx" + }, + { + "url": "http://www.pof.com/", + "status": 200, + "final_url": "https://www.pof.com/" + }, + { + "url": "http://www.politicalmoneyline.com/", + "status": 200, + "final_url": "https://info.cq.com/" + }, + { + "url": "http://www.politifact.com/", + "status": 200, + "final_url": "https://www.politifact.com/" + }, + { + "url": "http://www.pressreader.com/", + "status": 200, + "final_url": "https://www.pressreader.com/" + }, + { + "url": "http://www.projecthoneypot.org/", + "status": 200, + "final_url": "https://www.projecthoneypot.org/" + }, + { + "url": "http://www.projecthoneypot.org/list_of_ips.php", + "status": 200, + "final_url": "https://www.projecthoneypot.org/list_of_ips.php" + }, + { + "url": "http://www.quikr.com/", + "status": 200, + "final_url": "https://www.quikr.com/" + }, + { + "url": "http://www.readnotify.com/", + "status": 200, + "final_url": "https://www.readnotify.com/" + }, + { + "url": "http://www.sacbee.com/site-services/databases/state-pay/article2642161.html", + "status": 200, + "final_url": "https://www.sacbee.com/news/databases/state-pay/article229468549.html" + }, + { + "url": "http://www.sciencedirect.com/", + "status": 200, + "final_url": "https://www.sciencedirect.com/" + }, + { + "url": "http://www.searchalljunk.com/", + "status": 200, + "final_url": "https://www.searchalljunk.com/" + }, + { + "url": "http://www.searchenginecolossus.com/", + "status": 200, + "final_url": "https://www.searchenginecolossus.com/" + }, + { + "url": "http://www.searchftps.net/", + "status": 200, + "final_url": "https://www.searchftps.net/" + }, + { + "url": "http://www.searchtempest.com/", + "status": 200, + "final_url": "https://www.searchtempest.com/" + }, + { + "url": "http://www.sec.gov/edgar.shtml", + "status": 200, + "final_url": "https://www.sec.gov/submit-filings" + }, + { + "url": "http://www.shadowserver.org/wiki/pmwiki.php?n=Services/Reports", + "status": 200, + "final_url": "https://www.shadowserver.org/" + }, + { + "url": "http://www.shipais.com/", + "status": 200, + "final_url": "https://shipais.uk/" + }, + { + "url": "http://www.sitedossier.com/", + "status": 200, + "final_url": "http://www.sitedossier.com/audit/?40274" + }, + { + "url": "http://www.smugmug.com/search", + "status": 200, + "final_url": "https://www.smugmug.com/" + }, + { + "url": "http://www.sno.phy.queensu.ca/~phil/exiftool/", + "status": 200, + "final_url": "https://exiftool.org/" + }, + { + "url": "http://www.snopes.com/", + "status": 200, + "final_url": "https://www.snopes.com/" + }, + { + "url": "http://www.social-searcher.com/", + "status": 200, + "final_url": "https://www.social-searcher.com/" + }, + { + "url": "http://www.social-searcher.com/google-social-search/", + "status": 200, + "final_url": "https://www.social-searcher.com/google-social-search/" + }, + { + "url": "http://www.spydialer.com/default.aspx", + "status": 200, + "final_url": "https://www.spydialer.com:443/default.aspx" + }, + { + "url": "http://www.start.umd.edu/gtd/", + "status": 200, + "final_url": "https://www.start.umd.edu/data-tools/GTD" + }, + { + "url": "http://www.statscrop.com/", + "status": 200, + "final_url": "https://www.statscrop.com/" + }, + { + "url": "http://www.stopfake.org/en/category/tools/", + "status": 200, + "final_url": "https://www.stopfake.org/en/category/tools/" + }, + { + "url": "http://www.tcpiputils.com/domain-neighbors", + "status": 200, + "final_url": "https://dnslytics.com/" + }, + { + "url": "http://www.theinmatelocator.com/", + "status": 200, + "final_url": "https://theinmatelocator.com/" + }, + { + "url": "http://www.thescipub.com/", + "status": 200, + "final_url": "https://www.thescipub.com/" + }, + { + "url": "http://www.topix.com/search/article?q=", + "status": 200, + "final_url": "https://www.pch.com/quizzes" + }, + { + "url": "http://www.tracemyip.org/", + "status": 200, + "final_url": "https://www.tracemyip.org/" + }, + { + "url": "http://www.track-trace.com/", + "status": 200, + "final_url": "https://www.track-trace.com/" + }, + { + "url": "http://www.truecaller.com/", + "status": 200, + "final_url": "https://www.truecaller.com/" + }, + { + "url": "http://www.tubechop.com/", + "status": 200, + "final_url": "https://tubechop.com/" + }, + { + "url": "http://www.tumblr.com/tagged/search", + "status": 200, + "final_url": "https://www.tumblr.com/tagged/search" + }, + { + "url": "http://www.urbandictionary.com/", + "status": 200, + "final_url": "https://www.urbandictionary.com/" + }, + { + "url": "http://www.urlvoid.com/", + "status": 200, + "final_url": "https://www.urlvoid.com/" + }, + { + "url": "http://www.useragentstring.com/pages/useragentstring.php", + "status": 200, + "final_url": "https://www.useragentstring.com/pages/useragentstring.php" + }, + { + "url": "http://www.usnaviguide.com/", + "status": 200, + "final_url": "https://www.usnaviguide.com/" + }, + { + "url": "http://www.vault.com/", + "status": 200, + "final_url": "https://vault.com/" + }, + { + "url": "http://www.vesseltracker.com/app", + "status": 200, + "final_url": "https://www.vesseltracker.com/" + }, + { + "url": "http://www.vicenteaguileradiaz.com/tools/", + "status": 200, + "final_url": "https://www.vicenteaguileradiaz.com/tools/" + }, + { + "url": "http://www.visualsitemapper.com/", + "status": 200, + "final_url": "https://github.com/alentum/sitemapper-nodejs" + }, + { + "url": "http://www.wayp.com/", + "status": 200, + "final_url": "https://www.wayp.com/" + }, + { + "url": "http://www.webarchive.org.uk/ukwa/", + "status": 200, + "final_url": "https://www.webarchive.org.uk/ukwa/" + }, + { + "url": "http://www.whatbrowseramiusing.co/", + "status": 200, + "final_url": "https://www.whatbrowseramiusing.co/" + }, + { + "url": "http://www.whatsmybrowser.org/", + "status": 200, + "final_url": "https://www.whatsmybrowser.org/" + }, + { + "url": "http://www.yougotthenews.com/", + "status": 200, + "final_url": "https://yougotthenews.com/" + }, + { + "url": "http://zulu.zscaler.com/", + "status": 200, + "final_url": "https://zulu.zscaler.com/" + }, + { + "url": "https://academic.microsoft.com/", + "status": 200, + "final_url": "https://www.microsoft.com/en-us" + }, + { + "url": "https://archive.ics.uci.edu/ml/datasets/Spambase", + "status": 200, + "final_url": "https://archive.ics.uci.edu/dataset/94/spambase" + }, + { + "url": "https://archive.org/web/", + "status": 200, + "final_url": "https://web.archive.org/" + }, + { + "url": "https://beta.companieshouse.gov.uk/", + "status": 200, + "final_url": "https://find-and-update.company-information.service.gov.uk:443/" + }, + { + "url": "https://bhattsameer.github.io/numspy", + "status": 200, + "final_url": "https://bhattsameer.github.io/numspy/" + }, + { + "url": "https://ccsearch.creativecommons.org/", + "status": 200, + "final_url": "https://search.creativecommons.org/" + }, + { + "url": "https://censys.io/", + "status": 200, + "final_url": "https://censys.com/" + }, + { + "url": "https://chrome.google.com/webstore/detail/frame-by-frame-for-youtub/elkadbdicdciddfkdpmaolomehalghio?hl=en-GB", + "status": 200, + "final_url": "https://chromewebstore.google.com/detail/empty-title/elkadbdicdciddfkdpmaolomehalghio?hl=en-GB" + }, + { + "url": "https://chrome.google.com/webstore/detail/reveye-reverse-image-sear/keaaclcjhehbbapnphnmpiklalfhelgf?hl=en", + "status": 200, + "final_url": "https://chromewebstore.google.com/detail/reveye-reverse-image-sear/keaaclcjhehbbapnphnmpiklalfhelgf?hl=en" + }, + { + "url": "https://chrome.google.com/webstore/detail/wayback-machine/fpnmgdkabkmnadcjpehmlllkndpkmiak", + "status": 200, + "final_url": "https://chromewebstore.google.com/detail/wayback-machine/fpnmgdkabkmnadcjpehmlllkndpkmiak" + }, + { + "url": "https://cirt.net/passwords", + "status": 200, + "final_url": "https://cirt.net/passwords/" + }, + { + "url": "https://cse.google.com/cse/", + "status": 200, + "final_url": "https://programmablesearchengine.google.com/about/" + }, + { + "url": "https://cyber.gc.ca/", + "status": 200, + "final_url": "https://www.cyber.gc.ca/" + }, + { + "url": "https://databases.today/", + "status": 200, + "final_url": "https://snusbase.com/" + }, + { + "url": "https://dnstwister.report/", + "status": 200, + "final_url": "https://www.dnstwister.com/" + }, + { + "url": "https://domains.markmonitor.com/whois/", + "status": 200, + "final_url": "https://whois-webform.markmonitor.com:443/whois/" + }, + { + "url": "https://duckduckgo.com/aichat", + "status": 200, + "final_url": "https://duck.ai/chat" + }, + { + "url": "https://exonerator.torproject.org/", + "status": 200, + "final_url": "https://metrics.torproject.org/exonerator.html" + }, + { + "url": "https://flow.microsoft.com/en-us/", + "status": 200, + "final_url": "https://make.powerautomate.com/en-us/" + }, + { + "url": "https://freenetproject.org/pages/download.html", + "status": 200, + "final_url": "https://www.hyphanet.org/pages/download.html" + }, + { + "url": "https://geoguessr.com/", + "status": 200, + "final_url": "https://www.geoguessr.com/" + }, + { + "url": "https://getfirebug.com/downloads/", + "status": 200, + "final_url": "https://getfirebug.com/index.html" + }, + { + "url": "https://geti2p.net/en/", + "status": 200, + "final_url": "https://i2p.net/en/" + }, + { + "url": "https://github.com/InQuest/python-iocextract", + "status": 200, + "final_url": "https://github.com/InQuest/iocextract" + }, + { + "url": "https://github.com/T3hUb3rK1tten/TweetVacuum", + "status": 200, + "final_url": "https://github.com/UberKitten/TweetVacuum" + }, + { + "url": "https://github.com/leapsecurity/InSpy", + "status": 200, + "final_url": "https://github.com/jobroche/InSpy" + }, + { + "url": "https://github.com/vysec/LinkedInt", + "status": 200, + "final_url": "https://github.com/vysecurity/LinkedInt" + }, + { + "url": "https://github.com/zricethezav/gitleaks", + "status": 200, + "final_url": "https://github.com/gitleaks/gitleaks" + }, + { + "url": "https://hooktube.com/", + "status": 200, + "final_url": "https://cinemaphile.com/" + }, + { + "url": "https://hulbee.com/", + "status": 200, + "final_url": "https://hulbee.com/de" + }, + { + "url": "https://imgrab.com/", + "status": 200, + "final_url": "https://www.imgrab.com/" + }, + { + "url": "https://inflact.com/profiles/instagram-viewer/", + "status": 200, + "final_url": "https://inflact.com/instagram-viewer/profile/" + }, + { + "url": "https://iplogger.com/", + "status": 200, + "final_url": "https://iplogger.org/" + }, + { + "url": "https://ki.tc", + "status": 200, + "final_url": "https://ki.tc/login" + }, + { + "url": "https://lastpass.com/snapchat/", + "status": 200, + "final_url": "https://www.lastpass.com/features/dark-web-monitoring" + }, + { + "url": "https://moz.com/researchtools/ose/", + "status": 200, + "final_url": "https://moz.com/link-explorer" + }, + { + "url": "https://nerdydata.com/search", + "status": 200, + "final_url": "https://www.nerdydata.com/reports/new" + }, + { + "url": "https://news.google.com/news/advanced_news_search?", + "status": 200, + "final_url": "https://news.google.com/home?hl=en-US&gl=US&ceid=US:en" + }, + { + "url": "https://nextcaller.com/", + "status": 200, + "final_url": "https://www.pindrop.com/" + }, + { + "url": "https://openphish.com/feed.txt", + "status": 200, + "final_url": "https://raw.githubusercontent.com/openphish/public_feed/refs/heads/main/feed.txt" + }, + { + "url": "https://opensignal.com/", + "status": 200, + "final_url": "https://www.opensignal.com/" + }, + { + "url": "https://osint.industries/", + "status": 200, + "final_url": "https://www.osint.industries/" + }, + { + "url": "https://packetstormsecurity.com/", + "status": 200, + "final_url": "https://packetstorm.news/" + }, + { + "url": "https://panopticlick.eff.org/", + "status": 200, + "final_url": "https://coveryourtracks.eff.org/" + }, + { + "url": "https://portswigger.net/burp/", + "status": 200, + "final_url": "https://portswigger.net/burp" + }, + { + "url": "https://portswigger.net/burp/download.html", + "status": 200, + "final_url": "https://portswigger.net/burp" + }, + { + "url": "https://redditmetis.com/#", + "status": 200, + "final_url": "https://redditmetis.com/" + }, + { + "url": "https://riskdiscovery.com/honeydb/", + "status": 200, + "final_url": "https://honeydb.io/" + }, + { + "url": "https://secai.ai/research", + "status": 200, + "final_url": "https://i.secai.ai/research" + }, + { + "url": "https://secuniaresearch.flexerasoftware.com/community/research/", + "status": 200, + "final_url": "https://www.flexera.com/products/security/software-vulnerability-advisories?referrer=secunia" + }, + { + "url": "https://sitereview.bluecoat.com/sitereview.jsp", + "status": 200, + "final_url": "https://sitereview.bluecoat.com/#/" + }, + { + "url": "https://slangit.com/", + "status": 200, + "final_url": "https://slang.net/" + }, + { + "url": "https://swisscows.com/", + "status": 200, + "final_url": "https://swisscows.com/en" + }, + { + "url": "https://tails.boum.org/", + "status": 200, + "final_url": "https://tails.net/" + }, + { + "url": "https://thatoneprivacysite.net/", + "status": 200, + "final_url": "https://www.safetydetectives.com/best-vpns/" + }, + { + "url": "https://thatsthem.com/name-address-search", + "status": 200, + "final_url": "https://thatsthem.com/challenge?r=%2Freverse-address-lookup" + }, + { + "url": "https://thatsthem.com/reverse-email-lookup", + "status": 200, + "final_url": "https://thatsthem.com/challenge?r=%2Freverse-email-lookup" + }, + { + "url": "https://thatsthem.com/reverse-phone-lookup", + "status": 200, + "final_url": "https://thatsthem.com/challenge?r=%2Freverse-phone-lookup" + }, + { + "url": "https://thatsthem.com/vin-search", + "status": 200, + "final_url": "https://thatsthem.com/challenge?r=%2F" + }, + { + "url": "https://tools.digitalmethods.net/netvizz/youtube/", + "status": 200, + "final_url": "https://ytdt.digitalmethods.net/" + }, + { + "url": "https://tools.dnsstuff.com/", + "status": 200, + "final_url": "https://www.dnsstuff.com/freetools" + }, + { + "url": "https://totalhash.cymru.com/", + "status": 200, + "final_url": "https://www.team-cymru.com/mhr" + }, + { + "url": "https://twitter.com/quiztime", + "status": 200, + "final_url": "https://x.com/quiztime" + }, + { + "url": "https://twitter.com/search-advanced", + "status": 200, + "final_url": "https://x.com/search-advanced" + }, + { + "url": "https://twitter.com/search?q=%3Csearchterm%3E&src=typd&vertical=default&f=images", + "status": 200, + "final_url": "https://x.com/search?q=%3Csearchterm%3E&src=typd&vertical=default&f=images" + }, + { + "url": "https://twitter.com/search?q=SearchTerm%20since:2016-03-01%20until:2016-03-02", + "status": 200, + "final_url": "https://x.com/search?q=SearchTerm%20since:2016-03-01%20until:2016-03-02" + }, + { + "url": "https://twitter.com/search?q=geocode%3A36.1143855%2C-115.1727518%2C1km&src=typd", + "status": 200, + "final_url": "https://x.com/search?q=geocode%3A36.1143855%2C-115.1727518%2C1km&src=typd" + }, + { + "url": "https://vk.com/", + "status": 200, + "final_url": "https://m.vk.com/" + }, + { + "url": "https://wappalyzer.com/", + "status": 200, + "final_url": "https://www.wappalyzer.com/" + }, + { + "url": "https://web.skype.com/", + "status": 200, + "final_url": "https://teams.live.com/v2/unsupported-browser#env=life&browserOverrides={\"firefox\":{\"isSupported\":true,\"minVersion\":\"97.0\"},\"safari\":{\"isSupported\":true,\"minVersion\":\"16.0\"}}" + }, + { + "url": "https://www.apk-analyzer.net/", + "status": 200, + "final_url": "https://www.joesandbox.com/" + }, + { + "url": "https://www.ayi.com/index.php", + "status": 200, + "final_url": "https://www.areyouinterested.com/" + }, + { + "url": "https://www.binaryedge.io/", + "status": 200, + "final_url": "https://www.coalitioninc.com/security/coalition-signals-intelligence" + }, + { + "url": "https://www.certificate-transparency.org/known-logs", + "status": 200, + "final_url": "https://github.com/google/certificate-transparency-community-site/blob/master/docs/google/known-logs.md" + }, + { + "url": "https://www.crimereports.com/", + "status": 200, + "final_url": "https://cityprotect.com/" + }, + { + "url": "https://www.data24-7.com/signup.php", + "status": 200, + "final_url": "https://www.data247.com/register/main" + }, + { + "url": "https://www.deepl.com/", + "status": 200, + "final_url": "https://www.deepl.com/en" + }, + { + "url": "https://www.deteque.com/", + "status": 200, + "final_url": "https://www.spamhaus.com/" + }, + { + "url": "https://www.doogal.co.uk/BatchGeocoding.php", + "status": 200, + "final_url": "https://www.doogal.co.uk/BatchGeocoding" + }, + { + "url": "https://www.doogal.co.uk/BatchReverseGeocoding.php", + "status": 200, + "final_url": "https://www.doogal.co.uk/BatchReverseGeocoding" + }, + { + "url": "https://www.easywhois.com/", + "status": 200, + "final_url": "https://app.domainhelp.com/" + }, + { + "url": "https://www.emporis.com/", + "status": 200, + "final_url": "https://www.costar.com/" + }, + { + "url": "https://www.exploit-db.com/google-hacking-database/", + "status": 200, + "final_url": "https://www.exploit-db.com/google-hacking-database" + }, + { + "url": "https://www.facebook.com/livemap#", + "status": 200, + "final_url": "https://www.facebook.com/watch/live/?ref=live_delegate" + }, + { + "url": "https://www.file-analyzer.net/", + "status": 200, + "final_url": "https://www.joesandbox.com/" + }, + { + "url": "https://www.google.com/advanced_patent_search", + "status": 200, + "final_url": "https://patents.google.com/advanced" + }, + { + "url": "https://www.google.com/alerts#", + "status": 200, + "final_url": "https://www.google.com/alerts" + }, + { + "url": "https://www.google.com/earth/", + "status": 200, + "final_url": "https://earth.google.com/web/" + }, + { + "url": "https://www.google.com/finance", + "status": 200, + "final_url": "https://www.google.com/finance/" + }, + { + "url": "https://www.google.com/trends/", + "status": 200, + "final_url": "https://trends.google.com/trends/" + }, + { + "url": "https://www.gotinder.com/@%3Cusername%3E", + "status": 200, + "final_url": "https://tinder.com/@%3Cusername%3E" + }, + { + "url": "https://www.gpo.gov/fdsys/", + "status": 200, + "final_url": "https://www.govinfo.gov/" + }, + { + "url": "https://www.hlr-lookups.com/", + "status": 200, + "final_url": "https://www.hlr-lookups.com/en/start" + }, + { + "url": "https://www.hybrid-analysis.com/", + "status": 200, + "final_url": "https://hybrid-analysis.com/" + }, + { + "url": "https://www.inshorts.com/en/read", + "status": 200, + "final_url": "https://inshorts.com/en/read" + }, + { + "url": "https://www.liveleak.com/", + "status": 200, + "final_url": "https://www.itemfix.com/?r=ll" + }, + { + "url": "https://www.magnetforensics.com/free-tool-web-page-saver/", + "status": 200, + "final_url": "https://www.magnetforensics.com/resources/web-page-saver/" + }, + { + "url": "https://www.malwarepatrol.net/open-source.shtml", + "status": 200, + "final_url": "https://www.malwarepatrol.net/integrations-formats-threat-intelligence-feed-integration/" + }, + { + "url": "https://www.myfonts.com/WhatTheFont/", + "status": 200, + "final_url": "https://www.myfonts.com/pages/whatthefont" + }, + { + "url": "https://www.ncdc.noaa.gov/nexradinv/", + "status": 200, + "final_url": "https://www.ncei.noaa.gov/nexradinv/" + }, + { + "url": "https://www.nsopw.gov/en/Search", + "status": 200, + "final_url": "https://www.nsopw.gov/search-public-sex-offender-registries" + }, + { + "url": "https://www.numberway.com/", + "status": 200, + "final_url": "https://www.thisnumber.com/" + }, + { + "url": "https://www.opengov-muenchen.de/", + "status": 200, + "final_url": "https://opendata.muenchen.de/" + }, + { + "url": "https://www.overviewdocs.com/", + "status": 200, + "final_url": "https://github.com/overview/overview-local" + }, + { + "url": "https://www.owasp.org/index.php/Main_Page", + "status": 200, + "final_url": "https://owasp.org/" + }, + { + "url": "https://www.owler.com/", + "status": 200, + "final_url": "https://www.owler.com/corp" + }, + { + "url": "https://www.reddit.com/r/darknet", + "status": 200, + "final_url": "https://www.reddit.com/r/darknet/" + }, + { + "url": "https://www.reddit.com/r/deepweb", + "status": 200, + "final_url": "https://www.reddit.com/r/deepweb/" + }, + { + "url": "https://www.reddit.com/r/onions", + "status": 200, + "final_url": "https://www.reddit.com/r/onions/" + }, + { + "url": "https://www.registryfinder.com/", + "status": 200, + "final_url": "https://registryfinder.com:443/" + }, + { + "url": "https://www.robtex.com/", + "status": 200, + "final_url": "https://robtex.com/" + }, + { + "url": "https://www.sitesleuth.io/", + "status": 200, + "final_url": "https://zone.uk.net/" + }, + { + "url": "https://www.torproject.org/download/download-easy.html.en", + "status": 200, + "final_url": "https://www.torproject.org/download/" + }, + { + "url": "https://www.twilio.com/lookup", + "status": 200, + "final_url": "https://www.twilio.com/en-us/user-authentication-identity/lookup" + }, + { + "url": "https://www.url-analyzer.net/", + "status": 200, + "final_url": "https://www.joesandbox.com/" + }, + { + "url": "https://www.vigilante.pw/", + "status": 200, + "final_url": "https://dehashed.com/" + }, + { + "url": "https://www.virustotal.com/", + "status": 200, + "final_url": "https://www.virustotal.com/gui/" + }, + { + "url": "https://www.yandex.com/", + "status": 200, + "final_url": "https://yandex.com/" + }, + { + "url": "https://yasiv.com/youtube", + "status": 200, + "final_url": "https://yasiv.com/youtube/" + } + ], + "slow": [] +} \ No newline at end of file diff --git a/tools/link_checker.py b/tools/link_checker.py new file mode 100644 index 0000000..8195160 --- /dev/null +++ b/tools/link_checker.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +""" +Dead link checker for arf.json. +Extracts all URLs, checks HTTP status, and reports dead/redirected/slow links. +""" + +import json +import re +import sys +import time +import urllib.request +import urllib.error +import urllib.parse +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass, field +from typing import Optional +import ssl + +TIMEOUT = 30 +MAX_WORKERS = 20 +SLOW_THRESHOLD = 10 # seconds + +# Skip these domains known to block bots or require auth +SKIP_DOMAINS = set() + +# User agent to avoid some bot blocks +HEADERS = { + "User-Agent": "Mozilla/5.0 (compatible; OSINT-Framework-LinkChecker/1.0; +https://osintframework.com)" +} + + +@dataclass +class LinkResult: + url: str + status: Optional[int] = None + final_url: Optional[str] = None + elapsed: float = 0.0 + error: Optional[str] = None + + @property + def is_dead(self): + if self.error and "404" not in str(self.error): + return True + return self.status is not None and self.status >= 400 + + @property + def is_redirect(self): + return self.final_url and self.final_url != self.url + + @property + def is_slow(self): + return self.elapsed >= SLOW_THRESHOLD + + +def check_url(url: str) -> LinkResult: + result = LinkResult(url=url) + start = time.time() + try: + # Build request with headers + req = urllib.request.Request(url, headers=HEADERS) + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + + with urllib.request.urlopen(req, timeout=TIMEOUT, context=ctx) as resp: + result.status = resp.status + result.final_url = resp.url + except urllib.error.HTTPError as e: + result.status = e.code + result.error = str(e) + except urllib.error.URLError as e: + result.error = str(e.reason) + except Exception as e: + result.error = str(e) + finally: + result.elapsed = time.time() - start + return result + + +def extract_urls(json_path: str) -> list[str]: + with open(json_path) as f: + raw = f.read() + urls = re.findall(r'https?://[^\s"\'<>]+', raw) + # Clean trailing punctuation + cleaned = [] + for u in urls: + u = u.rstrip(".,;:)'\"") + cleaned.append(u) + return sorted(set(cleaned)) + + +def main(): + json_path = "public/arf.json" + urls = extract_urls(json_path) + total = len(urls) + print(f"Checking {total} unique URLs (timeout={TIMEOUT}s, workers={MAX_WORKERS})...\n") + + results = [] + completed = 0 + + with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: + future_to_url = {executor.submit(check_url, url): url for url in urls} + for future in as_completed(future_to_url): + result = future.result() + results.append(result) + completed += 1 + if completed % 50 == 0: + print(f" Progress: {completed}/{total}", flush=True) + + # Sort results by URL for consistent output + results.sort(key=lambda r: r.url) + + dead = [r for r in results if r.is_dead] + redirected = [r for r in results if r.is_redirect and not r.is_dead] + slow = [r for r in results if r.is_slow and not r.is_dead] + ok = [r for r in results if not r.is_dead and not r.is_slow] + + print("\n" + "=" * 70) + print(f"LINK AUDIT REPORT") + print("=" * 70) + print(f"Total URLs checked : {total}") + print(f"OK : {len(ok)}") + print(f"Dead (errors/404+) : {len(dead)}") + print(f"Redirected : {len(redirected)}") + print(f"Slow (>{SLOW_THRESHOLD}s) : {len(slow)}") + print() + + if dead: + print("=" * 70) + print(f"DEAD LINKS ({len(dead)})") + print("=" * 70) + for r in dead: + status_str = f"HTTP {r.status}" if r.status else f"ERROR: {r.error}" + print(f" [{status_str}] {r.url}") + print() + + if redirected: + print("=" * 70) + print(f"REDIRECTED LINKS ({len(redirected)})") + print("=" * 70) + for r in redirected: + print(f" [HTTP {r.status}] {r.url}") + print(f" -> {r.final_url}") + print() + + if slow: + print("=" * 70) + print(f"SLOW LINKS (>{SLOW_THRESHOLD}s) ({len(slow)})") + print("=" * 70) + for r in slow: + print(f" [{r.elapsed:.1f}s] {r.url}") + print() + + # Write JSON report + report = { + "summary": { + "total": total, + "ok": len(ok), + "dead": len(dead), + "redirected": len(redirected), + "slow": len(slow), + }, + "dead": [{"url": r.url, "status": r.status, "error": r.error} for r in dead], + "redirected": [{"url": r.url, "status": r.status, "final_url": r.final_url} for r in redirected], + "slow": [{"url": r.url, "elapsed": round(r.elapsed, 2)} for r in slow], + } + + report_path = "tools/link_audit_report.json" + with open(report_path, "w") as f: + json.dump(report, f, indent=2) + print(f"Full report written to: {report_path}") + + return len(dead) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/remove_dead_links.py b/tools/remove_dead_links.py new file mode 100644 index 0000000..d2d7574 --- /dev/null +++ b/tools/remove_dead_links.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +""" +Remove confirmed dead links from arf.json. +Targets: DNS failures, connection refused, HTTP 404. +Leaves alone: .onion, 403 (bot-block), 5xx (transient), timeouts. +""" + +import json +import sys + +REPORT_PATH = "tools/link_audit_report.json" +ARF_PATH = "public/arf.json" + + +def get_dead_urls(report_path: str) -> set: + with open(report_path) as f: + report = json.load(f) + + dead_urls = set() + for d in report["dead"]: + url = d.get("url", "") + err = d.get("error") or "" + status = d.get("status") + + # Skip .onion (Tor hidden services - expected failure from clearnet) + if ".onion" in url: + continue + # DNS failure = truly dead + if "nodename nor servname" in err: + dead_urls.add(url) + # Connection refused = truly dead + elif "Connection refused" in err: + dead_urls.add(url) + # 404 = page gone + elif status == 404: + dead_urls.add(url) + + return dead_urls + + +def remove_nodes(tree, dead_urls: set) -> tuple: + """Recursively remove leaf nodes whose URL is dead. Returns (cleaned_tree, removed_count).""" + removed = 0 + + if not isinstance(tree, dict): + return tree, 0 + + children = tree.get("children", []) + if not children: + # Leaf node - check URL + url = tree.get("url", "") + if url in dead_urls: + return None, 1 + return tree, 0 + + # Non-leaf: recurse into children + new_children = [] + for child in children: + cleaned, n = remove_nodes(child, dead_urls) + removed += n + if cleaned is not None: + new_children.append(cleaned) + + tree = dict(tree) + tree["children"] = new_children + return tree, removed + + +def main(): + dead_urls = get_dead_urls(REPORT_PATH) + print(f"Confirmed dead URLs to remove: {len(dead_urls)}") + + with open(ARF_PATH) as f: + data = json.load(f) + + cleaned, removed = remove_nodes(data, dead_urls) + + print(f"Removed {removed} nodes from arf.json") + + with open(ARF_PATH, "w") as f: + json.dump(cleaned, f, indent=2, separators=(",", ": ")) + f.write("\n") + + print(f"Wrote cleaned data to {ARF_PATH}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())