mirror of
https://github.com/lockfale/OSINT-Framework.git
synced 2026-08-25 07:12:24 +02:00
s0lrayandPaperclip
8c2de4d242
Rebased cleanly onto current master (resolving conflicts from community PRs #240-#471 merged while this branch was in flight). Changes: - Add GitHub Actions CI workflow (JSON lint, link check, smoke tests) - Remove 176 confirmed dead links from arf.json - Normalize 393 URLs from http:// to https:// - Fix invalid JSON (missing commas/braces from old format) - Remove Bower dependency; lock D3 to npm - Add link audit tooling (tools/) - Cherry-pick alive tools from community PR #391: FaceCheckID, Cisco Talos, VerifyEmail, YouTube Geofind, Whitepages, Hiya, Berla Vehicle Lookup, OpenAIP, Valkyrie File Analysis - Fix OpenInfrastructureMap URL (missing https://) Community PR #359 (Risk Toolbox / fraudfightersalliance.com): DNS does not resolve — all 27 proposed additions are dead, not merged. Co-Authored-By: Paperclip <noreply@paperclip.ing>
90 lines
2.2 KiB
Python
90 lines
2.2 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Remove confirmed dead links from arf.json.
|
|
Targets: DNS failures, connection refused, HTTP 404.
|
|
Leaves alone: .onion, 403 (bot-block), 5xx (transient), timeouts.
|
|
"""
|
|
|
|
import json
|
|
import sys
|
|
|
|
REPORT_PATH = "tools/link_audit_report.json"
|
|
ARF_PATH = "public/arf.json"
|
|
|
|
|
|
def get_dead_urls(report_path: str) -> set:
|
|
with open(report_path) as f:
|
|
report = json.load(f)
|
|
|
|
dead_urls = set()
|
|
for d in report["dead"]:
|
|
url = d.get("url", "")
|
|
err = d.get("error") or ""
|
|
status = d.get("status")
|
|
|
|
# Skip .onion (Tor hidden services - expected failure from clearnet)
|
|
if ".onion" in url:
|
|
continue
|
|
# DNS failure = truly dead
|
|
if "nodename nor servname" in err:
|
|
dead_urls.add(url)
|
|
# Connection refused = truly dead
|
|
elif "Connection refused" in err:
|
|
dead_urls.add(url)
|
|
# 404 = page gone
|
|
elif status == 404:
|
|
dead_urls.add(url)
|
|
|
|
return dead_urls
|
|
|
|
|
|
def remove_nodes(tree, dead_urls: set) -> tuple:
|
|
"""Recursively remove leaf nodes whose URL is dead. Returns (cleaned_tree, removed_count)."""
|
|
removed = 0
|
|
|
|
if not isinstance(tree, dict):
|
|
return tree, 0
|
|
|
|
children = tree.get("children", [])
|
|
if not children:
|
|
# Leaf node - check URL
|
|
url = tree.get("url", "")
|
|
if url in dead_urls:
|
|
return None, 1
|
|
return tree, 0
|
|
|
|
# Non-leaf: recurse into children
|
|
new_children = []
|
|
for child in children:
|
|
cleaned, n = remove_nodes(child, dead_urls)
|
|
removed += n
|
|
if cleaned is not None:
|
|
new_children.append(cleaned)
|
|
|
|
tree = dict(tree)
|
|
tree["children"] = new_children
|
|
return tree, removed
|
|
|
|
|
|
def main():
|
|
dead_urls = get_dead_urls(REPORT_PATH)
|
|
print(f"Confirmed dead URLs to remove: {len(dead_urls)}")
|
|
|
|
with open(ARF_PATH) as f:
|
|
data = json.load(f)
|
|
|
|
cleaned, removed = remove_nodes(data, dead_urls)
|
|
|
|
print(f"Removed {removed} nodes from arf.json")
|
|
|
|
with open(ARF_PATH, "w") as f:
|
|
json.dump(cleaned, f, indent=2, separators=(",", ": "))
|
|
f.write("\n")
|
|
|
|
print(f"Wrote cleaned data to {ARF_PATH}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|