mirror of
https://github.com/yogeshojha/rengine.git
synced 2026-10-20 07:11:28 +02:00
Yogesh OjhaandClaude Opus 5
7588fdc090
EndpointSource.FUZZ had a label, a rank and a SourceKind and no producer. stages/content_discovery/ is it: ffuf against every live site with a ranked wordlist, level 5 beside url_discovery, a capability a person chooses, and enabled=False by default so an existing engine does not silently gain a run that sends 25,000 requests. tools/data/content.txt ships with the image (6,000 words, SecLists raft small, directories interleaved before files at each rank, regenerated by scripts/fetch_content_wordlist.py and committed like the port registry so an offline box still has one). The order is the budget, exactly as the subdomain list's is. One ffuf process per site, never several sites through a second wordlist keyword. The clusterbomb form looks like the efficient shape and silently defeats the only filter that matters: measured on one soft-404 host, `-w hosts:HOSTW -w words:FUZZ -u HOSTW/FUZZ -ac -ach` returned 278 of 300 guessed paths, while the same host asked in its own process returned 0. ffuf cannot calibrate against a host it does not know until the payload is substituted. Sites run over a small thread pool and only the stage thread writes, the url_discovery rule. A second belt sits behind calibration because no calibration is perfect: a site answering to 25% or more of a ranked list is answering to anything, so its whole run is dropped and the sites are named in a warning rather than quietly filtered. -maxtime is the per-site budget, -sf gives up on a host returning 403 to everything, and a run that hit its budget, a site that was dropped and a site that could not be reached are each counted and reported. Rows store is_probed=True with the status ffuf observed — it made the request, so the status is an observation, not an inference. `input.FUZZ` is base64 in -json mode (plain in -of json, which is why vhost() is fine), so the word is read off `url`. Verified live: 3 sites x 300 words in 11.7s, 295 raw hits became 17 real ones — /admin, /login, /media, /modules and /index.php among them. 29 tests, each proven to fail against 10 mutations. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013dKUV96WkQQETZBMoAG98C
84 lines
2.7 KiB
Python
Executable File
84 lines
2.7 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Regenerate tools/data/content.txt from SecLists' web-content discovery list.
|
|
|
|
The list is MIT-licensed and slow-moving, so the generated file is committed rather
|
|
than fetched at build time — a scan on a box with no egress still gets a wordlist.
|
|
Run this when you want a newer cut:
|
|
|
|
python3 scripts/fetch_content_wordlist.py [--size 5000]
|
|
|
|
Words are ranked by how often the path was found in a real corpus, so the order is the
|
|
budget: ffuf walks the file top to bottom and a smaller wordlist is simply the first N
|
|
lines. Keep it that way.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import re
|
|
import sys
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
SOURCE = (
|
|
"https://raw.githubusercontent.com/danielmiessler/SecLists/master/"
|
|
"Discovery/Web-Content/raft-{size}-{kind}.txt"
|
|
)
|
|
KINDS = ("directories", "files")
|
|
SIZES = ("small", "medium", "large")
|
|
OUTPUT = Path(__file__).resolve().parent.parent / "tools" / "data" / "content.txt"
|
|
|
|
# one path segment: what ffuf appends to the base url, never a full path or a query
|
|
_WORD = re.compile(r"^[A-Za-z0-9._~!$&'()*+,;=:@%-]{1,100}$")
|
|
|
|
|
|
def fetch(size: str, kind: str) -> list[str]:
|
|
url = SOURCE.format(size=size, kind=kind)
|
|
with urllib.request.urlopen(url, timeout=60) as response: # noqa: S310
|
|
body = response.read().decode("utf-8", errors="replace")
|
|
out: list[str] = []
|
|
for raw in body.splitlines():
|
|
word = raw.strip().lstrip("/")
|
|
if word and not word.startswith("#") and _WORD.match(word):
|
|
out.append(word)
|
|
return out
|
|
|
|
|
|
def merge(size: str) -> list[str]:
|
|
"""Directories first at each rank, because a folder is worth more than a file."""
|
|
seen: set[str] = set()
|
|
ranked: list[str] = []
|
|
columns = [fetch(size, kind) for kind in KINDS]
|
|
for row in range(max(len(c) for c in columns)):
|
|
for column in columns:
|
|
if row >= len(column):
|
|
continue
|
|
word = column[row]
|
|
if word.lower() in seen:
|
|
continue
|
|
seen.add(word.lower())
|
|
ranked.append(word)
|
|
return ranked
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--size", choices=SIZES, default="small")
|
|
parser.add_argument("--limit", type=int, default=6000)
|
|
args = parser.parse_args()
|
|
|
|
try:
|
|
words = merge(args.size)
|
|
except OSError as e:
|
|
print(f"could not fetch the wordlist: {e}", file=sys.stderr)
|
|
return 1
|
|
|
|
kept = words[: args.limit]
|
|
OUTPUT.write_text("\n".join(kept) + "\n", encoding="utf-8")
|
|
print(f"wrote {len(kept):,} words to {OUTPUT}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|