mirror of
https://github.com/yogeshojha/rengine.git
synced 2026-10-18 22:33:24 +02:00
Yogesh OjhaandClaude Opus 5
47148f2b2e
Screenshots carry a perceptual hash, so hosts whose pages look alike are grouped in the gallery and the table. An exact hash matches only identical bytes, which leaves the same login across forty tenants in forty groups of one, so this is a 64-bit dHash taken from a fixed viewport band. Clustering follows the view's scope rather than being frozen at finalize: one run on a scan page, every target on the project-wide view. A cluster is every row within SCREENSHOT_DISTANCE of one representative, so its count and the screenshot: filter it opens are the same set by construction. Thresholds are measured over 9,890 stored screenshots on four estates. Distance 2 collapses the cPanel port variants and a default page's one-bit variants while 4 begins merging different skins. A hash setting fewer than four bits is a blank render and never represents a cluster; a variance floor let a one-bit hash sink 79 hosts into one group. Screenshots are held as WebP at native size, 18% of PNG, converted by a background task rather than the stage: httpx writes the PNG itself, so converting inline saves no peak disk and costs every scan serial time. Only settled scans are touched, or the path is rewritten under a stage still writing it. The hash is taken before the encode and never recomputed, so compression leaves the grouping untouched. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01U2BnU1s9wZg3KYgCHBxhuC
64 lines
1.7 KiB
Python
64 lines
1.7 KiB
Python
"""The perceptual hash of a rendered page."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
HASH_SIZE = 8
|
|
DIGEST_CHARS = 16
|
|
# the viewport band a full-page render is cut back to, as a fraction of its width
|
|
VIEWPORT_ASPECT = 0.625
|
|
_SIGN_BIT = 1 << 63
|
|
_WRAP = 1 << 64
|
|
_ROW = HASH_SIZE + 1
|
|
|
|
|
|
def to_signed(value: int) -> int:
|
|
"""A 64-bit hash as the bigint Postgres stores."""
|
|
return value - _WRAP if value >= _SIGN_BIT else value
|
|
|
|
|
|
def to_unsigned(value: int) -> int:
|
|
return value + _WRAP if value < 0 else value
|
|
|
|
|
|
def hex_digest(value: int) -> str:
|
|
return f"{to_unsigned(value):016x}"
|
|
|
|
|
|
def popcount(value: int) -> int:
|
|
return to_unsigned(value).bit_count()
|
|
|
|
|
|
def distance(left: int, right: int) -> int:
|
|
return (to_unsigned(left) ^ to_unsigned(right)).bit_count()
|
|
|
|
|
|
def phash_image(img) -> int:
|
|
"""The dHash of an already decoded render."""
|
|
from PIL import Image # noqa: PLC0415
|
|
|
|
width, height = img.size
|
|
band = min(height, max(1, round(width * VIEWPORT_ASPECT)))
|
|
grey = img.convert("L").crop((0, 0, width, band))
|
|
pixels = grey.resize((_ROW, HASH_SIZE), Image.Resampling.BOX).tobytes()
|
|
bits = 0
|
|
for y in range(HASH_SIZE):
|
|
base = y * _ROW
|
|
for x in range(HASH_SIZE):
|
|
bits = (bits << 1) | (pixels[base + x] > pixels[base + x + 1])
|
|
return to_signed(bits)
|
|
|
|
|
|
def phash_file(path: str | Path) -> int | None:
|
|
"""The dHash of a rendered page, or None when the file is not a readable image."""
|
|
from PIL import Image, UnidentifiedImageError # noqa: PLC0415
|
|
|
|
try:
|
|
if Path(path).stat().st_size == 0:
|
|
return None
|
|
with Image.open(path) as img:
|
|
return phash_image(img)
|
|
except (OSError, UnidentifiedImageError, ValueError):
|
|
return None
|