rengine/tools/httpx/parser.py

from __future__ import annotations

import re
from datetime import UTC, datetime
from typing import Any

from shared.services.scan_resolve import MASK
from shared.services.web_hygiene import evaluate as evaluate_hygiene
from shared.utils.datetime import utc_now
from shared.utils.net import split_host_port
from shared.utils.software import components_of
from shared.utils.tracking import tracking_ids
from tools.runner.fieldmap import F, parse_record

_REQUEST_SECRET_HEADER = re.compile(
    r"(?im)^((?:authorization|proxy-authorization|cookie|x-api-key|"
    r"[\w-]*(?:token|secret|api-?key)[\w-]*)\s*:\s*).+$"
)


def _redact_request(value: Any) -> str | None:
    if not value or not isinstance(value, str):
        return None
    return _REQUEST_SECRET_HEADER.sub(rf"\1{MASK}", value)


def _int(value: Any) -> int | None:
    try:
        return int(value)
    except (TypeError, ValueError):
        return None


def _str_list(value: Any) -> list[str]:
    if isinstance(value, list):
        return [str(v) for v in value if v]
    return [str(value)] if value else []


def _first(value: Any) -> str | None:
    if isinstance(value, list):
        return str(value[0]) if value else None
    return str(value) if value else None


def _asn_number(value: Any) -> int | None:
    if isinstance(value, int):
        return value
    try:
        return int(str(value).upper().replace("AS", "").strip())
    except (TypeError, ValueError):
        return None


def _parse_dt(value: Any) -> datetime | None:
    if not value or not isinstance(value, str):
        return None
    try:
        dt = datetime.fromisoformat(value.strip().replace("Z", "+00:00"))
    except ValueError:
        return None
    return dt.replace(tzinfo=UTC) if dt.tzinfo is None else dt


def _seconds(value: Any) -> float | None:
    text = str(value).strip()
    try:
        if text.endswith("ms"):
            return round(float(text[:-2]) / 1000, 4)
        if text.endswith("s"):
            return round(float(text[:-1]), 4)
        return round(float(text), 4)
    except ValueError:
        return None


def _host_of(record: dict) -> str:
    raw = (record.get("input") or record.get("url") or "").strip()
    if "://" in raw:
        raw = raw.split("://", 1)[1]
    return split_host_port(raw.split("/", 1)[0])[0].strip().lower()


def _tls_expired(record: dict) -> bool | None:
    not_after = _parse_dt((record.get("tls") or {}).get("not_after"))
    return not_after < utc_now() if not_after else None


def _tls_self_signed(record: dict) -> bool | None:
    tls = record.get("tls") or {}
    issuer, subject = tls.get("issuer_dn"), tls.get("subject_dn")
    return issuer == subject if (issuer and subject) else None


def _software(record: dict) -> list[dict]:
    return components_of(_str_list(record.get("tech")), record.get("webserver"))


HTTPX_FIELDS: dict[str, F] = {
    "url": F(lambda r: r.get("url") or r.get("input") or "", max_len=2000),
    "final_url": F("final_url", max_len=2000),
    "host": F(_host_of, max_len=500),
    "ip": F(lambda r: r.get("host_ip") or _first(r.get("a")), max_len=45),
    "a_records": F(lambda r: _str_list(r.get("a"))),
    "aaaa_records": F(lambda r: _str_list(r.get("aaaa"))),
    "port": F("port", _int),
    "scheme": F("scheme", str, max_len=8),
    "method": F("method", str, max_len=10),
    "path": F("path", str, max_len=2000),
    "status_code": F("status_code", _int),
    "chain_status_codes": F(lambda r: list(r.get("chain_status_codes") or [])),
    "title": F("title", str, max_len=1000),
    "webserver": F("webserver", str, max_len=255),
    "content_type": F("content_type", str, max_len=255),
    "content_length": F("content_length", _int),
    "location": F("location", str, max_len=2000),
    "response_time": F("time", _seconds),
    "words": F("words", _int),
    "lines": F("lines", _int),
    "tech": F(lambda r: _str_list(r.get("tech"))),
    "software": F(_software),
    "cpe": F(lambda r: list(r.get("cpe") or [])),
    "cname": F(lambda r: _first(r.get("cname")), max_len=500),
    "is_cdn": F(lambda r: bool(r.get("cdn"))),
    "cdn_name": F("cdn_name", str, max_len=100),
    "cdn_type": F("cdn_type", str, max_len=20),
    "supports_http2": F(lambda r: bool(r.get("http2"))),
    "supports_pipeline": F(lambda r: bool(r.get("pipeline"))),
    "jarm": F("jarm_hash", str, max_len=64),
    "favicon_hash": F("favicon", str, max_len=64),
    "favicon_path": F("favicon_path", str, max_len=2000),
    "content_hash": F("hash.body_sha256", str, max_len=80),
    "header_hash": F("hash.header_sha256", str, max_len=80),
    "tls_version": F("tls.tls_version", str, max_len=20),
    "tls_cipher": F("tls.cipher", str, max_len=100),
    "tls_subject_cn": F("tls.subject_cn", str, max_len=500),
    "tls_subject_dn": F("tls.subject_dn", str, max_len=1000),
    "tls_sans": F(lambda r: _str_list((r.get("tls") or {}).get("subject_an"))),
    "tls_issuer": F(lambda r: (r.get("tls") or {}).get("issuer_dn"), max_len=500),
    "tls_issuer_cn": F("tls.issuer_cn", str, max_len=500),
    "tls_issuer_org": F(
        lambda r: _first((r.get("tls") or {}).get("issuer_org")), max_len=500
    ),
    "tls_serial": F("tls.serial", str, max_len=200),
    "tls_fingerprint": F("tls.fingerprint_hash.sha256", str, max_len=80),
    "tls_not_before": F("tls.not_before", _parse_dt),
    "tls_not_after": F("tls.not_after", _parse_dt),
    "tls_expired": F(_tls_expired),
    "tls_self_signed": F(_tls_self_signed),
    "asn": F("asn.as_number", _asn_number),
    "asn_org": F("asn.as_name", str, max_len=255),
    "raw_request": F(lambda r: _redact_request(r.get("request")), max_len=65536),
    "raw_response_header": F("raw_header", str, max_len=65536),
    "response_headers": F(lambda r: r.get("header") or {}),
    "response_body": F("body", str, max_len=131072),
    "body_preview": F("body_preview", str, max_len=512),
}


def parse_httpx_record(record: dict) -> dict[str, Any]:
    fields = parse_record(record, HTTPX_FIELDS)
    hygiene = evaluate_hygiene(
        fields.get("response_headers"),
        fields.get("raw_response_header"),
        scheme=fields.get("scheme"),
        status_code=fields.get("status_code"),
        content_type=fields.get("content_type"),
        body=fields.get("response_body"),
    )
    fields["hygiene_issues"] = hygiene.issues
    fields["hygiene_checked"] = hygiene.checked
    fields["tracking_ids"] = tracking_ids(fields.get("response_body"))
    return fields