rengine/tools/nuclei/parser.py

"""Normalise one nuclei JSONL record into the shape reNgine stores."""

from __future__ import annotations

import hashlib
from dataclasses import dataclass, field
from datetime import datetime
from typing import Any
from urllib.parse import urlsplit

from shared.definitions.vulnerabilities import (
    MAX_EVIDENCE_BYTES,
    Protocol,
    Scanner,
    coerce_protocol,
    coerce_severity,
    is_kev,
)
from shared.utils.text import strip_control

_PROTOCOL_ALIASES = {
    "tcp": Protocol.NETWORK.value,
    "network": Protocol.NETWORK.value,
    "js": Protocol.JAVASCRIPT.value,
}


def _as_list(value: Any) -> list[str]:
    if value is None:
        return []
    if isinstance(value, str):
        parts = [strip_control(p).strip() for p in value.replace("\n", ",").split(",")]
        return [p for p in parts if p]
    if isinstance(value, (list, tuple, set)):
        out: list[str] = []
        for item in value:
            out.extend(_as_list(item))
        return out
    return [str(value)]


def _as_text(value: Any, limit: int = 0) -> str | None:
    if value is None:
        return None
    text = strip_control(str(value)).strip()
    if not text:
        return None
    return text[:limit] if limit else text


def _as_float(value: Any) -> float | None:
    try:
        return float(value)
    except (TypeError, ValueError):
        return None


def _as_int(value: Any) -> int | None:
    try:
        return int(str(value).strip())
    except (TypeError, ValueError):
        return None


def _timestamp(value: Any) -> datetime | None:
    text = _as_text(value)
    if not text:
        return None
    try:
        return datetime.fromisoformat(text.replace("Z", "+00:00"))
    except ValueError:
        return None


@dataclass
class Finding:
    fingerprint: str
    scanner: str
    template_id: str
    template_name: str
    template_path: str | None
    template_url: str | None
    severity: str
    protocol: str
    matcher_name: str | None
    extractor_name: str | None
    extracted_results: list[str]
    description: str | None
    impact: str | None
    remediation: str | None
    references: list[str]
    tags: list[str]
    authors: list[str]
    cve_ids: list[str]
    cwe_ids: list[str]
    cvss_metrics: str | None
    cvss_score: float | None
    epss_score: float | None
    epss_percentile: float | None
    cpe: str | None
    is_kev: bool
    matched_at: str
    host: str | None
    ip: str | None
    port: int | None
    scheme: str | None
    url: str | None
    path: str | None
    request: str | None
    response: str | None
    curl_command: str | None
    interaction: dict = field(default_factory=dict)
    extra: dict = field(default_factory=dict)
    observed_at: datetime | None = None


def fingerprint(
    scanner: str, template_id: str, matcher: str | None, matched_at: str
) -> str:
    raw = f"{scanner}|{template_id}|{matcher or ''}|{matched_at}"
    return hashlib.sha256(raw.encode("utf-8", "ignore")).hexdigest()


def _hostname(candidate: str | None) -> str | None:
    if not candidate:
        return None
    value = candidate.strip()
    if "://" in value:
        parsed = urlsplit(value)
        return parsed.hostname or None
    head = value.split("/", 1)[0]
    if head.count(":") == 1:
        head = head.split(":", 1)[0]
    return head or None


def _port_of(record: dict, url: str | None) -> int | None:
    direct = _as_int(record.get("port"))
    if direct:
        return direct
    if url and "://" in url:
        parsed = urlsplit(url)
        if parsed.port:
            return parsed.port
        if parsed.scheme == "https":
            return 443
        if parsed.scheme == "http":
            return 80
    host = record.get("host") or ""
    if isinstance(host, str) and host.count(":") == 1:
        return _as_int(host.rsplit(":", 1)[1])
    return None


def parse_finding(record: dict, scanner: str = Scanner.NUCLEI.value) -> Finding | None:
    template_id = _as_text(record.get("template-id"), 200)
    if not template_id:
        return None
    info = record.get("info") if isinstance(record.get("info"), dict) else {}
    classification = info.get("classification")
    classification = classification if isinstance(classification, dict) else {}
    metadata = info.get("metadata")
    metadata = metadata if isinstance(metadata, dict) else {}

    matched_at = (
        _as_text(record.get("matched-at"), 2000)
        or _as_text(record.get("url"), 2000)
        or _as_text(record.get("host"), 2000)
        or template_id
    )
    url = _as_text(record.get("url"), 2000) or (
        matched_at if "://" in matched_at else None
    )
    raw_type = (_as_text(record.get("type")) or "").lower()
    tags = _as_list(info.get("tags"))[:60]
    matcher = _as_text(record.get("matcher-name"), 200)

    return Finding(
        fingerprint=fingerprint(scanner, template_id, matcher, matched_at),
        scanner=scanner,
        template_id=template_id,
        template_name=_as_text(info.get("name"), 500) or template_id,
        template_path=_as_text(record.get("template"), 500)
        or _as_text(record.get("template-path"), 500),
        template_url=_as_text(record.get("template-url"), 1000),
        severity=coerce_severity(_as_text(info.get("severity"))),
        protocol=coerce_protocol(_PROTOCOL_ALIASES.get(raw_type, raw_type)),
        matcher_name=matcher,
        extractor_name=_as_text(record.get("extractor-name"), 200),
        extracted_results=_as_list(record.get("extracted-results"))[:50],
        description=_as_text(info.get("description")),
        impact=_as_text(info.get("impact")),
        remediation=_as_text(info.get("remediation")),
        references=_as_list(info.get("reference"))[:40],
        tags=tags,
        authors=_as_list(info.get("author"))[:20],
        cve_ids=[c.upper() for c in _as_list(classification.get("cve-id"))][:20],
        cwe_ids=[c.upper() for c in _as_list(classification.get("cwe-id"))][:20],
        cvss_metrics=_as_text(classification.get("cvss-metrics"), 200),
        cvss_score=_as_float(classification.get("cvss-score")),
        epss_score=_as_float(classification.get("epss-score")),
        epss_percentile=_as_float(classification.get("epss-percentile")),
        cpe=_as_text(classification.get("cpe"), 300),
        is_kev=is_kev(tags),
        matched_at=matched_at,
        host=_hostname(record.get("host")) or _hostname(matched_at),
        ip=_as_text(record.get("ip"), 45),
        port=_port_of(record, url),
        scheme=_as_text(record.get("scheme"), 16),
        url=url,
        path=_as_text(record.get("path"), 2000),
        request=_as_text(record.get("request"), MAX_EVIDENCE_BYTES),
        response=_as_text(record.get("response"), MAX_EVIDENCE_BYTES),
        curl_command=_as_text(record.get("curl-command"), MAX_EVIDENCE_BYTES),
        interaction=record.get("interaction") or {},
        extra={
            k: v for k, v in metadata.items() if isinstance(v, (str, int, float, bool))
        },
        observed_at=_timestamp(record.get("timestamp")),
    )