diff --git a/alembic/versions/2026_06_25_http_asset_final_url.py b/alembic/versions/2026_06_25_http_asset_final_url.py new file mode 100644 index 00000000..a4d9b822 --- /dev/null +++ b/alembic/versions/2026_06_25_http_asset_final_url.py @@ -0,0 +1,36 @@ +"""add http_assets.final_url (redirect destination) + +Revision ID: f1a3c5e7d9b2 +Revises: e7c9a1b3d5f8 +Create Date: 2026-06-25 16:00:00.000000+00:00 + +Keeps `host`/`url` as the probed key (correlation) and records where a +followed redirect actually landed. +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +import sqlmodel + +from alembic import op + +revision: str = "f1a3c5e7d9b2" +down_revision: str | None = "e7c9a1b3d5f8" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.add_column( + "http_assets", + sa.Column( + "final_url", + sqlmodel.sql.sqltypes.AutoString(length=2000), + nullable=True, + ), + ) + + +def downgrade() -> None: + op.drop_column("http_assets", "final_url") diff --git a/engines/http_probe/engine.py b/engines/http_probe/engine.py index 25ce4bc3..333c5b1f 100644 --- a/engines/http_probe/engine.py +++ b/engines/http_probe/engine.py @@ -143,6 +143,7 @@ class HttpProbeEngine(Engine): content_length=fields["content_length"], content_type=fields["content_type"], location=fields["location"], + final_url=fields["final_url"], tech=fields["tech"], ip=fields["ip"], cname=fields["cname"], diff --git a/shared/models/http_asset.py b/shared/models/http_asset.py index e158bac3..6947e53b 100644 --- a/shared/models/http_asset.py +++ b/shared/models/http_asset.py @@ -31,6 +31,7 @@ class HttpAsset(SQLModel, table=True): content_length: int | None = Field(default=None) content_type: str | None = Field(default=None, max_length=255) location: str | None = Field(default=None, max_length=2000) + final_url: str | None = Field(default=None, max_length=2000) tech: list = Field(default_factory=list, sa_column=Column(JSON, nullable=False)) ip: str | None = Field(default=None, max_length=45) @@ -72,6 +73,7 @@ class HttpAssetRead(BaseModel): content_length: int | None = None content_type: str | None = None location: str | None = None + final_url: str | None = None tech: list[str] = Field(default_factory=list) ip: str | None = None cname: str | None = None diff --git a/tools/httpx/parser.py b/tools/httpx/parser.py index df493771..90c5d360 100644 --- a/tools/httpx/parser.py +++ b/tools/httpx/parser.py @@ -3,7 +3,6 @@ from __future__ import annotations from datetime import UTC, datetime -from urllib.parse import urlsplit def _int(value) -> int | None: @@ -37,12 +36,16 @@ def _parse_dt(value) -> datetime | None: return dt.replace(tzinfo=UTC) if dt.tzinfo is None else dt +def _first(value) -> str | None: + return value[0] if isinstance(value, list) and value else None + + def _host_of(record: dict) -> str: - url = record.get("url") or "" - host = urlsplit(url).hostname if url else None - if host: - return host - return (record.get("input") or "").split(":")[0] + """The host we probed (the correlation key) — derived from httpx `input`.""" + raw = (record.get("input") or record.get("url") or "").strip() + if "://" in raw: + raw = raw.split("://", 1)[1] + return raw.split("/", 1)[0].rsplit(":", 1)[0].strip().lower() def parse_httpx_record(record: dict) -> dict: @@ -59,8 +62,9 @@ def parse_httpx_record(record: dict) -> dict: return { "url": record.get("url") or record.get("input") or "", + "final_url": _trunc(record.get("final_url"), 2000), "host": _host_of(record), - "ip": _trunc(record.get("host"), 45), + "ip": _trunc(record.get("host_ip") or _first(record.get("a")), 45), "port": _int(record.get("port")) or 0, "scheme": (record.get("scheme") or "https")[:8], "status_code": _int(record.get("status_code")),