import uuid from datetime import datetime from pydantic import BaseModel from pydantic import Field as PydanticField from sqlalchemy import Column, Text from sqlalchemy.types import JSON from sqlmodel import Field, SQLModel, UniqueConstraint from shared.definitions.asset_query import MAX_QUERY_LENGTH from shared.definitions.endpoints import ( MAX_FILENAME_LENGTH, MAX_HOST_LENGTH, MAX_PATH_LENGTH, MAX_URL_LENGTH, EndpointClass, EndpointSource, ) from shared.definitions.vulnerabilities import CoverageStatus from shared.models.asset_query import MatchEvidence, QueryError from shared.utils.datetime import utc_now def _json_list() -> Field: return Field(default_factory=list, sa_column=Column(JSON, nullable=False)) def _json_dict() -> Field: return Field(default_factory=dict, sa_column=Column(JSON, nullable=False)) def _text() -> Field: return Field(default=None, sa_column=Column(Text, nullable=True)) class Endpoint(SQLModel, table=True): __tablename__ = "endpoints" __table_args__ = ( UniqueConstraint("scan_id", "signature", name="uq_endpoint_scan_signature"), ) id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True, index=True) scan_id: uuid.UUID = Field(foreign_key="scans.id", index=True, ondelete="CASCADE") target_id: uuid.UUID = Field( foreign_key="targets.id", index=True, ondelete="CASCADE" ) project_id: uuid.UUID = Field(foreign_key="projects.id", index=True) # structural identity: path shape plus parameter names, never parameter values signature: str = Field(max_length=64, index=True) # where it is url: str = Field(max_length=MAX_URL_LENGTH) host: str = Field(max_length=MAX_HOST_LENGTH, index=True) port: int = Field(default=443) scheme: str = Field(default="https", max_length=8) path: str = Field(max_length=MAX_PATH_LENGTH) dir_path: str = Field(max_length=MAX_PATH_LENGTH, index=True) filename: str | None = Field(default=None, max_length=MAX_FILENAME_LENGTH) extension: str | None = Field(default=None, max_length=10, index=True) depth: int = Field(default=0, index=True) # what it accepts params: list = _json_list() param_count: int = Field(default=0, index=True) param_samples: list = _json_list() variants: int = Field(default=1) more_variants: bool = Field(default=False) methods: list = _json_list() # who says so sources: list = _json_list() primary_source: str = Field( default=EndpointSource.OTHER.value, max_length=24, index=True ) discovery: dict = _json_dict() found_on: str | None = Field(default=None, max_length=2000) # what it answered, if anything asked is_probed: bool = Field(default=False, index=True) status_code: int | None = Field(default=None, index=True) content_type: str | None = Field(default=None, max_length=255) content_length: int | None = Field(default=None) title: str | None = Field(default=None, max_length=1000) words: int | None = Field(default=None) lines: int | None = Field(default=None) response_time: float | None = Field(default=None) redirect_location: str | None = Field(default=None, max_length=2000) content_hash: str | None = Field(default=None, max_length=80, index=True) tech: list = _json_list() # how it reads endpoint_class: str = Field( default=EndpointClass.OTHER.value, max_length=16, index=True ) interest: list = _json_list() # correlation ids, resolved at write time; no FK so a row outlives its asset http_asset_id: uuid.UUID | None = Field(default=None, index=True) subdomain_id: uuid.UUID | None = Field(default=None, index=True) archive_last_seen: datetime | None = Field(default=None) discovered_at: datetime = Field(default_factory=utc_now, index=True) created_at: datetime = Field(default_factory=utc_now) class EndpointCoverage(SQLModel, table=True): """One provider's account of what it covered. A null count means unknown, never zero.""" __tablename__ = "endpoint_coverage" id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True, index=True) scan_id: uuid.UUID = Field(foreign_key="scans.id", index=True, ondelete="CASCADE") target_id: uuid.UUID = Field( foreign_key="targets.id", index=True, ondelete="CASCADE" ) project_id: uuid.UUID = Field(foreign_key="projects.id", index=True) source: str = Field(max_length=24, index=True) tool: str | None = Field(default=None, max_length=40) status: str = Field(default=CoverageStatus.COMPLETED.value, max_length=16) hosts_total: int = Field(default=0) hosts_scanned: int | None = Field(default=None) hosts_dropped: list = _json_list() urls_found: int | None = Field(default=None) urls_stored: int | None = Field(default=None) urls_probed: int | None = Field(default=None) pages_fetched: int | None = Field(default=None) depth_reached: int | None = Field(default=None) errors: int | None = Field(default=None) capped: bool = Field(default=False) cap_reason: str | None = Field(default=None, max_length=200) command: str | None = _text() error: str | None = Field(default=None, max_length=2000) started_at: datetime = Field(default_factory=utc_now) ended_at: datetime | None = Field(default=None) duration_seconds: float | None = Field(default=None) class SourceEvidence(BaseModel): """Why one provider believes this endpoint exists.""" source: str label: str kind: str detail: str | None = None found_on: str | None = None observed_at: datetime | None = None class EndpointRead(BaseModel): id: uuid.UUID scan_id: uuid.UUID target_id: uuid.UUID signature: str url: str host: str port: int scheme: str path: str dir_path: str filename: str | None = None extension: str | None = None depth: int params: list[str] = PydanticField(default_factory=list) param_count: int = 0 variants: int = 1 more_variants: bool = False methods: list[str] = PydanticField(default_factory=list) sources: list[str] = PydanticField(default_factory=list) primary_source: str evidence: list[SourceEvidence] = PydanticField(default_factory=list) found_on: str | None = None is_probed: bool = False status_code: int | None = None content_type: str | None = None content_length: int | None = None title: str | None = None words: int | None = None lines: int | None = None response_time: float | None = None redirect_location: str | None = None tech: list[str] = PydanticField(default_factory=list) endpoint_class: str interest: list[str] = PydanticField(default_factory=list) http_asset_id: uuid.UUID | None = None subdomain_id: uuid.UUID | None = None archive_last_seen: datetime | None = None discovered_at: datetime is_new: bool = False matched_in: list[MatchEvidence] = PydanticField(default_factory=list) class EndpointDetail(EndpointRead): param_samples: list = PydanticField(default_factory=list) discovery: dict = PydanticField(default_factory=dict) content_hash: str | None = None siblings: int = 0 class EndpointFilter(BaseModel): q: str | None = PydanticField(default=None, max_length=MAX_QUERY_LENGTH) host: str | None = None dir_path: str | None = None subtree: bool = True endpoint_class: str | None = None source: str | None = None interest: str | None = None status_class: str | None = None probed: bool | None = None new: bool = False hide_static: bool = False sort: str = "path" direction: str = "asc" page: int = 1 size: int = 50 def has_facets(self) -> bool: return any( ( self.host, self.dir_path, self.endpoint_class, self.source, self.interest, self.status_class, self.probed is not None, self.new, self.hide_static, ) ) class EndpointPage(BaseModel): items: list[EndpointRead] = PydanticField(default_factory=list) total: int = 0 total_capped: bool = False page: int = 1 size: int = 50 error: QueryError | None = None class EndpointFacet(BaseModel): value: str label: str count: int class EndpointFacets(BaseModel): endpoint_class: list[EndpointFacet] = PydanticField(default_factory=list) source: list[EndpointFacet] = PydanticField(default_factory=list) interest: list[EndpointFacet] = PydanticField(default_factory=list) status_class: list[EndpointFacet] = PydanticField(default_factory=list) extension: list[EndpointFacet] = PydanticField(default_factory=list) host: list[EndpointFacet] = PydanticField(default_factory=list) total: int = 0 static_total: int = 0 class TreeLeaf(BaseModel): """The one endpoint a folder holds when that folder is only its own index.""" id: uuid.UUID url: str host: str path: str params: list[str] = PydanticField(default_factory=list) param_count: int = 0 endpoint_class: str is_probed: bool = False status_code: int | None = None content_length: int | None = None sources: list[str] = PydanticField(default_factory=list) interest: list[str] = PydanticField(default_factory=list) class TreeNode(BaseModel): """One directory or host in the site tree, counted through the same filter as the table.""" key: str name: str path: str host: str | None = None kind: str = "directory" depth: int = 0 direct_count: int = 0 subtree_count: int = 0 child_count: int = 0 hosts: int = 1 status_mix: dict[str, int] = PydanticField(default_factory=dict) class_mix: dict[str, int] = PydanticField(default_factory=dict) sources: list[str] = PydanticField(default_factory=list) interest: list[str] = PydanticField(default_factory=list) has_params: bool = False params: int = 0 verified: int = 0 unprobed: int = 0 new_count: int = 0 gone_count: int = 0 anomaly: str | None = None archive_only: bool = False glyph: str = "folder" sample_url: str | None = None leaf: TreeLeaf | None = None query: str children: list["TreeNode"] = PydanticField(default_factory=list) lazy: bool = False folders: int = 0 top_folders: list[str] = PydanticField(default_factory=list) class HostPage(BaseModel): """One page of hosts with their rollups; a host's folders load when it is opened.""" items: list[TreeNode] = PydanticField(default_factory=list) total: int = 0 total_endpoints: int = 0 page: int = 1 size: int = 50 error: QueryError | None = None class MergedLeaf(BaseModel): """One path inside a folder, folded across every host that serves it.""" key: str path: str name: str params: list[str] = PydanticField(default_factory=list) param_count: int = 0 endpoint_class: str hosts: int = 0 endpoints: int = 0 status_mix: dict[str, int] = PydanticField(default_factory=dict) unprobed: int = 0 interest: list[str] = PydanticField(default_factory=list) sources: list[str] = PydanticField(default_factory=list) host_names: list[str] = PydanticField(default_factory=list) new_count: int = 0 sample_id: uuid.UUID sample_url: str sample_status: int | None = None query: str class MergedLeafPage(BaseModel): items: list[MergedLeaf] = PydanticField(default_factory=list) total: int = 0 truncated: bool = False class EndpointTree(BaseModel): mode: str = "host" nodes: list[TreeNode] = PydanticField(default_factory=list) total_endpoints: int = 0 total_nodes: int = 0 truncated: bool = False error: QueryError | None = None class CoverageRead(BaseModel): id: uuid.UUID source: str label: str tool: str | None = None status: str hosts_total: int = 0 hosts_scanned: int | None = None hosts_dropped: list[str] = PydanticField(default_factory=list) urls_found: int | None = None urls_stored: int | None = None urls_probed: int | None = None pages_fetched: int | None = None depth_reached: int | None = None errors: int | None = None capped: bool = False cap_reason: str | None = None error: str | None = None started_at: datetime ended_at: datetime | None = None duration_seconds: float | None = None class EndpointSummary(BaseModel): total: int = 0 probed: int = 0 live: int = 0 with_params: int = 0 interesting: int = 0 hosts: int = 0 new: int = 0 gone: int = 0 previous_scan_id: uuid.UUID | None = None previous_scan_at: datetime | None = None by_class: dict[str, int] = PydanticField(default_factory=dict) by_source: dict[str, int] = PydanticField(default_factory=dict) class GonePage(EndpointPage): """Endpoints the previous scan of this target recorded and this scan did not.""" previous_scan_id: uuid.UUID | None = None previous_scan_at: datetime | None = None class StructureFinding(BaseModel): """Something about the shape of the surface that is worth acting on.""" kind: str label: str detail: str count: int query: str samples: list[str] = PydanticField(default_factory=list) class PathSpread(BaseModel): path: str hosts: int endpoints: int query: str class StructureLine(BaseModel): key: str label: str detail: str | None = None count: int hosts: int = 0 query: str class ScanStructure(BaseModel): endpoints: int = 0 hosts: int = 0 probed: int = 0 directories: int = 0 max_depth: int = 0 with_params: int = 0 headline: str | None = None findings: list[StructureFinding] = PydanticField(default_factory=list) shared_paths: list[PathSpread] = PydanticField(default_factory=list) interest: list[StructureLine] = PydanticField(default_factory=list) by_class: list[StructureLine] = PydanticField(default_factory=list) by_source: list[StructureLine] = PydanticField(default_factory=list) class VerifyBranchRequest(BaseModel): host: str = PydanticField(max_length=500) dir_path: str | None = PydanticField(default=None, max_length=1500) limit: int = PydanticField(default=500, ge=1, le=2000) class VerifyBranchResponse(BaseModel): queued: int = 0 unverified: int = 0 accepted: bool = False