From 695c4803bb2ab93f2dee2be2fa9430dc3afa71f4 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Mon, 14 Sep 2026 12:26:27 +0530 Subject: [PATCH] refactor(engine): one intensity table, tiered fields, built-in Default engine Every stage config field is a decision; rate, concurrency, timeout and retries come from a per-tool intensity profile the stage reads as its transport. 173 fields become 83: 41 transport values, 40 constants and the dead ones leave the models, and the builder groups by capability with support stages as checkboxes and advanced fields behind one disclosure. Aggressive now changes what a tool is handed. Every project carries a built-in Default engine, seeded on project creation and on first list, undeletable: stage defaults, subfinder as the only passive source, vulnerability scan on with nuclei. From the tracing review: tool args can no longer restate a flag the stage set, the session check obeys the probe scheme, excluded scope reaches every stage that reads addresses, hosts or paths, the sitemap and source-map providers throttle at the transport rate, the blind tier obeys its switch, the fuzzing lanes split the budget, and the removed private-address and evidence switches are back as advanced fields. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Pq4xYJbgW5euUa9sCWBKFw --- alembic/versions/2026_09_14_builtin_engine.py | 37 +++ alembic/versions/2026_09_14_engine_knobs.py | 160 +++++++++++++ api/app/api/v1/onboarding.py | 2 + api/app/api/v1/projects.py | 2 + api/app/api/v1/scan_engines.py | 4 +- api/app/services/scan_engine/catalog.py | 20 +- api/app/services/scan_engine/effects.py | 13 +- api/app/services/scan_engine/service.py | 53 ++++- .../engines/engine-list-card.svelte | 17 +- .../components/engines/engine-topbar.svelte | 9 +- .../lib/components/engines/stage-field.svelte | 6 - .../lib/components/engines/stage-row.svelte | 201 +++++++++------- .../scans/launch/stage-config-popover.svelte | 45 +++- .../src/lib/stores/engine-catalog.svelte.ts | 9 + frontend/src/lib/types/scan-engine.ts | 37 ++- .../src/lib/utilities/engine-completion.ts | 4 +- .../src/lib/utilities/engine-summary.test.ts | 62 +++++ frontend/src/lib/utilities/engine-summary.ts | 19 +- .../(app)/automation/engines/+page.svelte | 5 +- .../automation/engines/[id]/+page.svelte | 215 +++++++----------- shared/definitions/default_engine.py | 36 +++ shared/definitions/intensity.py | 155 +++++++++++++ shared/models/scan_engine.py | 12 +- shared/services/scan_resolve.py | 59 ++--- stages/base.py | 27 +++ stages/config.py | 62 ++--- stages/content_discovery/config.py | 19 +- stages/content_discovery/stage.py | 13 +- stages/dast_scan/config.py | 108 ++------- stages/dast_scan/scanners/dalfox.py | 18 +- stages/dast_scan/scanners/nuclei.py | 11 +- stages/dast_scan/stage.py | 17 +- stages/endpoint_probe/config.py | 27 +-- stages/endpoint_probe/stage.py | 13 +- stages/host_discovery/config.py | 5 +- stages/host_discovery/stage.py | 17 +- stages/http_probe/config.py | 30 +-- stages/http_probe/stage.py | 21 +- stages/netblock_sweep/config.py | 34 +-- stages/netblock_sweep/stage.py | 44 ++-- stages/origin_probe/config.py | 19 +- stages/origin_probe/stage.py | 13 +- stages/passive_ports/config.py | 10 +- stages/passive_ports/stage.py | 5 +- stages/port_scan/config.py | 52 ++--- stages/port_scan/stage.py | 14 +- stages/registry.py | 29 ++- stages/reverse_dns/config.py | 4 +- stages/reverse_dns/stage.py | 17 +- stages/screenshot/config.py | 7 +- stages/screenshot/stage.py | 10 +- stages/seed_resolution/config.py | 7 +- stages/seed_resolution/stage.py | 6 +- stages/service_fingerprint/config.py | 13 +- stages/service_fingerprint/stage.py | 9 +- stages/session_check/config.py | 3 +- stages/session_check/stage.py | 12 +- stages/subdomain/config.py | 56 ++--- stages/subdomain/stage.py | 92 ++++---- stages/target_enrichment/config.py | 6 +- stages/target_enrichment/stage.py | 11 +- stages/url_discovery/config.py | 77 ++----- stages/url_discovery/providers/archive.py | 10 +- stages/url_discovery/providers/base.py | 39 ++++ stages/url_discovery/providers/katana.py | 11 +- stages/url_discovery/providers/known_files.py | 13 +- stages/url_discovery/providers/source_maps.py | 15 +- stages/url_discovery/stage.py | 3 + stages/vhost/config.py | 4 +- stages/vhost/stage.py | 10 +- stages/vulnerability_scan/config.py | 168 +++++--------- stages/vulnerability_scan/scanners/base.py | 15 ++ stages/vulnerability_scan/scanners/nuclei.py | 65 ++++-- stages/vulnerability_scan/stage.py | 6 +- tests/pipeline/test_dast_surface.py | 8 +- tests/pipeline/test_default_engine.py | 68 ++++++ tests/pipeline/test_nuclei_groups.py | 8 +- tests/pipeline/test_oast.py | 19 +- tests/pipeline/test_stage_knobs.py | 191 ++++++++++++++++ tests/pipeline/test_tool_args_precedence.py | 35 +++ tools/runner/executor.py | 43 +++- worker/app/tasks/endpoints.py | 19 +- 82 files changed, 1831 insertions(+), 1039 deletions(-) create mode 100644 alembic/versions/2026_09_14_builtin_engine.py create mode 100644 alembic/versions/2026_09_14_engine_knobs.py create mode 100644 frontend/src/lib/utilities/engine-summary.test.ts create mode 100644 shared/definitions/default_engine.py create mode 100644 shared/definitions/intensity.py create mode 100644 tests/pipeline/test_default_engine.py create mode 100644 tests/pipeline/test_stage_knobs.py create mode 100644 tests/pipeline/test_tool_args_precedence.py diff --git a/alembic/versions/2026_09_14_builtin_engine.py b/alembic/versions/2026_09_14_builtin_engine.py new file mode 100644 index 000000000..c113e5ed7 --- /dev/null +++ b/alembic/versions/2026_09_14_builtin_engine.py @@ -0,0 +1,37 @@ +"""One built-in engine per project. + +Revision ID: c9e4f2a71b58 +Revises: b7d3e1f04a26 +Create Date: 2026-09-14 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +revision: str = "c9e4f2a71b58" +down_revision: str | None = "b7d3e1f04a26" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "scan_engines", + sa.Column( + "builtin", sa.Boolean(), nullable=False, server_default=sa.false() + ), + ) + op.create_index( + "ux_scan_engines_builtin_project", + "scan_engines", + ["project_id"], + unique=True, + postgresql_where=sa.text("builtin"), + ) + + +def downgrade() -> None: + op.drop_index("ux_scan_engines_builtin_project", table_name="scan_engines") + op.drop_column("scan_engines", "builtin") diff --git a/alembic/versions/2026_09_14_engine_knobs.py b/alembic/versions/2026_09_14_engine_knobs.py new file mode 100644 index 000000000..8f60d8cdc --- /dev/null +++ b/alembic/versions/2026_09_14_engine_knobs.py @@ -0,0 +1,160 @@ +"""Drop the transport and constant fields from stored engines. + +Revision ID: b7d3e1f04a26 +Revises: a4c2e7d91f53 +Create Date: 2026-09-14 +""" + +from __future__ import annotations + +import json + +import sqlalchemy as sa +from alembic import op + +revision: str = "b7d3e1f04a26" +down_revision: str | None = "a4c2e7d91f53" +branch_labels = None +depends_on = None + +_REMOVED: dict[str, set[str]] = { + "content_discovery": {"threads", "rate", "timeout"}, + "dast_scan": { + "max_dirs_per_origin", + "dir_depth", + "fuzz_param_frequency", + "rate", + "threads", + "bulk_size", + "timeout", + "retries", + "max_host_error", + "waf_rate_divisor", + "honeypot_threshold", + "interactsh_server", + "oast_wait_minutes", + "store_evidence", + "max_evidence_recovery", + "max_findings", + }, + "endpoint_probe": {"threads", "timeout", "rate", "follow_redirects"}, + "host_discovery": {"rate", "threads", "timeout"}, + "http_probe": { + "threads", + "timeout", + "rate", + "follow_redirects", + "probe_open_ports", + "max_ports_per_host", + }, + "netblock_sweep": { + "min_addresses", + "max_asn_addresses", + "min_share", + "dns_threads", + "dns_timeout", + }, + "origin_probe": {"threads", "timeout", "rate", "max_ports_per_address"}, + "passive_ports": {"max_addresses"}, + "port_scan": {"rate", "threads", "timeout", "retries", "port_threshold"}, + "reverse_dns": {"dns_threads", "dns_timeout"}, + "screenshot": {"threads", "rate", "timeout"}, + "seed_resolution": {"cidr_skip_rfc1918"}, + "service_fingerprint": {"threads", "timeout", "include_unknown"}, + "session_check": {"timeout"}, + "subdomain_discovery": { + "permutation_seeds", + "permutation_limit", + "dns_threads", + "dns_batch_size", + "dns_batch_concurrency", + "dns_idle_timeout", + "dns_retry_silent", + }, + "target_enrichment": {"dns_threads", "dns_timeout"}, + "url_discovery": { + "threads", + "timeout", + "rate", + "crawl_javascript", + "max_urls", + "max_known_file_hosts", + "max_source_maps", + "max_archive_domains", + }, + "vhost": {"threads", "rate"}, + "vulnerability_scan": { + "rate", + "threads", + "bulk_size", + "timeout", + "retries", + "max_host_error", + "waf_rate_divisor", + "honeypot_threshold", + "oast_wait_minutes", + "max_evidence_recovery", + "max_findings", + }, +} + + +def _prune(stages: dict) -> tuple[dict, bool]: + changed = False + out: dict = {} + for name, config in (stages or {}).items(): + if not isinstance(config, dict): + out[name] = config + continue + gone = _REMOVED.get(name, set()) + kept = {k: v for k, v in config.items() if k not in gone} + if len(kept) != len(config): + changed = True + if kept: + out[name] = kept + elif config: + changed = True + return out, changed + + +def _yaml_mentions_removed(source: str | None) -> bool: + if not source: + return False + try: + import yaml # noqa: PLC0415 + + doc = yaml.safe_load(source) or {} + except Exception: # noqa: BLE001 + return False + stages = doc.get("stages") if isinstance(doc, dict) else None + if not isinstance(stages, dict): + return False + return any( + isinstance(cfg, dict) and set(cfg) & _REMOVED.get(name, set()) + for name, cfg in stages.items() + ) + + +def upgrade() -> None: + bind = op.get_bind() + rows = bind.execute( + sa.text("SELECT id, stages, yaml_source FROM scan_engines") + ).all() + for id_, stages, yaml_source in rows: + raw = stages if isinstance(stages, dict) else json.loads(stages or "{}") + pruned, changed = _prune(raw) + drop_yaml = _yaml_mentions_removed(yaml_source) + if not changed and not drop_yaml: + continue + bind.execute( + sa.text( + "UPDATE scan_engines SET stages = :stages, " + "yaml_source = CASE WHEN :drop THEN NULL ELSE yaml_source END " + "WHERE id = :id" + ), + {"stages": json.dumps(pruned), "drop": drop_yaml, "id": id_}, + ) + + +def downgrade() -> None: + pass diff --git a/api/app/api/v1/onboarding.py b/api/app/api/v1/onboarding.py index 1e05d0dff..9e9d703e8 100644 --- a/api/app/api/v1/onboarding.py +++ b/api/app/api/v1/onboarding.py @@ -8,6 +8,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.api.deps import CurrentSuperuser, CurrentUser from app.core.database import get_session from app.services.instance_settings import InstanceSettingsService +from app.services.scan_engine import ScanEngineService from shared.models.api_key import APIKey from shared.models.notification_channel import NotificationChannel from shared.models.project import Project, ProjectCreate, ProjectRead @@ -128,6 +129,7 @@ async def create_first_project( created_by=current_user.id, ) await add_with_unique_slug(session, project, data.name) + await ScanEngineService(session).ensure_builtin(project.id, current_user.id) await session.commit() await session.refresh(project) return project diff --git a/api/app/api/v1/projects.py b/api/app/api/v1/projects.py index 3a5dd311a..f5bd7a465 100644 --- a/api/app/api/v1/projects.py +++ b/api/app/api/v1/projects.py @@ -6,6 +6,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.api.deps import CurrentSuperuser, CurrentUser from app.core.database import get_session +from app.services.scan_engine import ScanEngineService from shared.models.organization import Organization from shared.models.project import Project, ProjectCreate, ProjectRead, ProjectSummary from shared.models.tag import Tag @@ -77,6 +78,7 @@ async def create_project( created_by=current_user.id, ) await add_with_unique_slug(session, project, project_in.name) + await ScanEngineService(session).ensure_builtin(project.id, current_user.id) await session.commit() await session.refresh(project) return project diff --git a/api/app/api/v1/scan_engines.py b/api/app/api/v1/scan_engines.py index ee8164464..82d610a8d 100644 --- a/api/app/api/v1/scan_engines.py +++ b/api/app/api/v1/scan_engines.py @@ -49,11 +49,11 @@ async def preview_engine_stages( @router.get("", response_model=list[ScanEngineRead]) async def list_engines( - _current_user: CurrentUser, + current_user: CurrentUser, service: Annotated[ScanEngineService, Depends(get_service)], project_id: Annotated[UUID, Query(description="Project ID")], ): - return await service.list(project_id) + return await service.list(project_id, created_by=current_user.id) @router.post("", response_model=ScanEngineRead, status_code=status.HTTP_201_CREATED) diff --git a/api/app/services/scan_engine/catalog.py b/api/app/services/scan_engine/catalog.py index b5f7afeca..0d65e2c90 100644 --- a/api/app/services/scan_engine/catalog.py +++ b/api/app/services/scan_engine/catalog.py @@ -1,6 +1,6 @@ from shared.definitions.launch import STAGE_GROUP_LABELS, seed_produces from shared.definitions.tools import SCAN_TOOLS -from shared.enums.scan import StageGroup +from shared.enums.scan import INTENSITIES, StageGroup from shared.enums.target import TargetType from shared.models.scan_engine import ( EngineCatalog, @@ -8,8 +8,10 @@ from shared.models.scan_engine import ( StageCatalogEntry, StageField, StageGroupEntry, + StageTransport, ToolOption, ) +from stages.config import FieldTier from stages.presets import PRESETS, preset_stages from stages.registry import phases, rate_tools, stages @@ -30,6 +32,7 @@ def _field_specs(spec) -> list[StageField]: defs = schema.get("$defs") or {} defaults = spec.defaults launch = set(spec.launch_fields) + tiers = spec.config_model.tiers() out: list[StageField] = [] for name, raw in (schema.get("properties") or {}).items(): prop = _resolve(raw, defs) @@ -45,7 +48,7 @@ def _field_specs(spec) -> list[StageField]: option_labels=prop.get("option_labels") or None, minimum=prop.get("minimum"), maximum=prop.get("maximum"), - scale=prop.get("scale"), + tier=tiers.get(name, FieldTier.BASIC.value), widget=prop.get("widget"), kind=prop.get("kind"), launch=name in launch, @@ -54,6 +57,18 @@ def _field_specs(spec) -> list[StageField]: return out +def _transport(spec) -> StageTransport | None: + if spec.transport_tool is None: + return None + by_intensity = {level: spec.transport(level) for level in INTENSITIES} + return StageTransport( + tool=spec.transport_tool, + rates={level: t.rate for level, t in by_intensity.items()}, + threads={level: t.threads for level, t in by_intensity.items()}, + timeout=by_intensity[INTENSITIES[0]].timeout, + ) + + def build_catalog() -> EngineCatalog: return EngineCatalog( phases=[phase for phase, _ in phases()], @@ -75,6 +90,7 @@ def build_catalog() -> EngineCatalog: role=spec.role, consumes=sorted(spec.consumes), produces=sorted(spec.produces), + transport=_transport(spec), defaults=spec.defaults, fields=_field_specs(spec), ) diff --git a/api/app/services/scan_engine/effects.py b/api/app/services/scan_engine/effects.py index 2a12b91d4..71f8a45b9 100644 --- a/api/app/services/scan_engine/effects.py +++ b/api/app/services/scan_engine/effects.py @@ -13,7 +13,6 @@ from shared.models.scan_context import ScanContext from shared.models.scan_engine import EnginePreviewResult, PreviewResolved from shared.models.scan_preview import PreviewPhase, PreviewTool, PreviewToolStatus from shared.services.scan_resolve import merge_engine_context -from stages.config import Scale from stages.registry import ordered_levels, phases @@ -88,18 +87,14 @@ def _stage_status( f"{spec.title}: no API key for {', '.join(missing)}. Reduced coverage." ) - scaled = spec.config_model.scaled_fields() - - def _scaled(kind: Scale): - return next((values[f] for f, (s, _) in scaled.items() if s is kind), None) - + transport = (resolved.transports or {}).get(spec.name) or {} return PreviewTool( capability=spec.name, label=spec.title, status=PreviewToolStatus.WILL_RUN, - rate=_scaled(Scale.RATE), - threads=_scaled(Scale.THREADS), - timeout=_scaled(Scale.TIMEOUT), + rate=transport.get("rate"), + threads=transport.get("threads"), + timeout=transport.get("timeout"), ) diff --git a/api/app/services/scan_engine/service.py b/api/app/services/scan_engine/service.py index 666e822f4..db9bb5149 100644 --- a/api/app/services/scan_engine/service.py +++ b/api/app/services/scan_engine/service.py @@ -3,6 +3,7 @@ from uuid import UUID import yaml from fastapi import HTTPException, status from sqlalchemy import func, select +from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession from app.services.scan_engine.validation import ( @@ -18,6 +19,12 @@ from app.services.scan_engine.validation import ( _validate_tool_options, _validate_yaml_source, ) +from shared.definitions.default_engine import ( + DEFAULT_ENGINE_DESCRIPTION, + DEFAULT_ENGINE_INTENSITY, + DEFAULT_ENGINE_NAME, + default_engine_stages, +) from shared.enums.scan import SCAN_OPEN_STATUSES from shared.models.scan import Scan from shared.models.scan_engine import ( @@ -59,6 +66,7 @@ def _to_read(engine: ScanEngine, usage: EngineUsage | None = None) -> ScanEngine stages=dict(engine.stages or {}), yaml_source=engine.yaml_source, tool_options=_mask_tool_options(engine.tool_options), + builtin=bool(engine.builtin), created_at=engine.created_at, updated_at=engine.updated_at, last_used_at=engine.last_used_at, @@ -121,7 +129,45 @@ class ScanEngineService: await self.session.refresh(engine) return _to_read(engine) - async def list(self, project_id: UUID) -> list[ScanEngineRead]: + async def ensure_builtin(self, project_id: UUID, created_by: UUID) -> ScanEngine: + """The project's built-in engine, created on first sight.""" + existing = await self._builtin_for(project_id) + if existing is not None: + return existing + engine = ScanEngine( + project_id=project_id, + created_by=created_by, + name=DEFAULT_ENGINE_NAME, + description=DEFAULT_ENGINE_DESCRIPTION, + intensity=DEFAULT_ENGINE_INTENSITY, + stages=_validate_stages(default_engine_stages()), + builtin=True, + ) + try: + async with self.session.begin_nested(): + self.session.add(engine) + await self.session.flush() + except IntegrityError: + raced = await self._builtin_for(project_id) + if raced is None: + raise + return raced + return engine + + async def _builtin_for(self, project_id: UUID) -> ScanEngine | None: + result = await self.session.execute( + select(ScanEngine).where( + ScanEngine.project_id == project_id, ScanEngine.builtin.is_(True) + ) + ) + return result.scalar_one_or_none() + + async def list( + self, project_id: UUID, created_by: UUID | None = None + ) -> list[ScanEngineRead]: + if created_by is not None: + await self.ensure_builtin(project_id, created_by) + await self.session.commit() result = await self.session.execute( select(ScanEngine) .where(ScanEngine.project_id == project_id) @@ -173,6 +219,11 @@ class ScanEngineService: async def delete(self, id: UUID, project_id: UUID) -> bool: engine = await self._get_or_404(id, project_id) + if engine.builtin: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=f"{engine.name} is built in and is not deleted.", + ) running = await _running_scans_for(self.session, engine.id) if running: raise HTTPException( diff --git a/frontend/src/lib/components/engines/engine-list-card.svelte b/frontend/src/lib/components/engines/engine-list-card.svelte index 1fcf53bea..5deee9936 100644 --- a/frontend/src/lib/components/engines/engine-list-card.svelte +++ b/frontend/src/lib/components/engines/engine-list-card.svelte @@ -73,7 +73,7 @@ data-selected={isSelected} >
- {#if onSelect} + {#if onSelect && !engine.builtin}
{INTENSITY_LABELS[engine.intensity] ?? engine.intensity} + {#if engine.builtin} + Built in + {/if}
{#if engine.description}

{engine.description}

@@ -128,11 +131,13 @@ Export YAML - - onDelete?.()}> - - Delete - + {#if !engine.builtin} + + onDelete?.()}> + + Delete + + {/if}
diff --git a/frontend/src/lib/components/engines/engine-topbar.svelte b/frontend/src/lib/components/engines/engine-topbar.svelte index a2889ea20..0a82c5ff7 100644 --- a/frontend/src/lib/components/engines/engine-topbar.svelte +++ b/frontend/src/lib/components/engines/engine-topbar.svelte @@ -161,9 +161,16 @@ Duplicate - onDelete?.()}> + onDelete?.()} + > Delete + {#if engine.builtin} + Built in + {/if} diff --git a/frontend/src/lib/components/engines/stage-field.svelte b/frontend/src/lib/components/engines/stage-field.svelte index 5f83eda35..4c3ec173e 100644 --- a/frontend/src/lib/components/engines/stage-field.svelte +++ b/frontend/src/lib/components/engines/stage-field.svelte @@ -8,7 +8,6 @@ import { Button } from '$lib/components/ui/button'; import RotateCcw from '@lucide/svelte/icons/rotate-ccw'; import type { StageField } from '$lib/types/scan-engine'; - import { SCALE_HELP } from '$lib/types/scan-engine'; import { parseCsv } from '$lib/utilities/parse'; import CustomTemplatesField from './custom-templates-field.svelte'; import WordlistField from './wordlist-field.svelte'; @@ -72,11 +71,6 @@ {#if field.description} {field.description} {/if} - {#if field.scale} - - {SCALE_HELP[field.scale]} - - {/if}
diff --git a/frontend/src/lib/components/engines/stage-row.svelte b/frontend/src/lib/components/engines/stage-row.svelte index c60597537..468d69b51 100644 --- a/frontend/src/lib/components/engines/stage-row.svelte +++ b/frontend/src/lib/components/engines/stage-row.svelte @@ -1,6 +1,7 @@ {routeLabels.engines} · reNgine @@ -471,7 +472,7 @@