diff --git a/alembic/versions/2026_09_14_builtin_engine.py b/alembic/versions/2026_09_14_builtin_engine.py new file mode 100644 index 000000000..c113e5ed7 --- /dev/null +++ b/alembic/versions/2026_09_14_builtin_engine.py @@ -0,0 +1,37 @@ +"""One built-in engine per project. + +Revision ID: c9e4f2a71b58 +Revises: b7d3e1f04a26 +Create Date: 2026-09-14 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +revision: str = "c9e4f2a71b58" +down_revision: str | None = "b7d3e1f04a26" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "scan_engines", + sa.Column( + "builtin", sa.Boolean(), nullable=False, server_default=sa.false() + ), + ) + op.create_index( + "ux_scan_engines_builtin_project", + "scan_engines", + ["project_id"], + unique=True, + postgresql_where=sa.text("builtin"), + ) + + +def downgrade() -> None: + op.drop_index("ux_scan_engines_builtin_project", table_name="scan_engines") + op.drop_column("scan_engines", "builtin") diff --git a/alembic/versions/2026_09_14_engine_knobs.py b/alembic/versions/2026_09_14_engine_knobs.py new file mode 100644 index 000000000..8f60d8cdc --- /dev/null +++ b/alembic/versions/2026_09_14_engine_knobs.py @@ -0,0 +1,160 @@ +"""Drop the transport and constant fields from stored engines. + +Revision ID: b7d3e1f04a26 +Revises: a4c2e7d91f53 +Create Date: 2026-09-14 +""" + +from __future__ import annotations + +import json + +import sqlalchemy as sa +from alembic import op + +revision: str = "b7d3e1f04a26" +down_revision: str | None = "a4c2e7d91f53" +branch_labels = None +depends_on = None + +_REMOVED: dict[str, set[str]] = { + "content_discovery": {"threads", "rate", "timeout"}, + "dast_scan": { + "max_dirs_per_origin", + "dir_depth", + "fuzz_param_frequency", + "rate", + "threads", + "bulk_size", + "timeout", + "retries", + "max_host_error", + "waf_rate_divisor", + "honeypot_threshold", + "interactsh_server", + "oast_wait_minutes", + "store_evidence", + "max_evidence_recovery", + "max_findings", + }, + "endpoint_probe": {"threads", "timeout", "rate", "follow_redirects"}, + "host_discovery": {"rate", "threads", "timeout"}, + "http_probe": { + "threads", + "timeout", + "rate", + "follow_redirects", + "probe_open_ports", + "max_ports_per_host", + }, + "netblock_sweep": { + "min_addresses", + "max_asn_addresses", + "min_share", + "dns_threads", + "dns_timeout", + }, + "origin_probe": {"threads", "timeout", "rate", "max_ports_per_address"}, + "passive_ports": {"max_addresses"}, + "port_scan": {"rate", "threads", "timeout", "retries", "port_threshold"}, + "reverse_dns": {"dns_threads", "dns_timeout"}, + "screenshot": {"threads", "rate", "timeout"}, + "seed_resolution": {"cidr_skip_rfc1918"}, + "service_fingerprint": {"threads", "timeout", "include_unknown"}, + "session_check": {"timeout"}, + "subdomain_discovery": { + "permutation_seeds", + "permutation_limit", + "dns_threads", + "dns_batch_size", + "dns_batch_concurrency", + "dns_idle_timeout", + "dns_retry_silent", + }, + "target_enrichment": {"dns_threads", "dns_timeout"}, + "url_discovery": { + "threads", + "timeout", + "rate", + "crawl_javascript", + "max_urls", + "max_known_file_hosts", + "max_source_maps", + "max_archive_domains", + }, + "vhost": {"threads", "rate"}, + "vulnerability_scan": { + "rate", + "threads", + "bulk_size", + "timeout", + "retries", + "max_host_error", + "waf_rate_divisor", + "honeypot_threshold", + "oast_wait_minutes", + "max_evidence_recovery", + "max_findings", + }, +} + + +def _prune(stages: dict) -> tuple[dict, bool]: + changed = False + out: dict = {} + for name, config in (stages or {}).items(): + if not isinstance(config, dict): + out[name] = config + continue + gone = _REMOVED.get(name, set()) + kept = {k: v for k, v in config.items() if k not in gone} + if len(kept) != len(config): + changed = True + if kept: + out[name] = kept + elif config: + changed = True + return out, changed + + +def _yaml_mentions_removed(source: str | None) -> bool: + if not source: + return False + try: + import yaml # noqa: PLC0415 + + doc = yaml.safe_load(source) or {} + except Exception: # noqa: BLE001 + return False + stages = doc.get("stages") if isinstance(doc, dict) else None + if not isinstance(stages, dict): + return False + return any( + isinstance(cfg, dict) and set(cfg) & _REMOVED.get(name, set()) + for name, cfg in stages.items() + ) + + +def upgrade() -> None: + bind = op.get_bind() + rows = bind.execute( + sa.text("SELECT id, stages, yaml_source FROM scan_engines") + ).all() + for id_, stages, yaml_source in rows: + raw = stages if isinstance(stages, dict) else json.loads(stages or "{}") + pruned, changed = _prune(raw) + drop_yaml = _yaml_mentions_removed(yaml_source) + if not changed and not drop_yaml: + continue + bind.execute( + sa.text( + "UPDATE scan_engines SET stages = :stages, " + "yaml_source = CASE WHEN :drop THEN NULL ELSE yaml_source END " + "WHERE id = :id" + ), + {"stages": json.dumps(pruned), "drop": drop_yaml, "id": id_}, + ) + + +def downgrade() -> None: + pass diff --git a/api/app/api/v1/onboarding.py b/api/app/api/v1/onboarding.py index 1e05d0dff..9e9d703e8 100644 --- a/api/app/api/v1/onboarding.py +++ b/api/app/api/v1/onboarding.py @@ -8,6 +8,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.api.deps import CurrentSuperuser, CurrentUser from app.core.database import get_session from app.services.instance_settings import InstanceSettingsService +from app.services.scan_engine import ScanEngineService from shared.models.api_key import APIKey from shared.models.notification_channel import NotificationChannel from shared.models.project import Project, ProjectCreate, ProjectRead @@ -128,6 +129,7 @@ async def create_first_project( created_by=current_user.id, ) await add_with_unique_slug(session, project, data.name) + await ScanEngineService(session).ensure_builtin(project.id, current_user.id) await session.commit() await session.refresh(project) return project diff --git a/api/app/api/v1/projects.py b/api/app/api/v1/projects.py index 3a5dd311a..f5bd7a465 100644 --- a/api/app/api/v1/projects.py +++ b/api/app/api/v1/projects.py @@ -6,6 +6,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.api.deps import CurrentSuperuser, CurrentUser from app.core.database import get_session +from app.services.scan_engine import ScanEngineService from shared.models.organization import Organization from shared.models.project import Project, ProjectCreate, ProjectRead, ProjectSummary from shared.models.tag import Tag @@ -77,6 +78,7 @@ async def create_project( created_by=current_user.id, ) await add_with_unique_slug(session, project, project_in.name) + await ScanEngineService(session).ensure_builtin(project.id, current_user.id) await session.commit() await session.refresh(project) return project diff --git a/api/app/api/v1/scan_engines.py b/api/app/api/v1/scan_engines.py index ee8164464..82d610a8d 100644 --- a/api/app/api/v1/scan_engines.py +++ b/api/app/api/v1/scan_engines.py @@ -49,11 +49,11 @@ async def preview_engine_stages( @router.get("", response_model=list[ScanEngineRead]) async def list_engines( - _current_user: CurrentUser, + current_user: CurrentUser, service: Annotated[ScanEngineService, Depends(get_service)], project_id: Annotated[UUID, Query(description="Project ID")], ): - return await service.list(project_id) + return await service.list(project_id, created_by=current_user.id) @router.post("", response_model=ScanEngineRead, status_code=status.HTTP_201_CREATED) diff --git a/api/app/services/scan_engine/catalog.py b/api/app/services/scan_engine/catalog.py index b5f7afeca..0d65e2c90 100644 --- a/api/app/services/scan_engine/catalog.py +++ b/api/app/services/scan_engine/catalog.py @@ -1,6 +1,6 @@ from shared.definitions.launch import STAGE_GROUP_LABELS, seed_produces from shared.definitions.tools import SCAN_TOOLS -from shared.enums.scan import StageGroup +from shared.enums.scan import INTENSITIES, StageGroup from shared.enums.target import TargetType from shared.models.scan_engine import ( EngineCatalog, @@ -8,8 +8,10 @@ from shared.models.scan_engine import ( StageCatalogEntry, StageField, StageGroupEntry, + StageTransport, ToolOption, ) +from stages.config import FieldTier from stages.presets import PRESETS, preset_stages from stages.registry import phases, rate_tools, stages @@ -30,6 +32,7 @@ def _field_specs(spec) -> list[StageField]: defs = schema.get("$defs") or {} defaults = spec.defaults launch = set(spec.launch_fields) + tiers = spec.config_model.tiers() out: list[StageField] = [] for name, raw in (schema.get("properties") or {}).items(): prop = _resolve(raw, defs) @@ -45,7 +48,7 @@ def _field_specs(spec) -> list[StageField]: option_labels=prop.get("option_labels") or None, minimum=prop.get("minimum"), maximum=prop.get("maximum"), - scale=prop.get("scale"), + tier=tiers.get(name, FieldTier.BASIC.value), widget=prop.get("widget"), kind=prop.get("kind"), launch=name in launch, @@ -54,6 +57,18 @@ def _field_specs(spec) -> list[StageField]: return out +def _transport(spec) -> StageTransport | None: + if spec.transport_tool is None: + return None + by_intensity = {level: spec.transport(level) for level in INTENSITIES} + return StageTransport( + tool=spec.transport_tool, + rates={level: t.rate for level, t in by_intensity.items()}, + threads={level: t.threads for level, t in by_intensity.items()}, + timeout=by_intensity[INTENSITIES[0]].timeout, + ) + + def build_catalog() -> EngineCatalog: return EngineCatalog( phases=[phase for phase, _ in phases()], @@ -75,6 +90,7 @@ def build_catalog() -> EngineCatalog: role=spec.role, consumes=sorted(spec.consumes), produces=sorted(spec.produces), + transport=_transport(spec), defaults=spec.defaults, fields=_field_specs(spec), ) diff --git a/api/app/services/scan_engine/effects.py b/api/app/services/scan_engine/effects.py index 2a12b91d4..71f8a45b9 100644 --- a/api/app/services/scan_engine/effects.py +++ b/api/app/services/scan_engine/effects.py @@ -13,7 +13,6 @@ from shared.models.scan_context import ScanContext from shared.models.scan_engine import EnginePreviewResult, PreviewResolved from shared.models.scan_preview import PreviewPhase, PreviewTool, PreviewToolStatus from shared.services.scan_resolve import merge_engine_context -from stages.config import Scale from stages.registry import ordered_levels, phases @@ -88,18 +87,14 @@ def _stage_status( f"{spec.title}: no API key for {', '.join(missing)}. Reduced coverage." ) - scaled = spec.config_model.scaled_fields() - - def _scaled(kind: Scale): - return next((values[f] for f, (s, _) in scaled.items() if s is kind), None) - + transport = (resolved.transports or {}).get(spec.name) or {} return PreviewTool( capability=spec.name, label=spec.title, status=PreviewToolStatus.WILL_RUN, - rate=_scaled(Scale.RATE), - threads=_scaled(Scale.THREADS), - timeout=_scaled(Scale.TIMEOUT), + rate=transport.get("rate"), + threads=transport.get("threads"), + timeout=transport.get("timeout"), ) diff --git a/api/app/services/scan_engine/service.py b/api/app/services/scan_engine/service.py index 666e822f4..db9bb5149 100644 --- a/api/app/services/scan_engine/service.py +++ b/api/app/services/scan_engine/service.py @@ -3,6 +3,7 @@ from uuid import UUID import yaml from fastapi import HTTPException, status from sqlalchemy import func, select +from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession from app.services.scan_engine.validation import ( @@ -18,6 +19,12 @@ from app.services.scan_engine.validation import ( _validate_tool_options, _validate_yaml_source, ) +from shared.definitions.default_engine import ( + DEFAULT_ENGINE_DESCRIPTION, + DEFAULT_ENGINE_INTENSITY, + DEFAULT_ENGINE_NAME, + default_engine_stages, +) from shared.enums.scan import SCAN_OPEN_STATUSES from shared.models.scan import Scan from shared.models.scan_engine import ( @@ -59,6 +66,7 @@ def _to_read(engine: ScanEngine, usage: EngineUsage | None = None) -> ScanEngine stages=dict(engine.stages or {}), yaml_source=engine.yaml_source, tool_options=_mask_tool_options(engine.tool_options), + builtin=bool(engine.builtin), created_at=engine.created_at, updated_at=engine.updated_at, last_used_at=engine.last_used_at, @@ -121,7 +129,45 @@ class ScanEngineService: await self.session.refresh(engine) return _to_read(engine) - async def list(self, project_id: UUID) -> list[ScanEngineRead]: + async def ensure_builtin(self, project_id: UUID, created_by: UUID) -> ScanEngine: + """The project's built-in engine, created on first sight.""" + existing = await self._builtin_for(project_id) + if existing is not None: + return existing + engine = ScanEngine( + project_id=project_id, + created_by=created_by, + name=DEFAULT_ENGINE_NAME, + description=DEFAULT_ENGINE_DESCRIPTION, + intensity=DEFAULT_ENGINE_INTENSITY, + stages=_validate_stages(default_engine_stages()), + builtin=True, + ) + try: + async with self.session.begin_nested(): + self.session.add(engine) + await self.session.flush() + except IntegrityError: + raced = await self._builtin_for(project_id) + if raced is None: + raise + return raced + return engine + + async def _builtin_for(self, project_id: UUID) -> ScanEngine | None: + result = await self.session.execute( + select(ScanEngine).where( + ScanEngine.project_id == project_id, ScanEngine.builtin.is_(True) + ) + ) + return result.scalar_one_or_none() + + async def list( + self, project_id: UUID, created_by: UUID | None = None + ) -> list[ScanEngineRead]: + if created_by is not None: + await self.ensure_builtin(project_id, created_by) + await self.session.commit() result = await self.session.execute( select(ScanEngine) .where(ScanEngine.project_id == project_id) @@ -173,6 +219,11 @@ class ScanEngineService: async def delete(self, id: UUID, project_id: UUID) -> bool: engine = await self._get_or_404(id, project_id) + if engine.builtin: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=f"{engine.name} is built in and is not deleted.", + ) running = await _running_scans_for(self.session, engine.id) if running: raise HTTPException( diff --git a/frontend/src/lib/components/engines/engine-list-card.svelte b/frontend/src/lib/components/engines/engine-list-card.svelte index 1fcf53bea..5deee9936 100644 --- a/frontend/src/lib/components/engines/engine-list-card.svelte +++ b/frontend/src/lib/components/engines/engine-list-card.svelte @@ -73,7 +73,7 @@ data-selected={isSelected} >
- {#if onSelect} + {#if onSelect && !engine.builtin}
{INTENSITY_LABELS[engine.intensity] ?? engine.intensity} + {#if engine.builtin} + Built in + {/if}
{#if engine.description}

{engine.description}

@@ -128,11 +131,13 @@ Export YAML - - onDelete?.()}> - - Delete - + {#if !engine.builtin} + + onDelete?.()}> + + Delete + + {/if}
diff --git a/frontend/src/lib/components/engines/engine-topbar.svelte b/frontend/src/lib/components/engines/engine-topbar.svelte index a2889ea20..0a82c5ff7 100644 --- a/frontend/src/lib/components/engines/engine-topbar.svelte +++ b/frontend/src/lib/components/engines/engine-topbar.svelte @@ -161,9 +161,16 @@ Duplicate - onDelete?.()}> + onDelete?.()} + > Delete + {#if engine.builtin} + Built in + {/if} diff --git a/frontend/src/lib/components/engines/stage-field.svelte b/frontend/src/lib/components/engines/stage-field.svelte index 5f83eda35..4c3ec173e 100644 --- a/frontend/src/lib/components/engines/stage-field.svelte +++ b/frontend/src/lib/components/engines/stage-field.svelte @@ -8,7 +8,6 @@ import { Button } from '$lib/components/ui/button'; import RotateCcw from '@lucide/svelte/icons/rotate-ccw'; import type { StageField } from '$lib/types/scan-engine'; - import { SCALE_HELP } from '$lib/types/scan-engine'; import { parseCsv } from '$lib/utilities/parse'; import CustomTemplatesField from './custom-templates-field.svelte'; import WordlistField from './wordlist-field.svelte'; @@ -72,11 +71,6 @@ {#if field.description} {field.description} {/if} - {#if field.scale} - - {SCALE_HELP[field.scale]} - - {/if}
diff --git a/frontend/src/lib/components/engines/stage-row.svelte b/frontend/src/lib/components/engines/stage-row.svelte index c60597537..468d69b51 100644 --- a/frontend/src/lib/components/engines/stage-row.svelte +++ b/frontend/src/lib/components/engines/stage-row.svelte @@ -1,6 +1,7 @@ {routeLabels.engines} · reNgine @@ -471,7 +472,7 @@