Files
rengine/shared/services/orchestrator/tracking.py
T
Yogesh OjhaandClaude Opus 5 e821bd330d fix(scans): one NUL byte in a response no longer costs a 66-minute stage
A gov.np HTTP probe ran 66 minutes over 25,916 host and port pairs, then lost every
row: a captured response body carried a NUL byte, postgres refuses NUL in text and in
json alike, and the failing flush rolled back the whole stage transaction. Recording
that failure then failed too — psycopg puts the offending parameters in its error
message, so the NUL travelled into `scan_activities.error` and took the failure path
down with it, leaving the activity RUNNING and the scan unable to finalize.

`shared/utils/text.py` is now the one place that names what postgres will not store.
`strip_nul` runs through every string, list and dict a tool record yields, applied in
`tools/runner/fieldmap.py` so every tool that maps a record is covered rather than
httpx alone. `strip_control` replaces the two identical local regexes in the nuclei
parser and the template indexer, and now also guards the activity log, the activity
timeline and the command recorder — the three places that write a tool's own words.

The HTTP probe's batch flush runs in a savepoint: a row postgres refuses costs that
row, retried one at a time, and the count comes back as a PARTIAL stage naming how
many responses could not be stored. Recording a stage failure is wrapped too, so a
second error while writing the first can never leave a scan running forever.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01DPueZXQMbjLXMokXeA8LFP
2026-09-06 03:59:33 +05:30

173 lines
5.7 KiB
Python

import logging
import uuid
from collections.abc import Callable
from sqlalchemy import update
from sqlalchemy.orm import Session
from shared.definitions.constants import MAX_COMMAND_OUTPUT
from shared.enums.scan import ScanActivityStatus
from shared.models.scan import Scan
from shared.models.scan_activity import ScanActivity
from shared.models.scan_command import ScanCommand
from shared.services.orchestrator.events import ScanEventPublisher
from shared.services.scan_resolve import redact_command
from shared.utils.datetime import utc_now
from shared.utils.text import strip_control
logger = logging.getLogger(__name__)
class ScanActivityService:
"""Per-stage activity timeline (create RUNNING, transition to terminal)."""
def __init__(self, session: Session) -> None:
self.session = session
def create(
self,
scan: Scan,
*,
name: str,
title: str,
celery_task_id: str | None = None,
) -> ScanActivity:
activity = ScanActivity(
scan_id=scan.id,
project_id=scan.project_id,
name=name,
title=title,
status=ScanActivityStatus.RUNNING.value,
celery_task_id=celery_task_id,
started_at=utc_now(),
)
self.session.add(activity)
self.session.commit()
self.session.refresh(activity)
return activity
def finish(
self,
activity: ScanActivity,
*,
status: ScanActivityStatus,
result: dict | None = None,
error: str | None = None,
traceback: str | None = None,
) -> None:
# Conditional update: only transition from RUNNING, so a concurrent cancel
# that flipped this row to ABORTED is never clobbered back to success/failed.
self.session.execute(
update(ScanActivity)
.where(
ScanActivity.id == activity.id,
ScanActivity.status == ScanActivityStatus.RUNNING.value,
)
.values(
status=status.value,
result=result or {},
error=strip_control(redact_command(error))[:2000] if error else None,
traceback=strip_control(redact_command(traceback))
if traceback
else None,
completed_at=utc_now(),
)
)
self.session.commit()
class ScanCommandRecorder:
"""Registers every command + captures its log; thread-safe (fresh session per write)."""
def __init__(
self,
*,
session_factory: Callable[[], Session],
scan_id: uuid.UUID,
project_id: uuid.UUID,
activity_id: uuid.UUID | None = None,
events: ScanEventPublisher | None = None,
) -> None:
self._session_factory = session_factory
self._scan_id = scan_id
self._project_id = project_id
self._activity_id = activity_id
self._events = events
def start(self, tool: str, command: str) -> uuid.UUID:
cmd = ScanCommand(
scan_id=self._scan_id,
project_id=self._project_id,
activity_id=self._activity_id,
tool=tool,
command=redact_command(command),
status=ScanActivityStatus.RUNNING.value,
started_at=utc_now(),
)
with self._session_factory() as session:
session.add(cmd)
session.commit()
session.refresh(cmd)
command_id = cmd.id
stored_command = cmd.command
if self._events is not None:
try:
self._events.command_started(
command_id=command_id,
activity_id=self._activity_id,
tool=tool,
command=stored_command,
)
except Exception:
logger.debug("command_started event emit failed", exc_info=True)
return command_id
def finish(
self,
command_id: uuid.UUID,
*,
return_code: int,
output: str,
error: str | None,
duration_seconds: float,
) -> None:
status = (
ScanActivityStatus.SUCCESS
if return_code == 0
else ScanActivityStatus.FAILED
)
try:
with self._session_factory() as session:
cmd = session.get(ScanCommand, command_id)
# skip if a cancel already flipped this command to a terminal status
if cmd is None or cmd.status != ScanActivityStatus.RUNNING.value:
return
cmd.status = status.value
cmd.return_code = return_code
cmd.output = strip_control(redact_command(output or ""))[
:MAX_COMMAND_OUTPUT
]
cmd.error = (
strip_control(redact_command(error))[:2000] if error else None
)
cmd.duration_seconds = duration_seconds
cmd.completed_at = utc_now()
session.add(cmd)
session.commit()
tool = cmd.tool
except Exception:
logger.warning("scan command finish failed", exc_info=True)
return
if self._events is not None:
try:
self._events.command_finished(
command_id=command_id,
activity_id=self._activity_id,
tool=tool,
return_code=return_code,
status=status.value,
duration_seconds=duration_seconds,
)
except Exception:
logger.debug("command_finished event emit failed", exc_info=True)