From 0d2ab5a0b3e012ea9e4169d3e8a56c12fba4c564 Mon Sep 17 00:00:00 2001 From: Matt <36310667+NotoriousRebel@users.noreply.github.com> Date: Mon, 10 Aug 2026 00:08:05 -0400 Subject: [PATCH] Add the HarvestView web application (#2512) * feat: add HarvestView operator UI * Use JSONL-only HarvestView file flows * Drive HarvestView activities from the API catalog * Add result action controls * Render shared DNS resolver defaults * Clarify resolver cardinality in HarvestView * Load Tabulator from CDNjs * Fix HarvestView wiki link * Align HarvestView with execution status contract * Add HarvestView import and action controls * Harden HarvestView browser assertions * Simplify HarvestView run selection * Show truthful HarvestView execution outcomes * refactor: show canonical URL results in HarvestView * Show canonical hostname results in HarvestView * test(harvestview): remove browser error race --- .github/workflows/harvestview-e2e.yml | 62 ++ .gitignore | 36 +- DESIGN.md | 57 + PRODUCT.md | 33 + README.md | 21 +- docs/wiki/Configuration-and-API-Keys.md | 2 +- docs/wiki/Installation.md | 18 + docs/wiki/Responsible-Use-and-Scope.md | 2 +- docs/wiki/Rest-API.md | 7 +- docs/wiki/Troubleshooting.md | 2 +- pyproject.toml | 4 +- tests/e2e/conftest.py | 179 ++++ tests/e2e/test_harvestview.py | 961 +++++++++++++++++ tests/lib/test_api_v1.py | 1 - tests/lib/test_harvestview_ui.py | 135 +++ tests/test_workflow_contract.py | 13 + theHarvester/lib/api/api.py | 2 + theHarvester/lib/api/auth.py | 23 +- theHarvester/lib/api/harvestview.py | 62 ++ .../lib/api/static/harvestview/app.css | 773 ++++++++++++++ .../lib/api/static/harvestview/app.js | 990 ++++++++++++++++++ .../lib/api/static/harvestview/index.html | 315 ++++++ .../lib/api/static/harvestview/logo.webp | Bin 0 -> 7774 bytes uv.lock | 51 + 24 files changed, 3736 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/harvestview-e2e.yml create mode 100644 DESIGN.md create mode 100644 PRODUCT.md create mode 100644 tests/e2e/conftest.py create mode 100644 tests/e2e/test_harvestview.py create mode 100644 tests/lib/test_harvestview_ui.py create mode 100644 theHarvester/lib/api/harvestview.py create mode 100644 theHarvester/lib/api/static/harvestview/app.css create mode 100644 theHarvester/lib/api/static/harvestview/app.js create mode 100644 theHarvester/lib/api/static/harvestview/index.html create mode 100644 theHarvester/lib/api/static/harvestview/logo.webp diff --git a/.github/workflows/harvestview-e2e.yml b/.github/workflows/harvestview-e2e.yml new file mode 100644 index 00000000..c349dc21 --- /dev/null +++ b/.github/workflows/harvestview-e2e.yml @@ -0,0 +1,62 @@ +name: HarvestView browser E2E + +on: + workflow_dispatch: + pull_request: + paths: + - '.github/workflows/harvestview-e2e.yml' + - 'pyproject.toml' + - 'uv.lock' + - 'theHarvester/lib/api/**' + - 'tests/e2e/**' + - 'tests/lib/test_api_v1.py' + - 'tests/lib/test_harvestview_ui.py' + +permissions: + contents: read + +jobs: + harvestview-e2e: + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - name: Harden the runner + uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 + with: + egress-policy: audit + + - name: Check out repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Install uv and Python + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + python-version: '3.13' + enable-cache: true + cache-dependency-glob: uv.lock + + - name: Install project dependencies + run: uv sync --all-groups --frozen + + - name: Install Chromium + run: uv run playwright install --with-deps chromium + + - name: Run HarvestView browser tests + run: >- + uv run pytest -m harvestview_e2e + --browser chromium + --tracing=retain-on-failure + --screenshot=only-on-failure + --output=test-results + + - name: Upload browser test artifacts + if: ${{ failure() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: harvestview-browser-test-results + path: test-results/ + if-no-files-found: warn + retention-days: 7 diff --git a/.gitignore b/.gitignore index 5158b7a3..84942119 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ *.pyc *.sqlite *.html +!theHarvester/lib/api/static/harvestview/index.html *.htm *.vscode *.xml @@ -18,4 +19,37 @@ api-keys.yaml .venv .venv/** .pyre -.junie \ No newline at end of file +.junie +.env +.env.* +.secrets/ +test-results/ + +# impeccable-ignore-start +# Ephemeral output, runtime state, and per-dev overrides. +# Unanchored: .impeccable may sit at the repo root or under a nested +# workspace (apps/web/.impeccable/...); anchored patterns would miss it. +# Shared artifacts stay tracked: config.json, live/config.json, +# design.json, critique/*.md. +**/.impeccable/config.local.json +**/.impeccable/hook.cache.json +**/.impeccable/hook.pending.json +**/.impeccable/*.png +**/.impeccable/live/server.json +**/.impeccable/live/sessions/ +**/.impeccable/live/previews/ +**/.impeccable/live/annotations/ +**/.impeccable/live/cache/ +**/.impeccable/live/manual-edit-apply-transaction.json +**/.impeccable/live/manual-edit-events.jsonl +**/.impeccable/live/manual-edit-evidence/ +**/.impeccable/live/pending-manual-edits.json +**/.impeccable/live/deferred-svelte-component-accepts.json +**/.impeccable/live/*.png +# impeccable-ignore-end + +# Shared Impeccable artifacts override the repository-wide *.json rule. +!**/.impeccable/config.json +!**/.impeccable/live/config.json +!**/.impeccable/design.json +!**/.impeccable/critique/*.md diff --git a/DESIGN.md b/DESIGN.md new file mode 100644 index 00000000..9b53dd41 --- /dev/null +++ b/DESIGN.md @@ -0,0 +1,57 @@ +# HarvestView design system + +## Direction + +Use the approved “Run desk” direction: a dark mineral navigation rail, warm neutral work surface, teal operational accent, amber warnings, and restrained red failures. The interface should feel like a field notebook crossed with a reliable control room-not a generic SaaS card grid. + +## Typography + +Use the local system stack only. Display and headings use a compact humanist sans stack; evidence values, IDs, timestamps, and status metadata use the system monospace stack. Body text stays at 16px on small screens and line length stays below 72 characters where prose appears. + +## Layout + +Desktop uses a fixed app header, a history rail, and one flexible evidence workbench. Result routes use a single table surface rather than nested cards. Tablet collapses secondary metadata. Mobile stacks history above evidence, preserves all actions, and keeps touch targets at least 44px. + +## Color tokens + +Use OKLCH tokens for background, surface, ink, muted text, line, teal accent, amber warning, red danger, and blue information. Light and dark themes must both meet WCAG AA contrast. Status always includes text or an icon as well as color. + +## Interaction + +Use native dialogs, buttons, inputs, details, and file controls. Motion is limited to short opacity/transform transitions for dialogs, notices, and selection; reduced-motion removes transforms and durations. Focus rings are never suppressed. Dynamic status changes use a polite live region. + +## Tables and evidence + +Use the pinned CDNjs Tabulator browser build for sorting, filtering, selection, and pagination. Load only its default table theme, with HarvestView's own stylesheet controlling the visual system. DNS status uses resolved, no-answer, disputed, uncertain, and not-captured labels. Long values wrap or truncate with a title; they never break the viewport. + +## CSS architecture + +HarvestView uses its own `app.css` and native HTML controls. A general UI framework +would not make the interface better by itself; it would replace the existing Run +Desk visual language with framework defaults or require the same custom overrides +again. + +| Option | Benefit | Cost for HarvestView | Decision | +| --- | --- | --- | --- | +| Custom CSS | Keeps the existing visual system, native controls, and zero-build workflow. | HarvestView owns its small reset and component rules. | Use. | +| Bootstrap | Mature components, utilities, and documentation. | No Bootstrap APIs are used; adding them would duplicate 232 KB of styles and make the interface more generic. | Remove. | +| Pico | Small class-light API and sensible semantic defaults. | Its global element styles compete with the existing native-control and theme rules. | Do not add. | +| Bulma | CSS-only component classes. | Requires a markup rewrite and adds a larger stylesheet without improving the evidence workflow. | Do not add. | +| Tailwind | Strong utility workflow and small compiled output when a build step is used. | Requires a markup rewrite and build pipeline; its browser CDN is development-only. | Do not add. | + +Tabulator is the exception because it supplies table behavior HarvestView actually +uses. Its JavaScript and pinned default theme load from CDNjs by default with +Subresource Integrity. Isolated deployments can self-host those exact assets by +following the installation wiki. `app.css` owns the visual treatment on top of +that structural theme. + +Primary references: [Bootstrap 5.3 installation](https://getbootstrap.com/docs/5.3/getting-started/introduction/), +[Tabulator 6.x installation](https://tabulator.info/docs/6.x/install), +[Tabulator 6.x themes](https://tabulator.info/docs/6.x/theme), +[Pico quick start](https://github.com/picocss/pico#quick-start), +[Bulma quick install](https://github.com/jgthms/bulma#quick-install), and +[Tailwind Play CDN guidance](https://tailwindcss.com/docs/installation/play-cdn). + +## Voice + +Use precise operator language: “Start enumeration,” “Request cancellation,” “Import result file,” and “No runs yet.” Errors state what failed and the next action. Avoid scan, session, job, and vague success/error labels where the glossary has a precise term. diff --git a/PRODUCT.md b/PRODUCT.md new file mode 100644 index 00000000..011ae26c --- /dev/null +++ b/PRODUCT.md @@ -0,0 +1,33 @@ +# HarvestView product context + +## Product + +HarvestView is the local web application for running theHarvester and analyzing its results. It turns finite enumeration runs and imported result files into durable, searchable evidence without turning theHarvester into a monitoring service. + +## Operator + +The operator is a technically capable security practitioner working at a desk, often under time pressure. They need to see exactly what was authorized, what ran, what failed, and what evidence remains. + +## Core jobs + +- Launch one explicitly authorized enumeration with P0/P1/P2 boundaries visible before execution. +- Cancel work and know whether cancellation is requested, in progress, or complete. +- Reopen prior runs and compare route-specific evidence without rerunning reconnaissance. +- Import existing theHarvester JSONL evidence. +- Export normalized results and inspect managed screenshots. +- Start a screenshot or DNS brute-force action from a subdomain result without changing the parent evidence. + +## Product principles + +- Local-first and fail-closed. +- Evidence before decoration. +- Passive by default; active work is explicit. +- One finite run at a time. +- Honest state: lifecycle status and evidence completeness are different facts. +- Dense enough for an expert, calm enough for long sessions. + +## Platform + +web + +The app is served by the existing local FastAPI application. Desktop is primary; tablet and mobile remain fully operable. diff --git a/README.md b/README.md index de352077..8255a33b 100644 --- a/README.md +++ b/README.md @@ -79,19 +79,30 @@ uv run theHarvester -h Options such as DNS brute force (`-c`), bounded recursive DNS (`--dns-recursive-depth`), reverse DNS lookup (`-n`), takeover checks (`-t`), API endpoint scanning (`-a`), DNS resolution (`-r`), and screenshots (`--screenshot`) generate additional network activity. Use them only within an explicitly authorized scope. -Recursive DNS requires exactly three distinct resolver IPs through `--dns-resolvers` or the compatible `--dns-resolve` value. It advances only names with two-vantage address consensus that are distinguishable from closest-encloser wildcard controls. Depth, DNS record query, and runtime limits are configurable through the three `--dns-recursive-*` options; the default query ceiling is 3,000 record queries across resolver vantages, and three consecutive zero-yield batches also stop recursion. PTR names for current addresses are retained as secondary evidence, but they do not establish current addressability or become recursion seeds. `POST /api/v1/runs` exposes the same controls. +Recursive DNS requires exactly three distinct resolver IPs through `--dns-resolvers` or the compatible `--dns-resolve` value. It advances only names with two-vantage address consensus that are distinguishable from closest-encloser wildcard controls. Depth, DNS record query, and runtime limits are configurable through the three `--dns-recursive-*` options; the default query ceiling is 3,000 record queries across resolver vantages, and three consecutive zero-yield batches also stop recursion. PTR names for current addresses are retained as secondary evidence, but they do not establish current addressability or become recursion seeds. HarvestView and `POST /api/v1/runs` expose the same controls. Screenshot capture also requires a Playwright-compatible browser; see the installation guide for setup. -## REST API +## HarvestView and REST API `restfulHarvest` starts a FastAPI service on `127.0.0.1:5000` by default: ```bash +export THEHARVESTER_API_KEY='replace-with-a-long-random-value' uv run restfulHarvest ``` -Open [http://127.0.0.1:5000/docs](http://127.0.0.1:5000/docs) for interactive Swagger documentation or [http://127.0.0.1:5000/redoc](http://127.0.0.1:5000/redoc) for ReDoc. +Open [HarvestView](http://127.0.0.1:5000/) to run and inspect finite enumerations in the local web app. The server gives the local browser a derived HttpOnly session cookie, so the API key is never entered into or stored by HarvestView. + +HarvestView uses its own `app.css` rather than a general UI framework. Bootstrap, +Bulma, Pico, and Tailwind would duplicate the existing design layer or require a +markup and build-pipeline rewrite. Tabulator 6.5.2's table behavior and default +theme load from pinned CDNjs URLs with Subresource Integrity. HarvestView +therefore needs network access to CDNjs by default. See the +[self-hosting instructions](docs/wiki/Installation.md) for +an isolated deployment. + +Open [Swagger](http://127.0.0.1:5000/docs) or [ReDoc](http://127.0.0.1:5000/redoc) for the automation contract. | Route | Purpose | | --- | --- | @@ -104,7 +115,9 @@ Open [http://127.0.0.1:5000/docs](http://127.0.0.1:5000/docs) for interactive Sw | `POST /api/v1/runs/import-database` | Import completed runs from a theHarvester SQLite database. | | `GET /api/v1/runs/{run_id}/export` | Export normalized evidence as JSONL. | -Every `/api/v1/*` route requires `THEHARVESTER_API_KEY` in the `X-API-Key` header. Provider credentials stay in server-side configuration and cannot be supplied in a request. Keep the service bound to localhost. If you require remote access, add network access controls and TLS. +HarvestView can start a screenshot or DNS brute-force run directly from a hostname result. These actions create a separate run record for that hostname and leave the parent evidence unchanged. Resolver addresses may be entered directly or loaded from a text file with one IP address per line. Ordinary DNS actions accept one or more resolvers; recursive DNS requires exactly three. + +API clients send `THEHARVESTER_API_KEY` in the `X-API-Key` header; HarvestView uses its derived browser cookie. Provider credentials stay in server-side configuration and cannot be supplied in a request. Keep the service bound to localhost. If you require remote access, add network access controls and TLS. When `--proxies` and `--take-over` are combined, supported discovery and takeover requests use the configured proxies. diff --git a/docs/wiki/Configuration-and-API-Keys.md b/docs/wiki/Configuration-and-API-Keys.md index 4558e38a..0255cb3b 100644 --- a/docs/wiki/Configuration-and-API-Keys.md +++ b/docs/wiki/Configuration-and-API-Keys.md @@ -72,4 +72,4 @@ export THEHARVESTER_API_KEY='replace-with-a-long-random-value' uv run restfulHarvest ``` -API clients send the same value in the `X-API-Key` header. Provider credentials remain in `api-keys.yaml` and cannot be supplied through an API request. +API clients send the same value in the `X-API-Key` header. HarvestView receives a derived HttpOnly browser cookie when it is opened locally, so the key is never entered into or stored by the web app. Provider credentials remain in `api-keys.yaml` and cannot be supplied through an API request. diff --git a/docs/wiki/Installation.md b/docs/wiki/Installation.md index b2155401..8ef7eaf6 100644 --- a/docs/wiki/Installation.md +++ b/docs/wiki/Installation.md @@ -35,6 +35,24 @@ uv run pytest The supported console commands are `theHarvester` and `restfulHarvest`. There is no root `theHarvester.py` launcher. +### Self-host Tabulator + +HarvestView loads Tabulator 6.5.2 from CDNjs by default. For an isolated deployment, download the same pinned files in a connected environment and copy them into `theHarvester/lib/api/static/harvestview/`: + +| File | CDNjs source | SRI | +| --- | --- | --- | +| `tabulator.min.css` | `https://cdnjs.cloudflare.com/ajax/libs/tabulator-tables/6.5.2/css/tabulator.min.css` | `sha512-t8I/asqzdu/MRgVLxVanQ/c5bhUA1qZ/zA432a/3nUh0kkd7P8Qch35wQvTODivf9D6Xv3h7F8p7ezcUyBOQrQ==` | +| `tabulator.min.js` | `https://cdnjs.cloudflare.com/ajax/libs/tabulator-tables/6.5.2/js/tabulator.min.js` | `sha512-AF0YMSgc0Ui4IJPb4hJNSi16wFidZEQa6ZTCAeguF3h5glVnAPuz/JT2ai9ypKhsc9n6CEXBB+tMdxsv1q+rxg==` | + +Then replace the two CDNjs tags in `theHarvester/lib/api/static/harvestview/index.html` with same-origin references: + +```html + + +``` + +Remove the CDN-only `integrity`, `crossorigin`, and `referrerpolicy` attributes from those local tags. Rebuild the package or container after copying the assets. + ### Screenshot support The screenshot option requires a Playwright-compatible Chromium browser: diff --git a/docs/wiki/Responsible-Use-and-Scope.md b/docs/wiki/Responsible-Use-and-Scope.md index 741ee155..ad72e58d 100644 --- a/docs/wiki/Responsible-Use-and-Scope.md +++ b/docs/wiki/Responsible-Use-and-Scope.md @@ -37,6 +37,6 @@ Results may contain private infrastructure, employee addresses, account identifi ## API exposure -Every `/api/v1/*` route requires `THEHARVESTER_API_KEY`. Provider credentials remain server-side. +Every `/api/v1/*` route requires `THEHARVESTER_API_KEY`. HarvestView is restricted to a loopback browser origin and uses a derived HttpOnly cookie for those same routes. Provider credentials remain server-side. Keep the service on localhost. If you require remote access, add network controls and TLS in front of the existing API authentication. diff --git a/docs/wiki/Rest-API.md b/docs/wiki/Rest-API.md index 3b3163a1..ddb78b80 100644 --- a/docs/wiki/Rest-API.md +++ b/docs/wiki/Rest-API.md @@ -1,6 +1,6 @@ # REST API -`restfulHarvest` serves one versioned API for local automation. +`restfulHarvest` serves HarvestView at `/` and one versioned API for local automation. ## Start the service @@ -15,6 +15,7 @@ The service binds to `127.0.0.1:5000` by default. Use `uv run restfulHarvest -h` Open: +- HarvestView: [http://127.0.0.1:5000/](http://127.0.0.1:5000/) - Swagger UI: [http://127.0.0.1:5000/docs](http://127.0.0.1:5000/docs) - ReDoc: [http://127.0.0.1:5000/redoc](http://127.0.0.1:5000/redoc) @@ -48,6 +49,8 @@ curl -s http://127.0.0.1:5000/api/v1/sources \ | jq ``` +HarvestView receives a derived HttpOnly browser-session cookie when loaded from localhost. The browser never stores or displays the configured API key. Cookie-authenticated mutations also require a matching same-origin request. + Provider credentials remain in theHarvester's server-side configuration. Requests cannot supply provider API keys. ## Submit and inspect a run @@ -108,6 +111,8 @@ The action catalog and run request use the same names. For example, set `takeove Every custom API scan entry must be a URL path beginning with `/`. The API does not accept a server-side file path. +HarvestView's subdomain action buttons call this route and create a separate run without changing the completed parent run. + ## Import and export Import records existing evidence and never contacts the target. For one run, send the same JSONL written by `theHarvester -f NAME`: diff --git a/docs/wiki/Troubleshooting.md b/docs/wiki/Troubleshooting.md index c90975d8..4c77b685 100644 --- a/docs/wiki/Troubleshooting.md +++ b/docs/wiki/Troubleshooting.md @@ -82,7 +82,7 @@ uv run restfulHarvest --log-level debug Then open [http://127.0.0.1:5000/docs](http://127.0.0.1:5000/docs). -- `401` on `/api/v1/*`: the `X-API-Key` header does not match. +- `401` on `/api/v1/*`: the `X-API-Key` header or HarvestView browser session does not match. - `503` on `/api/v1/*`: `THEHARVESTER_API_KEY` was not configured before startup. - `429`: a reverse proxy or remote provider applied its own rate limit. `restfulHarvest` has no built-in request limiter. - `503` when creating a run: the execution worker is disabled or unavailable. diff --git a/pyproject.toml b/pyproject.toml index 04dcf99e..2911222b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ dev = [ "mypy-extensions==1.1.0", "pytest==9.1.1", "pytest-asyncio==1.4.0", + "pytest-playwright==0.8.0", "types-certifi==2021.10.8.3", "types-chardet==5.0.4.6", "types-python-dateutil==2.9.0.20260518", @@ -69,9 +70,10 @@ restfulHarvest = "theHarvester.restfulHarvest:main" minversion = "8.3.3" asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "function" -addopts = "--no-header --strict-markers" +addopts = "--no-header --strict-markers -m 'not harvestview_e2e'" markers = [ "live_network: contacts an external service and runs only with --run-live-network", + "harvestview_e2e: real-browser tests against an isolated local HarvestView server", ] testpaths = ["tests"] diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py new file mode 100644 index 00000000..07247318 --- /dev/null +++ b/tests/e2e/conftest.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +import os +import subprocess +import sys +import time +from collections import Counter +from ipaddress import ip_address +from pathlib import Path +from typing import TYPE_CHECKING, TextIO +from urllib.parse import urlsplit + +import httpx +import pytest + +if TYPE_CHECKING: + from playwright.sync_api import Page, Response, Route + +CDNJS_TABULATOR_ASSETS = { + 'https://cdnjs.cloudflare.com/ajax/libs/tabulator-tables/6.5.2/css/tabulator.min.css', + 'https://cdnjs.cloudflare.com/ajax/libs/tabulator-tables/6.5.2/js/tabulator.min.js', +} + + +class HarvestViewServer: + def __init__(self, repo_root: Path, port: int, environment: dict[str, str], server_log: Path) -> None: + self.repo_root = repo_root + self.port = port + self.environment = environment + self.server_log = server_log + self.url = f'http://127.0.0.1:{port}' + self._output: TextIO | None = None + self._process: subprocess.Popen[str] | None = None + + def start(self) -> None: + self.server_log.parent.mkdir(parents=True, exist_ok=True) + self._output = self.server_log.open('a', encoding='utf-8') + self._process = subprocess.Popen( + [ + sys.executable, + '-m', + 'uvicorn', + 'theHarvester.lib.api.api:app', + '--host', + '127.0.0.1', + '--port', + str(self.port), + '--log-level', + 'warning', + ], + cwd=self.repo_root, + env=self.environment, + stdout=self._output, + stderr=subprocess.STDOUT, + text=True, + ) + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + if self._process.poll() is not None: + self.stop() + pytest.fail(f'theHarvester test server exited during startup; see {self.server_log}') + try: + if httpx.get(f'{self.url}/openapi.json', timeout=0.25).status_code == 200: + return + except httpx.HTTPError: + time.sleep(0.05) + self.stop() + pytest.fail(f'theHarvester test server did not become ready; see {self.server_log}') + + def stop(self) -> None: + if self._process is not None and self._process.poll() is None: + self._process.terminate() + try: + self._process.wait(timeout=5) + except subprocess.TimeoutExpired: + self._process.kill() + self._process.wait(timeout=5) + self._process = None + if self._output is not None: + self._output.close() + self._output = None + + def restart(self) -> None: + self.stop() + self.start() + + +class BrowserFailures: + def __init__(self, page: Page) -> None: + self.console_errors: Counter[str] = Counter() + self.local_failures: Counter[tuple[str, int, str]] = Counter() + self.external_requests: list[str] = [] + self.allowed_console_errors: Counter[str] = Counter() + self.allowed_responses: Counter[tuple[str, int, str]] = Counter() + page.on('console', self._record_console_message) + page.on('response', self._record_response) + page.route('**/*', self._guard_request) + + def _record_console_message(self, message) -> None: + if message.type == 'error': + self.console_errors[message.text] += 1 + + def _guard_request(self, route: Route) -> None: + request = route.request + parsed = urlsplit(request.url) + if request.url in CDNJS_TABULATOR_ASSETS: + route.continue_() + return + if parsed.scheme not in {'http', 'https'} or parsed.hostname == 'localhost': + route.continue_() + return + try: + is_loopback = parsed.hostname is not None and ip_address(parsed.hostname).is_loopback + except ValueError: + is_loopback = False + if is_loopback: + route.continue_() + return + self.external_requests.append(f'{request.method} {request.url}') + route.abort('blockedbyclient') + + def _record_response(self, response: Response) -> None: + path = urlsplit(response.url).path + if response.status >= 500 or (path.startswith('/api/v1/') and response.status == 401): + self.local_failures[(response.request.method, response.status, path)] += 1 + + def allow_response(self, method: str, status_code: int, path: str) -> None: + self.allowed_responses[(method, status_code, path)] += 1 + + def allow_console_error(self, message: str) -> None: + self.allowed_console_errors[message] += 1 + + def assert_clean(self) -> None: + assert self.console_errors - self.allowed_console_errors == Counter() + assert self.local_failures == self.allowed_responses + assert self.external_requests == [] + + +@pytest.fixture(autouse=True) +def browser_failures(page: Page) -> BrowserFailures: + failures = BrowserFailures(page) + yield failures + failures.assert_clean() + + +@pytest.fixture +def harvestview_server(tmp_path: Path, unused_tcp_port: int) -> HarvestViewServer: + repo_root = Path(__file__).parents[2] + artifact_dir = repo_root / 'test-results' + artifact_dir.mkdir(exist_ok=True) + server_log = artifact_dir / f'harvestview-server-{unused_tcp_port}.log' + environment = os.environ.copy() + environment.update( + { + 'THEHARVESTER_API_KEY': 'harvestview-e2e-key', + 'THEHARVESTER_RUN_ARTIFACTS': str(tmp_path / 'artifacts'), + 'THEHARVESTER_RUN_DB': str(tmp_path / 'runs.sqlite'), + 'THEHARVESTER_RUN_WORKER': 'disabled', + 'ALL_PROXY': 'http://127.0.0.1:9', + 'HTTPS_PROXY': 'http://127.0.0.1:9', + 'HTTP_PROXY': 'http://127.0.0.1:9', + 'NO_PROXY': '127.0.0.1,localhost', + 'all_proxy': 'http://127.0.0.1:9', + 'https_proxy': 'http://127.0.0.1:9', + 'http_proxy': 'http://127.0.0.1:9', + 'no_proxy': '127.0.0.1,localhost', + } + ) + server = HarvestViewServer(repo_root, unused_tcp_port, environment, server_log) + server.start() + try: + yield server + finally: + server.stop() + + +@pytest.fixture +def harvestview_server_url(harvestview_server: HarvestViewServer) -> str: + return harvestview_server.url diff --git a/tests/e2e/test_harvestview.py b/tests/e2e/test_harvestview.py new file mode 100644 index 00000000..9c17b404 --- /dev/null +++ b/tests/e2e/test_harvestview.py @@ -0,0 +1,961 @@ +from __future__ import annotations + +import asyncio +import json +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +from datetime import UTC, datetime +from pathlib import Path + +import pytest +from playwright.sync_api import Page, Route, expect + +pytestmark = pytest.mark.harvestview_e2e + + +def write_jsonl_evidence(path: Path, evidence: dict[str, object]) -> None: + results = evidence.get('results', []) + assert isinstance(results, list) + counts = Counter(str(result['type']) for result in results if isinstance(result, dict)) + summary = { + 'type': 'summary', + 'run_id': evidence['run_id'], + 'target': evidence['target'], + 'started_at': evidence['started_at'], + 'completed_at': evidence['completed_at'], + 'evidence_status': evidence['status'], + 'source_executions': evidence.get('source_executions', []), + 'action_executions': evidence.get('action_executions', []), + 'artifacts': evidence.get('artifacts', []), + 'result_count': len(results), + 'counts': dict(sorted(counts.items())), + } + records = [ + summary, + *[ + {**result, 'sources': result.get('sources', []), 'actions': result.get('actions', [])} + for result in results + if isinstance(result, dict) + ], + ] + path.write_text(''.join(json.dumps(record, sort_keys=True) + '\n' for record in records), encoding='utf-8') + + +def record_api_statuses(page: Page, server_url: str) -> dict[str, int]: + api_statuses: dict[str, int] = {} + + def record_response(response) -> None: + if response.url.startswith(f'{server_url}/api/v1/'): + api_statuses[response.url.removeprefix(server_url)] = response.status + + page.on('response', record_response) + return api_statuses + + +def test_fresh_browser_session_authenticates_through_harvestview( + harvestview_server_url: str, + page: Page, +) -> None: + api_statuses = record_api_statuses(page, harvestview_server_url) + page.goto(f'{harvestview_server_url}/') + + expect(page.get_by_role('heading', name='No enumeration runs yet')).to_be_visible() + assert api_statuses['/api/v1/sources'] == 200 + assert api_statuses['/api/v1/runs'] == 200 + expect(page.get_by_text('Invalid API key')).to_have_count(0) + + +def test_browser_session_stays_authenticated_after_server_restart( + harvestview_server, + page: Page, +) -> None: + page.goto(f'{harvestview_server.url}/') + expect(page.get_by_role('heading', name='No enumeration runs yet')).to_be_visible() + + harvestview_server.restart() + page.reload() + + expect(page.get_by_role('heading', name='No enumeration runs yet')).to_be_visible() + expect(page.get_by_text('Invalid API key')).to_have_count(0) + + +@pytest.mark.parametrize('viewport', [{'width': 1440, 'height': 900}, {'width': 390, 'height': 844}]) +def test_versioned_assets_and_tooltips_work_at_supported_viewports( + harvestview_server_url: str, + page: Page, + viewport: dict[str, int], +) -> None: + page.set_viewport_size(viewport) + page.goto(f'{harvestview_server_url}/') + + stylesheet = page.locator('link[href*="/static/harvestview/app.css"]') + script = page.locator('script[src*="/static/harvestview/app.js"]') + assert '?v=' in (stylesheet.get_attribute('href') or '') + assert '?v=' in (script.get_attribute('src') or '') + assert '{{' not in (stylesheet.get_attribute('href') or '') + assert '{{' not in (script.get_attribute('src') or '') + + page.get_by_role('button', name='Start enumeration').first.click() + tooltip = page.get_by_role('button', name='Explain discovery sources') + tooltip.hover() + page.wait_for_function( + "node => getComputedStyle(node, '::after').opacity === '1'", + arg=tooltip.element_handle(), + ) + tooltip_content = tooltip.evaluate("node => getComputedStyle(node, '::after').content") + + assert 'Credential warnings identify sources that cannot start' in tooltip_content + assert 'Credential warnings identify sources that cannot start' in tooltip.get_attribute('aria-description') + assert tooltip.bounding_box()['width'] >= 44 + assert tooltip.bounding_box()['height'] >= 44 + assert page.locator('#submit-run-button').evaluate( + """button => { + const buttonBox = button.getBoundingClientRect(); + const dialogBox = button.closest('dialog').getBoundingClientRect(); + return buttonBox.top >= dialogBox.top && buttonBox.bottom <= dialogBox.bottom && buttonBox.bottom <= innerHeight; + }""" + ) + assert page.evaluate('document.documentElement.scrollWidth <= document.documentElement.clientWidth') + + +def test_disabled_worker_rejects_submission_without_creating_a_run( + harvestview_server_url: str, + page: Page, + browser_failures, +) -> None: + browser_failures.allow_response('POST', 503, '/api/v1/runs') + browser_failures.allow_console_error( + 'Failed to load resource: the server responded with a status of 503 (Service Unavailable)' + ) + page.goto(f'{harvestview_server_url}/') + page.get_by_role('button', name='Start enumeration').first.click() + page.locator('#run-target').fill('example.com') + + with page.expect_response( + lambda response: response.url == f'{harvestview_server_url}/api/v1/runs' and response.request.method == 'POST' + ) as submission: + page.locator('#submit-run-button').click() + + assert submission.value.status == 503 + expect(page.locator('#new-run-error')).to_have_text('theHarvester execution worker is disabled') + history = page.context.request.get(f'{harvestview_server_url}/api/v1/runs') + assert history.status == 200 + assert history.json() == [] + + +def test_harvestview_can_submit_overridable_execution_controls( + harvestview_server_url: str, + page: Page, + browser_failures, + tmp_path: Path, +) -> None: + browser_failures.allow_response('POST', 503, '/api/v1/runs') + browser_failures.allow_console_error( + 'Failed to load resource: the server responded with a status of 503 (Service Unavailable)' + ) + captured: dict[str, object] = {} + + def capture_submission(route: Route) -> None: + if route.request.method != 'POST': + route.continue_() + return + captured.update(route.request.post_data_json) + route.fulfill(status=503, json={'detail': 'Controls captured'}) + + page.route(f'{harvestview_server_url}/api/v1/runs', capture_submission) + page.goto(f'{harvestview_server_url}/') + page.set_default_timeout(2_000) + page.get_by_role('button', name='Start enumeration').first.click() + page.get_by_role('button', name='Clear', exact=True).click() + page.locator('#run-target').fill('example.com') + page.get_by_text('Advanced execution controls', exact=True).click() + page.locator('#run-start').fill('25') + page.locator('#run-deadline').fill('86400') + page.locator('[name="proxies"]').check() + page.locator('[name="shodan"]').check() + page.locator('[name="dns_lookup"]').check() + page.locator('[name="takeover"]').check() + page.locator('[name="api_scan"]').check() + page.locator('#api-scan-paths').fill('/api/v2\n/health') + page.locator('#dns-recursive-depth').fill('3') + page.locator('#dns-recursive-query-limit').fill('1234') + page.locator('#dns-recursive-runtime-seconds').fill('12.5') + resolver_file = tmp_path / 'resolvers.txt' + resolver_file.write_text('192.0.2.53\n198.51.100.53\n203.0.113.53\n', encoding='utf-8') + page.locator('#dns-resolver-file').set_input_files(resolver_file) + expect(page.locator('#dns-resolvers')).to_have_value('192.0.2.53,198.51.100.53,203.0.113.53') + + expect(page.locator('#activity-summary')).to_have_text('P0 selected · P1 selected · P2 selected') + page.locator('[data-activity="P0"] input[value="crtsh"]').check() + page.locator('#submit-run-button').click() + expect(page.locator('#new-run-error')).to_have_text('Controls captured') + + assert captured == { + 'target': 'example.com', + 'sources': ['crtsh'], + 'limit': 500, + 'start': 25, + 'deadline_seconds': 86_400, + 'proxies': True, + 'dns_lookup': True, + 'dns_resolve': False, + 'dns_resolvers': ['192.0.2.53', '198.51.100.53', '203.0.113.53'], + 'dns_recursive_depth': 3, + 'dns_recursive_query_limit': 1_234, + 'dns_recursive_runtime_seconds': 12.5, + 'dns_brute': False, + 'shodan': True, + 'screenshot': False, + 'takeover': True, + 'api_scan': True, + 'api_scan_paths': ['/api/v2', '/health'], + } + + +def test_harvestview_submits_a_target_only_api_scan( + harvestview_server_url: str, + page: Page, + browser_failures, +) -> None: + browser_failures.allow_response('POST', 503, '/api/v1/runs') + browser_failures.allow_console_error( + 'Failed to load resource: the server responded with a status of 503 (Service Unavailable)' + ) + captured: dict[str, object] = {} + + def capture_submission(route: Route) -> None: + if route.request.method != 'POST': + route.continue_() + return + captured.update(route.request.post_data_json) + route.fulfill(status=503, json={'detail': 'Target-only action captured'}) + + page.route(f'{harvestview_server_url}/api/v1/runs', capture_submission) + page.goto(f'{harvestview_server_url}/') + page.get_by_role('button', name='Start enumeration').first.click() + page.get_by_role('button', name='Clear', exact=True).click() + page.locator('#run-target').fill('api.example.test') + page.locator('[name="api_scan"]').check() + page.locator('details.advanced-execution summary').click() + page.locator('#api-scan-paths').fill('/api/v2\n/health') + page.locator('#submit-run-button').click() + + expect(page.locator('#new-run-error')).to_have_text('Target-only action captured') + assert captured['sources'] == [] + assert captured['api_scan'] is True + assert captured['api_scan_paths'] == ['/api/v2', '/health'] + + +def test_hostname_actions_queue_isolated_runs( + harvestview_server_url: str, + page: Page, + browser_failures, +) -> None: + browser_failures.allow_response('POST', 503, '/api/v1/runs') + browser_failures.allow_response('POST', 503, '/api/v1/runs') + browser_failures.allow_console_error( + 'Failed to load resource: the server responded with a status of 503 (Service Unavailable)' + ) + browser_failures.allow_console_error( + 'Failed to load resource: the server responded with a status of 503 (Service Unavailable)' + ) + run = { + 'run_id': 'parent-run', + 'target': 'example.com', + 'status': 'completed', + 'origin': 'local', + 'created_at': '2026-08-05T12:00:00+00:00', + 'started_at': '2026-08-05T12:00:01+00:00', + 'completed_at': '2026-08-05T12:00:05+00:00', + 'cancellation_requested_at': None, + 'evidence_status': 'complete', + 'result_count': 1, + 'activities': ['P0'], + 'sources': ['crtsh'], + 'request': { + 'sources': ['crtsh'], + 'limit': 25, + 'deadline_seconds': 300, + 'dns_resolvers': ['192.0.2.53'], + }, + 'source_executions': [], + 'action_executions': [ + { + 'action': 'dns-brute', + 'status': 'failed', + 'result_count': 0, + 'duration_ms': 125, + 'error_type': 'TimeoutError', + 'stop_reason': 'query-errors', + } + ], + 'results': [{'type': 'hostname', 'value': 'api.example.com'}], + 'screenshots': [], + 'log': '', + 'error': None, + } + submissions: list[dict[str, object]] = [] + + def route_runs(route: Route) -> None: + if route.request.method == 'POST': + submission = route.request.post_data_json + submissions.append(submission) + detail = 'Screenshot captured' if submission.get('screenshot') else 'DNS brute captured' + route.fulfill(status=503, json={'detail': detail}) + else: + route.fulfill(json=[run]) + + page.route(f'{harvestview_server_url}/api/v1/runs', route_runs) + page.route(f'{harvestview_server_url}/api/v1/runs/parent-run', lambda route: route.fulfill(json=run)) + page.goto(f'{harvestview_server_url}/') + + expect(page.get_by_role('button', name='Hostnames 1')).to_be_enabled() + + page.locator('#provider-details summary').click() + expect(page.locator('#provider-title')).to_have_text('Execution outcomes') + action_row = page.locator('#provider-body tr').filter(has_text='dns-brute') + expect(action_row).to_contain_text('Action') + expect(action_row).to_contain_text('failed') + expect(action_row).to_contain_text('TimeoutError') + + page.get_by_role('button', name='Take screenshot of api.example.com (P2)').click() + expect(page.locator('#toast')).to_contain_text('Screenshot captured') + page.get_by_role('button', name='DNS brute force api.example.com (P1)').click() + expect(page.locator('#toast')).to_contain_text('DNS brute captured') + + assert submissions == [ + {'target': 'api.example.com', 'sources': [], 'screenshot': True}, + { + 'target': 'api.example.com', + 'sources': [], + 'dns_brute': True, + 'dns_resolvers': ['192.0.2.53'], + }, + ] + + +def test_accepted_result_action_is_not_reported_as_failed_when_refresh_fails( + harvestview_server_url: str, + page: Page, + browser_failures, +) -> None: + browser_failures.allow_response('GET', 503, '/api/v1/runs') + browser_failures.allow_console_error( + 'Failed to load resource: the server responded with a status of 503 (Service Unavailable)' + ) + run = { + 'run_id': 'parent-run', + 'target': 'example.com', + 'status': 'running', + 'origin': 'local', + 'created_at': '2026-08-05T12:00:00+00:00', + 'started_at': '2026-08-05T12:00:01+00:00', + 'completed_at': None, + 'cancellation_requested_at': None, + 'evidence_status': 'partial', + 'result_count': 1, + 'activities': ['P0'], + 'sources': ['crtsh'], + 'request': {'sources': ['crtsh'], 'limit': 25, 'deadline_seconds': 300}, + 'source_executions': [], + 'action_executions': [], + 'results': [{'type': 'hostname', 'value': 'api.example.com'}], + 'screenshots': [], + 'log': '', + 'error': None, + } + list_calls = 0 + post_calls = 0 + cancelled_ids: list[str] = [] + + def route_runs(route: Route) -> None: + nonlocal list_calls, post_calls + if route.request.method == 'POST': + post_calls += 1 + route.fulfill(status=201, json={'run_id': 'queued-action', 'target': 'api.example.com', 'status': 'queued'}) + return + list_calls += 1 + if list_calls == 1: + route.fulfill(json=[run]) + elif list_calls == 2: + route.fulfill(status=503, json={'detail': 'Refresh unavailable'}) + else: + route.fulfill(json=[{**run, 'status': 'cancelled', 'completed_at': '2026-08-05T12:00:06+00:00'}]) + + def route_cancel(route: Route, run_id: str) -> None: + cancelled_ids.append(run_id) + route.fulfill(json={**run, 'status': 'cancelled', 'completed_at': '2026-08-05T12:00:06+00:00'}) + + page.route(f'{harvestview_server_url}/api/v1/runs', route_runs) + page.route( + f'{harvestview_server_url}/api/v1/runs/parent-run/cancel', + lambda route: route_cancel(route, 'parent-run'), + ) + page.route( + f'{harvestview_server_url}/api/v1/runs/queued-action/cancel', + lambda route: route_cancel(route, 'queued-action'), + ) + page.route(f'{harvestview_server_url}/api/v1/runs/parent-run', lambda route: route.fulfill(json=run)) + page.goto(f'{harvestview_server_url}/') + + page.get_by_role('button', name='Take screenshot of api.example.com (P2)').click() + + expect(page.locator('#toast')).to_contain_text('was queued, but the run view could not refresh') + expect(page.locator('#toast')).to_contain_text('Do not submit it again') + expect(page.locator('#detail-run-id')).to_have_text('parent-run') + page.locator('#cancel-run-button').click() + expect(page.locator('#toast')).to_have_text('Queued enumeration cancelled.') + assert post_calls == 1 + assert cancelled_ids == ['parent-run'] + + +def test_accepted_cancellation_is_not_reported_as_failed_when_history_refresh_fails( + harvestview_server_url: str, + page: Page, + browser_failures, +) -> None: + browser_failures.allow_response('GET', 503, '/api/v1/runs') + browser_failures.allow_console_error( + 'Failed to load resource: the server responded with a status of 503 (Service Unavailable)' + ) + run = { + 'run_id': 'running-run', + 'target': 'example.com', + 'status': 'running', + 'origin': 'local', + 'created_at': '2026-08-05T12:00:00+00:00', + 'started_at': '2026-08-05T12:00:01+00:00', + 'completed_at': None, + 'cancellation_requested_at': None, + 'evidence_status': 'partial', + 'result_count': 0, + 'activities': ['P0'], + 'sources': ['crtsh'], + 'request': {'sources': ['crtsh'], 'limit': 25, 'deadline_seconds': 300}, + 'source_executions': [], + 'action_executions': [], + 'results': [], + 'screenshots': [], + 'log': '', + 'error': None, + } + list_calls = 0 + cancel_calls = 0 + + def route_runs(route: Route) -> None: + nonlocal list_calls + list_calls += 1 + if list_calls == 1: + route.fulfill(json=[run]) + else: + route.fulfill(status=503, json={'detail': 'Refresh unavailable'}) + + def route_cancel(route: Route) -> None: + nonlocal cancel_calls + cancel_calls += 1 + route.fulfill(json={**run, 'status': 'cancelling', 'cancellation_requested_at': '2026-08-05T12:00:02+00:00'}) + + page.route(f'{harvestview_server_url}/api/v1/runs', route_runs) + page.route(f'{harvestview_server_url}/api/v1/runs/running-run/cancel', route_cancel) + page.route(f'{harvestview_server_url}/api/v1/runs/running-run', lambda route: route.fulfill(json=run)) + page.goto(f'{harvestview_server_url}/') + + page.locator('#cancel-run-button').click() + + expect(page.locator('#toast')).to_contain_text('Cancellation was accepted, but run history could not refresh') + expect(page.locator('#toast')).to_contain_text('Do not request it again') + assert cancel_calls == 1 + + +def test_harvestview_can_select_sources_by_result_capability(harvestview_server_url: str, page: Page) -> None: + page.goto(f'{harvestview_server_url}/') + catalog = page.context.request.get(f'{harvestview_server_url}/api/v1/sources').json() + expected = {source['name'] for source in catalog['sources'] if 'ips' in source['capabilities']} + + page.get_by_role('button', name='Start enumeration').first.click() + page.get_by_role('button', name='Clear', exact=True).click() + page.locator('#source-capability').select_option('ips') + page.get_by_role('button', name='Add sources').click() + + selected = set(page.locator('#source-groups input:checked').evaluate_all('(inputs) => inputs.map(input => input.value)')) + assert selected == expected + + +def test_polling_recovers_after_one_transient_refresh_failure( + harvestview_server_url: str, + page: Page, + browser_failures, +) -> None: + browser_failures.allow_response('GET', 503, '/api/v1/runs/retry-run') + browser_failures.allow_console_error( + 'Failed to load resource: the server responded with a status of 503 (Service Unavailable)' + ) + run = { + 'run_id': 'retry-run', + 'target': 'example.com', + 'status': 'running', + 'origin': 'local', + 'created_at': '2026-08-05T12:00:00+00:00', + 'started_at': '2026-08-05T12:00:01+00:00', + 'completed_at': None, + 'cancellation_requested_at': None, + 'evidence_status': 'partial', + 'result_count': 0, + 'activities': ['P0'], + 'sources': ['crtsh'], + 'request': {'sources': ['crtsh'], 'limit': 25, 'deadline_seconds': 300}, + 'source_executions': [], + 'results': [], + 'screenshots': [], + 'log': '', + 'error': None, + } + detail_calls = 0 + + def route_runs(route: Route) -> None: + route.fulfill(json=[run]) + + def route_detail(route: Route) -> None: + nonlocal detail_calls + detail_calls += 1 + if detail_calls == 2: + route.fulfill(status=503, json={'detail': 'Temporary refresh failure'}) + else: + completed = detail_calls >= 3 + route.fulfill( + json={ + **run, + 'status': 'completed' if completed else 'running', + 'completed_at': '2026-08-05T12:00:05+00:00' if completed else None, + 'evidence_status': 'complete' if completed else 'partial', + } + ) + + page.route(f'{harvestview_server_url}/api/v1/runs', route_runs) + page.route(f'{harvestview_server_url}/api/v1/runs/retry-run', route_detail) + page.goto(f'{harvestview_server_url}/') + + expect(page.locator('#status-chips')).to_contain_text('completed', timeout=7_000) + assert detail_calls >= 3 + + +def test_workspace_startup_failure_has_an_inline_retry( + harvestview_server_url: str, + page: Page, + browser_failures, +) -> None: + browser_failures.allow_response('GET', 503, '/api/v1/sources') + browser_failures.allow_console_error( + 'Failed to load resource: the server responded with a status of 503 (Service Unavailable)' + ) + source_calls = 0 + + def fail_once(route: Route) -> None: + nonlocal source_calls + source_calls += 1 + if source_calls == 1: + route.fulfill(status=503, json={'detail': 'Temporary startup failure'}) + else: + route.continue_() + + page.route(f'{harvestview_server_url}/api/v1/sources', fail_once) + page.goto(f'{harvestview_server_url}/') + page.set_default_timeout(3_000) + + expect(page.locator('#workspace-error')).to_be_visible() + expect(page.locator('#workspace-error-message')).to_have_text('Temporary startup failure') + page.locator('#retry-workspace-button').click() + + expect(page.get_by_role('heading', name='No enumeration runs yet')).to_be_visible() + assert source_calls == 2 + + +def test_unchanged_poll_keeps_result_table_filters(harvestview_server_url: str, page: Page) -> None: + run = { + 'run_id': 'stable-run', + 'target': 'example.com', + 'status': 'running', + 'origin': 'local', + 'created_at': '2026-08-05T12:00:00+00:00', + 'started_at': '2026-08-05T12:00:01+00:00', + 'completed_at': None, + 'cancellation_requested_at': None, + 'evidence_status': 'partial', + 'result_count': 1, + 'activities': ['P0', 'P2'], + 'sources': ['crtsh'], + 'request': { + 'sources': ['crtsh'], + 'limit': 25, + 'deadline_seconds': 300, + 'proxies': True, + 'dns_lookup': True, + 'takeover': True, + }, + 'source_executions': [], + 'results': [{'type': 'hostname', 'value': 'api.example.com'}], + 'screenshots': [], + 'log': '', + 'error': None, + } + + page.route(f'{harvestview_server_url}/api/v1/runs', lambda route: route.fulfill(json=[run])) + page.route(f'{harvestview_server_url}/api/v1/runs/stable-run', lambda route: route.fulfill(json=run)) + page.goto(f'{harvestview_server_url}/') + expect(page.locator('#request-options')).to_contain_text('Proxy transportSelected') + expect(page.locator('#request-options')).to_contain_text('DNS lookup (/24 reverse expansion)Selected') + expect(page.locator('#request-options')).to_contain_text('Takeover transportConfigured proxy') + value_filter = page.locator('.tabulator-col[tabulator-field="value"] .tabulator-header-filter input') + value_filter.fill('api') + + page.wait_for_timeout(1_600) + + expect(value_filter).to_have_value('api') + + +@pytest.mark.parametrize('initial_detail_status', ['queued', 'running']) +def test_submission_notification_matches_the_loaded_lifecycle( + harvestview_server_url: str, + page: Page, + initial_detail_status: str, +) -> None: + page.goto(f'{harvestview_server_url}/') + queued_run = { + 'run_id': 'toast-run', + 'target': 'example.com', + 'status': 'queued', + 'origin': 'local', + 'created_at': '2026-08-05T12:00:00+00:00', + 'started_at': None, + 'completed_at': None, + 'cancellation_requested_at': None, + 'evidence_status': 'partial', + 'result_count': 0, + 'activities': ['P0'], + 'sources': ['crtsh'], + 'request': {'sources': ['crtsh'], 'limit': 25, 'deadline_seconds': 300}, + 'source_executions': [], + 'results': [], + 'screenshots': [], + 'log': '', + 'error': None, + } + detail_status = initial_detail_status + + def route_runs(route: Route) -> None: + if route.request.method == 'POST': + route.fulfill(status=201, json=queued_run) + else: + route.fulfill(json=[{**queued_run, 'status': 'running', 'started_at': '2026-08-05T12:00:01+00:00'}]) + + def route_detail(route: Route) -> None: + if detail_status == 'queued': + run = queued_run + elif detail_status == 'running': + run = { + **queued_run, + 'status': 'running', + 'started_at': '2026-08-05T12:00:01+00:00', + } + else: + run = { + **queued_run, + 'status': 'cancelling', + 'started_at': '2026-08-05T12:00:01+00:00', + 'cancellation_requested_at': '2026-08-05T12:00:02+00:00', + } + route.fulfill(json=run) + + page.route(f'{harvestview_server_url}/api/v1/runs', route_runs) + page.route(f'{harvestview_server_url}/api/v1/runs/toast-run', route_detail) + page.get_by_role('button', name='Start enumeration').first.click() + page.locator('#run-target').fill('example.com') + page.locator('#run-limit').fill('25') + page.locator('#run-deadline').fill('300') + page.locator('#submit-run-button').click() + + if initial_detail_status == 'queued': + expect(page.locator('#toast')).to_have_text('Enumeration for example.com is queued.') + detail_status = 'running' + expect(page.locator('#status-chips')).to_contain_text('running') + expect(page.locator('#submit-run-button')).to_be_enabled() + expect(page.locator('#toast')).to_be_hidden(timeout=2500) + detail_status = 'cancelling' + expect(page.locator('#lifecycle-track strong')).to_contain_text(['Cancellation requested']) + + +@pytest.mark.parametrize( + ('terminal_status', 'title', 'copy'), + [ + ('cancelled', 'Enumeration cancelled', 'The enumeration was cancelled.'), + ('failed', 'Enumeration failed', 'Provider process exited.'), + ], +) +def test_empty_terminal_run_explains_its_lifecycle( + harvestview_server_url: str, + page: Page, + terminal_status: str, + title: str, + copy: str, +) -> None: + run = { + 'run_id': f'{terminal_status}-run', + 'target': 'example.com', + 'status': terminal_status, + 'origin': 'local', + 'created_at': '2026-08-05T12:00:00+00:00', + 'started_at': '2026-08-05T12:00:01+00:00', + 'completed_at': '2026-08-05T12:00:02+00:00', + 'cancellation_requested_at': None, + 'evidence_status': 'partial', + 'result_count': 0, + 'activities': ['P0'], + 'sources': ['crtsh'], + 'request': {'sources': ['crtsh'], 'limit': 25, 'deadline_seconds': 300}, + 'source_executions': [], + 'results': [], + 'screenshots': [], + 'log': '', + 'error': 'Provider process exited.' if terminal_status == 'failed' else None, + } + page.route(f'{harvestview_server_url}/api/v1/runs', lambda route: route.fulfill(json=[run])) + page.route(f'{harvestview_server_url}/api/v1/runs/{run["run_id"]}', lambda route: route.fulfill(json=run)) + + page.goto(f'{harvestview_server_url}/') + + expect(page.locator('#results-empty-title')).to_have_text(title) + expect(page.locator('#results-empty-copy')).to_contain_text(copy) + expect(page.locator('#results-empty-copy')).to_contain_text('The retained evidence record is partial.') + + +def test_completed_empty_import_explains_terminal_outcome( + harvestview_server_url: str, + page: Page, + tmp_path: Path, +) -> None: + evidence_file = tmp_path / 'empty-run.jsonl' + write_jsonl_evidence( + evidence_file, + { + 'run_id': '4ce79bb1-91a1-4456-8589-e5d82b55f2b4', + 'target': 'example.com', + 'started_at': '2026-08-05T12:00:00+00:00', + 'completed_at': '2026-08-05T12:00:25+00:00', + 'status': 'complete', + 'source_executions': [ + {'source': 'crtsh', 'status': 'completed', 'result_count': 0, 'duration_ms': 25000}, + ], + 'results': [], + }, + ) + + page.goto(f'{harvestview_server_url}/') + page.get_by_role('button', name='Import result file').first.click() + page.locator('#result-file').set_input_files(evidence_file) + page.locator('#submit-import-button').click() + + expect(page.locator('#results-empty-title')).to_have_text('Enumeration completed') + expect(page.locator('#results-summary')).to_have_text( + '0 normalized results · 1 completed (1 zero-result) / 0 partial / 0 skipped / 0 failed.' + ) + expect(page.locator('#results-empty-copy')).to_have_text( + 'crtsh returned no normalized evidence. The retained evidence record is complete.' + ) + expect(page.locator('#lifecycle-track strong')).to_have_text(['Submitted', 'Started', 'Completed']) + expect(page.get_by_role('button', name='All JSONL')).to_be_enabled() + with page.expect_download() as jsonl_download: + page.get_by_role('button', name='All JSONL').click() + exported_records = [json.loads(line) for line in Path(jsonl_download.value.path()).read_text(encoding='utf-8').splitlines()] + assert len(exported_records) == 1 + assert exported_records[0]['evidence_status'] == 'complete' + assert exported_records[0]['result_count'] == 0 + assert exported_records[0]['source_executions'][0]['status'] == 'completed' + + +def test_harvestview_imports_completed_runs_from_sqlite( + harvestview_server_url: str, + page: Page, + tmp_path: Path, +) -> None: + from theHarvester.lib.completed_result import CompletedResult + from theHarvester.lib.database import ResultStore, dispose_sqlite_databases + + database = tmp_path / 'completed-runs.sqlite' + now = datetime.now(UTC) + completed = CompletedResult.finish( + target='sqlite.example.test', + started_at=now, + completed_at=now, + groups={'hostname': ['api.sqlite.example.test']}, + ) + + async def prepare_database() -> None: + store = ResultStore(database) + await store.initialize() + await store.save_run(completed) + await dispose_sqlite_databases() + + with ThreadPoolExecutor(max_workers=1) as executor: + executor.submit(lambda: asyncio.run(prepare_database())).result() + + page.goto(f'{harvestview_server_url}/') + page.get_by_role('button', name='Import result file').first.click() + page.locator('#result-file').set_input_files(database) + page.locator('#submit-import-button').click() + + expect(page.locator('#detail-target')).to_have_text('sqlite.example.test') + expect(page.locator('#run-count')).to_have_text('1') + expect(page.locator('#toast')).to_have_text('Imported 1 run from completed-runs.sqlite; 0 already present.') + expect(page.get_by_role('button', name='Hostnames 1')).to_be_enabled() + + +def test_harvestview_can_import_and_analyze_fixture_evidence_through_the_real_ui( + harvestview_server_url: str, + page: Page, + tmp_path: Path, +) -> None: + page.set_viewport_size({'width': 1440, 'height': 900}) + page.context.grant_permissions(['clipboard-read', 'clipboard-write'], origin=harvestview_server_url) + page.goto(f'{harvestview_server_url}/') + catalog = page.context.request.get(f'{harvestview_server_url}/api/v1/sources').json() + passive_sources = [source for source in catalog['sources'] if source['activity'] == 'P0'] + credentialed = [source for source in passive_sources if source['credentials']] + uncredentialed = [source for source in passive_sources if not source['credentials']] + ordered_sources = uncredentialed[:23] + credentialed[:2] + uncredentialed[23:] + credentialed[2:] + assert len(ordered_sources) == len(passive_sources) + assert ordered_sources[23]['credentials'] + assert ordered_sources[24]['credentials'] + + evidence = { + 'run_id': '7bb74ee1-c81c-4ccd-8ec7-e8e496490f53', + 'target': 'example.com', + 'started_at': '2026-08-04T12:00:01+00:00', + 'completed_at': '2026-08-04T12:03:21+00:00', + 'status': 'partial', + 'source_executions': [ + { + 'source': source['name'], + 'status': 'completed' if index < 23 else 'skipped', + 'result_count': 1 if index < 3 else 0, + 'duration_ms': index + 1, + **({'error_type': 'SourceDidNotStart'} if index >= 23 else {}), + **({'stop_reason': 'missing-credentials'} if index == 23 else {}), + } + for index, source in enumerate(ordered_sources) + ], + 'results': [ + { + 'type': result_type, + 'value': '192.0.2.10' if result_type == 'ip' else f'{result_type}.example.com', + 'sources': [ordered_sources[index]['name']] if index < 3 else [], + } + for index, result_type in enumerate(('hostname', 'ip', 'asn', 'email', 'url', 'framework', 'person', 'language')) + ] + + [ + { + 'type': 'ip', + 'value': '198.51.100.10', + } + ], + } + evidence_file = tmp_path / 'broad-run.jsonl' + write_jsonl_evidence(evidence_file, evidence) + + page.get_by_role('button', name='Import result file').first.click() + page.locator('#result-file').set_input_files(evidence_file) + page.locator('#submit-import-button').click() + + expect(page.locator('#provider-outcome-summary')).to_have_text( + f'23 completed (20 zero-result) / 0 partial / {len(ordered_sources) - 23} skipped / 0 failed' + ) + expect(page.locator('#run-count')).to_have_text('1') + expect(page.locator('#run-list button')).to_have_count(1) + assert page.locator('#results-title').evaluate( + "node => Boolean(node.compareDocumentPosition(document.querySelector('#provider-title')) & Node.DOCUMENT_POSITION_FOLLOWING)" + ) + assert page.locator('#results-title').evaluate( + "node => Boolean(node.compareDocumentPosition(document.querySelector('#lifecycle-title')) & Node.DOCUMENT_POSITION_FOLLOWING)" + ) + assert page.locator('#results-title').evaluate( + "node => Boolean(node.compareDocumentPosition(document.querySelector('#run-facts')) & Node.DOCUMENT_POSITION_FOLLOWING)" + ) + + page.locator('.provider-details summary').click() + missing_credentials_row = page.locator('#provider-body tr').filter(has_text=ordered_sources[23]['name']) + expect(missing_credentials_row.locator('td').last).to_have_text( + 'Required credentials were not configured; add them, then retry.' + ) + unexpected_skip_reason = page.locator('#provider-body tr').filter(has_text=ordered_sources[24]['name']).locator('td').last + expect(unexpected_skip_reason).to_contain_text('Credentials required:') + expect(unexpected_skip_reason).to_contain_text('Source did not start') + + page.locator('#history-search').fill('not-present') + expect(page.locator('#history-empty')).to_be_visible() + page.locator('#history-search').fill('') + page.get_by_role('button', name='IP addresses 2').click() + expect(page.locator('#route-count')).to_have_text('2') + value_column = page.locator('.tabulator-col[tabulator-field="value"]') + value_filter = value_column.locator('.tabulator-header-filter input') + dns_filter = page.locator('.tabulator-col[tabulator-field="dns_status"] .tabulator-header-filter input') + expect(value_filter).to_have_attribute('placeholder', 'Filter values') + expect(dns_filter).to_have_attribute('placeholder', 'Filter DNS') + + value_filter.press_sequentially('198.51') + expect(page.locator('.tabulator-row:visible')).to_have_count(1) + expect(page.locator('.tabulator-row:visible')).to_contain_text('198.51.100.10') + value_filter.press('ControlOrMeta+a') + value_filter.press('Backspace') + expect(page.locator('.tabulator-row:visible')).to_have_count(2) + dns_filter.press_sequentially('not captured') + expect(page.locator('.tabulator-row:visible')).to_have_count(2) + dns_filter.press('ControlOrMeta+a') + dns_filter.press('Backspace') + expect(page.locator('.tabulator-row:visible')).to_have_count(2) + + resize_handles = page.locator('.tabulator-header .tabulator-col-resize-handle') + expect(resize_handles).to_have_count(2) + selection_column_width = page.locator('.tabulator-header .tabulator-row-header').bounding_box()['width'] + assert 40 <= selection_column_width <= 56 + resize_handle = resize_handles.first + initial_width = value_column.bounding_box()['width'] + handle_box = resize_handle.bounding_box() + page.mouse.move(handle_box['x'] + handle_box['width'] / 2, handle_box['y'] + handle_box['height'] / 2) + page.mouse.down() + page.mouse.move(handle_box['x'] + handle_box['width'] / 2 + 80, handle_box['y'] + handle_box['height'] / 2) + page.mouse.up() + assert value_column.bounding_box()['width'] >= initial_width + 60 + + page.locator('.tabulator-row').first.click() + expect(page.locator('#copy-route-button')).to_have_text('Copy selected (1)') + page.locator('#copy-route-button').click() + expect(page.locator('#toast')).to_have_text('Copied 1 selected IP addresses.') + assert page.evaluate('navigator.clipboard.readText()') == '192.0.2.10' + + with page.expect_download() as jsonl_download: + page.get_by_role('button', name='All JSONL').click() + exported_records = [json.loads(line) for line in Path(jsonl_download.value.path()).read_text(encoding='utf-8').splitlines()] + assert exported_records[0]['type'] == 'summary' + assert exported_records[0]['evidence_status'] == 'partial' + assert any(record['type'] == 'ip' and record['value'] == '192.0.2.10' for record in exported_records[1:]) + + page.get_by_role('button', name='Start enumeration').first.click() + expect(page.locator('#new-run-dialog').get_by_text('Credentials required:', exact=False).first).to_be_visible() + page.get_by_role('button', name='Select all P0').click() + assert page.locator('[data-activity="P0"] input:checked').count() == len(passive_sources) + assert page.locator('[data-activity="P1"] input:checked, [data-activity="P2"] input:checked').count() == 0 + assert page.locator('.action-grid input:checked').count() == 0 + page.get_by_role('button', name='Clear', exact=True).click() + assert page.locator('[data-activity="P0"] input:checked').count() == 0 + expect(page.locator('#activity-summary')).to_have_text('P0 off · P1 off · P2 off') + + page.keyboard.press('Escape') + expect(page.get_by_role('button', name='Start enumeration').first).to_be_focused() + page.set_viewport_size({'width': 390, 'height': 844}) + expect(page.locator('#provider-outcome-summary')).to_be_visible() + expect(page.get_by_role('columnheader', name='Outcome')).to_be_visible() + expect(page.get_by_role('columnheader', name='Results')).to_be_hidden() + expect(page.locator('#route-overflow-cue')).to_be_visible() + value_filter.scroll_into_view_if_needed() + expect(value_filter).to_be_visible() + assert value_filter.bounding_box()['height'] >= 44 + assert page.locator('#route-tabs').evaluate('node => node.scrollWidth > node.clientWidth') diff --git a/tests/lib/test_api_v1.py b/tests/lib/test_api_v1.py index 55d9fcf3..f3d93c34 100644 --- a/tests/lib/test_api_v1.py +++ b/tests/lib/test_api_v1.py @@ -81,7 +81,6 @@ def test_api_exposes_one_fresh_run_contract(tmp_path, monkeypatch) -> None: schema = client.get('/openapi.json').json() paths = set(schema['paths']) old_responses = [ - client.get('/'), client.get('/query?domain=example.test&source=crtsh'), client.get('/sources'), client.get('/dnsbrute?domain=example.test'), diff --git a/tests/lib/test_harvestview_ui.py b/tests/lib/test_harvestview_ui.py new file mode 100644 index 00000000..26ce1293 --- /dev/null +++ b/tests/lib/test_harvestview_ui.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +import ipaddress + +from fastapi.testclient import TestClient + + +def test_harvestview_owns_root_and_issues_an_http_only_session(tmp_path, monkeypatch) -> None: + from theHarvester.lib.api import api + from theHarvester.lib.resolver_selection import DEFAULT_DNS_RESOLVERS + + monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key') + monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite')) + monkeypatch.setenv('THEHARVESTER_RUN_WORKER', 'disabled') + + with TestClient(api.app, base_url='http://127.0.0.1', client=('127.0.0.1', 50000)) as client: + root = client.get('/') + legacy = client.get('/app') + runs = client.get('/api/v1/runs') + + assert root.status_code == 200 + assert 'HarvestView' in root.text + assert f'value="{",".join(DEFAULT_DNS_RESOLVERS)}"' in root.text + assert 'Resolve with the configured resolver addresses.' in root.text + assert legacy.status_code == 404 + cookie = root.headers['set-cookie'] + assert 'theharvester-api-key=' in cookie + assert 'test-key' not in cookie + assert 'HttpOnly' in cookie + assert 'SameSite=strict' in cookie + assert 'Path=/api/v1' in cookie + assert runs.status_code == 200 + assert runs.json() == [] + + +def test_harvestview_assets_load_outside_the_repository_directory(tmp_path, monkeypatch) -> None: + from theHarvester.lib.api import api + + monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key') + monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite')) + monkeypatch.setenv('THEHARVESTER_RUN_WORKER', 'disabled') + monkeypatch.chdir(tmp_path) + + with TestClient(api.app, base_url='http://127.0.0.1', client=('127.0.0.1', 50000)) as client: + response = client.get('/static/harvestview/app.js') + + assert response.status_code == 200 + assert 'function renderResults' in response.text + + +def test_harvestview_offers_jsonl_and_sqlite_imports_with_jsonl_export(tmp_path, monkeypatch) -> None: + from theHarvester.lib.api import api + + monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key') + monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite')) + monkeypatch.setenv('THEHARVESTER_RUN_WORKER', 'disabled') + + with TestClient(api.app, base_url='http://127.0.0.1', client=('127.0.0.1', 50000)) as client: + root = client.get('/') + script = client.get('/static/harvestview/app.js') + + assert 'accept=".jsonl,.sqlite,.sqlite3,.db,application/x-ndjson,application/vnd.sqlite3"' in root.text + assert 'id="export-jsonl-button"' in root.text + assert 'id="route-csv-button"' not in root.text + assert 'id="export-json-button"' not in root.text + assert 'id="export-csv-button"' not in root.text + assert '/export' in script.text + assert "fileKind === 'jsonl' ? '/api/v1/runs/import' : '/api/v1/runs/import-database'" in script.text + assert '/exports/' not in script.text + assert 'text/csv' not in script.text + assert 'versioned JSONL' not in root.text + assert 'interesting-url' not in script.text + assert 'api-endpoint' not in script.text + assert 'linkedin-link' not in script.text + + +def test_harvestview_loads_pinned_tabulator_from_cdnjs(tmp_path, monkeypatch) -> None: + from theHarvester.lib.api import api + + monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key') + monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite')) + monkeypatch.setenv('THEHARVESTER_RUN_WORKER', 'disabled') + + with TestClient(api.app, base_url='http://127.0.0.1', client=('127.0.0.1', 50000)) as client: + root = client.get('/') + theme = client.get('/static/harvestview/tabulator.min.css') + script = client.get('/static/harvestview/tabulator.min.js') + license_file = client.get('/static/harvestview/TABULATOR-LICENSE') + bootstrap = client.get('/static/harvestview/bootstrap.min.css') + old_theme = client.get('/static/harvestview/tabulator_bootstrap5.min.css') + + assert root.status_code == 200 + assert 'bootstrap.min.css' not in root.text + assert 'tabulator_bootstrap5.min.css' not in root.text + assert ( + '' + ) in root.text + assert ( + '' + ) in root.text + assert 'https://unpkg.com' not in root.text + assert theme.status_code == 404 + assert script.status_code == 404 + assert license_file.status_code == 404 + assert bootstrap.status_code == 404 + assert old_theme.status_code == 404 + + +def test_docker_mode_trusts_only_the_detected_gateway(tmp_path, monkeypatch) -> None: + from theHarvester.lib.api import api, harvestview + + monkeypatch.setenv('THEHARVESTER_API_KEY', 'test-key') + monkeypatch.setenv('THEHARVESTER_RUN_DB', str(tmp_path / 'runs.sqlite')) + monkeypatch.setenv('THEHARVESTER_RUN_WORKER', 'disabled') + monkeypatch.setattr(harvestview, '_docker_gateway', lambda: ipaddress.ip_address('172.18.0.1')) + + with TestClient(api.app, base_url='http://127.0.0.1', client=('172.18.0.1', 50000)) as client: + disabled = client.get('/') + + monkeypatch.setenv('THEHARVESTER_HARVESTVIEW_LOCAL_PROXY', 'enabled') + with TestClient(api.app, base_url='http://127.0.0.1', client=('172.18.0.1', 50000)) as client: + gateway = client.get('/') + with TestClient(api.app, base_url='http://127.0.0.1', client=('172.18.0.2', 50000)) as client: + sibling = client.get('/') + with TestClient(api.app, base_url='http://attacker.example', client=('172.18.0.1', 50000)) as client: + rebound = client.get('/') + + assert disabled.status_code == 403 + assert gateway.status_code == 200 + assert sibling.status_code == 403 + assert rebound.status_code == 403 diff --git a/tests/test_workflow_contract.py b/tests/test_workflow_contract.py index e3596bb7..dcfd5efa 100644 --- a/tests/test_workflow_contract.py +++ b/tests/test_workflow_contract.py @@ -8,6 +8,7 @@ import yaml WORKFLOW_DIR = Path(__file__).parents[1] / '.github' / 'workflows' CI_WORKFLOW_PATH = WORKFLOW_DIR / 'theHarvester.yml' SMOKE_WORKFLOW_PATH = WORKFLOW_DIR / 'provider-smoke.yml' +HARVESTVIEW_WORKFLOW_PATH = WORKFLOW_DIR / 'harvestview-e2e.yml' def _workflow(path: Path) -> dict[str, Any]: @@ -35,3 +36,15 @@ def test_live_provider_smoke_requires_manual_dispatch() -> None: assert workflow['permissions'] == {'contents': 'read'} assert smoke_job['env']['SMOKE_TEST_DOMAIN'] == 'mozilla.org' assert 'pytest --run-live-network -m live_network' in commands + + +def test_harvestview_browser_failures_keep_only_targeted_diagnostics() -> None: + workflow = _workflow(HARVESTVIEW_WORKFLOW_PATH) + steps = workflow['jobs']['harvestview-e2e']['steps'] + test_command = next(step['run'] for step in steps if step['name'] == 'Run HarvestView browser tests') + upload = next(step for step in steps if step['name'] == 'Upload browser test artifacts') + + assert '--video' not in test_command + assert '--tracing=retain-on-failure' in test_command + assert '--screenshot=only-on-failure' in test_command + assert upload['if'] == '${{ failure() }}' diff --git a/theHarvester/lib/api/api.py b/theHarvester/lib/api/api.py index e53b26d7..01d6d39a 100644 --- a/theHarvester/lib/api/api.py +++ b/theHarvester/lib/api/api.py @@ -8,6 +8,7 @@ from fastapi import FastAPI from starlette.staticfiles import StaticFiles from theHarvester import __version__ +from theHarvester.lib.api.harvestview import router as harvestview_router from theHarvester.lib.api.run_worker import start_worker, stop_worker from theHarvester.lib.api.runs import router as api_router from theHarvester.lib.database import dispose_sqlite_databases @@ -36,6 +37,7 @@ app = FastAPI( redoc_url='/redoc', lifespan=lifespan, ) +app.include_router(harvestview_router) app.include_router(api_router) STATIC_DIRECTORY = Path(__file__).resolve().parent / 'static' app.mount('/static', StaticFiles(directory=STATIC_DIRECTORY), name='static') diff --git a/theHarvester/lib/api/auth.py b/theHarvester/lib/api/auth.py index 95579b47..0c83c7ec 100644 --- a/theHarvester/lib/api/auth.py +++ b/theHarvester/lib/api/auth.py @@ -1,12 +1,15 @@ +import hmac import os import secrets from pathlib import Path from typing import Annotated -from fastapi import Header, HTTPException, status +from fastapi import Cookie, Header, HTTPException, Request, status API_KEY_ENV_VAR = 'THEHARVESTER_API_KEY' API_KEY_FILE_ENV_VAR = f'{API_KEY_ENV_VAR}_FILE' +API_KEY_COOKIE_NAME = 'theharvester-api-key' +BROWSER_SESSION_CONTEXT = b'theharvester-browser-session' def _configured_api_key() -> str | None: @@ -22,8 +25,14 @@ def _configured_api_key() -> str | None: return None +def browser_session_token(configured_api_key: str) -> str: + return hmac.digest(configured_api_key.encode(), BROWSER_SESSION_CONTEXT, 'sha256').hex() + + def get_api_key( + request: Request, x_api_key: Annotated[str | None, Header(alias='X-API-Key')] = None, + api_key_cookie: Annotated[str | None, Cookie(alias=API_KEY_COOKIE_NAME)] = None, ) -> str: """Validate the API key used by protected API routes.""" configured_api_key = _configured_api_key() @@ -33,10 +42,20 @@ def get_api_key( detail=f'{API_KEY_ENV_VAR} is not configured', ) - if x_api_key is None or not secrets.compare_digest(x_api_key, configured_api_key): + valid_header = x_api_key is not None and secrets.compare_digest(x_api_key, configured_api_key) + expected_cookie = browser_session_token(configured_api_key) + valid_cookie = api_key_cookie is not None and secrets.compare_digest(api_key_cookie, expected_cookie) + if not valid_header and not valid_cookie: raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail='Invalid API key', ) + if valid_cookie and not valid_header and request.method not in {'GET', 'HEAD', 'OPTIONS'}: + expected_origin = str(request.base_url).rstrip('/') + if not secrets.compare_digest(request.headers.get('origin', ''), expected_origin): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail='Invalid request origin', + ) return configured_api_key diff --git a/theHarvester/lib/api/harvestview.py b/theHarvester/lib/api/harvestview.py new file mode 100644 index 00000000..da119ede --- /dev/null +++ b/theHarvester/lib/api/harvestview.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import ipaddress +import os +from pathlib import Path + +from fastapi import APIRouter, HTTPException, Request, status +from fastapi.responses import HTMLResponse + +from theHarvester import __version__ +from theHarvester.lib.api.auth import API_KEY_COOKIE_NAME, _configured_api_key, browser_session_token +from theHarvester.lib.resolver_selection import DEFAULT_DNS_RESOLVERS + +router = APIRouter() + + +def _docker_gateway() -> ipaddress.IPv4Address | None: + try: + for line in Path('/proc/net/route').read_text(encoding='ascii').splitlines()[1:]: + fields = line.split() + if len(fields) >= 3 and fields[1] == '00000000': + return ipaddress.IPv4Address(bytes.fromhex(fields[2])[::-1]) + except (OSError, ValueError): + pass + return None + + +@router.get('/', include_in_schema=False) +async def harvestview_app(request: Request) -> HTMLResponse: + try: + client_address = ipaddress.ip_address(request.client.host) if request.client is not None else None + trusted_gateway = ( + _docker_gateway() if os.getenv('THEHARVESTER_HARVESTVIEW_LOCAL_PROXY', '').casefold() == 'enabled' else None + ) + is_local_client = client_address is not None and (client_address.is_loopback or client_address == trusted_gateway) + except ValueError: + is_local_client = False + try: + hostname = request.url.hostname + is_loopback_host = hostname == 'localhost' or (hostname is not None and ipaddress.ip_address(hostname).is_loopback) + except ValueError: + is_loopback_host = False + if not is_local_client or not is_loopback_host: + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail='theHarvester is available only on localhost') + + static_dir = Path(__file__).parent / 'static' / 'harvestview' + template = (static_dir / 'index.html').read_text(encoding='utf-8') + asset_version = max((static_dir / name).stat().st_mtime_ns for name in ('app.css', 'app.js')) + response = HTMLResponse( + template.replace('{{VERSION}}', __version__) + .replace('{{ASSET_VERSION}}', str(asset_version)) + .replace('{{DNS_RESOLVERS}}', ','.join(DEFAULT_DNS_RESOLVERS)) + ) + if configured_api_key := _configured_api_key(): + response.set_cookie( + API_KEY_COOKIE_NAME, + browser_session_token(configured_api_key), + httponly=True, + samesite='strict', + path='/api/v1', + ) + return response diff --git a/theHarvester/lib/api/static/harvestview/app.css b/theHarvester/lib/api/static/harvestview/app.css new file mode 100644 index 00000000..96649541 --- /dev/null +++ b/theHarvester/lib/api/static/harvestview/app.css @@ -0,0 +1,773 @@ +:root { + color-scheme: light; + --bg: oklch(0.955 0.012 78); + --surface: oklch(0.985 0.008 78); + --surface-raised: oklch(1 0 0); + --surface-muted: oklch(0.93 0.015 78); + --ink: oklch(0.22 0.022 220); + --ink-soft: oklch(0.38 0.025 220); + --muted: oklch(0.47 0.02 220); + --line: oklch(0.82 0.018 80); + --line-strong: oklch(0.68 0.025 220); + --nav: oklch(0.205 0.028 220); + --nav-raised: oklch(0.245 0.03 220); + --nav-line: oklch(0.37 0.028 220); + --nav-ink: oklch(0.95 0.012 82); + --nav-muted: oklch(0.75 0.023 210); + --accent: oklch(0.6 0.115 182); + --accent-strong: oklch(0.4 0.105 182); + --accent-text: oklch(0.4 0.105 182); + --accent-soft: oklch(0.91 0.052 182); + --focus: oklch(0.58 0.14 182); + --info: oklch(0.43 0.12 245); + --info-soft: oklch(0.92 0.04 245); + --warning: oklch(0.42 0.12 74); + --warning-soft: oklch(0.93 0.06 80); + --danger: oklch(0.47 0.18 28); + --danger-soft: oklch(0.93 0.045 28); + --success: oklch(0.4 0.12 155); + --success-soft: oklch(0.92 0.045 155); + --shadow: 0 18px 60px oklch(0.18 0.02 220 / 0.16); + --font-sans: Inter, ui-sans-serif, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; + --font-mono: "SFMono-Regular", Consolas, "Liberation Mono", monospace; + --z-sticky: 20; + --z-modal: 50; + --z-toast: 70; + --z-tooltip: 80; +} + +html[data-theme="dark"] { + color-scheme: dark; + --bg: oklch(0.17 0.018 220); + --surface: oklch(0.205 0.02 220); + --surface-raised: oklch(0.245 0.022 220); + --surface-muted: oklch(0.275 0.022 220); + --ink: oklch(0.94 0.012 80); + --ink-soft: oklch(0.81 0.015 80); + --muted: oklch(0.7 0.02 210); + --line: oklch(0.36 0.024 220); + --line-strong: oklch(0.49 0.03 210); + --nav: oklch(0.135 0.023 220); + --nav-raised: oklch(0.18 0.026 220); + --nav-line: oklch(0.32 0.027 220); + --nav-ink: oklch(0.95 0.012 82); + --nav-muted: oklch(0.72 0.023 210); + --accent: oklch(0.72 0.12 182); + --accent-strong: oklch(0.4 0.105 182); + --accent-text: oklch(0.82 0.1 182); + --accent-soft: oklch(0.28 0.055 182); + --focus: oklch(0.78 0.13 182); + --info: oklch(0.72 0.12 245); + --info-soft: oklch(0.28 0.055 245); + --warning: oklch(0.78 0.13 78); + --warning-soft: oklch(0.3 0.06 78); + --danger: oklch(0.73 0.16 28); + --danger-soft: oklch(0.29 0.065 28); + --success: oklch(0.72 0.13 155); + --success-soft: oklch(0.28 0.055 155); + --shadow: 0 22px 72px oklch(0.05 0.02 220 / 0.42); +} + +@media (prefers-color-scheme: dark) { + html[data-theme="system"] { + color-scheme: dark; + --bg: oklch(0.17 0.018 220); + --surface: oklch(0.205 0.02 220); + --surface-raised: oklch(0.245 0.022 220); + --surface-muted: oklch(0.275 0.022 220); + --ink: oklch(0.94 0.012 80); + --ink-soft: oklch(0.81 0.015 80); + --muted: oklch(0.7 0.02 210); + --line: oklch(0.36 0.024 220); + --line-strong: oklch(0.49 0.03 210); + --nav: oklch(0.135 0.023 220); + --nav-raised: oklch(0.18 0.026 220); + --nav-line: oklch(0.32 0.027 220); + --accent: oklch(0.72 0.12 182); + --accent-strong: oklch(0.4 0.105 182); + --accent-text: oklch(0.82 0.1 182); + --accent-soft: oklch(0.28 0.055 182); + --focus: oklch(0.78 0.13 182); + --info: oklch(0.72 0.12 245); + --info-soft: oklch(0.28 0.055 245); + --warning: oklch(0.78 0.13 78); + --warning-soft: oklch(0.3 0.06 78); + --danger: oklch(0.73 0.16 28); + --danger-soft: oklch(0.29 0.065 28); + --success: oklch(0.72 0.13 155); + --success-soft: oklch(0.28 0.055 155); + --shadow: 0 22px 72px oklch(0.05 0.02 220 / 0.42); + } +} + +* { box-sizing: border-box; } + +html { min-width: 320px; background: var(--nav); } + +h1, +h2, +h3, +p { margin-block-start: 0; } + +fieldset { + min-width: 0; + margin: 0; + padding: 0; + border: 0; +} + +table { border-collapse: collapse; } + +img { + max-width: 100%; + vertical-align: middle; +} + +body { + min-height: 100vh; + margin: 0; + color: var(--ink); + background: var(--bg); + font-family: var(--font-sans); + font-size: 15px; + line-height: 1.5; + text-rendering: optimizeLegibility; +} + +button, +input, +select, +textarea { font: inherit; } + +button { touch-action: manipulation; } + +button:focus-visible, +input:focus-visible, +select:focus-visible, +summary:focus-visible, +a:focus-visible { + outline: 3px solid var(--accent); + outline-offset: 3px; +} + +[hidden] { display: none !important; } + +.sr-only { + position: absolute !important; + width: 1px !important; + height: 1px !important; + padding: 0 !important; + margin: -1px !important; + overflow: hidden !important; + clip: rect(0, 0, 0, 0) !important; + white-space: nowrap !important; + border: 0 !important; +} + +.skip-link { + position: fixed; + z-index: calc(var(--z-toast) + 1); + inset: 12px auto auto 12px; + padding: 10px 14px; + color: var(--nav); + background: var(--accent); + border-radius: 6px; + transform: translateY(-160%); +} + +.skip-link:focus { transform: translateY(0); } + +.mono { font-family: var(--font-mono); } + +.eyebrow { + margin: 0 0 5px; + color: var(--muted); + font: 700 11px/1.2 var(--font-mono); + letter-spacing: 0.1em; + text-transform: uppercase; +} + +.app-header { + position: sticky; + z-index: var(--z-sticky); + inset-block-start: 0; + display: flex; + min-height: 72px; + align-items: center; + gap: 28px; + padding: 10px 22px; + color: var(--nav-ink); + background: var(--nav); + border-block-end: 1px solid var(--nav-line); +} + +.brand { + display: inline-flex; + min-width: 265px; + align-items: center; + gap: 12px; + color: inherit; + text-decoration: none; +} + +.brand img { width: 44px; height: 44px; object-fit: contain; } +.brand-copy { display: grid; gap: 1px; } +.brand strong { font-size: 17px; line-height: 1.1; letter-spacing: -0.015em; } +.brand small { color: var(--nav-muted); font: 11px/1.3 var(--font-mono); } +.version-badge { + padding: 4px 7px; + color: var(--nav-muted); + border: 1px solid var(--nav-line); + border-radius: 999px; + font: 700 10px/1 var(--font-mono); + white-space: nowrap; +} + +.header-context { + display: flex; + align-items: center; + gap: 9px; + color: var(--nav-muted); + font-size: 12px; +} + +.signal-dot { + width: 8px; + height: 8px; + background: var(--accent); + border-radius: 50%; + box-shadow: 0 0 0 4px oklch(0.72 0.12 182 / 0.14); +} + +.header-actions { + display: flex; + align-items: center; + gap: 8px; + margin-inline-start: auto; +} + +.header-button, +.icon-button, +.button { + min-height: 44px; + padding: 8px 13px; + color: var(--ink); + background: var(--surface-raised); + border: 1px solid var(--line-strong); + border-radius: 7px; + font-weight: 700; + transition: border-color 140ms ease-out, background-color 140ms ease-out, color 140ms ease-out, transform 140ms ease-out; +} + +.header-button { + color: var(--nav-ink); + background: var(--nav-raised); + border-color: var(--nav-line); + font-size: 13px; +} + +.header-button:hover, +.icon-button:hover { border-color: var(--accent); } +.header-button.primary, +.button.primary { color: oklch(0.15 0.025 220); background: var(--accent); border-color: var(--accent); } +.header-button.primary:hover, +.button.primary:hover { background: var(--accent-strong); color: white; } +.button:hover { border-color: var(--accent-strong); } +.button:active, +.header-button:active { transform: translateY(1px); } +.button.small { min-height: 44px; padding: 8px 10px; font-size: 12px; } +.button.danger { color: var(--danger); background: var(--danger-soft); border-color: var(--danger); } +.button:disabled { cursor: wait; opacity: 0.58; } + +.icon-button { + display: inline-grid; + width: 44px; + padding: 0; + place-items: center; + color: var(--nav-ink); + background: var(--nav-raised); + border-color: var(--nav-line); + font-size: 22px; +} + +.app-shell { + display: grid; + min-height: calc(100vh - 72px); + grid-template-columns: minmax(285px, 335px) minmax(0, 1fr); +} + +.history-panel { + position: sticky; + inset-block-start: 72px; + min-width: 0; + height: calc(100vh - 72px); + padding: 24px 17px 36px; + overflow-y: auto; + color: var(--nav-ink); + background: var(--nav); + border-inline-end: 1px solid var(--nav-line); +} + +.history-panel .eyebrow { color: var(--nav-muted); } +.history-heading { display: flex; align-items: end; justify-content: space-between; gap: 14px; margin: 0 5px 17px; } +.history-heading h1 { margin: 0; font-size: 20px; line-height: 1.15; letter-spacing: -0.025em; } + +.count-badge { + display: inline-flex; + min-width: 28px; + min-height: 24px; + align-items: center; + justify-content: center; + padding: 3px 8px; + color: var(--nav); + background: var(--accent); + border-radius: 999px; + font: 700 11px/1 var(--font-mono); +} + +.count-badge.quiet { color: var(--ink); background: var(--surface-muted); } + +.search-field { + display: flex; + min-height: 46px; + align-items: center; + gap: 8px; + padding: 0 11px; + color: var(--nav-muted); + background: var(--nav-raised); + border: 1px solid var(--nav-line); + border-radius: 8px; +} + +.search-field:focus-within { border-color: var(--accent); } +.search-field input { width: 100%; min-width: 0; min-height: 44px; color: var(--nav-ink); background: transparent; border: 0; outline: 0; } +.search-field input::placeholder { color: var(--nav-muted); opacity: 1; } + +.run-list { display: grid; gap: 7px; margin-block-start: 14px; } + +.run-item { + position: relative; + display: grid; + width: 100%; + min-height: 78px; + grid-template-columns: minmax(0, 1fr) auto; + gap: 5px 10px; + padding: 13px 12px 12px 16px; + overflow: hidden; + color: var(--nav-ink); + text-align: start; + background: transparent; + border: 1px solid transparent; + border-radius: 8px; +} + +.run-item::before { content: ""; position: absolute; inset: 10px auto 10px 0; width: 3px; background: var(--nav-line); border-radius: 0 3px 3px 0; } +.run-item:hover { background: var(--nav-raised); border-color: var(--nav-line); } +.run-item.selected { background: var(--nav-raised); border-color: var(--accent); } +.run-item.selected::before { background: var(--accent); } +.run-target { min-width: 0; overflow: hidden; font-weight: 760; text-overflow: ellipsis; white-space: nowrap; } +.run-meta { min-width: 0; overflow: hidden; color: var(--nav-muted); font: 11px/1.4 var(--font-mono); text-overflow: ellipsis; white-space: nowrap; } +.run-results { align-self: end; color: var(--nav-muted); font: 11px/1 var(--font-mono); } + +.history-empty { padding: 42px 10px; color: var(--nav-muted); text-align: center; } +.history-empty span { display: block; font-size: 26px; } + +.workbench { min-width: 0; padding: clamp(22px, 3.2vw, 48px); background: var(--bg); } + +.center-state { + display: grid; + min-height: min(620px, calc(100vh - 150px)); + max-width: 690px; + align-content: center; + justify-items: center; + margin: auto; + text-align: center; +} + +.center-state h2 { max-width: 15ch; margin: 8px 0 9px; font-size: clamp(28px, 4vw, 46px); line-height: 1.02; letter-spacing: -0.035em; text-wrap: balance; } +.center-state > p:not(.eyebrow) { max-width: 62ch; color: var(--ink-soft); text-wrap: pretty; } +.loader { width: 34px; height: 34px; border: 3px solid var(--line); border-block-start-color: var(--accent); border-radius: 50%; animation: spin 700ms linear infinite; } +@keyframes spin { to { transform: rotate(360deg); } } + +.empty-mark { display: grid; width: 74px; height: 74px; margin-block-end: 20px; place-items: center; color: var(--accent-text); border: 1px solid var(--accent); border-radius: 50%; font: 800 30px/1 var(--font-mono); } +.empty-actions { display: flex; flex-wrap: wrap; justify-content: center; gap: 9px; margin-block: 18px 32px; } +.empty-guardrails { display: grid; width: 100%; grid-template-columns: repeat(3, 1fr); margin: 0; border-block: 1px solid var(--line); } +.empty-guardrails div { padding: 16px; text-align: start; border-inline-end: 1px solid var(--line); } +.empty-guardrails div:last-child { border: 0; } +.empty-guardrails dt { color: var(--accent-text); font: 800 12px/1 var(--font-mono); } +.empty-guardrails dd { margin: 6px 0 0; color: var(--muted); font-size: 12px; } + +.run-detail { width: min(1500px, 100%); margin-inline: auto; } +.detail-header { display: flex; align-items: start; justify-content: space-between; gap: 24px; padding-block-end: 25px; border-block-end: 1px solid var(--line-strong); } +.detail-identity { min-width: 0; } +.detail-identity h2 { margin: 2px 0 5px; overflow-wrap: anywhere; font-size: clamp(31px, 4vw, 58px); line-height: 1; letter-spacing: -0.038em; text-wrap: balance; } +.detail-run-id { margin: 0; overflow-wrap: anywhere; color: var(--muted); font-size: 11px; } +.detail-status { display: flex; flex: 0 0 auto; flex-direction: column; align-items: end; gap: 12px; } +.status-chips { display: flex; flex-wrap: wrap; justify-content: end; gap: 6px; } +.provider-outcome-summary { margin: -4px 0 0; color: var(--muted); font: 11px/1.4 var(--font-mono); } + +.status-chip { + display: inline-flex; + min-height: 28px; + align-items: center; + gap: 6px; + padding: 4px 9px; + color: var(--ink-soft); + background: var(--surface-muted); + border: 1px solid var(--line); + border-radius: 999px; + font: 750 11px/1 var(--font-mono); + text-transform: capitalize; +} + +.status-chip::before { content: ""; width: 7px; height: 7px; background: currentColor; border-radius: 50%; } +.status-chip.completed, .status-chip.complete, .status-chip.succeeded { color: var(--success); background: var(--success-soft); } +.status-chip.running { color: var(--info); background: var(--info-soft); } +.status-chip.queued, .status-chip.cancelling, .status-chip.partial, .status-chip.rate-limited { color: var(--warning); background: var(--warning-soft); } +.status-chip.failed, .status-chip.cancelled { color: var(--danger); background: var(--danger-soft); } + +.run-facts { display: grid; grid-template-columns: repeat(5, minmax(120px, 1fr)); margin: 0; border-block-end: 1px solid var(--line); } +.run-facts div { min-width: 0; padding: 17px 15px; border-inline-end: 1px solid var(--line); } +.run-facts div:first-child { padding-inline-start: 0; } +.run-facts div:last-child { border: 0; } +.run-facts dt { color: var(--muted); font: 700 10px/1.2 var(--font-mono); letter-spacing: 0.08em; text-transform: uppercase; } +.run-facts dd { margin: 6px 0 0; overflow-wrap: anywhere; font-weight: 720; } + +.section-heading { display: flex; align-items: end; justify-content: space-between; gap: 18px; margin-block-end: 16px; } +.section-heading.compact { align-items: start; margin-block-end: 13px; } +.section-heading h3 { margin: 0; font-size: 22px; line-height: 1.1; letter-spacing: -0.025em; } +.heading-with-help { display: flex; align-items: center; gap: 7px; } +.help-tip { + position: relative; + display: inline-grid; + width: 45px; + height: 45px; + flex: 0 0 45px; + padding: 0; + place-items: center; + color: var(--accent-text); + background: var(--surface-muted); + border: 1px solid var(--line-strong); + border-radius: 50%; + font: 800 12px/1 var(--font-mono); +} +.help-tip:hover, .help-tip:focus { color: var(--nav); background: var(--accent); border-color: var(--accent); } +.help-tip::after { + position: absolute; + z-index: var(--z-tooltip); + inset: calc(100% + 8px) auto auto 0; + width: min(300px, calc(100vw - 40px)); + padding: 9px 11px; + visibility: hidden; + color: var(--nav-ink); + text-align: start; + content: attr(data-tooltip); + background: var(--nav); + border: 1px solid var(--nav-line); + border-radius: 7px; + box-shadow: 0 4px 8px oklch(0.08 0.02 220 / 0.22); + opacity: 0; + font: 12px/1.45 var(--font-sans); + pointer-events: none; + transform: translateY(-4px); + transition: opacity 150ms ease, transform 150ms ease, visibility 150ms; +} +.help-tip:hover::after, .help-tip:focus::after { visibility: visible; opacity: 1; transform: translateY(0); } +.section-note { max-width: 65ch; margin: 4px 0 0; color: var(--muted); font-size: 12px; text-wrap: pretty; } +.section-actions { display: flex; flex-wrap: wrap; justify-content: end; gap: 6px; } + +.lifecycle-section { padding-block: 24px 20px; } +.lifecycle-track { display: grid; grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); margin: 0; padding: 0; list-style: none; } +.lifecycle-step { position: relative; min-width: 0; padding: 20px 14px 0 0; border-block-start: 2px solid var(--line); } +.lifecycle-step::before { content: ""; position: absolute; inset: -6px auto auto 0; width: 10px; height: 10px; background: var(--surface-raised); border: 2px solid var(--line-strong); border-radius: 50%; } +.lifecycle-step.reached { border-color: var(--accent); } +.lifecycle-step.reached::before { background: var(--accent); border-color: var(--accent); } +.lifecycle-step strong { display: block; overflow: hidden; font: 750 11px/1.3 var(--font-mono); text-overflow: ellipsis; text-transform: uppercase; white-space: nowrap; } +.lifecycle-step span { display: block; margin-block-start: 5px; color: var(--muted); font-size: 11px; } + +.overview-grid { display: grid; grid-template-columns: 1fr; border: 1px solid var(--line-strong); } +.overview-panel { min-width: 0; padding: 20px; background: var(--surface); } + +.activity-bands { display: grid; grid-template-columns: repeat(3, 1fr); gap: 7px; margin-block-end: 18px; } +.activity-band { min-width: 0; padding: 10px; color: var(--muted); background: var(--surface-muted); border-block-start: 3px solid var(--line-strong); } +.activity-band.active.p0 { color: var(--accent-text); border-color: var(--accent); background: var(--accent-soft); } +.activity-band.active.p1 { color: var(--warning); border-color: var(--warning); background: var(--warning-soft); } +.activity-band.active.p2 { color: var(--danger); border-color: var(--danger); background: var(--danger-soft); } +.activity-band strong { display: block; font: 800 11px/1 var(--font-mono); } +.activity-band span { display: block; margin-block-start: 5px; font-size: 11px; } + +.request-options { display: grid; grid-template-columns: 1fr 1fr; margin: 0; } +.request-options div { min-width: 0; padding: 8px 10px 8px 0; border-block-start: 1px solid var(--line); } +.request-options dt { color: var(--muted); font-size: 11px; } +.request-options dd { margin: 2px 0 0; overflow-wrap: anywhere; font: 12px/1.4 var(--font-mono); } + +.provider-table-wrap { max-height: 264px; overflow: auto; } +.provider-details { margin-block-start: 28px; padding: 0 20px 20px; background: var(--surface); border: 1px solid var(--line-strong); } +.provider-details summary { display: flex; min-height: 58px; align-items: center; justify-content: space-between; gap: 12px; cursor: pointer; font-size: 18px; font-weight: 760; } +.provider-details summary::before { content: "›"; color: var(--accent-text); font-size: 24px; } +.provider-details[open] summary::before { transform: rotate(90deg); } +.provider-details summary #provider-title { margin-inline-end: auto; } +.provider-table { width: 100%; border-collapse: collapse; } +.provider-table th, .provider-table td { padding: 9px 8px; text-align: start; border-block-end: 1px solid var(--line); } +.provider-table th { position: sticky; inset-block-start: 0; z-index: 1; color: var(--muted); background: var(--surface); font: 700 10px/1.2 var(--font-mono); letter-spacing: 0.06em; text-transform: uppercase; } +.provider-table td:first-child { font-weight: 720; } +.provider-table td:nth-child(4), .provider-table th:nth-child(4), .provider-table td:nth-child(5), .provider-table th:nth-child(5) { text-align: end; } +.panel-empty { margin: 25px 0; color: var(--muted); text-align: center; } + +.results-section { margin-block-start: 36px; } +.results-heading { align-items: start; } +.route-tabs { display: flex; overflow-x: auto; border-block-end: 1px solid var(--line-strong); scrollbar-width: thin; } +.route-overflow-cue { display: none; margin: 7px 0 0; color: var(--muted); font-size: 12px; } +.route-tab { display: inline-flex; min-height: 46px; flex: 0 0 auto; align-items: center; gap: 8px; padding: 8px 13px; color: var(--muted); background: transparent; border: 0; border-block-end: 3px solid transparent; font-weight: 720; } +.route-tab:hover { color: var(--ink); background: var(--surface-muted); } +.route-tab.active { color: var(--accent-text); border-color: var(--accent); } +.route-tab .count-badge { min-width: 23px; min-height: 21px; color: var(--ink); background: var(--surface-muted); } + +.result-workbench { margin-block-start: 14px; border: 1px solid var(--line-strong); background: var(--surface); } +.table-toolbar { display: flex; min-height: 60px; align-items: center; justify-content: space-between; gap: 16px; padding: 9px 12px; border-block-end: 1px solid var(--line); } +.table-toolbar label { display: flex; align-items: center; gap: 9px; color: var(--muted); font-size: 12px; } +.table-toolbar input { width: min(340px, 42vw); min-height: 44px; padding: 7px 10px; color: var(--ink); background: var(--surface-raised); border: 1px solid var(--line); border-radius: 6px; } +.table-guidance { max-width: 34ch; margin: 0; color: var(--muted); font-size: 11px; line-height: 1.4; } +.results-empty { padding: 54px 20px; color: var(--muted); text-align: center; border-block-end: 1px solid var(--line); } +.results-empty span { font: 34px/1 var(--font-mono); } +.results-empty h4 { margin: 11px 0 4px; color: var(--ink); } +.results-empty p { max-width: 72ch; margin: 0 auto; } + +.tabulator { + color: var(--ink); + background: var(--surface); + border: 0; + font-family: var(--font-sans); + font-size: 13px; +} + +.tabulator .tabulator-header { color: var(--ink-soft); background: var(--surface-muted); border-color: var(--line); } +.tabulator .tabulator-header .tabulator-col { min-height: 44px; background: var(--surface-muted); border-color: var(--line); } +.tabulator .tabulator-header .tabulator-header-filter { margin-block-start: 7px; } +.tabulator .tabulator-header .tabulator-col .tabulator-header-filter input { + width: 100%; + min-height: 34px; + padding: 6px 8px; + color: var(--ink); + background: var(--surface-raised); + border: 1px solid var(--line-strong); + border-radius: 5px; + font: 12px/1.2 var(--font-sans); +} +.tabulator .tabulator-header .tabulator-col .tabulator-header-filter input::placeholder { color: var(--muted); opacity: 1; } +.tabulator .tabulator-header .tabulator-col .tabulator-header-filter input:focus-visible { color: var(--ink); background: var(--surface-raised); border-color: var(--accent); outline: 3px solid var(--focus); outline-offset: -2px; } +.tabulator .tabulator-header .tabulator-col-resize-handle { width: 10px; margin-inline: -5px; } +.tabulator .tabulator-header .tabulator-col-resize-handle::after { + position: absolute; + inset: 8px auto 8px 50%; + width: 1px; + content: ""; + background: var(--line-strong); +} +.tabulator .tabulator-header .tabulator-col-resize-handle:hover::after { width: 2px; background: var(--accent-strong); } +.tabulator .tabulator-col-resize-guide { width: 2px; background: var(--accent-strong); opacity: 0.9; } +.tabulator-row { min-height: 45px; color: var(--ink); background: var(--surface); border-color: var(--line); } +.tabulator-row.tabulator-row-even { background: var(--surface-raised); } +.tabulator-row:hover { background: var(--accent-soft); } +.tabulator-row.tabulator-selected { color: var(--ink); background: var(--accent-soft); } +.tabulator-row .tabulator-cell { padding: 11px 10px; border-color: var(--line); } +.tabulator .tabulator-row .tabulator-cell.tabulator-row-header { background: inherit; } +.tabulator .tabulator-footer { color: var(--ink-soft); background: var(--surface-muted); border-color: var(--line); } +.tabulator .tabulator-footer .tabulator-page { min-width: 44px; min-height: 44px; color: var(--ink); background: var(--surface-raised); border-color: var(--line); } +.tabulator .tabulator-footer .tabulator-page-size { min-height: 44px; } +.tabulator .tabulator-footer .tabulator-page.active { color: var(--nav); background: var(--accent); } +.value-cell { overflow-wrap: anywhere; font-family: var(--font-mono); font-size: 12px; } +.result-actions { display: flex; flex-wrap: wrap; gap: 6px; } +.result-actions .button { flex: 1 1 118px; white-space: nowrap; } +.dns-label { display: inline-flex; min-width: 82px; align-items: center; gap: 6px; font-size: 11px; font-weight: 750; } +.dns-label::before { content: ""; width: 7px; height: 7px; background: currentColor; border-radius: 50%; } +.dns-label.resolved { color: var(--success); } +.dns-label.disputed, .dns-label.uncertain { color: var(--warning); } +.dns-label.no-answer { color: var(--danger); } +.dns-label.not-captured { color: var(--muted); } + +.screenshot-section { margin-block-start: 38px; padding-block-start: 28px; border-block-start: 1px solid var(--line-strong); } +.screenshot-gallery { display: grid; grid-template-columns: repeat(auto-fit, minmax(230px, 1fr)); gap: 10px; } +.screenshot-card { min-width: 0; padding: 0; overflow: hidden; color: var(--ink); text-align: start; background: var(--surface); border: 1px solid var(--line-strong); border-radius: 8px; } +.screenshot-card:hover { border-color: var(--accent); } +.screenshot-frame { display: grid; aspect-ratio: 16 / 10; overflow: hidden; place-items: center; color: var(--muted); background: var(--surface-muted); } +.screenshot-frame img { width: 100%; height: 100%; object-fit: cover; } +.screenshot-card strong { display: block; padding: 10px 12px 2px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.screenshot-card small { display: block; padding: 0 12px 11px; color: var(--muted); } + +.run-log { margin-block-start: 28px; color: var(--ink-soft); border-block: 1px solid var(--line); } +.run-log summary { padding: 14px 4px; cursor: pointer; font-weight: 730; } +.run-log pre { max-height: 360px; margin: 0 0 15px; padding: 15px; overflow: auto; color: var(--ink-soft); background: var(--surface-muted); font: 11px/1.5 var(--font-mono); white-space: pre-wrap; } + +dialog { + max-width: calc(100vw - 28px); + max-height: calc(100vh - 28px); + padding: 0; + overflow: auto; + color: var(--ink); + background: var(--surface-raised); + border: 1px solid var(--line-strong); + border-radius: 12px; + box-shadow: var(--shadow); +} + +dialog::backdrop { background: oklch(0.08 0.02 220 / 0.72); backdrop-filter: blur(3px); } +dialog[open] { animation: dialog-in 180ms cubic-bezier(0.22, 1, 0.36, 1); } +@keyframes dialog-in { from { opacity: 0; transform: translateY(10px) scale(0.985); } } + +.form-dialog { width: min(650px, calc(100vw - 28px)); } +.wide-dialog { width: min(980px, calc(100vw - 28px)); } +.dialog-content { padding: clamp(22px, 4vw, 34px); } +.dialog-header { display: flex; align-items: start; justify-content: space-between; gap: 15px; } +.dialog-content h2 { margin: 0; font-size: clamp(26px, 4vw, 36px); line-height: 1.05; letter-spacing: -0.032em; text-wrap: balance; } +.dialog-content > p { max-width: 70ch; color: var(--ink-soft); } +.dialog-content code { color: var(--accent-text); } +.dialog-content label:not(.action-choice, .source-choice, .file-drop) { display: grid; gap: 6px; font-weight: 720; } +.dialog-content input:not([type="checkbox"], [type="file"]), .dialog-content select, .dialog-content textarea { width: 100%; min-height: 44px; padding: 9px 11px; color: var(--ink); background: var(--surface); border: 1px solid var(--line-strong); border-radius: 7px; } +.dialog-content input::placeholder { color: var(--muted); opacity: 1; } +.dialog-content textarea { min-height: 88px; resize: vertical; } +.dialog-content small { color: var(--muted); font-weight: 450; } +.dialog-close { color: var(--ink); background: transparent; border-color: var(--line); } +.form-grid { display: grid; gap: 14px; margin-block: 23px; } +.form-grid.three { grid-template-columns: minmax(220px, 1.4fr) minmax(130px, 0.6fr) minmax(160px, 0.7fr); } +.form-grid.three .api-scan-paths { grid-column: 1 / -1; } +.advanced-execution { margin-block: 18px 0; padding: 12px 14px; background: var(--surface); border: 1px solid var(--line); border-radius: 7px; } +.advanced-execution summary { min-height: 32px; align-content: center; font-weight: 780; cursor: pointer; } +.advanced-execution .form-grid { margin-block-end: 0; } +.dialog-content .inline-choice { display: flex; min-height: 44px; align-items: center; align-self: end; gap: 8px; } +.inline-choice input { width: 17px; height: 17px; accent-color: var(--accent-strong); } + +.source-fieldset, .action-fieldset { margin: 24px 0 0; padding: 0; border: 0; } +.source-fieldset legend, .action-fieldset legend { margin-block-end: 10px; font-size: 16px; font-weight: 780; } +.source-tools { display: flex; align-items: end; justify-content: space-between; gap: 12px; margin-block-end: 10px; } +.source-search { max-width: 390px; margin-block-end: 10px; } +.source-tools .source-search { width: min(390px, 100%); margin-block-end: 0; } +.source-actions { display: flex; flex: 0 0 auto; flex-wrap: wrap; align-items: end; gap: 6px; } +.source-groups { display: grid; max-height: 285px; grid-template-columns: repeat(3, 1fr); gap: 10px; overflow: auto; } +.source-group { min-width: 0; padding: 10px; background: var(--surface); border-block-start: 3px solid var(--line-strong); } +.source-group[data-activity="P0"] { border-color: var(--accent); } +.source-group[data-activity="P1"] { border-color: var(--warning); } +.source-group[data-activity="P2"] { border-color: var(--danger); } +.source-group h3 { margin: 0 0 8px; font: 800 11px/1 var(--font-mono); } +.source-choice { display: flex; min-height: 44px; align-items: start; gap: 8px; padding: 9px 5px; border-block-start: 1px solid var(--line); font-weight: 650; } +.source-choice input, .action-choice input { flex: 0 0 auto; width: 17px; height: 17px; margin-block-start: 2px; accent-color: var(--accent-strong); } +.source-choice span { min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.source-choice small { display: block; overflow: hidden; font: 10px/1.35 var(--font-mono); text-overflow: ellipsis; white-space: nowrap; } +.source-choice .credential-note { color: var(--warning); } +.source-group-empty { color: var(--muted); font-size: 12px; } + +.action-grid { display: grid; grid-template-columns: repeat(3, 1fr); gap: 7px; } +.action-choice { display: flex; min-width: 0; min-height: 76px; align-items: start; gap: 9px; padding: 10px; background: var(--surface); border: 1px solid var(--line); border-block-start-width: 3px; border-radius: 6px; } +.action-choice.p0 { border-block-start-color: var(--accent); } +.action-choice.p1 { border-block-start-color: var(--warning); } +.action-choice.p2 { border-block-start-color: var(--danger); } +.action-choice strong, .action-choice small { display: block; } +.action-choice strong { line-height: 1.3; } +.action-choice small { margin-block-start: 3px; font-size: 10px; line-height: 1.35; } +.activity-summary { padding: 11px 13px; color: var(--ink) !important; background: var(--accent-soft); border: 1px solid var(--accent); font: 700 12px/1.4 var(--font-mono); } +.form-error { margin: 12px 0 0; padding: 10px 12px; color: var(--danger) !important; background: var(--danger-soft); border: 1px solid var(--danger); font-weight: 650; } +.dialog-actions { display: flex; justify-content: end; gap: 8px; margin-block-start: 24px; } +#new-run-dialog .dialog-actions { + position: sticky; + z-index: var(--z-sticky); + inset-block-end: 0; + padding-block: 14px; + background: var(--surface-raised); + border-block-start: 1px solid var(--line); +} + +.file-drop { position: relative; display: grid; min-height: 210px; margin-block-start: 22px; padding: 28px; place-items: center; align-content: center; text-align: center; background: var(--surface); border: 2px dashed var(--line-strong); border-radius: 9px; cursor: pointer; } +.file-drop:hover, .file-drop:focus-within { border-color: var(--accent); background: var(--accent-soft); } +.file-drop input { position: absolute; width: 1px; height: 1px; overflow: hidden; opacity: 0; } +.file-mark { margin-block-end: 9px; color: var(--accent-text); font: 34px/1 var(--font-mono); } +.file-drop strong, .file-drop small { display: block; } + +.screenshot-dialog { width: min(1180px, calc(100vw - 28px)); } +.screenshot-dialog-header { display: flex; align-items: start; justify-content: space-between; gap: 20px; padding: 18px 20px; border-block-end: 1px solid var(--line); } +.screenshot-dialog-header h2 { margin: 0; overflow-wrap: anywhere; font-size: 24px; } +.screenshot-dialog img { display: block; width: 100%; max-height: calc(100vh - 150px); object-fit: contain; background: var(--surface-muted); } + +.toast-message { position: fixed; z-index: var(--z-toast); inset: auto 22px 22px auto; max-width: min(430px, calc(100vw - 44px)); padding: 12px 15px; color: var(--nav-ink); background: var(--nav); border: 1px solid var(--accent); border-radius: 8px; box-shadow: var(--shadow); } +.toast-message.error { border-color: var(--danger); } + +@media (max-width: 1080px) { + .header-context { display: none; } + .app-shell { grid-template-columns: 290px minmax(0, 1fr); } + .workbench { padding: 28px 24px; } + .run-facts { grid-template-columns: repeat(3, 1fr); } + .run-facts div:nth-child(3) { border: 0; } + .run-facts div:nth-child(n+4) { border-block-start: 1px solid var(--line); } + .overview-grid { grid-template-columns: 1fr; } + .form-grid.three { grid-template-columns: 1.4fr 0.6fr; } + .form-grid.three label:last-child { grid-column: 1 / -1; } +} + +@media (max-width: 780px) { + body { font-size: 16px; } + .help-tip::after { position: fixed; inset: auto 16px 16px; width: auto; } + .app-header { position: relative; flex-wrap: wrap; min-height: auto; gap: 9px; padding: 10px 14px; } + .brand { min-width: 0; } + .brand small { display: none; } + .header-actions { width: 100%; flex-wrap: wrap; order: 2; } + .header-button { flex: 0 0 auto; } + .app-shell { display: block; } + .history-panel { position: relative; inset-block-start: auto; height: auto; padding: 18px 14px; overflow: visible; border-inline-end: 0; border-block-end: 1px solid var(--nav-line); } + .run-list { display: flex; padding-block-end: 4px; overflow-x: auto; } + .run-item { min-width: 235px; flex: 0 0 235px; } + .workbench { padding: 25px 16px 48px; } + .detail-header, .section-heading, .results-heading { align-items: start; flex-direction: column; } + .detail-status { width: 100%; align-items: start; } + .status-chips { justify-content: start; } + .route-overflow-cue { display: block; } + .run-facts { grid-template-columns: 1fr 1fr; } + .run-facts div, .run-facts div:nth-child(3), .run-facts div:nth-child(n+4) { padding: 12px 10px; border: 0; border-block-end: 1px solid var(--line); } + .run-facts div:nth-child(odd) { padding-inline-start: 0; border-inline-end: 1px solid var(--line); } + .lifecycle-track { grid-template-columns: 1fr; } + .lifecycle-step { min-height: 65px; padding: 0 0 18px 24px; border-block-start: 0; border-inline-start: 2px solid var(--line); } + .lifecycle-step::before { inset: 0 auto auto -6px; } + .activity-bands { grid-template-columns: 1fr; } + .request-options { grid-template-columns: 1fr; } + .source-groups { grid-template-columns: 1fr; } + .source-tools { align-items: stretch; flex-direction: column; } + .source-actions .button { flex: 1; } + .action-grid { grid-template-columns: 1fr 1fr; } + .form-grid.three { grid-template-columns: 1fr; } + .form-grid.three label:last-child { grid-column: auto; } + .section-actions { justify-content: start; } + .table-toolbar { align-items: stretch; flex-direction: column; } + .table-toolbar label { align-items: stretch; flex-direction: column; } + .table-toolbar input { width: 100%; } + .table-guidance { max-width: none; } + .tabulator .tabulator-header .tabulator-col .tabulator-header-filter input { min-height: 44px; } + .provider-table th:nth-child(5), .provider-table td:nth-child(5) { display: none; } + .empty-guardrails { grid-template-columns: 1fr; } + .empty-guardrails div { border-inline-end: 0; border-block-end: 1px solid var(--line); } + .empty-guardrails div:last-child { border-block-end: 0; } +} + +@media (max-width: 500px) { + .brand img { width: 38px; height: 38px; } + .brand strong { font-size: 15px; } + .version-badge { padding: 3px 6px; font-size: 9px; } + .header-button { font-size: 12px; } + #theme-button { white-space: nowrap; } + .detail-identity h2 { font-size: 34px; } + .action-grid { grid-template-columns: 1fr; } + .dialog-content { padding: 20px 16px; } + .dialog-actions { align-items: stretch; flex-direction: column-reverse; } + .dialog-actions .button { width: 100%; } + .provider-table th:nth-child(4), .provider-table td:nth-child(4) { display: none; } +} + +@media (prefers-reduced-motion: reduce) { + *, *::before, *::after { + scroll-behavior: auto !important; + animation-duration: 0.01ms !important; + animation-iteration-count: 1 !important; + transition-duration: 0.01ms !important; + } +} + +@media (forced-colors: active) { + .signal-dot, .status-chip::before, .dns-label::before, .lifecycle-step::before { forced-color-adjust: none; } +} diff --git a/theHarvester/lib/api/static/harvestview/app.js b/theHarvester/lib/api/static/harvestview/app.js new file mode 100644 index 00000000..23190c13 --- /dev/null +++ b/theHarvester/lib/api/static/harvestview/app.js @@ -0,0 +1,990 @@ +(() => { + 'use strict'; + + const $ = selector => document.querySelector(selector); + const $$ = selector => [...document.querySelectorAll(selector)]; + const ROUTE_ORDER = [ + 'hostname', 'ip', 'asn', 'email', 'url', 'person', 'person-link', 'takeover', 'shodan', + 'scope-extension', 'external-relationship', 'other' + ]; + const ROUTE_LABELS = { + hostname: 'Hostnames', ip: 'IP addresses', asn: 'ASNs', email: 'Emails', url: 'URLs', + person: 'People', 'person-link': 'People links', takeover: 'Takeover evidence', shodan: 'Shodan evidence', + 'scope-extension': 'Scope extensions', 'external-relationship': 'External relationships', other: 'Other' + }; + const ACTION_FIELDS = {'dns-recursive': 'dns_recursive_depth'}; + const SQLITE_SUFFIXES = ['.sqlite', '.sqlite3', '.db']; + const state = { + runs: [], + selectedId: null, + detail: null, + sources: [], + actions: [], + selectedSources: new Set(['crtsh']), + route: null, + theme: localStorage.getItem('runs-theme') || 'system', + pollTimer: null, + pollErrorShown: false, + resultTable: null, + screenshotUrls: new Map(), + }; + + const nodes = { + themeButton: $('#theme-button'), importButton: $('#import-button'), newRunButton: $('#new-run-button'), + loading: $('#loading-state'), empty: $('#empty-state'), detail: $('#run-detail'), + workspaceError: $('#workspace-error'), workspaceErrorMessage: $('#workspace-error-message'), + retryWorkspace: $('#retry-workspace-button'), + runCount: $('#run-count'), historySearch: $('#history-search'), runList: $('#run-list'), historyEmpty: $('#history-empty'), + detailTarget: $('#detail-target'), detailRunId: $('#detail-run-id'), statusChips: $('#status-chips'), cancel: $('#cancel-run-button'), + runFacts: $('#run-facts'), lifecycleTrack: $('#lifecycle-track'), lifecycleNote: $('#lifecycle-note'), + resultsSection: $('#results-section'), + activityBands: $('#activity-bands'), requestOptions: $('#request-options'), providerBody: $('#provider-body'), + providerSummary: $('#provider-summary'), providerOutcomeSummary: $('#provider-outcome-summary'), providerEmpty: $('#provider-empty'), + providerDetails: $('#provider-details'), resultsSummary: $('#results-summary'), + routeTabs: $('#route-tabs'), resultsEmpty: $('#results-empty'), resultsEmptyTitle: $('#results-empty-title'), + resultsEmptyCopy: $('#results-empty-copy'), resultWorkbench: $('#result-workbench'), + routeOverflowCue: $('#route-overflow-cue'), + resultSearch: $('#result-search'), routeCount: $('#route-count'), copySelected: $('#copy-route-button'), + exportJsonl: $('#export-jsonl-button'), screenshotSection: $('#screenshot-section'), + screenshotGallery: $('#screenshot-gallery'), logSection: $('#log-section'), logOutput: $('#run-log-output'), + newRunDialog: $('#new-run-dialog'), newRunForm: $('#new-run-form'), sourceSearch: $('#source-search'), sourceGroups: $('#source-groups'), + sourceCapability: $('#source-capability'), selectCapability: $('#select-capability-button'), + selectP0: $('#select-p0-button'), clearP0: $('#clear-p0-button'), + dnsResolvers: $('#dns-resolvers'), dnsResolverFile: $('#dns-resolver-file'), + activitySummary: $('#activity-summary'), newRunError: $('#new-run-error'), submitRun: $('#submit-run-button'), + importDialog: $('#import-dialog'), importForm: $('#import-form'), resultFile: $('#result-file'), fileLabel: $('#file-label'), + importError: $('#import-error'), submitImport: $('#submit-import-button'), screenshotDialog: $('#screenshot-dialog'), + screenshotDialogTitle: $('#screenshot-dialog-title'), screenshotDialogImage: $('#screenshot-dialog-image'), + toast: $('#toast'), announcer: $('#announcer'), + }; + + for (const tip of $$('.help-tip')) tip.setAttribute('aria-description', tip.dataset.tooltip); + + function escapeHtml(value) { + return String(value ?? '').replace(/[&<>'"]/g, character => ({ + '&': '&', '<': '<', '>': '>', "'": ''', '"': '"' + })[character]); + } + + function safeClass(value) { + return String(value ?? '').toLowerCase().replace(/[^a-z0-9-]/g, '-'); + } + + function errorMessage(payload, fallback) { + const detail = payload?.detail; + if (typeof detail === 'string') return detail; + if (Array.isArray(detail)) return detail.map(item => item.msg || 'Invalid value').join('. '); + return fallback; + } + + async function api(path, options = {}) { + const headers = new Headers(options.headers || {}); + const response = await fetch(path, {...options, headers}); + if (!response.ok) { + let payload = null; + try { payload = await response.json(); } catch { /* response has no JSON body */ } + const error = new Error(errorMessage(payload, `${response.status} ${response.statusText}`)); + error.status = response.status; + throw error; + } + return response; + } + + function announce(message) { + nodes.announcer.textContent = ''; + requestAnimationFrame(() => { nodes.announcer.textContent = message; }); + } + + let toastTimer = null; + function toast(message, isError = false) { + clearTimeout(toastTimer); + nodes.toast.textContent = message; + nodes.toast.classList.toggle('error', isError); + nodes.toast.hidden = false; + toastTimer = setTimeout(() => { nodes.toast.hidden = true; }, 4200); + } + + function dismissToast() { + clearTimeout(toastTimer); + nodes.toast.hidden = true; + } + + function setBusy(button, busy, label) { + if (!button.dataset.idleLabel) button.dataset.idleLabel = button.textContent; + button.disabled = busy; + button.textContent = busy ? label : button.dataset.idleLabel; + } + + function showFormError(node, message) { + node.textContent = message; + node.hidden = !message; + } + + function formatDate(value) { + if (!value) return 'Not yet'; + const date = new Date(value); + if (Number.isNaN(date.getTime())) return String(value); + return new Intl.DateTimeFormat(undefined, { + month: 'short', day: 'numeric', year: date.getFullYear() !== new Date().getFullYear() ? 'numeric' : undefined, + hour: 'numeric', minute: '2-digit', second: '2-digit' + }).format(date); + } + + function formatDuration(started, completed) { + if (!started) return 'Not started'; + if (!completed) return 'In progress'; + const seconds = Math.max(0, Math.round((new Date(completed) - new Date(started)) / 1000)); + if (!Number.isFinite(seconds)) return 'Unknown'; + if (seconds < 60) return `${seconds} sec`; + const minutes = Math.floor(seconds / 60); + return `${minutes} min ${seconds % 60} sec`; + } + + function statusChip(status, prefix = '') { + if (!status) return ''; + const label = prefix ? `${prefix}: ${status}` : status; + return `${escapeHtml(label)}`; + } + + function applyTheme() { + if (!['system', 'light', 'dark'].includes(state.theme)) state.theme = 'system'; + document.documentElement.dataset.theme = state.theme; + nodes.themeButton.textContent = `Theme: ${state.theme}`; + nodes.themeButton.setAttribute('aria-label', `Color theme is ${state.theme}. Change color theme`); + localStorage.setItem('runs-theme', state.theme); + } + + function cycleTheme() { + const themes = ['system', 'light', 'dark']; + state.theme = themes[(themes.indexOf(state.theme) + 1) % themes.length]; + applyTheme(); + toast(`${state.theme[0].toUpperCase()}${state.theme.slice(1)} theme selected.`); + } + + function openDialog(dialog, focusSelector) { + const formError = dialog.querySelector('.form-error'); + if (formError) showFormError(formError, ''); + if (!dialog.open) dialog.showModal(); + const focusTarget = focusSelector ? dialog.querySelector(focusSelector) : dialog.querySelector('input, button'); + requestAnimationFrame(() => focusTarget?.focus()); + } + + function closeDialog(dialog) { + if (dialog?.open) dialog.close(); + } + + async function loadWorkspace() { + nodes.newRunButton.disabled = true; + nodes.loading.hidden = false; + nodes.empty.hidden = true; + nodes.detail.hidden = true; + nodes.workspaceError.hidden = true; + const [catalogResponse, runsResponse] = await Promise.all([ + api('/api/v1/sources'), api('/api/v1/runs') + ]); + const catalog = await catalogResponse.json(); + state.sources = catalog.sources; + state.actions = catalog.actions; + nodes.newRunButton.disabled = false; + const capabilities = [...new Set(state.sources.flatMap(source => source.capabilities || []))].sort(); + nodes.sourceCapability.innerHTML = '' + capabilities.map(capability => ``).join(''); + state.runs = await runsResponse.json(); + nodes.loading.hidden = true; + renderHistory(); + if (!state.runs.length) { + state.selectedId = null; + nodes.empty.hidden = false; + return; + } + const preferred = state.runs.some(run => run.run_id === state.selectedId) ? state.selectedId : state.runs[0].run_id; + await selectRun(preferred); + } + + function filteredRuns() { + const query = nodes.historySearch.value.trim().toLowerCase(); + if (!query) return state.runs; + return state.runs.filter(run => [run.target, run.run_id, run.status, run.origin, ...(run.activities || [])].some(value => String(value).toLowerCase().includes(query))); + } + + function renderHistory() { + const focusedRunId = document.activeElement?.dataset.runId; + nodes.runCount.textContent = state.runs.length; + const runs = filteredRuns(); + nodes.historyEmpty.hidden = runs.length > 0; + nodes.historyEmpty.querySelector('p').textContent = state.runs.length ? 'No runs match this search.' : 'No saved runs yet.'; + nodes.runList.innerHTML = runs.map(run => ` + `).join(''); + if (focusedRunId) [...nodes.runList.children].find(button => button.dataset.runId === focusedRunId)?.focus({preventScroll: true}); + } + + function formatCount(value, singular, plural = `${singular}s`) { + const count = Number(value || 0); + return `${count.toLocaleString()} ${count === 1 ? singular : plural}`; + } + + function isTerminalStatus(status) { + return ['completed', 'failed', 'cancelled'].includes(status); + } + + async function selectRun(runId) { + stopPolling(); + state.selectedId = runId; + state.detail = null; + nodes.loading.hidden = false; + nodes.detail.hidden = true; + nodes.cancel.hidden = true; + renderHistory(); + try { + const response = await api(`/api/v1/runs/${encodeURIComponent(runId)}`); + const detail = await response.json(); + if (state.selectedId !== runId) return null; + state.detail = detail; + renderDetail(); + if (!isTerminalStatus(state.detail.status)) startPolling(); + return null; + } catch (error) { + if (state.selectedId !== runId) return null; + nodes.loading.hidden = true; + toast(`Could not load the run: ${error.message}. Select it again to retry.`, true); + return error; + } + } + + function renderFacts(run) { + const facts = [ + ['Origin', run.origin], ['Submitted', formatDate(run.created_at)], ['Started', formatDate(run.started_at)], + ['Duration', formatDuration(run.started_at, run.completed_at)], ['Results', Number(run.result_count || 0).toLocaleString()] + ]; + nodes.runFacts.innerHTML = facts.map(([label, value]) => `
${escapeHtml(label)}
${escapeHtml(value)}
`).join(''); + } + + function renderLifecycle(run) { + const terminalLabel = run.status === 'completed' ? 'Completed' : run.status === 'failed' ? 'Failed' : run.status === 'cancelled' ? 'Cancelled' : 'Terminal'; + const steps = [['Submitted', run.created_at], ['Started', run.started_at]]; + if (run.cancellation_requested_at) steps.push(['Cancellation requested', run.cancellation_requested_at]); + steps.push([terminalLabel, run.completed_at]); + nodes.lifecycleTrack.innerHTML = steps.map(([label, time]) => ` +
  • ${escapeHtml(label)}${escapeHtml(formatDate(time))}
    • `).join(''); + const notes = { + queued: 'Waiting for the single local worker.', running: 'The isolated child process owns the finite execution.', + cancelling: 'Termination requested; forced termination follows after the grace period.', cancelled: 'The child can no longer produce work.', + completed: 'Lifecycle is terminal. Evidence completeness is reported separately.', failed: run.error || 'The run ended without a successful lifecycle completion.' + }; + nodes.lifecycleNote.textContent = notes[run.status] || ''; + } + + function renderAuthorization(run) { + const active = new Set(run.activities || []); + nodes.activityBands.innerHTML = ['P0', 'P1', 'P2'].map(activity => ` +
    + ${activity}${active.has(activity) ? 'Selected' : 'Off'} +
    `).join(''); + const request = run.request || {}; + const sources = request.sources?.join(', ') || 'Not recorded'; + const options = [ + ['Sources', sources], ['Result limit', request.limit ?? 'Imported evidence'], + ['Result start offset', request.start ?? 'Not recorded'], + ['Whole-run deadline', request.deadline_seconds ? `${request.deadline_seconds} seconds` : 'Not applicable'], + ['Proxy transport', request.proxies ? 'Selected' : 'Off'], + ['DNS lookup (/24 reverse expansion)', request.dns_lookup ? 'Selected' : 'Off'], + ['DNS resolution', request.dns_resolve ? 'Selected' : 'Off'], ['DNS brute force', request.dns_brute ? 'Selected' : 'Off'], + ['DNS resolver vantages', request.dns_resolvers?.join(', ') || 'Not recorded'], + ['Recursive DNS depth', request.dns_recursive_depth ?? 'Not recorded'], + ['Recursive DNS query budget', request.dns_recursive_query_limit ?? 'Not recorded'], + ['Recursive DNS runtime', request.dns_recursive_runtime_seconds ? `${request.dns_recursive_runtime_seconds} seconds` : 'Not recorded'], + ['Screenshots', request.screenshot ? 'Selected' : 'Off'], + ['Takeover transport', request.takeover ? (request.proxies ? 'Configured proxy' : 'Direct') : 'Off'], + ['API endpoint interaction', request.api_scan ? 'Selected' : 'Off'] + ]; + if (request.filename) options.unshift(['Imported file', request.filename]); + nodes.requestOptions.innerHTML = options.map(([label, value]) => `
    ${escapeHtml(label)}
    ${escapeHtml(value)}
    `).join(''); + } + + function sourceName(execution) { return execution.source || 'Unknown source'; } + function executionName(execution) { return execution.source || execution.action || 'Unknown producer'; } + function executionKind(execution) { return execution.source ? 'Source' : execution.action ? 'Action' : 'Unknown'; } + function credentialRequirement(source) { + const credentials = source?.credentials || []; + if (!credentials.length) return ''; + const labels = credentials.map(value => value.replaceAll('-', ' ').replace(/^api /, 'API ')); + return `Credentials required: ${labels.join(', ')}`; + } + + function executionReason(execution) { + const errorType = execution.error_type; + if (execution.stop_reason === 'missing-credentials') { + return 'Required credentials were not configured; add them, then retry.'; + } + if (execution.source && execution.status === 'skipped' && errorType === 'SourceDidNotStart') { + const requirement = credentialRequirement(state.sources.find(source => source.name === sourceName(execution))); + return requirement + ? `${requirement}. Source did not start; verify configuration or inspect the child log, then retry.` + : 'Source did not start; inspect the child log, then retry.'; + } + return errorType || execution.stop_reason?.replaceAll('-', ' ') || '-'; + } + + function renderExecutions(run) { + const executions = [...(run.source_executions || []), ...(run.action_executions || [])]; + const counts = {completed: 0, partial: 0, skipped: 0, failed: 0, 'rate-limited': 0}; + let zeroResultCount = 0; + for (const execution of executions) { + if (Object.hasOwn(counts, execution.status)) counts[execution.status] += 1; + if (execution.status === 'completed' && Number(execution.result_count || 0) === 0) zeroResultCount += 1; + } + nodes.providerSummary.textContent = executions.length; + nodes.providerOutcomeSummary.hidden = executions.length === 0; + nodes.providerOutcomeSummary.textContent = `${counts.completed} completed (${zeroResultCount} zero-result) / ${counts.partial} partial / ${counts.skipped} skipped / ${counts.failed} failed${counts['rate-limited'] ? ` / ${counts['rate-limited']} rate-limited` : ''}`; + nodes.providerEmpty.hidden = executions.length > 0; + nodes.providerBody.innerHTML = executions.map(execution => ` + ${escapeHtml(executionName(execution))}${executionKind(execution)}${statusChip(execution.status || 'unknown')} + ${Number(execution.result_count || 0).toLocaleString()}${execution.duration_ms == null ? '-' : `${Math.round(execution.duration_ms).toLocaleString()} ms`} + ${escapeHtml(executionReason(execution))}`).join(''); + } + + function groupedResults() { + const groups = new Map(); + for (const result of state.detail?.results || []) { + const type = result.type || 'other'; + if (!groups.has(type)) groups.set(type, []); + groups.get(type).push(result); + } + return [...groups.entries()].sort(([left], [right]) => { + const leftIndex = ROUTE_ORDER.indexOf(left); + const rightIndex = ROUTE_ORDER.indexOf(right); + return (leftIndex < 0 ? 999 : leftIndex) - (rightIndex < 0 ? 999 : rightIndex) || left.localeCompare(right); + }); + } + + function dnsFormatter(cell) { + const value = cell.getValue() || 'not-captured'; + return `${escapeHtml(value.replaceAll('-', ' '))}`; + } + + function columnTextFilter(headerValue, rowValue) { + const query = String(headerValue || '').trim().toLowerCase().replaceAll('-', ' '); + const value = Array.isArray(rowValue) ? rowValue.join(' ') : rowValue || 'not-captured'; + return String(value).toLowerCase().replaceAll('-', ' ').includes(query); + } + + function resultActionFormatter(cell) { + const target = escapeHtml(cell.getRow().getData().value); + return `
    + + +
    `; + } + + function mountResultTable(rows) { + state.resultTable?.destroy(); + nodes.copySelected.disabled = true; + nodes.copySelected.textContent = 'Copy selected'; + const columns = [ + {title: 'Value', field: 'value', formatter: cell => `${escapeHtml(cell.getValue())}`, minWidth: 260, widthGrow: 2, headerFilter: 'input', headerFilterFunc: columnTextFilter, headerFilterPlaceholder: 'Filter values'}, + {title: 'DNS', field: 'dns_status', formatter: dnsFormatter, width: 130, responsive: 1, headerFilter: 'input', headerFilterFunc: columnTextFilter, headerFilterPlaceholder: 'Filter DNS'}, + ]; + if (state.route === 'hostname') { + columns.push({ + title: 'Actions', field: 'value', formatter: resultActionFormatter, headerSort: false, + minWidth: 265, width: 265, responsive: 0, resizable: false, + cellClick: (event, cell) => { + const button = event.target.closest('[data-run-action]'); + if (!button) return; + event.stopPropagation(); + queueResultAction(button.dataset.runAction, cell.getRow().getData().value, button); + }, + }); + } + state.resultTable = new Tabulator(nodes.resultWorkbench.querySelector('#result-grid'), { + data: rows, + layout: 'fitColumns', + responsiveLayout: 'collapse', + resizableColumnGuide: true, + columnDefaults: {resizable: true}, + selectableRows: true, + rowHeader: { + formatter: 'rowSelection', titleFormatter: 'rowSelection', headerSort: false, + width: 48, widthGrow: 0, resizable: false, frozen: true, headerHozAlign: 'center', hozAlign: 'center' + }, + maxHeight: 590, + placeholder: 'No results match this filter.', + pagination: true, + paginationMode: 'local', + paginationSize: 15, + paginationSizeSelector: [15, 30, 60, 120], + paginationCounter: 'rows', + initialSort: [{column: 'value', dir: 'asc'}], + columns + }); + state.resultTable.on('rowSelectionChanged', selected => { + nodes.copySelected.disabled = selected.length === 0; + nodes.copySelected.textContent = selected.length ? `Copy selected (${selected.length})` : 'Copy selected'; + }); + } + + function renderResults(run) { + const focusedRoute = document.activeElement?.dataset.route; + const groups = groupedResults(); + const total = (run.results || []).length; + if (total) { + nodes.resultsSummary.textContent = `${formatCount(total, 'normalized result')} across ${formatCount(groups.length, 'route')}.`; + } else if (run.status === 'completed') { + const emptySources = (run.source_executions || []) + .filter(execution => execution.status === 'completed' && Number(execution.result_count || 0) === 0) + .map(sourceName); + let sourceSummary = 'No normalized evidence was returned.'; + if (emptySources.length === 1) sourceSummary = `${emptySources[0]} returned no normalized evidence.`; + if (emptySources.length > 1) sourceSummary = `${emptySources.length} selected sources returned no normalized evidence.`; + const outcomeSummary = nodes.providerOutcomeSummary.hidden ? '' : ` · ${nodes.providerOutcomeSummary.textContent}`; + nodes.resultsSummary.textContent = `0 normalized results${outcomeSummary}.`; + nodes.resultsEmptyTitle.textContent = 'Enumeration completed'; + nodes.resultsEmptyCopy.textContent = `${sourceSummary} The retained evidence record is ${run.evidence_status || 'not recorded'}.`; + } else if (run.status === 'failed' || run.status === 'cancelled') { + const evidenceStatus = run.evidence_status || 'not recorded'; + nodes.resultsSummary.textContent = '0 normalized results.'; + nodes.resultsEmptyTitle.textContent = run.status === 'failed' ? 'Enumeration failed' : 'Enumeration cancelled'; + nodes.resultsEmptyCopy.textContent = run.status === 'failed' + ? `${run.error || 'The enumeration failed.'} The retained evidence record is ${evidenceStatus}.` + : `The enumeration was cancelled. The retained evidence record is ${evidenceStatus}.`; + } else { + nodes.resultsSummary.textContent = 'Queued and running records remain visible before terminal evidence exists.'; + nodes.resultsEmptyTitle.textContent = 'No normalized evidence yet'; + nodes.resultsEmptyCopy.textContent = nodes.resultsSummary.textContent; + } + nodes.resultsEmpty.hidden = total > 0; + nodes.resultWorkbench.hidden = total === 0; + nodes.routeTabs.hidden = total === 0; + nodes.routeOverflowCue.hidden = total === 0; + nodes.exportJsonl.disabled = !run.evidence_status; + nodes.copySelected.disabled = true; + if (!total) { + nodes.routeTabs.innerHTML = ''; + state.resultTable?.destroy(); + state.resultTable = null; + return; + } + if (!groups.some(([type]) => type === state.route)) state.route = groups[0][0]; + nodes.routeTabs.innerHTML = groups.map(([type, results]) => ` + `).join(''); + const rows = groups.find(([type]) => type === state.route)?.[1] || []; + nodes.routeCount.textContent = rows.length; + nodes.resultSearch.value = ''; + mountResultTable(rows); + if (focusedRoute) { + requestAnimationFrame(() => nodes.routeTabs.querySelector(`[data-route="${CSS.escape(focusedRoute)}"]`)?.focus({preventScroll: true})); + } + } + + function revokeScreenshots() { + for (const url of state.screenshotUrls.values()) URL.revokeObjectURL(url); + state.screenshotUrls.clear(); + } + + async function loadScreenshot(screenshot, frame) { + try { + const response = await api(screenshot.url); + const objectUrl = URL.createObjectURL(await response.blob()); + state.screenshotUrls.set(screenshot.url, objectUrl); + frame.innerHTML = `Screenshot preview of ${escapeHtml(screenshot.target)}`; + } catch { + frame.textContent = 'Preview unavailable. Reload the run to retry.'; + } + } + + function renderScreenshots(run) { + revokeScreenshots(); + const screenshots = run.screenshots || []; + nodes.screenshotSection.hidden = screenshots.length === 0; + nodes.screenshotGallery.innerHTML = screenshots.map((screenshot, index) => ` + `).join(''); + screenshots.forEach((screenshot, index) => loadScreenshot(screenshot, nodes.screenshotGallery.children[index].querySelector('.screenshot-frame'))); + } + + function renderDetail(previousRun = null) { + const run = state.detail; + nodes.loading.hidden = true; + nodes.empty.hidden = true; + nodes.detail.hidden = false; + nodes.detailTarget.textContent = run.target; + nodes.detailRunId.textContent = run.run_id; + nodes.statusChips.innerHTML = `${statusChip(run.status, 'Lifecycle')}${statusChip(run.evidence_status, 'Evidence')}`; + nodes.cancel.hidden = !['queued', 'running', 'cancelling'].includes(run.status); + nodes.cancel.disabled = run.status === 'cancelling'; + nodes.cancel.textContent = run.status === 'cancelling' ? 'Cancellation in progress' : 'Request cancellation'; + if (isTerminalStatus(run.status)) { + nodes.detail.insertBefore(nodes.resultsSection, nodes.runFacts); + } else { + nodes.detail.insertBefore(nodes.resultsSection, nodes.providerDetails); + } + renderFacts(run); + renderLifecycle(run); + renderAuthorization(run); + renderExecutions(run); + if (!previousRun || previousRun.status !== run.status || JSON.stringify(previousRun.results) !== JSON.stringify(run.results)) { + renderResults(run); + } + if (!previousRun || JSON.stringify(previousRun.screenshots) !== JSON.stringify(run.screenshots)) { + renderScreenshots(run); + } + nodes.logSection.hidden = !run.log; + nodes.logOutput.textContent = run.log || ''; + } + + async function refreshSelected() { + if (!state.selectedId) return; + const selectedId = state.selectedId; + try { + const [detailResponse, runsResponse] = await Promise.all([ + api(`/api/v1/runs/${encodeURIComponent(selectedId)}`), api('/api/v1/runs') + ]); + const previousDetail = state.detail; + const previousStatus = previousDetail?.status; + const [detail, runs] = await Promise.all([detailResponse.json(), runsResponse.json()]); + if (state.selectedId !== selectedId) return; + state.detail = detail; + state.runs = runs; + state.pollErrorShown = false; + renderHistory(); + renderDetail(previousDetail); + if (state.detail.status !== previousStatus) { + if (previousStatus === 'queued') dismissToast(); + announce(`Run lifecycle is now ${state.detail.status}.`); + } + if (isTerminalStatus(state.detail.status)) stopPolling(); + } catch (error) { + if (state.selectedId !== selectedId) return; + if (!state.pollErrorShown) { + state.pollErrorShown = true; + toast(`Could not refresh the run: ${error.message}. Retrying automatically.`, true); + } + } + } + + async function pollSelected() { + state.pollTimer = null; + await refreshSelected(); + if (state.selectedId && state.detail && !isTerminalStatus(state.detail.status)) { + state.pollTimer = setTimeout(pollSelected, 1200); + } + } + + function startPolling() { + stopPolling(); + state.pollTimer = setTimeout(pollSelected, 1200); + } + + function stopPolling() { + if (state.pollTimer) clearTimeout(state.pollTimer); + state.pollTimer = null; + state.pollErrorShown = false; + } + + function renderSourceGroups(filter = '') { + const query = filter.trim().toLowerCase(); + const groups = ['P0', 'P1', 'P2'].map(activity => [activity, state.sources.filter(source => { + if (source.activity !== activity) return false; + const haystack = [source.name, ...(source.capabilities || [])].join(' ').toLowerCase(); + return !query || haystack.includes(query); + })]); + nodes.sourceGroups.innerHTML = groups.map(([activity, sources]) => ` +
    +

    ${activity} · ${activity === 'P0' ? 'Passive' : activity === 'P1' ? 'DNS interaction' : 'Direct interaction'}

    + ${sources.length ? sources.map(source => ` + `).join('') : '

    No matching sources.

    '} +
    `).join(''); + } + + function setP0Selection(selected) { + for (const source of state.sources.filter(source => source.activity === 'P0')) { + if (selected) state.selectedSources.add(source.name); + else state.selectedSources.delete(source.name); + } + renderSourceGroups(nodes.sourceSearch.value); + updateActivitySummary(); + announce(selected ? 'All passive P0 sources selected.' : 'Passive P0 sources cleared.'); + } + + function selectCapability() { + const capability = nodes.sourceCapability.value; + if (!capability) return; + for (const source of state.sources) { + if ((source.capabilities || []).includes(capability)) state.selectedSources.add(source.name); + } + renderSourceGroups(nodes.sourceSearch.value); + updateActivitySummary(); + announce(`Sources providing ${capability} selected.`); + } + + function selectedActivities() { + const activities = new Set(); + for (const source of state.sources) if (state.selectedSources.has(source.name)) activities.add(source.activity); + for (const action of state.actions) { + const field = nodes.newRunForm.elements[ACTION_FIELDS[action.name] || action.name.replaceAll('-', '_')]; + const selected = action.name === 'dns-recursive' + ? Number(field?.value) > 0 + : field?.checked; + if (selected) activities.add(action.activity); + } + return activities; + } + + function updateActivitySummary() { + const activities = selectedActivities(); + nodes.activitySummary.textContent = `P0 ${activities.has('P0') ? 'selected' : 'off'} · P1 ${activities.has('P1') ? 'selected' : 'off'} · P2 ${activities.has('P2') ? 'selected' : 'off'}`; + nodes.activitySummary.style.borderColor = activities.has('P2') ? 'var(--danger)' : activities.has('P1') ? 'var(--warning)' : 'var(--accent)'; + } + + function openNewRun() { + nodes.newRunForm.reset(); + nodes.newRunForm.elements.limit.value = 500; + nodes.newRunForm.elements.deadline_seconds.value = 1800; + state.selectedSources = new Set(state.sources.some(source => source.name === 'crtsh') ? ['crtsh'] : [state.sources[0]?.name].filter(Boolean)); + nodes.sourceSearch.value = ''; + nodes.sourceCapability.value = ''; + renderSourceGroups(); + updateActivitySummary(); + openDialog(nodes.newRunDialog, '#run-target'); + } + + function openImport() { + nodes.importForm.reset(); + nodes.fileLabel.textContent = 'Choose a JSONL or SQLite file'; + openDialog(nodes.importDialog, '#result-file'); + } + + async function focusCreatedRun(runId) { + state.selectedId = runId; + const runsResponse = await api('/api/v1/runs'); + state.runs = await runsResponse.json(); + renderHistory(); + const loadError = await selectRun(runId); + if (loadError) throw loadError; + return state.selectedId === runId && state.detail?.run_id === runId; + } + + async function focusAcceptedRun(runId, acceptedMessage) { + const previousSelectedId = state.selectedId; + const previousDetail = state.detail; + try { + if (!await focusCreatedRun(runId)) { + toast(`${acceptedMessage}.`); + return false; + } + return true; + } catch (error) { + if (state.selectedId === runId) { + state.selectedId = previousSelectedId; + state.detail = previousDetail; + renderHistory(); + if (previousDetail) { + renderDetail(); + if (!isTerminalStatus(previousDetail.status)) startPolling(); + } else { + nodes.loading.hidden = true; + nodes.detail.hidden = true; + } + } + toast(`${acceptedMessage}, but the run view could not refresh: ${error.message}. Do not submit it again; reload the page to view it.`, true); + return false; + } + } + + async function queueResultAction(action, target, button) { + const label = action === 'screenshot' ? 'Screenshot' : 'DNS brute force'; + const payload = {target, sources: [], [action]: true}; + const resolvers = state.detail?.request?.dns_resolvers; + if (action === 'dns_brute' && Array.isArray(resolvers) && resolvers.length) { + payload.dns_resolvers = resolvers; + } + setBusy(button, true, 'Starting…'); + try { + const response = await api('/api/v1/runs', { + method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify(payload) + }); + const run = await response.json(); + if (await focusAcceptedRun(run.run_id, `${label} for ${target} was queued`)) { + toast(`${label} for ${target} is ${state.detail?.status || 'submitted'}.`); + } + } catch (error) { + toast(`Could not start ${label.toLowerCase()}: ${error.message}.`, true); + } finally { + setBusy(button, false, ''); + } + } + + async function submitRun(event) { + event.preventDefault(); + showFormError(nodes.newRunError, ''); + const form = new FormData(nodes.newRunForm); + const actionSelected = state.actions.some(action => { + const field = ACTION_FIELDS[action.name] || action.name.replaceAll('-', '_'); + return action.name === 'dns-recursive' ? Number(form.get(field)) > 0 : form.has(field); + }); + if (!state.selectedSources.size && !actionSelected) { + showFormError(nodes.newRunError, 'Select at least one discovery source or additional activity.'); + return; + } + const payload = { + target: form.get('target'), sources: [...state.selectedSources], limit: Number(form.get('limit')), + start: Number(form.get('start')), deadline_seconds: Number(form.get('deadline_seconds')), + proxies: form.has('proxies'), dns_lookup: form.has('dns_lookup'), dns_resolve: form.has('dns_resolve'), + dns_resolvers: String(form.get('dns_resolvers')).split(',').map(value => value.trim()), + dns_recursive_depth: Number(form.get('dns_recursive_depth')), + dns_recursive_query_limit: Number(form.get('dns_recursive_query_limit')), + dns_recursive_runtime_seconds: Number(form.get('dns_recursive_runtime_seconds')), + dns_brute: form.has('dns_brute'), shodan: form.has('shodan'), screenshot: form.has('screenshot'), + takeover: form.has('takeover'), api_scan: form.has('api_scan'), + api_scan_paths: form.has('api_scan') + ? String(form.get('api_scan_paths')).split(/\r?\n/).map(value => value.trim()).filter(Boolean) + : [] + }; + setBusy(nodes.submitRun, true, 'Submitting…'); + try { + const response = await api('/api/v1/runs', { + method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify(payload) + }); + const run = await response.json(); + closeDialog(nodes.newRunDialog); + const focused = await focusAcceptedRun(run.run_id, `Enumeration for ${run.target} was created`); + if (focused && state.detail?.status === 'queued') { + toast(`Enumeration for ${run.target} is queued.`); + } + } catch (error) { + showFormError(nodes.newRunError, error.message); + } finally { + setBusy(nodes.submitRun, false, ''); + } + } + + async function submitImport(event) { + event.preventDefault(); + showFormError(nodes.importError, ''); + const file = nodes.resultFile.files[0]; + if (!file) { + showFormError(nodes.importError, 'Choose a JSONL or SQLite result file.'); + return; + } + const lowerName = file.name.toLowerCase(); + const fileKind = lowerName.endsWith('.jsonl') ? 'jsonl' : SQLITE_SUFFIXES.some(suffix => lowerName.endsWith(suffix)) ? 'sqlite' : null; + if (!fileKind) { + showFormError(nodes.importError, 'Choose a .jsonl, .sqlite, .sqlite3, or .db file.'); + return; + } + if (fileKind === 'jsonl' && file.size > 10 * 1024 * 1024) { + showFormError(nodes.importError, 'JSONL file exceeds the 10 MiB limit.'); + return; + } + setBusy(nodes.submitImport, true, 'Importing…'); + try { + const path = fileKind === 'jsonl' ? '/api/v1/runs/import' : '/api/v1/runs/import-database'; + const contentType = fileKind === 'jsonl' ? 'application/x-ndjson' : 'application/vnd.sqlite3'; + const response = await api(`${path}?filename=${encodeURIComponent(file.name)}`, { + method: 'POST', headers: {'Content-Type': contentType}, body: file + }); + const imported = await response.json(); + closeDialog(nodes.importDialog); + if (fileKind === 'jsonl') { + if (await focusAcceptedRun(imported.run_id, `${file.name} was imported`)) { + toast(`Imported ${file.name} without executing discovery.`); + } + } else { + const importedIds = imported.imported_run_ids || []; + const skippedIds = imported.skipped_run_ids || []; + const selectedId = importedIds[0] || skippedIds[0]; + if (selectedId) { + if (!await focusAcceptedRun(selectedId, `${file.name} was imported`)) return; + } else { + try { + const runsResponse = await api('/api/v1/runs'); + state.runs = await runsResponse.json(); + renderHistory(); + } catch (error) { + toast(`${file.name} was imported, but run history could not refresh: ${error.message}. Do not import it again; reload the page to view it.`, true); + return; + } + } + toast(`Imported ${formatCount(importedIds.length, 'run')} from ${file.name}; ${skippedIds.length} already present.`); + } + } catch (error) { + showFormError(nodes.importError, error.message); + } finally { + setBusy(nodes.submitImport, false, ''); + } + } + + async function requestCancellation() { + if (!state.selectedId) return; + const selectedId = state.selectedId; + nodes.cancel.disabled = true; + let detail; + try { + const response = await api(`/api/v1/runs/${encodeURIComponent(selectedId)}/cancel`, {method: 'POST'}); + detail = await response.json(); + } catch (error) { + if (state.selectedId !== selectedId) return; + nodes.cancel.disabled = false; + toast(`Could not request cancellation: ${error.message}. Refresh the run state and try again.`, true); + return; + } + if (state.selectedId !== selectedId) return; + state.detail = detail; + renderDetail(); + if (state.detail.status === 'cancelling') startPolling(); + else stopPolling(); + try { + const runsResponse = await api('/api/v1/runs'); + const runs = await runsResponse.json(); + if (state.selectedId !== selectedId) return; + state.runs = runs; + renderHistory(); + } catch (error) { + if (state.selectedId !== selectedId) return; + toast(`Cancellation was accepted, but run history could not refresh: ${error.message}. Do not request it again; reload the page to confirm it.`, true); + return; + } + toast(state.detail.status === 'cancelled' ? 'Queued enumeration cancelled.' : 'Cancellation requested.'); + } + + async function downloadServerExport() { + try { + const response = await api(`/api/v1/runs/${encodeURIComponent(state.selectedId)}/export`); + const disposition = response.headers.get('Content-Disposition') || ''; + const match = disposition.match(/filename="([^"]+)"/); + downloadBlob(await response.blob(), match?.[1] || 'harvestview-results.jsonl'); + } catch (error) { + toast(`Could not export results: ${error.message}. Keep the run open and try again.`, true); + } + } + + function downloadBlob(blob, filename) { + const url = URL.createObjectURL(blob); + const link = Object.assign(document.createElement('a'), {href: url, download: filename}); + document.body.append(link); + link.click(); + link.remove(); + setTimeout(() => URL.revokeObjectURL(url), 0); + toast(`Downloaded ${filename}.`); + } + + async function copySelected() { + const selected = state.resultTable?.getSelectedRows().map(row => row.getData()) || []; + if (!selected.length) return; + const text = selected.map(result => result.value).join('\n'); + try { + await navigator.clipboard.writeText(text); + toast(`Copied ${selected.length} selected ${ROUTE_LABELS[state.route] || state.route}.`); + } catch { + toast('Clipboard access was unavailable. Use the JSONL export instead.', true); + } + } + + function openScreenshot(index) { + const screenshot = state.detail?.screenshots?.[index]; + const objectUrl = state.screenshotUrls.get(screenshot?.url); + if (!screenshot || !objectUrl) { + toast('The screenshot preview is not available. Reload the run to retry.', true); + return; + } + nodes.screenshotDialogTitle.textContent = screenshot.target; + nodes.screenshotDialogImage.src = objectUrl; + nodes.screenshotDialogImage.alt = `Screenshot of ${screenshot.target}`; + openDialog(nodes.screenshotDialog, '[data-close-dialog]'); + } + + nodes.themeButton.addEventListener('click', cycleTheme); + nodes.retryWorkspace.addEventListener('click', start); + nodes.newRunButton.addEventListener('click', openNewRun); + nodes.importButton.addEventListener('click', openImport); + nodes.historySearch.addEventListener('input', renderHistory); + nodes.newRunForm.addEventListener('submit', submitRun); + nodes.importForm.addEventListener('submit', submitImport); + nodes.cancel.addEventListener('click', requestCancellation); + nodes.exportJsonl.addEventListener('click', downloadServerExport); + nodes.copySelected.addEventListener('click', copySelected); + nodes.resultSearch.addEventListener('input', event => { + const query = event.target.value.trim().toLowerCase(); + state.resultTable?.setFilter(row => !query || row.value.toLowerCase().includes(query)); + }); + nodes.sourceSearch.addEventListener('input', event => renderSourceGroups(event.target.value)); + nodes.selectCapability.addEventListener('click', selectCapability); + nodes.selectP0.addEventListener('click', () => setP0Selection(true)); + nodes.clearP0.addEventListener('click', () => setP0Selection(false)); + nodes.sourceGroups.addEventListener('change', event => { + if (!event.target.matches('input[type="checkbox"]')) return; + if (event.target.checked) state.selectedSources.add(event.target.value); + else state.selectedSources.delete(event.target.value); + updateActivitySummary(); + }); + nodes.newRunForm.addEventListener('change', updateActivitySummary); + nodes.dnsResolverFile.addEventListener('change', async () => { + const file = nodes.dnsResolverFile.files[0]; + if (!file) return; + try { + const resolvers = (await file.text()).split(/\r?\n/).map(value => value.trim()).filter(Boolean); + nodes.dnsResolvers.value = resolvers.join(','); + announce(`${formatCount(resolvers.length, 'resolver address')} loaded from ${file.name}.`); + } catch { + toast(`Could not read ${file.name}. Choose a plain text resolver file and try again.`, true); + } + }); + nodes.resultFile.addEventListener('change', () => { + const file = nodes.resultFile.files[0]; + nodes.fileLabel.textContent = file ? `${file.name} · ${(file.size / 1024).toLocaleString(undefined, {maximumFractionDigits: 1})} KiB` : 'Choose a JSONL or SQLite file'; + }); + + document.addEventListener('click', event => { + const runButton = event.target.closest('[data-run-id]'); + if (runButton) selectRun(runButton.dataset.runId); + const routeButton = event.target.closest('[data-route]'); + if (routeButton) { + state.route = routeButton.dataset.route; + renderResults(state.detail); + announce(`${ROUTE_LABELS[state.route] || state.route} route selected.`); + } + const screenshotButton = event.target.closest('[data-screenshot-index]'); + if (screenshotButton) openScreenshot(Number(screenshotButton.dataset.screenshotIndex)); + if (event.target.closest('[data-action="new-run"]')) openNewRun(); + if (event.target.closest('[data-action="import"]')) openImport(); + const closeButton = event.target.closest('[data-close-dialog]'); + if (closeButton) closeDialog(closeButton.closest('dialog')); + }); + + for (const dialog of [nodes.newRunDialog, nodes.importDialog, nodes.screenshotDialog]) { + dialog.addEventListener('click', event => { + if (event.target === dialog) closeDialog(dialog); + }); + } + + window.addEventListener('beforeunload', () => { + stopPolling(); + revokeScreenshots(); + }); + + async function start() { + applyTheme(); + try { + await loadWorkspace(); + } catch (error) { + nodes.loading.hidden = true; + nodes.workspaceErrorMessage.textContent = error.message; + nodes.workspaceError.hidden = false; + } + } + + start(); +})(); diff --git a/theHarvester/lib/api/static/harvestview/index.html b/theHarvester/lib/api/static/harvestview/index.html new file mode 100644 index 00000000..096cbf19 --- /dev/null +++ b/theHarvester/lib/api/static/harvestview/index.html @@ -0,0 +1,315 @@ + + + + + + + HarvestView + + + + + + + +
    + + + HarvestViewAnalysis workspace for theHarvester + v{{VERSION}} + + + +
    + +
    + + +
    +
    + +

    Opening the run desk

    +

    Loading local lifecycle and evidence records.

    +
    + + + + + + +
    +
    + + +
    +
    +

    Start an enumeration

    + +
    +

    Choose only work you are authorized to perform. Passive sources are P0; DNS and direct interaction remain off until explicitly selected.

    + +
    + + + +
    + +
    + Advanced execution controls +
    + + + + + + + + +
    +
    + +
    + Discovery sources +
    + +
    + + + + +
    +
    +
    +
    + +
    + Additional activity +
    + + + + + + + +
    +
    + +

    P0 selected · P1 off · P2 off

    + + +
    +     + + +
    +
    +

    Import a result file

    + +
    +

    Choose theHarvester JSONL evidence or a closed, checkpointed SQLite database. Import never contacts the target or replays a run. SQLite stores run evidence but does not embed screenshot files.

    + + + +
    +     + + +
    +

    Screenshot

    + +
    + +     + + +
    + + + + + diff --git a/theHarvester/lib/api/static/harvestview/logo.webp b/theHarvester/lib/api/static/harvestview/logo.webp new file mode 100644 index 0000000000000000000000000000000000000000..05c8d9d65f4d9f8ab4b87be907da15b28ee6e580 GIT binary patch literal 7774 zcmV-k9--k;-3w|_2mn2&NKsHisjcuFh*tTuk zW<9oTtF}99^R#boJNq5ce_6DxgspVi_^2z^OqwVPlKHj;YujmCPIq$0E8Dhh+qRu_ zY^!72wr$%^+8x_QW6f{QwRYE9buQ1Y*ekYmW}M@5V%tf(uWWn8$*Mo*7u!~2#kO^- zR#n)J&E^-IyK;4QwQXy~ZevCVS8cpOBeresyXFeo_Ka<-v8&EiU&T%uXd+3?_TYfQ z0<=*OB~cOe(SHYy9cn~mn)>3;kSaZ%S&t`hn z&SD3%a$8Fk*zL_#4B8(8k$X^SAK0$K=eaSx$F`T`wwBvmQPd7+=CRSLaEaYJv;ze2 zgm8{6w&bbP>S$i-ypl&jF-)Q9E@ub}uuPJ91=QF;MjdXGk%lS=S%Rl$c4-wtx2LQptRkD%T z^yxpShUIlimaN#iVk(O1ZExPV_e`!=TNr^JYX^?lVhcX=qkgDpUTcQxti-Fx_M~&$ zn$_uSP7Iqk`-rZw_ltiWuKq7Snh6Hk)~h}8%iitnKuSM!;s;=q!W_frI_1nvig5bqFn zY5turpX0u45pS=dr}fZ9`9FfB2>X=1XsbS7NB-?Ideqb2!*QLRA7H2O|GK z(5<;GaI$%u%cNG}wTyYLCoN)V8?MEkLZRXZ1!4Yn9hcbkp~4))cIKLbBdj~X9xzP0K-_P!7W6^uZvA+;U+$(8=2`S-* z=~qS<8n#Isq%5UAtY-CW{_u zomXI}v`U>cKkAk}8#A~L2t_>IdXr~%IfHviGs&^gp%PBLkg?7B9RMcKO9EBSwNPuk zwiiRBj;6gPDfD{z6;|)u0$E!J2izCt3ycpI+eSLM$$G289ySrxROq?^jNhScMe?gZ zUqIl9qpmSGZx$CFPg`E%aBbhDvpO5LD$$Y2)=IYOzSzEtM!rTR8&q4L`DNa>zD)3< z0jzub@U-yZzkmx~qw(8wnDV_eC6qF029&{GbozV*V5h3MGs_iKN6+S1mPn0BGg6Y4 z-MABL5YdC;CL&vTG4zgRyzilFf*^RA?i*DxNXerVDzD~C$y!ds(GWSXQQEWqI?{`B zC7nEDwy)r~2Hk@90bwFK2f%$`ubbgF%`-%=x;RGEk*_f?w@B8BUrdqY^JT6lr5c`r zLMY-0-G(%Hq%Aom%QOROlHFy*J~4>Jh|W2knxL6pU+yxS_lFQS6@HJUv|lvGahFWIzAoeHfKZI7v8ciK)FlMGrb6B#lb&zw84-jsAcno@*|C!84P-8>$$=>0DwP5G zLq)pmrRvD}K-u6!A#$b-4R&ljO8AowB~g=ma_P%gPuCvxmk- z$tpR_(0yS=YtQ^hDe%Gi3>#3!FO!4)b>Zl zl7d$;+FGi>HYx)x@cS4faKtIQj7W%9U5xJ=gh=P^pJ)%=pn&{P*Bxc#SXF>PxMh&D zWU5gx+9VSdv&RHRS}o?YLXGXcK7G=f>rCI1KG{ZAM+n`j-nvpxIHOyiPTsb*T$Hx9 ztVgg$nFOpqc%xE3rSe%fK4IF*%8y2#nd%G~qCMAne?>;XB%5ko|23&%&{dEFOXRJ) zOjPyf**$wLf6l2NupLK4V8>Ps z?YiRDhBNbMJYT%n7EtyjP)noY-U%wMA<^*d#7C_27w)a!#bQ%h?i`q z&T((usr5)j&wbPajU-5VtDRfc2~VMIA(Gt6|1KhtGlPbC5;3%~Zx*+T(sPjwYS0lz z;{Fc#oAX`J#Hrg{4v-DE7CV1RQ&#*5#R*C5noI0Dc|6hX1viLmw;I0B*&XJQ1~Ijy zK{j$$BsB=6+%@DC5?X!GA&JK2&E%0*Ts2yOYC2JOXT%sx`J&x~Vmx3l1(5^B)CeS~ zPOO(PhwzO@xjYfh$-UqOVll!>DrRyeJ5ks}$k$QOMF6Thz`@n;2GDqvDE1I72q5mt z?6-e=OhgD42{0AF_)SFJtO7gyW?e+aBUD!a4*)xP*WpzHBj{rXWFZ`1)Y6h=Yuf*| z;Dg)^Ec%Pw(n*)wP>x$R-N)Do5d>*-`Rok)(Bkg@);Ys8A9c-Tw6wtL3FaSWI)6K3 zN}aG$ipE6F&II>?UBpnSY?^ZE!iT2}0tOck;HW98RGgd}NQM9}OH zxy+XH`FrTWU$o5T7S)&fnVPDYfwVPe=OeqQktYPlEqb+7Wm#+K$gZwDBGT$$&Q#9u zp1O%qIJqG^Xc(;v0C*5^SK(oCab;aZ#sj#4;x-6;ooU1P^gvWXgfJ>g5KIsNv#S7y z%0__=IJm}45QV^TV&S7R!xBGxhc?KOCX-Cqgqu?v$iW4BGEl_8IS&QbZ(e`%J_C+` zfDGj8Tmqa6T{zfu)F~0c#RoBP;Y_+OVIU#yB!DfBzuIdIybjDj>av@T<>G{HI^bq$ zc^@bnJHj1JyYUKY7(}$&aJ*Kyp=*X(I~zHj2*GijvK|6@CsQx4b08>yXQAb2Sy2={ zv4$XJuK1y}fT^hMET%jQxo2USCt5ZYv%UG$;as4UI0>Twz}qM4mSU+jkH=(=;SSw} zS+ecxRjcPJAz4nNi=>;d50H;;S?w+;-Ut?xc2l>CbrbeZ@1jmPX;MSvC_w6R8(dJY zGiIKoSuKY}KnCMXB47_%WyvyHR$tIFBDyFhlpx@ki|^^{d>QLYIslBVY|%R?I4UAs z1rdbs2EvMI5b!EPo&4;G--k@40^9u$bjQp&x&_tp0J=pKdv}5JTykut5R}z9mdo)I zp((n@U$TlexW_;ULUY9)4d=d@ZwkCsQ1u>lCLBJE$jZ_syv8>orc85pg zz-$`k5tQQu5x^0%*OVK5K3gG-(aLC@(>L$y$mBH4{SX(FNikwo!q5q=33T05h?g12 z_#o*Udq(6z!&vNLkm#TT09$Ove?b#>O6ZhnxmpP<`^rv8xH$w#lF4j1Aiu{XX(@(g z61!15a8Y@5sFIj?z?!><&A43)V6M|{~8^G+yBl>fBRt7$02xmV@b(U#s?pCgB$ zD#lnK)KSSjYo}?Nq7WC@q0$>IfKrebpq=cDYexU-HzS6E=YazjZaQFowD(t7KGP&N zNiNTc+)!M-qb;N54|1p@N!O)DzRJK=9GM;lKrG&t(*UgsQ#ziqNoPG<9zNtSV&`pYm=j-7U|?70n;`jPd&i#ijnX}sCV1Hye7Tw zLXwNZzV&p79GJ@s$3qY+RVAGAq^b1s3e_k{n?H5a{`-d`a>~hWr`GWy87>z7+j&aG zpjrr=vFIYXwNm)bu(m(a!%LM)N>ai086uvfgo7P|KIlhWtvQ@N#3MO zZNYng2m~&2^+3ZjgtWCJTT~NNkT%4&T)-i)aCB_oges{RWS8eyE|yoQStT2fo>FXw zL8`FvT-1H^|H`Vxq!yvf{xJNNyEj4%1x9kobGw2AHXd-1L+`7a$0mK}7)IjF){=cg zAxXqa;LPe2Njvk~=@#KnEM-NW+xY{rk&v##UBD5?SxMzZpA@jXWF94(UD8zU% z23lu7r7kpse-_K2!voA`<^tnw~JH9u`1!APfC;xdNp(M;TJO2P_Pm}ey&3{ z+zUydMY=w)@%aA2JY6I8Xw84`th;?aBCw2=E_LO&-Gota{Qp5;+|Htl73_9g^0|d8 z0jxXFU#2ED7{rCiIQwry=;5=eYh^hbi>ZYmh{_&b_-q8)v*ztSEr3E+V#-?TY>sc= zjBHQ^xM@pS_hpplWYnn5`5&kUT=;NOf{*^W)VKQf3l390yGT|D#l?pwFLJOp(MqIC z{wHOP)ia9H38uVMuQxW$GYode>pPbO*#9P?yydvMkE}yIT+4GKXC9Yy!L7ceRQD>T z8A!PNqWdR|Ax3?|M0iUy#q?!n_;Pe@qn&+|7xqW4KJ)r=9cEsJav0tP!x9$(YP&js9lGH!T!4mi;Zh{LDuXSz1!;t128n<__Du!(Tnajj(gJ^ zSGjn2SzLdW$lz97(wk)F#wIkn!XuC~te6hE7$RVNRYEnqn-RAF<9A^9VIh@)vm=T> zG$=-FQs+>eF~9SM$l3Qr<+xo%Aa|`90uO>U67EkDH;!l)j~pqEwftIEAt&7xLtQtI zL15kjw~vj!yIiomI!0m2al>M;S)VM@gA;&7$x4SfOU+n){+<~2K-PPpz8dYTjm%V-3^vF;#0cWSSyu!in? zEzuRyWOJzD6CZv<#9|130ndi|g-h1-sV0HO3=ptl&^vtSRklgHtZc_6bQw_#kq0jc zM;+sQF5)4{ISU!P6*EA@;XWxT*+sDAB_AYhzEux510Oid+1aD!Be+{Cd$mQWnLZccQX)TlNu5pd85FLJBi^cu$jCCu zg(pVsNmp9&`EDsxFje!mmb6hK5ZtTe+Oda{J$Qc!r?vMW{@r1HO z1sT=D?*B_}m^5&>cX?B)9AnVVnR_LPesiZ$|21jzSus?~Kw?iCIe{3LJ(ko74;tc5 z^IZ2^UP71D8(jC3YKRQx16XU({Y9NK6!i>W`b)^g-n9AKc3N`hdulUeiw;tjnfgi> z!77<0>o+toTb?6MWe?h+GWa7nrtb`pTZ!SgeZtUc_}4nL6b0rF+E z^v;n3^Wv;_zlQwQdH1Fxq>s1Mw(BO)Zg;Z%@<(mPU&w(>B=f*9pv)ODq0rs^7XOZsC6TlkPO7Beva2@zB0rczp?X_}t=$E~tB}pGbn5Y3yXLSvGZdlz-QQQ4L%S#i>?B{sAlIu005HGC zr~O)vpHh)EId#%?WJDl_GB7uYrgJvy=L@H!@7lh|v;lsRQmBYp>eq=Y>~Mw-wl1fH zj5Uti9hKQdfC0{1M>=sa4T zq*A}?C7tZFkT#bcF%-w=8Zs_vsS`}s(v@Jx9i*hzORD|$f8~fxj9_%5-95BD(0w*C1AT@6 z(Yb>09_+D&4Ei`rcM7H@=e&=4iCx3wyRiOwz|4JMC$3#4$(m&JlH_}}2dczvbJcey z=En9K9eK*`aJ1wxg<|Aea~pE>1&#mim~Kd(t#@hshdCSpP&c;e++zAb_1RqhAWHE1g!WWTs6 z>$Q1=&6%cYkchEmV>1~cJ&X++vYBwJ4k8}V_$=}7GRC3nz9JOllCdE5w97! zIdLNd-Wi)5ujgT7+4^ZsutlOn*ffpiOd`W%otuQI2ph2>BQ<*vG14Y_p#ssQGP^J% zi-A6;a+$-^H*sNuiCYVu4QG31Q{42%3Ju2E0XnD4p*WlG*fmB%2RmP4I0C)eI3^b3 zYS=TA#Bx?27Pp^s!|U>z#Tked*@!G{0>Ok8wGu0{4nF8J&6)elij}oVkchzWWo+dx zy;}DIM7x*!x`h#1s}+di^;tiXDxxKLN1|%uR_{=?trMVI!ptnJ!l70rXlmkP*Q5Ou zH3!rB?+Bvm(Ip?MRT>94?V|;|$}PGQxLs4h>I}tL4O1Ts-&hRVazMOQWdLh4Ny9R5 zwGL2K&!gd@{cjG*scId9Fy|G3RchAYl9@J-d3Iom@?aZ(+8v z0nqiMxoLc3z_6EYbbdMjd%obTd>0G|g3lVdQ>8rBTV|jDFy^Ya(W|vTMr%YS>Um&6 zNR7A%D}4lvkUd9ReapuW__HIp>B@bm2W5orBc3%vWuBTUQD1{F&YAyo zg-FMm7gZJ*T<_FbFtETVIA9!5Ef9!ZS}hfrap{z5(TEQ3q_lWPh{l=K)nHKdo>}vy37Hn}a zZR(|(023dXmpo&&nxgxNp2tMjJFHX_#De(4lf4$acEqHK4>6CX;>BxhtTc;P<~aD& z-ZYW-iqsM(5SfL^x~YSeQYFJAy{3JY7X(_{r^Cdew&;<%>k1^ibr=5 z(qf*G`)?jJo3(3y1WYb|Ocf0Y)%#64&AU6F%13wAE$MC?Oh&8yRTgv>J#24vkR;yk z-*>1>YZm?czbr%NU6X~*ARn{Sc+kh?0mgZT!w-=?w5XpN@7PJ#rr7hN zUTLs&5>D0cF*|`JK+pD9Cvc1 zu&~o+2;4e23g>S-RjNl+^F}8KHtGxoJNQ}3++ZU1f}_xOJ|u{qcmAn`^MY~5+$w0S z_eNfR&$IbzKR>$VIB-;f^*np9K#kEBKmvE0|6pBChWp zf1%z=ghBq^xHA|(>z`IpNlpbr0N*aRFU~%25wcE%@1jbxgbj$V62W)@3A;N3+40r)B#xI za6(^JSg+3m7^I)Ubm>m$kJ|{r)^xDT?-3d2RNp6wG-22Kqkrp~v0mHjj}X{u&GbWu z_o>`qiY^AmnUlnasYQsARa1l?dCA&)WEBNluTRJwabpfu>#1Qoqq}$}*%O=xARS^2 z{sMtn@0FamVl=HI)w54KAh%XiR0&K&2f?l$-O(kM3{2iF6?OnWNT1F9=^FwRslKEO z=><(@$${^KIr+d2&N*m}2dKW+yfk<1Lhjw{YASmM3w-%Jivs(D*LFDJcc}YT0wDZK z72|-D^z1d_H5Hc&5#*(I`)`4@t73BcQR5o#Qy0<$$SkP?JP)*YRjV3{vH|u#OZg$b z3e=&z^x=6hMHn5ars(3l1}aG;1LkM9F>aLr;d;04#tyKHS1s20eG}o6Uz@43QY5zl zf$zwwo<<7Z+wD{$TXXg7A!J1STF*ungQu#4ZNIb>r~1)36RE4g#+lWg4G^@diqUly zLAAG2ZC|s7M>J4x5G)9Et@coE!Cmh`HJdvGjK~I_(rk;T1_Nb{+9CJ5)py)Y?e{fc z-1NiU-4X7KAot6sN|KkrI@F>BvRN= km$;~wbtMsTO`Hq$V`nm4s+brTLIiPei|ape{pVw60w9YSUjP6A literal 0 HcmV?d00001 diff --git a/uv.lock b/uv.lock index 302bfbeb..bb88468c 100644 --- a/uv.lock +++ b/uv.lock @@ -1399,6 +1399,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, ] +[[package]] +name = "pytest-base-url" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ae/1a/b64ac368de6b993135cb70ca4e5d958a5c268094a3a2a4cac6f0021b6c4f/pytest_base_url-2.1.0.tar.gz", hash = "sha256:02748589a54f9e63fcbe62301d6b0496da0d10231b753e950c63e03aee745d45", size = 6702, upload-time = "2024-01-31T22:43:00.81Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/1c/b00940ab9eb8ede7897443b771987f2f4a76f06be02f1b3f01eb7567e24a/pytest_base_url-2.1.0-py3-none-any.whl", hash = "sha256:3ad15611778764d451927b2a53240c1a7a591b521ea44cebfe45849d2d2812e6", size = 5302, upload-time = "2024-01-31T22:42:58.897Z" }, +] + +[[package]] +name = "pytest-playwright" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "playwright" }, + { name = "pytest" }, + { name = "pytest-base-url" }, + { name = "python-slugify" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/58/ef/172eb8e23c80491fc72f1401c72f9305663873649351306a38b18406b0c9/pytest_playwright-0.8.0.tar.gz", hash = "sha256:7888d4a2443160c82e0c506c437076679f86b36d1910427250d90fbf1843a981", size = 17132, upload-time = "2026-05-18T10:16:15.919Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/71/1c545fac6a9054b52b3771238fb2dc6e8f1d0ccec116e1c7786ec191887c/pytest_playwright-0.8.0-py3-none-any.whl", hash = "sha256:856aae6efd4bc055f2ef229c647768760bcaad5cd3a5983c314ac260a974a933", size = 17143, upload-time = "2026-05-18T10:16:18.226Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -1411,6 +1439,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, ] +[[package]] +name = "python-slugify" +version = "8.0.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "text-unidecode" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/87/c7/5e1547c44e31da50a460df93af11a535ace568ef89d7a811069ead340c4a/python-slugify-8.0.4.tar.gz", hash = "sha256:59202371d1d05b54a9e7720c5e038f928f45daaffe41dd10822f3907b937c856", size = 10921, upload-time = "2024-02-08T18:32:45.488Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/62/02da182e544a51a5c3ccf4b03ab79df279f9c60c5e82d5e8bec7ca26ac11/python_slugify-8.0.4-py2.py3-none-any.whl", hash = "sha256:276540b79961052b66b7d116620b36518847f52d5fd9e3a70164fc8c50faa6b8", size = 10051, upload-time = "2024-02-08T18:32:43.911Z" }, +] + [[package]] name = "python-socks" version = "2.8.1" @@ -1626,6 +1666,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/bb/2799cc2ede3ed41131f8975621e7213dfc7ef4acbbaadfa440f32500c370/starlette-1.3.1-py3-none-any.whl", hash = "sha256:c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6", size = 73632, upload-time = "2026-06-12T09:23:10.017Z" }, ] +[[package]] +name = "text-unidecode" +version = "1.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ab/e2/e9a00f0ccb71718418230718b3d900e71a5d16e701a3dae079a21e9cd8f8/text-unidecode-1.3.tar.gz", hash = "sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93", size = 76885, upload-time = "2019-08-30T21:36:45.405Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/a5/c0b6468d3824fe3fde30dbb5e1f687b291608f9473681bbf7dabbf5a87d7/text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8", size = 78154, upload-time = "2019-08-30T21:37:03.543Z" }, +] + [[package]] name = "theharvester" source = { editable = "." } @@ -1662,6 +1711,7 @@ dev = [ { name = "mypy-extensions" }, { name = "pytest" }, { name = "pytest-asyncio" }, + { name = "pytest-playwright" }, { name = "ruff" }, { name = "ty" }, { name = "types-certifi" }, @@ -1706,6 +1756,7 @@ dev = [ { name = "mypy-extensions", specifier = "==1.1.0" }, { name = "pytest", specifier = "==9.1.1" }, { name = "pytest-asyncio", specifier = "==1.4.0" }, + { name = "pytest-playwright", specifier = "==0.8.0" }, { name = "ruff", specifier = "==0.15.20" }, { name = "ty", specifier = "==0.0.54" }, { name = "types-certifi", specifier = "==2021.10.8.3" },