From d766c7c59a8cfe8d91c7f6b5f533bd90f341d017 Mon Sep 17 00:00:00 2001 From: Matt <36310667+NotoriousRebel@users.noreply.github.com> Date: Mon, 10 Aug 2026 00:19:00 -0400 Subject: [PATCH] Package HarvestView for local deployment (#2513) * feat: package HarvestView for local deployment * Align container smoke with JSONL API * Align container smoke with unversioned JSONL * refactor: rename web launcher to harvestview --- .dockerignore | 26 +++++- .github/workflows/dockerci.yml | 2 +- .github/workflows/harvestview-container.yml | 90 +++++++++++++++++++ CHANGELOG.md | 12 +-- Dockerfile | 55 ++++++++---- PRODUCT.md | 4 +- README.md | 23 ++++- bin/{restfulHarvest => harvestview} | 2 +- docker-compose.yml | 39 +++++--- docs/wiki/Configuration-and-API-Keys.md | 2 +- docs/wiki/Home.md | 5 +- docs/wiki/Installation.md | 24 ++--- docs/wiki/Rest-API.md | 6 +- docs/wiki/Troubleshooting.md | 6 +- pyproject.toml | 2 +- tests/test_docker.py | 63 +++++++++++++ tests/test_harvestview_launcher.py | 26 ++++++ tests/test_restful_harvest.py | 15 ---- .../{restfulHarvest.py => harvestview.py} | 0 19 files changed, 327 insertions(+), 75 deletions(-) create mode 100644 .github/workflows/harvestview-container.yml rename bin/{restfulHarvest => harvestview} (57%) create mode 100644 tests/test_docker.py create mode 100644 tests/test_harvestview_launcher.py delete mode 100644 tests/test_restful_harvest.py rename theHarvester/{restfulHarvest.py => harvestview.py} (100%) diff --git a/.dockerignore b/.dockerignore index cd4b949a..043854ae 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,10 +1,28 @@ -.github/* +.git +.github/ +.gitignore .gitattributes .git-blame-ignore-revs +.env +.env.* +.secrets/ +.impeccable/ +theHarvester/data/api-keys.yaml +theHarvester/data/proxies.yaml .idea/ -.pytest_cache -.mypy_cache -tests/* +.venv/ +.uv-cache/ +__pycache__/ +*.py[cod] +*.sqlite* +*.db +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +build/ +dist/ +test-results/ +tests/ README/ bin/ theHarvester-logo.png diff --git a/.github/workflows/dockerci.yml b/.github/workflows/dockerci.yml index bc30ab0d..551f2bba 100644 --- a/.github/workflows/dockerci.yml +++ b/.github/workflows/dockerci.yml @@ -20,4 +20,4 @@ jobs: - name: Build the Docker image run: docker build --tag theharvester . - name: Smoke test - run: docker run --rm theharvester --help | grep restfulHarvest + run: docker run --rm theharvester --help | grep harvestview diff --git a/.github/workflows/harvestview-container.yml b/.github/workflows/harvestview-container.yml new file mode 100644 index 00000000..776bc0ac --- /dev/null +++ b/.github/workflows/harvestview-container.yml @@ -0,0 +1,90 @@ +name: HarvestView container smoke + +on: + workflow_dispatch: + pull_request: + paths: + - '.dockerignore' + - '.github/workflows/harvestview-container.yml' + - 'Dockerfile' + - 'docker-compose.yml' + - 'pyproject.toml' + - 'theHarvester/lib/api/**' + - 'tests/test_docker.py' + - 'uv.lock' + +permissions: + contents: read + +jobs: + container-smoke: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Harden runner + uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 + with: + egress-policy: audit + + - name: Check out repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Prepare operator secret + run: | + install -d -m 0700 .secrets + printf '%s\n' 'harvestview-container-smoke-key' > .secrets/operator-api-key + chmod 0444 .secrets/operator-api-key + + - name: Build and start container + env: + THEHARVESTER_PORT: '8769' + run: | + docker compose -p harvestview-container-smoke build theharvester.svc.local + docker compose -p harvestview-container-smoke up -d --no-build + + - name: Verify routes, Chromium, and restart persistence + env: + THEHARVESTER_PORT: '8769' + run: | + for attempt in $(seq 1 60); do + if curl --fail --silent --show-error http://127.0.0.1:8769/ > /tmp/harvestview.html; then + break + fi + sleep 1 + done + grep --fixed-strings 'HarvestView' /tmp/harvestview.html + test "$(curl --silent --output /dev/null --write-out '%{http_code}' http://127.0.0.1:8769/app)" = '404' + test "$(curl --silent --output /dev/null --write-out '%{http_code}' http://127.0.0.1:8769/api/v1/runs)" = '401' + curl --fail --silent --show-error \ + --header 'X-API-Key: harvestview-container-smoke-key' \ + http://127.0.0.1:8769/api/v1/runs > /dev/null + printf '%s\n' \ + '{"completed_at":"2026-08-08T01:01:00Z","counts":{"hostname":1,"ip":1},"evidence_status":"complete","result_count":2,"run_id":"9f9b4383-6cc4-4f3f-80a4-c8d21930dc2d","started_at":"2026-08-08T01:00:00Z","target":"example.test","type":"summary"}' \ + '{"type":"hostname","value":"www.example.test"}' \ + '{"type":"ip","value":"192.0.2.10"}' > /tmp/smoke.jsonl + curl --fail --silent --show-error \ + --request POST \ + --header 'Content-Type: application/x-ndjson' \ + --header 'X-API-Key: harvestview-container-smoke-key' \ + --data-binary @/tmp/smoke.jsonl \ + 'http://127.0.0.1:8769/api/v1/runs/import?filename=smoke.jsonl' > /tmp/imported-run.json + run_id="$(python3 -c "import json; print(json.load(open('/tmp/imported-run.json', encoding='utf-8'))['run_id'])")" + curl --fail --silent --show-error \ + --header 'X-API-Key: harvestview-container-smoke-key' \ + "http://127.0.0.1:8769/api/v1/runs/${run_id}/export" \ + | python3 -c "import json, sys; rows = [json.loads(line) for line in sys.stdin if line.strip()]; assert rows[0]['type'] == 'summary'; assert any(row.get('type') == 'ip' for row in rows[1:]); assert len(rows) == 3" + docker compose -p harvestview-container-smoke exec -T theharvester.svc.local \ + python -c "from playwright.sync_api import sync_playwright; p = sync_playwright().start(); browser = p.chromium.launch(headless=True); browser.close(); p.stop()" + docker compose -p harvestview-container-smoke restart theharvester.svc.local + curl --retry 30 --retry-all-errors --retry-delay 1 --fail --silent --show-error \ + --header 'X-API-Key: harvestview-container-smoke-key' \ + "http://127.0.0.1:8769/api/v1/runs/${run_id}" > /tmp/persisted-run.json + python3 -c "import json; data = json.load(open('/tmp/persisted-run.json', encoding='utf-8')); assert data['status'] == 'completed'; assert len(data['results']) == 2" + + - name: Stop container + if: ${{ always() }} + env: + THEHARVESTER_PORT: '8769' + run: docker compose -p harvestview-container-smoke down --volumes diff --git a/CHANGELOG.md b/CHANGELOG.md index 96434d70..bbecac70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- Added HarvestView, an authenticated local browser workspace backed by a durable single-worker `/api/v1` run lifecycle with cancellation, deadlines, JSONL-only file interchange, retained partial evidence, and real-browser regression coverage. +- Added a pinned, non-root Docker Compose deployment for HarvestView and the REST API with localhost-only publishing, file-secret authentication, private durable run storage, and an authenticated API health check. - Added bounded recursive DNS discovery with three-vantage consensus, closest-encloser wildcard controls, exact-address PTR evidence, and hard query, depth, runtime, and zero-yield limits. -- Added authenticated REST routes for listing completed enumeration runs and retrieving their normalized evidence. -- Added an authenticated HIBP verified-domain source for CLI and conditionally authenticated REST queries that retains normalized account emails and stable breach names without retaining the raw account mapping. +- Added an authenticated HIBP verified-domain source for CLI and `/api/v1` runs that retains normalized account emails and stable breach names without retaining the raw account mapping. - Added keyless Shodan Certificate Transparency hostname discovery with bounded requests and offline response contracts. - Added bounded, keyless subdomain discovery through Arquivo.pt's public CDX API with offline response contracts. - Added transactional SQLite storage and loading for completed full-pipeline runs without changing legacy result rows. @@ -27,16 +28,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added root contributor and security policies, structured issue forms, repository agent guidance, discovery terminology, and an operator-focused documentation wiki ([d090a29a](https://github.com/laramies/theHarvester/commit/d090a29a), [7c491ef5](https://github.com/laramies/theHarvester/commit/7c491ef5), [8b9d420b](https://github.com/laramies/theHarvester/commit/8b9d420b)). ### Changed +- Made `harvestview` the sole launcher for the local web application and REST API. - Standardized SQLite, JSONL, API, and HarvestView result names on `hostname` and `ip` without a presentation alias. - Standardized URL-producing adapters, JSON, JSONL, SQLite, and API evidence on one `url` result kind while preserving producer provenance. +- Replaced the unversioned and provider-specific REST routes with one authenticated `/api/v1` source and run contract shared with CLI execution semantics. - Fixed proxied POST requests so they retain the request method, body, and query parameters. - Migrated Pentest-Tools discovery to its API v2 Bearer-authenticated scan, status, and output endpoints. -- Included HIBP verified-domain in `all` and matching capability selectors like every other P0 source, with REST operator authentication applied after source expansion when its provider key is configured. -- Allowed REST `/query` requests to select discovery sources by result capability, matching the CLI's union semantics while preserving explicit source selection. +- Included HIBP verified-domain in `all` and matching capability selectors like every other P0 source. - Changed `-b all` to select every cataloged P0 passive source once while leaving P1 DNS and P2 direct sources available through explicit selection. - Expanded Common Crawl discovery to use every unique crawl ending within one year of the newest catalog entry, validate catalog endpoints, batch requests, cap each query at 100 pages, and enforce the CLI result limit across page requests ([249ce64b](https://github.com/laramies/theHarvester/commit/249ce64b), [70470cd8](https://github.com/laramies/theHarvester/commit/70470cd8)). - Completed bounded pagination for Wayback Archive and Cert Spotter, including continuation handling, truncation diagnostics, and preservation of partial results on provider failures ([df6ff2c9](https://github.com/laramies/theHarvester/commit/df6ff2c9), [f85a08ff](https://github.com/laramies/theHarvester/commit/f85a08ff)). -- Routed operator messages and diagnostics through logging, preserved host logging policy and existing handlers, and configured logging for the standalone API example ([8a7b8b71](https://github.com/laramies/theHarvester/commit/8a7b8b71)). +- Routed operator messages and diagnostics through logging, preserved host logging policy and existing handlers, and configured logging for the API service ([8a7b8b71](https://github.com/laramies/theHarvester/commit/8a7b8b71)). - Added credential configuration adapters, deferred proxy configuration loading until required, and retained compatibility accessors such as `Core.brave_key()` ([ccd91176](https://github.com/laramies/theHarvester/commit/ccd91176)). - Centralized hostname scope normalization for parser and storage boundaries, including case-insensitive targets, trailing dots, optional `www.` prefixes, and exact DNS-label matching ([c0a0b653](https://github.com/laramies/theHarvester/commit/c0a0b653), [a474f086](https://github.com/laramies/theHarvester/commit/a474f086), [70470cd8](https://github.com/laramies/theHarvester/commit/70470cd8)). - Replaced deprecated hostname resolution with `getaddrinfo`-based handling ([6a847435](https://github.com/laramies/theHarvester/commit/6a847435)). diff --git a/Dockerfile b/Dockerfile index 0a2dcd62..ad4d7ed5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,25 +1,50 @@ -FROM python:3.14-slim-trixie@sha256:cea0e6040540fb2b965b6e7fb5ffa00871e632eef63719f0ea54bca189ce14a6 +ARG PYTHON_IMAGE=python:3.14-slim-trixie@sha256:cea0e6040540fb2b965b6e7fb5ffa00871e632eef63719f0ea54bca189ce14a6 +FROM ${PYTHON_IMAGE} AS builder + +ENV UV_COMPILE_BYTECODE=1 \ + UV_LINK_MODE=copy \ + UV_PROJECT_ENVIRONMENT=/opt/venv + +WORKDIR /app +COPY pyproject.toml uv.lock README.md ./ +COPY theHarvester ./theHarvester + +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=from=ghcr.io/astral-sh/uv:0.12.1@sha256:cf4eedcaa81655197f625739489effcbe71b61ceb1506f332c3facae5deceded,source=/uv,target=/usr/local/bin/uv \ + uv sync --locked --no-dev --no-editable --reinstall-package theharvester + +FROM ${PYTHON_IMAGE} AS runtime LABEL maintainer="@jay_townsend1 & @NotoriousRebel1" +LABEL org.opencontainers.image.title="theHarvester" \ + org.opencontainers.image.description="theHarvester API and HarvestView web application" \ + org.opencontainers.image.licenses="GPL-2.0-only" \ + org.opencontainers.image.source="https://github.com/laramies/theHarvester" -RUN useradd -m -u 1000 -s /bin/bash theharvester +ENV HOME=/var/lib/theharvester \ + PATH=/opt/venv/bin:$PATH \ + PLAYWRIGHT_BROWSERS_PATH=/opt/ms-playwright \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + THEHARVESTER_RUN_ARTIFACTS=/var/lib/theharvester/run-artifacts \ + THEHARVESTER_RUN_DB=/var/lib/theharvester/runs.sqlite -RUN apt-get update && apt-get upgrade -yqq && apt-get clean && \ - rm -rf /var/lib/apt/lists/* -# Set workdir and copy project files WORKDIR /app -COPY . /app +COPY --from=builder /opt/venv /opt/venv -# Create and sync environment using uv -# Compile bytecode for faster startup and install to system site-packages -RUN --mount=from=ghcr.io/astral-sh/uv,source=/uv,target=/bin/uv \ - UV_PROJECT_ENVIRONMENT=/usr/local uv sync --no-dev --no-cache --compile-bytecode +RUN groupadd --gid 10001 theharvester \ + && useradd --uid 10001 --gid 10001 --home-dir "$HOME" --shell /usr/sbin/nologin theharvester \ + && install -d -o 10001 -g 10001 -m 0700 "$HOME" "$THEHARVESTER_RUN_ARTIFACTS" \ + && playwright install --with-deps --only-shell chromium \ + && chmod -R a+rX "$PLAYWRIGHT_BROWSERS_PATH" \ + && rm -rf /var/lib/apt/lists/* -# Use non-root user USER theharvester -# Expose port if the service listens on 80 -EXPOSE 80 +EXPOSE 8000 -# Run the application as theharvester user -ENTRYPOINT ["restfulHarvest", "-H", "0.0.0.0", "-p", "80"] +HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ + CMD ["python", "-c", "from urllib.request import Request, urlopen; from theHarvester.lib.api.auth import _configured_api_key; key = _configured_api_key(); assert key; urlopen(Request('http://127.0.0.1:8000/api/v1/runs', headers={'X-API-Key': key}), timeout=3).close()"] + +ENTRYPOINT ["harvestview"] +CMD ["-H", "0.0.0.0", "-p", "8000"] diff --git a/PRODUCT.md b/PRODUCT.md index 011ae26c..df46e6f6 100644 --- a/PRODUCT.md +++ b/PRODUCT.md @@ -14,8 +14,8 @@ The operator is a technically capable security practitioner working at a desk, o - Cancel work and know whether cancellation is requested, in progress, or complete. - Reopen prior runs and compare route-specific evidence without rerunning reconnaissance. - Import existing theHarvester JSONL evidence. -- Export normalized results and inspect managed screenshots. -- Start a screenshot or DNS brute-force action from a subdomain result without changing the parent evidence. +- Export normalized JSONL results and inspect managed screenshots. +- Start a screenshot or DNS brute-force action from a hostname result without changing the parent evidence. ## Product principles diff --git a/README.md b/README.md index 8255a33b..195f6be7 100644 --- a/README.md +++ b/README.md @@ -85,11 +85,11 @@ Screenshot capture also requires a Playwright-compatible browser; see the instal ## HarvestView and REST API -`restfulHarvest` starts a FastAPI service on `127.0.0.1:5000` by default: +`harvestview` starts the local web application and API on `127.0.0.1:5000` by default: ```bash export THEHARVESTER_API_KEY='replace-with-a-long-random-value' -uv run restfulHarvest +uv run harvestview ``` Open [HarvestView](http://127.0.0.1:5000/) to run and inspect finite enumerations in the local web app. The server gives the local browser a derived HttpOnly session cookie, so the API key is never entered into or stored by HarvestView. @@ -104,6 +104,25 @@ an isolated deployment. Open [Swagger](http://127.0.0.1:5000/docs) or [ReDoc](http://127.0.0.1:5000/redoc) for the automation contract. +### Docker Compose + +The supplied Compose service runs as an unprivileged user, stores run records in a named volume, loads the operator key from a file secret, and publishes only to host loopback. Create the secret before the first start: + +```bash +install -d -m 0700 .secrets +openssl rand -hex 32 > .secrets/operator-api-key +chmod 0444 .secrets/operator-api-key +docker compose up --build -d +docker compose ps +``` + +The `0700` directory protects the secret on the host, while the read-only `0444` file lets the unprivileged container process read its bind-mounted copy. Open [HarvestView](http://127.0.0.1:5000/). The image includes Chromium for optional screenshots. Provider keys and proxies remain in the existing read-only YAML mounts and are excluded from the image build context. + +```bash +docker compose logs -f theharvester.svc.local +docker compose down +``` + | Route | Purpose | | --- | --- | | `GET /api/v1/sources` | List registered discovery sources and capabilities. | diff --git a/bin/restfulHarvest b/bin/harvestview similarity index 57% rename from bin/restfulHarvest rename to bin/harvestview index e772df6f..29e9d329 100755 --- a/bin/restfulHarvest +++ b/bin/harvestview @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -from theHarvester.restfulHarvest import main +from theHarvester.harvestview import main if __name__ == '__main__': main() diff --git a/docker-compose.yml b/docker-compose.yml index c6e5e0c4..a5a704e0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,15 +1,34 @@ services: theharvester.svc.local: - container_name: theHarvester + image: theharvester:local + build: + context: . + init: true + restart: unless-stopped + read_only: true + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + shm_size: 1gb + tmpfs: + - /tmp:rw,noexec,nosuid,nodev,size=128m + environment: + THEHARVESTER_API_KEY_FILE: /run/secrets/operator-api-key + THEHARVESTER_HARVESTVIEW_LOCAL_PROXY: enabled + secrets: + - source: operator-api-key + target: operator-api-key volumes: - - ./theHarvester/data/api-keys.yaml:/root/.theHarvester/api-keys.yaml - - ./theHarvester/data/api-keys.yaml:/etc/theHarvester/api-keys.yaml - - ./theHarvester/data/proxies.yaml:/etc/theHarvester/proxies.yaml - - ./theHarvester/data/proxies.yaml:/root/.theHarvester/proxies.yaml - build: . + - theharvester-data:/var/lib/theharvester + - ./theHarvester/data/api-keys.yaml:/etc/theHarvester/api-keys.yaml:ro + - ./theHarvester/data/proxies.yaml:/etc/theHarvester/proxies.yaml:ro ports: - - "5000:80" + - "127.0.0.1:${THEHARVESTER_PORT:-5000}:8000" -networks: - default: - name: app_theHarvester_network +volumes: + theharvester-data: + +secrets: + operator-api-key: + file: ${THEHARVESTER_API_KEY_FILE:-./.secrets/operator-api-key} diff --git a/docs/wiki/Configuration-and-API-Keys.md b/docs/wiki/Configuration-and-API-Keys.md index 0255cb3b..7bcfd178 100644 --- a/docs/wiki/Configuration-and-API-Keys.md +++ b/docs/wiki/Configuration-and-API-Keys.md @@ -69,7 +69,7 @@ Every `/api/v1/*` route requires a server-side key: ```bash export THEHARVESTER_API_KEY='replace-with-a-long-random-value' -uv run restfulHarvest +uv run harvestview ``` API clients send the same value in the `X-API-Key` header. HarvestView receives a derived HttpOnly browser cookie when it is opened locally, so the key is never entered into or stored by the web app. Provider credentials remain in `api-keys.yaml` and cannot be supplied through an API request. diff --git a/docs/wiki/Home.md b/docs/wiki/Home.md index 102cd591..f7ebbbe6 100644 --- a/docs/wiki/Home.md +++ b/docs/wiki/Home.md @@ -17,8 +17,9 @@ DNS brute force, DNS resolution, takeover checks, screenshots, and API-path scan ## Choose an interface - **Command line:** best for interactive reconnaissance and report generation. -- **REST API:** best for local integrations and browser-accessible Swagger/ReDoc documentation. See [REST API](Rest-API). -- **Docker Compose:** runs the REST API service, not the normal interactive CLI. +- **HarvestView:** best for creating and inspecting durable local enumeration runs in a browser. See [REST API](Rest-API). +- **REST API:** best for authenticated local integrations and Swagger/ReDoc documentation. See [REST API](Rest-API). +- **Docker Compose:** packages HarvestView and the REST API, not the normal interactive CLI. The repository [README](https://github.com/laramies/theHarvester) owns the current feature summary and source/result matrix. The live `theHarvester -h` output owns the complete CLI reference. diff --git a/docs/wiki/Installation.md b/docs/wiki/Installation.md index 8ef7eaf6..c5cac5ac 100644 --- a/docs/wiki/Installation.md +++ b/docs/wiki/Installation.md @@ -33,7 +33,7 @@ uv sync --all-groups uv run pytest ``` -The supported console commands are `theHarvester` and `restfulHarvest`. There is no root `theHarvester.py` launcher. +The supported console commands are `theHarvester` and `harvestview`. There is no root `theHarvester.py` launcher. ### Self-host Tabulator @@ -63,26 +63,30 @@ uv run playwright install chromium On Linux, Playwright may report missing system libraries. Follow the host-specific dependency instructions printed by Playwright, then rerun the browser installation. -## Docker Compose API service +## Docker Compose HarvestView service -The Docker image starts `restfulHarvest`; it does not open an interactive theHarvester CLI: +The Docker image starts `harvestview`; it does not open an interactive theHarvester CLI. Create the operator-key secret before the first start: ```bash git clone https://github.com/laramies/theHarvester.git cd theHarvester -docker compose up --build +install -d -m 0700 .secrets +openssl rand -hex 32 > .secrets/operator-api-key +chmod 0444 .secrets/operator-api-key +docker compose up --build -d +docker compose ps ``` -Open [http://127.0.0.1:5000/docs](http://127.0.0.1:5000/docs) after the service starts. +The `0700` directory protects the secret on the host, while the read-only `0444` file lets the unprivileged container process read its bind-mounted copy. Open [HarvestView](http://127.0.0.1:5000/) after the service starts. Swagger remains available at [http://127.0.0.1:5000/docs](http://127.0.0.1:5000/docs). -The supplied Compose mapping publishes port `5000` on every host interface. For a local-only service, change it to: +The supplied configuration publishes container port `8000` only on host `127.0.0.1:5000`, runs as an unprivileged user with a read-only root filesystem, and stores run records in the `theharvester-data` volume. The image includes Chromium for optional screenshot capture. -```yaml -ports: - - "127.0.0.1:5000:80" +```bash +docker compose logs -f theharvester.svc.local +docker compose down ``` -Do not expose the service directly to an untrusted network. Core query routes are not authenticated. +Every `/api/v1/*` route is authenticated. Do not change the loopback port mapping or expose the service directly to an untrusted network without adding TLS and network access controls. ## Next step diff --git a/docs/wiki/Rest-API.md b/docs/wiki/Rest-API.md index ddb78b80..44ba86c1 100644 --- a/docs/wiki/Rest-API.md +++ b/docs/wiki/Rest-API.md @@ -1,6 +1,6 @@ # REST API -`restfulHarvest` serves HarvestView at `/` and one versioned API for local automation. +`harvestview` serves the web application at `/` and one versioned API for local automation. ## Start the service @@ -8,10 +8,10 @@ Set a long random API key before startup: ```bash export THEHARVESTER_API_KEY='replace-with-a-long-random-value' -uv run restfulHarvest +uv run harvestview ``` -The service binds to `127.0.0.1:5000` by default. Use `uv run restfulHarvest -h` for launcher options. +The service binds to `127.0.0.1:5000` by default. Use `uv run harvestview -h` for launcher options. Open: diff --git a/docs/wiki/Troubleshooting.md b/docs/wiki/Troubleshooting.md index 4c77b685..6e5622e8 100644 --- a/docs/wiki/Troubleshooting.md +++ b/docs/wiki/Troubleshooting.md @@ -77,14 +77,14 @@ If Chromium reports missing Linux libraries, install the host dependencies recom Start with: ```bash -uv run restfulHarvest --log-level debug +uv run harvestview --log-level debug ``` Then open [http://127.0.0.1:5000/docs](http://127.0.0.1:5000/docs). - `401` on `/api/v1/*`: the `X-API-Key` header or HarvestView browser session does not match. - `503` on `/api/v1/*`: `THEHARVESTER_API_KEY` was not configured before startup. -- `429`: a reverse proxy or remote provider applied its own rate limit. `restfulHarvest` has no built-in request limiter. +- `429`: a reverse proxy or remote provider applied its own rate limit. `harvestview` has no built-in request limiter. - `503` when creating a run: the execution worker is disabled or unavailable. ## Docker @@ -94,7 +94,7 @@ docker compose ps docker compose logs theharvester.svc.local ``` -The container runs the REST API on container port `8000`, published as host port `5000` by the supplied Compose file. +The container runs HarvestView and the REST API on container port `8000`, published only as `127.0.0.1:5000` by the supplied Compose file. If startup reports a missing secret, create `.secrets/operator-api-key` as shown in the installation guide. ## File an actionable issue diff --git a/pyproject.toml b/pyproject.toml index 2911222b..3b824f78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,7 @@ dev = [ [project.scripts] theHarvester = "theHarvester.theHarvester:main" -restfulHarvest = "theHarvester.restfulHarvest:main" +harvestview = "theHarvester.harvestview:main" [tool.pytest.ini_options] minversion = "8.3.3" diff --git a/tests/test_docker.py b/tests/test_docker.py new file mode 100644 index 00000000..eb4653b8 --- /dev/null +++ b/tests/test_docker.py @@ -0,0 +1,63 @@ +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).parents[1] + + +def test_container_starts_and_checks_harvestview_on_port_8000() -> None: + dockerfile = (REPO_ROOT / 'Dockerfile').read_text(encoding='utf-8') + + assert 'EXPOSE 8000' in dockerfile + assert 'from theHarvester.lib.api.auth import _configured_api_key' in dockerfile + assert 'key = _configured_api_key(); assert key' in dockerfile + assert 'http://127.0.0.1:8000/api/v1/runs' in dockerfile + assert "headers={'X-API-Key': key}" in dockerfile + assert 'ENTRYPOINT ["harvestview"]' in dockerfile + assert 'CMD ["-H", "0.0.0.0", "-p", "8000"]' in dockerfile + assert '127.0.0.1:8000/app' not in dockerfile + assert 'COPY --chown=10001:10001 theHarvester ./theHarvester' not in dockerfile + + +def test_container_rebuilds_the_local_package_when_source_changes() -> None: + dockerfile = (REPO_ROOT / 'Dockerfile').read_text(encoding='utf-8') + + assert '--reinstall-package theharvester' in dockerfile + + +def test_container_smoke_uses_the_unversioned_jsonl_contract() -> None: + workflow = (REPO_ROOT / '.github/workflows/harvestview-container.yml').read_text(encoding='utf-8') + + assert '"evidence_status":"complete"' in workflow + assert '"type":"ip"' in workflow + assert 'ip-address' not in workflow + assert 'schema_version' not in workflow + + +def test_compose_keeps_harvestview_local_and_persists_private_run_data() -> None: + compose = yaml.safe_load((REPO_ROOT / 'docker-compose.yml').read_text(encoding='utf-8')) + service = compose['services']['theharvester.svc.local'] + + assert service['ports'] == ['127.0.0.1:${THEHARVESTER_PORT:-5000}:8000'] + assert service['environment']['THEHARVESTER_API_KEY_FILE'] == '/run/secrets/operator-api-key' + assert service['environment']['THEHARVESTER_HARVESTVIEW_LOCAL_PROXY'] == 'enabled' + assert service['read_only'] is True + assert service['cap_drop'] == ['ALL'] + assert 'theharvester-data:/var/lib/theharvester' in service['volumes'] + assert compose['secrets']['operator-api-key']['file'] == '${THEHARVESTER_API_KEY_FILE:-./.secrets/operator-api-key}' + + +def test_container_build_context_excludes_local_secrets_and_run_data() -> None: + ignored = set((REPO_ROOT / '.dockerignore').read_text(encoding='utf-8').splitlines()) + + assert { + '.git', + '.env', + '.env.*', + '.secrets/', + '.venv/', + 'test-results/', + '*.sqlite*', + 'theHarvester/data/api-keys.yaml', + 'theHarvester/data/proxies.yaml', + } <= ignored diff --git a/tests/test_harvestview_launcher.py b/tests/test_harvestview_launcher.py new file mode 100644 index 00000000..95fb02ca --- /dev/null +++ b/tests/test_harvestview_launcher.py @@ -0,0 +1,26 @@ +import sys +import tomllib +from pathlib import Path + +import pytest + +from theHarvester import harvestview + + +def test_project_scripts_expose_harvestview() -> None: + scripts = tomllib.loads(Path('pyproject.toml').read_text(encoding='utf-8'))['project']['scripts'] + + assert scripts == { + 'theHarvester': 'theHarvester.theHarvester:main', + 'harvestview': 'theHarvester.harvestview:main', + } + + +def test_help_does_not_offer_rate_limit_configuration(monkeypatch, capsys): + monkeypatch.setattr(sys, 'argv', ['harvestview', '--help']) + + with pytest.raises(SystemExit) as exit_info: + harvestview.main() + + assert exit_info.value.code == 0 + assert '--rate-limit' not in capsys.readouterr().out diff --git a/tests/test_restful_harvest.py b/tests/test_restful_harvest.py deleted file mode 100644 index 58b3d7f6..00000000 --- a/tests/test_restful_harvest.py +++ /dev/null @@ -1,15 +0,0 @@ -import sys - -import pytest - -from theHarvester import restfulHarvest - - -def test_help_does_not_offer_rate_limit_configuration(monkeypatch, capsys): - monkeypatch.setattr(sys, 'argv', ['restfulHarvest', '--help']) - - with pytest.raises(SystemExit) as exit_info: - restfulHarvest.main() - - assert exit_info.value.code == 0 - assert '--rate-limit' not in capsys.readouterr().out diff --git a/theHarvester/restfulHarvest.py b/theHarvester/harvestview.py similarity index 100% rename from theHarvester/restfulHarvest.py rename to theHarvester/harvestview.py