WEBDIENSTE/theHarvester
1
0
Fork 0
mirror of https://github.com/laramies/theHarvester.git synced 2026-08-17 19:35:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

Package HarvestView for local deployment (#2513)

Browse Source 
* feat: package HarvestView for local deployment

* Align container smoke with JSONL API

* Align container smoke with unversioned JSONL

* refactor: rename web launcher to harvestview
This commit is contained in:
Matt
2026-08-10 00:19:00 -04:00
committed by GitHub
parent 0d2ab5a0b3
commit d766c7c59a
19 changed files with 327 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.dockerignore
+22 -4
View File
@@ -1,10 +1,28 @@
.github/*
.git
.github/
.gitignore
.gitattributes
.git-blame-ignore-revs
.env
.env.*
.secrets/
.impeccable/
theHarvester/data/api-keys.yaml
theHarvester/data/proxies.yaml
.idea/
.pytest_cache
.mypy_cache
tests/*
.venv/
.uv-cache/
__pycache__/
*.py[cod]
*.sqlite*
*.db
.pytest_cache/
.mypy_cache/
.ruff_cache/
build/
dist/
test-results/
tests/
README/
bin/
theHarvester-logo.png
.github/workflows/dockerci.yml
+1 -1
View File
@@ -20,4 +20,4 @@ jobs:
- name: Build the Docker image
run: docker build --tag theharvester .
- name: Smoke test
run: docker run --rm theharvester --help | grep restfulHarvest
run: docker run --rm theharvester --help | grep harvestview
.github/workflows/harvestview-container.yml
+90
View File
@@ -0,0 +1,90 @@
name: HarvestView container smoke
on:
workflow_dispatch:
pull_request:
paths:
- '.dockerignore'
- '.github/workflows/harvestview-container.yml'
- 'Dockerfile'
- 'docker-compose.yml'
- 'pyproject.toml'
- 'theHarvester/lib/api/**'
- 'tests/test_docker.py'
- 'uv.lock'
permissions:
contents: read
jobs:
container-smoke:
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- name: Harden runner
uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
with:
egress-policy: audit
- name: Check out repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: Prepare operator secret
run: |
install -d -m 0700 .secrets
printf '%s\n' 'harvestview-container-smoke-key' > .secrets/operator-api-key
chmod 0444 .secrets/operator-api-key
- name: Build and start container
env:
THEHARVESTER_PORT: '8769'
run: |
docker compose -p harvestview-container-smoke build theharvester.svc.local
docker compose -p harvestview-container-smoke up -d --no-build
- name: Verify routes, Chromium, and restart persistence
env:
THEHARVESTER_PORT: '8769'
run: |
for attempt in $(seq 1 60); do
if curl --fail --silent --show-error http://127.0.0.1:8769/ > /tmp/harvestview.html; then
break
fi
sleep 1
done
grep --fixed-strings '<title>HarvestView</title>' /tmp/harvestview.html
test "$(curl --silent --output /dev/null --write-out '%{http_code}' http://127.0.0.1:8769/app)" = '404'
test "$(curl --silent --output /dev/null --write-out '%{http_code}' http://127.0.0.1:8769/api/v1/runs)" = '401'
curl --fail --silent --show-error \
--header 'X-API-Key: harvestview-container-smoke-key' \
http://127.0.0.1:8769/api/v1/runs > /dev/null
printf '%s\n' \
'{"completed_at":"2026-08-08T01:01:00Z","counts":{"hostname":1,"ip":1},"evidence_status":"complete","result_count":2,"run_id":"9f9b4383-6cc4-4f3f-80a4-c8d21930dc2d","started_at":"2026-08-08T01:00:00Z","target":"example.test","type":"summary"}' \
'{"type":"hostname","value":"www.example.test"}' \
'{"type":"ip","value":"192.0.2.10"}' > /tmp/smoke.jsonl
curl --fail --silent --show-error \
--request POST \
--header 'Content-Type: application/x-ndjson' \
--header 'X-API-Key: harvestview-container-smoke-key' \
--data-binary @/tmp/smoke.jsonl \
'http://127.0.0.1:8769/api/v1/runs/import?filename=smoke.jsonl' > /tmp/imported-run.json
run_id="$(python3 -c "import json; print(json.load(open('/tmp/imported-run.json', encoding='utf-8'))['run_id'])")"
curl --fail --silent --show-error \
--header 'X-API-Key: harvestview-container-smoke-key' \
"http://127.0.0.1:8769/api/v1/runs/${run_id}/export" \
| python3 -c "import json, sys; rows = [json.loads(line) for line in sys.stdin if line.strip()]; assert rows[0]['type'] == 'summary'; assert any(row.get('type') == 'ip' for row in rows[1:]); assert len(rows) == 3"
docker compose -p harvestview-container-smoke exec -T theharvester.svc.local \
python -c "from playwright.sync_api import sync_playwright; p = sync_playwright().start(); browser = p.chromium.launch(headless=True); browser.close(); p.stop()"
docker compose -p harvestview-container-smoke restart theharvester.svc.local
curl --retry 30 --retry-all-errors --retry-delay 1 --fail --silent --show-error \
--header 'X-API-Key: harvestview-container-smoke-key' \
"http://127.0.0.1:8769/api/v1/runs/${run_id}" > /tmp/persisted-run.json
python3 -c "import json; data = json.load(open('/tmp/persisted-run.json', encoding='utf-8')); assert data['status'] == 'completed'; assert len(data['results']) == 2"
- name: Stop container
if: ${{ always() }}
env:
THEHARVESTER_PORT: '8769'
run: docker compose -p harvestview-container-smoke down --volumes
CHANGELOG.md
+7 -5
View File
@@ -8,9 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Added
- Added HarvestView, an authenticated local browser workspace backed by a durable single-worker `/api/v1` run lifecycle with cancellation, deadlines, JSONL-only file interchange, retained partial evidence, and real-browser regression coverage.
- Added a pinned, non-root Docker Compose deployment for HarvestView and the REST API with localhost-only publishing, file-secret authentication, private durable run storage, and an authenticated API health check.
- Added bounded recursive DNS discovery with three-vantage consensus, closest-encloser wildcard controls, exact-address PTR evidence, and hard query, depth, runtime, and zero-yield limits.
- Added authenticated REST routes for listing completed enumeration runs and retrieving their normalized evidence.
- Added an authenticated HIBP verified-domain source for CLI and conditionally authenticated REST queries that retains normalized account emails and stable breach names without retaining the raw account mapping.
- Added an authenticated HIBP verified-domain source for CLI and `/api/v1` runs that retains normalized account emails and stable breach names without retaining the raw account mapping.
- Added keyless Shodan Certificate Transparency hostname discovery with bounded requests and offline response contracts.
- Added bounded, keyless subdomain discovery through Arquivo.pt's public CDX API with offline response contracts.
- Added transactional SQLite storage and loading for completed full-pipeline runs without changing legacy result rows.
@@ -27,16 +28,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Added root contributor and security policies, structured issue forms, repository agent guidance, discovery terminology, and an operator-focused documentation wiki ([d090a29a](https://github.com/laramies/theHarvester/commit/d090a29a), [7c491ef5](https://github.com/laramies/theHarvester/commit/7c491ef5), [8b9d420b](https://github.com/laramies/theHarvester/commit/8b9d420b)).
### Changed
- Made `harvestview` the sole launcher for the local web application and REST API.
- Standardized SQLite, JSONL, API, and HarvestView result names on `hostname` and `ip` without a presentation alias.
- Standardized URL-producing adapters, JSON, JSONL, SQLite, and API evidence on one `url` result kind while preserving producer provenance.
- Replaced the unversioned and provider-specific REST routes with one authenticated `/api/v1` source and run contract shared with CLI execution semantics.
- Fixed proxied POST requests so they retain the request method, body, and query parameters.
- Migrated Pentest-Tools discovery to its API v2 Bearer-authenticated scan, status, and output endpoints.
- Included HIBP verified-domain in `all` and matching capability selectors like every other P0 source, with REST operator authentication applied after source expansion when its provider key is configured.
- Allowed REST `/query` requests to select discovery sources by result capability, matching the CLI's union semantics while preserving explicit source selection.
- Included HIBP verified-domain in `all` and matching capability selectors like every other P0 source.
- Changed `-b all` to select every cataloged P0 passive source once while leaving P1 DNS and P2 direct sources available through explicit selection.
- Expanded Common Crawl discovery to use every unique crawl ending within one year of the newest catalog entry, validate catalog endpoints, batch requests, cap each query at 100 pages, and enforce the CLI result limit across page requests ([249ce64b](https://github.com/laramies/theHarvester/commit/249ce64b), [70470cd8](https://github.com/laramies/theHarvester/commit/70470cd8)).
- Completed bounded pagination for Wayback Archive and Cert Spotter, including continuation handling, truncation diagnostics, and preservation of partial results on provider failures ([df6ff2c9](https://github.com/laramies/theHarvester/commit/df6ff2c9), [f85a08ff](https://github.com/laramies/theHarvester/commit/f85a08ff)).
- Routed operator messages and diagnostics through logging, preserved host logging policy and existing handlers, and configured logging for the standalone API example ([8a7b8b71](https://github.com/laramies/theHarvester/commit/8a7b8b71)).
- Routed operator messages and diagnostics through logging, preserved host logging policy and existing handlers, and configured logging for the API service ([8a7b8b71](https://github.com/laramies/theHarvester/commit/8a7b8b71)).
- Added credential configuration adapters, deferred proxy configuration loading until required, and retained compatibility accessors such as `Core.brave_key()` ([ccd91176](https://github.com/laramies/theHarvester/commit/ccd91176)).
- Centralized hostname scope normalization for parser and storage boundaries, including case-insensitive targets, trailing dots, optional `www.` prefixes, and exact DNS-label matching ([c0a0b653](https://github.com/laramies/theHarvester/commit/c0a0b653), [a474f086](https://github.com/laramies/theHarvester/commit/a474f086), [70470cd8](https://github.com/laramies/theHarvester/commit/70470cd8)).
- Replaced deprecated hostname resolution with `getaddrinfo`-based handling ([6a847435](https://github.com/laramies/theHarvester/commit/6a847435)).
Dockerfile
+40 -15
View File
@@ -1,25 +1,50 @@
FROM python:3.14-slim-trixie@sha256:cea0e6040540fb2b965b6e7fb5ffa00871e632eef63719f0ea54bca189ce14a6
ARG PYTHON_IMAGE=python:3.14-slim-trixie@sha256:cea0e6040540fb2b965b6e7fb5ffa00871e632eef63719f0ea54bca189ce14a6
FROM ${PYTHON_IMAGE} AS builder
ENV UV_COMPILE_BYTECODE=1 \
UV_LINK_MODE=copy \
UV_PROJECT_ENVIRONMENT=/opt/venv
WORKDIR /app
COPY pyproject.toml uv.lock README.md ./
COPY theHarvester ./theHarvester
RUN --mount=type=cache,target=/root/.cache/uv \
--mount=from=ghcr.io/astral-sh/uv:0.12.1@sha256:cf4eedcaa81655197f625739489effcbe71b61ceb1506f332c3facae5deceded,source=/uv,target=/usr/local/bin/uv \
uv sync --locked --no-dev --no-editable --reinstall-package theharvester
FROM ${PYTHON_IMAGE} AS runtime
LABEL maintainer="@jay_townsend1 & @NotoriousRebel1"
LABEL org.opencontainers.image.title="theHarvester" \
org.opencontainers.image.description="theHarvester API and HarvestView web application" \
org.opencontainers.image.licenses="GPL-2.0-only" \
org.opencontainers.image.source="https://github.com/laramies/theHarvester"
RUN useradd -m -u 1000 -s /bin/bash theharvester
ENV HOME=/var/lib/theharvester \
PATH=/opt/venv/bin:$PATH \
PLAYWRIGHT_BROWSERS_PATH=/opt/ms-playwright \
PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
THEHARVESTER_RUN_ARTIFACTS=/var/lib/theharvester/run-artifacts \
THEHARVESTER_RUN_DB=/var/lib/theharvester/runs.sqlite
RUN apt-get update && apt-get upgrade -yqq && apt-get clean && \
rm -rf /var/lib/apt/lists/*
# Set workdir and copy project files
WORKDIR /app
COPY . /app
COPY --from=builder /opt/venv /opt/venv
# Create and sync environment using uv
# Compile bytecode for faster startup and install to system site-packages
RUN --mount=from=ghcr.io/astral-sh/uv,source=/uv,target=/bin/uv \
UV_PROJECT_ENVIRONMENT=/usr/local uv sync --no-dev --no-cache --compile-bytecode
RUN groupadd --gid 10001 theharvester \
&& useradd --uid 10001 --gid 10001 --home-dir "$HOME" --shell /usr/sbin/nologin theharvester \
&& install -d -o 10001 -g 10001 -m 0700 "$HOME" "$THEHARVESTER_RUN_ARTIFACTS" \
&& playwright install --with-deps --only-shell chromium \
&& chmod -R a+rX "$PLAYWRIGHT_BROWSERS_PATH" \
&& rm -rf /var/lib/apt/lists/*
# Use non-root user
USER theharvester
# Expose port if the service listens on 80
EXPOSE 80
EXPOSE 8000
# Run the application as theharvester user
ENTRYPOINT ["restfulHarvest", "-H", "0.0.0.0", "-p", "80"]
HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
CMD ["python", "-c", "from urllib.request import Request, urlopen; from theHarvester.lib.api.auth import _configured_api_key; key = _configured_api_key(); assert key; urlopen(Request('http://127.0.0.1:8000/api/v1/runs', headers={'X-API-Key': key}), timeout=3).close()"]
ENTRYPOINT ["harvestview"]
CMD ["-H", "0.0.0.0", "-p", "8000"]
PRODUCT.md
+2 -2
View File
@@ -14,8 +14,8 @@ The operator is a technically capable security practitioner working at a desk, o
- Cancel work and know whether cancellation is requested, in progress, or complete.
- Reopen prior runs and compare route-specific evidence without rerunning reconnaissance.
- Import existing theHarvester JSONL evidence.
- Export normalized results and inspect managed screenshots.
- Start a screenshot or DNS brute-force action from a subdomain result without changing the parent evidence.
- Export normalized JSONL results and inspect managed screenshots.
- Start a screenshot or DNS brute-force action from a hostname result without changing the parent evidence.
## Product principles
README.md
+21 -2
View File
@@ -85,11 +85,11 @@ Screenshot capture also requires a Playwright-compatible browser; see the instal
## HarvestView and REST API
`restfulHarvest` starts a FastAPI service on `127.0.0.1:5000` by default:
`harvestview` starts the local web application and API on `127.0.0.1:5000` by default:
```bash
export THEHARVESTER_API_KEY='replace-with-a-long-random-value'
uv run restfulHarvest
uv run harvestview
```
Open [HarvestView](http://127.0.0.1:5000/) to run and inspect finite enumerations in the local web app. The server gives the local browser a derived HttpOnly session cookie, so the API key is never entered into or stored by HarvestView.
@@ -104,6 +104,25 @@ an isolated deployment.
Open [Swagger](http://127.0.0.1:5000/docs) or [ReDoc](http://127.0.0.1:5000/redoc) for the automation contract.
### Docker Compose
The supplied Compose service runs as an unprivileged user, stores run records in a named volume, loads the operator key from a file secret, and publishes only to host loopback. Create the secret before the first start:
```bash
install -d -m 0700 .secrets
openssl rand -hex 32 > .secrets/operator-api-key
chmod 0444 .secrets/operator-api-key
docker compose up --build -d
docker compose ps
```
The `0700` directory protects the secret on the host, while the read-only `0444` file lets the unprivileged container process read its bind-mounted copy. Open [HarvestView](http://127.0.0.1:5000/). The image includes Chromium for optional screenshots. Provider keys and proxies remain in the existing read-only YAML mounts and are excluded from the image build context.
```bash
docker compose logs -f theharvester.svc.local
docker compose down
```
| Route | Purpose |
| --- | --- |
| `GET /api/v1/sources` | List registered discovery sources and capabilities. |
bin/restfulHarvest → bin/harvestview
+1 -1
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
from theHarvester.restfulHarvest import main
from theHarvester.harvestview import main
if __name__ == '__main__':
main()
docker-compose.yml
+29 -10
View File
@@ -1,15 +1,34 @@
services:
theharvester.svc.local:
container_name: theHarvester
image: theharvester:local
build:
context: .
init: true
restart: unless-stopped
read_only: true
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
shm_size: 1gb
tmpfs:
- /tmp:rw,noexec,nosuid,nodev,size=128m
environment:
THEHARVESTER_API_KEY_FILE: /run/secrets/operator-api-key
THEHARVESTER_HARVESTVIEW_LOCAL_PROXY: enabled
secrets:
- source: operator-api-key
target: operator-api-key
volumes:
- ./theHarvester/data/api-keys.yaml:/root/.theHarvester/api-keys.yaml
- ./theHarvester/data/api-keys.yaml:/etc/theHarvester/api-keys.yaml
- ./theHarvester/data/proxies.yaml:/etc/theHarvester/proxies.yaml
- ./theHarvester/data/proxies.yaml:/root/.theHarvester/proxies.yaml
build: .
- theharvester-data:/var/lib/theharvester
- ./theHarvester/data/api-keys.yaml:/etc/theHarvester/api-keys.yaml:ro
- ./theHarvester/data/proxies.yaml:/etc/theHarvester/proxies.yaml:ro
ports:
- "5000:80"
- "127.0.0.1:${THEHARVESTER_PORT:-5000}:8000"
networks:
default:
name: app_theHarvester_network
volumes:
theharvester-data:
secrets:
operator-api-key:
file: ${THEHARVESTER_API_KEY_FILE:-./.secrets/operator-api-key}
docs/wiki/Configuration-and-API-Keys.md
+1 -1
View File
@@ -69,7 +69,7 @@ Every `/api/v1/*` route requires a server-side key:
```bash
export THEHARVESTER_API_KEY='replace-with-a-long-random-value'
uv run restfulHarvest
uv run harvestview
```
API clients send the same value in the `X-API-Key` header. HarvestView receives a derived HttpOnly browser cookie when it is opened locally, so the key is never entered into or stored by the web app. Provider credentials remain in `api-keys.yaml` and cannot be supplied through an API request.
docs/wiki/Home.md
+3 -2
View File
@@ -17,8 +17,9 @@ DNS brute force, DNS resolution, takeover checks, screenshots, and API-path scan
## Choose an interface
- **Command line:** best for interactive reconnaissance and report generation.
- **REST API:** best for local integrations and browser-accessible Swagger/ReDoc documentation. See [REST API](Rest-API).
- **Docker Compose:** runs the REST API service, not the normal interactive CLI.
- **HarvestView:** best for creating and inspecting durable local enumeration runs in a browser. See [REST API](Rest-API).
- **REST API:** best for authenticated local integrations and Swagger/ReDoc documentation. See [REST API](Rest-API).
- **Docker Compose:** packages HarvestView and the REST API, not the normal interactive CLI.
The repository [README](https://github.com/laramies/theHarvester) owns the current feature summary and source/result matrix. The live `theHarvester -h` output owns the complete CLI reference.
docs/wiki/Installation.md
+14 -10
View File
@@ -33,7 +33,7 @@ uv sync --all-groups
uv run pytest
```
The supported console commands are `theHarvester` and `restfulHarvest`. There is no root `theHarvester.py` launcher.
The supported console commands are `theHarvester` and `harvestview`. There is no root `theHarvester.py` launcher.
### Self-host Tabulator
@@ -63,26 +63,30 @@ uv run playwright install chromium
On Linux, Playwright may report missing system libraries. Follow the host-specific dependency instructions printed by Playwright, then rerun the browser installation.
## Docker Compose API service
## Docker Compose HarvestView service
The Docker image starts `restfulHarvest`; it does not open an interactive theHarvester CLI:
The Docker image starts `harvestview`; it does not open an interactive theHarvester CLI. Create the operator-key secret before the first start:
```bash
git clone https://github.com/laramies/theHarvester.git
cd theHarvester
docker compose up --build
install -d -m 0700 .secrets
openssl rand -hex 32 > .secrets/operator-api-key
chmod 0444 .secrets/operator-api-key
docker compose up --build -d
docker compose ps
```
Open [http://127.0.0.1:5000/docs](http://127.0.0.1:5000/docs) after the service starts.
The `0700` directory protects the secret on the host, while the read-only `0444` file lets the unprivileged container process read its bind-mounted copy. Open [HarvestView](http://127.0.0.1:5000/) after the service starts. Swagger remains available at [http://127.0.0.1:5000/docs](http://127.0.0.1:5000/docs).
The supplied Compose mapping publishes port `5000` on every host interface. For a local-only service, change it to:
The supplied configuration publishes container port `8000` only on host `127.0.0.1:5000`, runs as an unprivileged user with a read-only root filesystem, and stores run records in the `theharvester-data` volume. The image includes Chromium for optional screenshot capture.
```yaml
ports:
- "127.0.0.1:5000:80"
```bash
docker compose logs -f theharvester.svc.local
docker compose down
```
Do not expose the service directly to an untrusted network. Core query routes are not authenticated.
Every `/api/v1/*` route is authenticated. Do not change the loopback port mapping or expose the service directly to an untrusted network without adding TLS and network access controls.
## Next step
docs/wiki/Rest-API.md
+3 -3
View File
@@ -1,6 +1,6 @@
# REST API
`restfulHarvest` serves HarvestView at `/` and one versioned API for local automation.
`harvestview` serves the web application at `/` and one versioned API for local automation.
## Start the service
@@ -8,10 +8,10 @@ Set a long random API key before startup:
```bash
export THEHARVESTER_API_KEY='replace-with-a-long-random-value'
uv run restfulHarvest
uv run harvestview
```
The service binds to `127.0.0.1:5000` by default. Use `uv run restfulHarvest -h` for launcher options.
The service binds to `127.0.0.1:5000` by default. Use `uv run harvestview -h` for launcher options.
Open:
docs/wiki/Troubleshooting.md
+3 -3
View File
@@ -77,14 +77,14 @@ If Chromium reports missing Linux libraries, install the host dependencies recom
Start with:
```bash
uv run restfulHarvest --log-level debug
uv run harvestview --log-level debug
```
Then open [http://127.0.0.1:5000/docs](http://127.0.0.1:5000/docs).
- `401` on `/api/v1/*`: the `X-API-Key` header or HarvestView browser session does not match.
- `503` on `/api/v1/*`: `THEHARVESTER_API_KEY` was not configured before startup.
- `429`: a reverse proxy or remote provider applied its own rate limit. `restfulHarvest` has no built-in request limiter.
- `429`: a reverse proxy or remote provider applied its own rate limit. `harvestview` has no built-in request limiter.
- `503` when creating a run: the execution worker is disabled or unavailable.
## Docker
@@ -94,7 +94,7 @@ docker compose ps
docker compose logs theharvester.svc.local
```
The container runs the REST API on container port `8000`, published as host port `5000` by the supplied Compose file.
The container runs HarvestView and the REST API on container port `8000`, published only as `127.0.0.1:5000` by the supplied Compose file. If startup reports a missing secret, create `.secrets/operator-api-key` as shown in the installation guide.
## File an actionable issue
pyproject.toml
+1 -1
View File
@@ -64,7 +64,7 @@ dev = [
[project.scripts]
theHarvester = "theHarvester.theHarvester:main"
restfulHarvest = "theHarvester.restfulHarvest:main"
harvestview = "theHarvester.harvestview:main"
[tool.pytest.ini_options]
minversion = "8.3.3"
tests/test_docker.py
+63
View File
@@ -0,0 +1,63 @@
from pathlib import Path
import yaml
REPO_ROOT = Path(__file__).parents[1]
def test_container_starts_and_checks_harvestview_on_port_8000() -> None:
dockerfile = (REPO_ROOT / 'Dockerfile').read_text(encoding='utf-8')
assert 'EXPOSE 8000' in dockerfile
assert 'from theHarvester.lib.api.auth import _configured_api_key' in dockerfile
assert 'key = _configured_api_key(); assert key' in dockerfile
assert 'http://127.0.0.1:8000/api/v1/runs' in dockerfile
assert "headers={'X-API-Key': key}" in dockerfile
assert 'ENTRYPOINT ["harvestview"]' in dockerfile
assert 'CMD ["-H", "0.0.0.0", "-p", "8000"]' in dockerfile
assert '127.0.0.1:8000/app' not in dockerfile
assert 'COPY --chown=10001:10001 theHarvester ./theHarvester' not in dockerfile
def test_container_rebuilds_the_local_package_when_source_changes() -> None:
dockerfile = (REPO_ROOT / 'Dockerfile').read_text(encoding='utf-8')
assert '--reinstall-package theharvester' in dockerfile
def test_container_smoke_uses_the_unversioned_jsonl_contract() -> None:
workflow = (REPO_ROOT / '.github/workflows/harvestview-container.yml').read_text(encoding='utf-8')
assert '"evidence_status":"complete"' in workflow
assert '"type":"ip"' in workflow
assert 'ip-address' not in workflow
assert 'schema_version' not in workflow
def test_compose_keeps_harvestview_local_and_persists_private_run_data() -> None:
compose = yaml.safe_load((REPO_ROOT / 'docker-compose.yml').read_text(encoding='utf-8'))
service = compose['services']['theharvester.svc.local']
assert service['ports'] == ['127.0.0.1:${THEHARVESTER_PORT:-5000}:8000']
assert service['environment']['THEHARVESTER_API_KEY_FILE'] == '/run/secrets/operator-api-key'
assert service['environment']['THEHARVESTER_HARVESTVIEW_LOCAL_PROXY'] == 'enabled'
assert service['read_only'] is True
assert service['cap_drop'] == ['ALL']
assert 'theharvester-data:/var/lib/theharvester' in service['volumes']
assert compose['secrets']['operator-api-key']['file'] == '${THEHARVESTER_API_KEY_FILE:-./.secrets/operator-api-key}'
def test_container_build_context_excludes_local_secrets_and_run_data() -> None:
ignored = set((REPO_ROOT / '.dockerignore').read_text(encoding='utf-8').splitlines())
assert {
'.git',
'.env',
'.env.*',
'.secrets/',
'.venv/',
'test-results/',
'*.sqlite*',
'theHarvester/data/api-keys.yaml',
'theHarvester/data/proxies.yaml',
} <= ignored
tests/test_harvestview_launcher.py
+26
View File
@@ -0,0 +1,26 @@
import sys
import tomllib
from pathlib import Path
import pytest
from theHarvester import harvestview
def test_project_scripts_expose_harvestview() -> None:
scripts = tomllib.loads(Path('pyproject.toml').read_text(encoding='utf-8'))['project']['scripts']
assert scripts == {
'theHarvester': 'theHarvester.theHarvester:main',
'harvestview': 'theHarvester.harvestview:main',
}
def test_help_does_not_offer_rate_limit_configuration(monkeypatch, capsys):
monkeypatch.setattr(sys, 'argv', ['harvestview', '--help'])
with pytest.raises(SystemExit) as exit_info:
harvestview.main()
assert exit_info.value.code == 0
assert '--rate-limit' not in capsys.readouterr().out
tests/test_restful_harvest.py
-15
View File
@@ -1,15 +0,0 @@
import sys
import pytest
from theHarvester import restfulHarvest
def test_help_does_not_offer_rate_limit_configuration(monkeypatch, capsys):
monkeypatch.setattr(sys, 'argv', ['restfulHarvest', '--help'])
with pytest.raises(SystemExit) as exit_info:
restfulHarvest.main()
assert exit_info.value.code == 0
assert '--rate-limit' not in capsys.readouterr().out
theHarvester/restfulHarvest.py → theHarvester/harvestview.py
View File