mirror of
https://github.com/yogeshojha/rengine.git
synced 2026-10-20 07:11:28 +02:00
A full static review of the 3.0 tree, run as two fan-out passes with every
finding verified against the running stack rather than taken on inspection.
Correctness and security
- instance settings encrypted the mask over the stored AI key: `!= MASK` never
matched `••••••••abcd`, so saving settings destroyed the key and `test_ai`
posted the mask as a bearer token
- the MFA token was replayable for its full five-minute life; the gate now
checks revocation and consumes the token after a successful exchange
- logout revoked nothing for a bearer client while answering "Logged out"
- `POST /api-keys/{id}/test` returned the plaintext key inside httpx's error
string, which embeds the whole URL
- an interest rule could be edited or deleted through another project's id
- redis ran the celery broker under `allkeys-lru`, free to evict a queued task
or a chord counter; now `noeviction`
- nuclei's stored command carried the OAST token and every auth header
- a finalize step could poison the session and leave a scan RUNNING for ever
- `is:issue` raised TypeError on every search
- offset paging repeated and skipped rows in four dimensions
- a cancelled run kept a stale `scans.*_found` rollup
- every PDF report raised `KeyError: 'software'`
- `version_key` truncated a wide numeric run and inverted release ordering
- psycopg raises a NUL as a bare ValueError, above StatementError, so every
savepoint guard was blind to the case it was written for
Performance
- endpoints.host had no trigram index: 35.9ms seq scan -> 0.389ms
- free text compiled to one OR, not the prescribed UNION ALL: 132ms -> 41ms
- `/http-assets/summary` and `/ips/summary` materialised whole tables
- the cross-link `ip` kind ran an estate query per page and drew nothing
Frontend
- `$effect` loaders that read the state they write, an unbounded request loop
on a failing endpoint
- error states that rendered as empty states
- 2FA backup codes were unusable while the copy said they worked
- a "Scan history" dialog that always reported none
- ~400 duplicated lines across five filter bars, now one component
Build and tooling
- every scanner installed at @latest; all 13 pinned to the measured versions
- CI built `web/`, gone since 2.x, with tags that cannot be valid
- the prettier pre-commit hook discarded its filenames and always passed
- the test suite destroyed itself when run twice
Also: subdomains from public code, via a pinned github-subdomains source and a
GitHub API key that subfinder receives too; sublist3r and oneforall removed
after measuring one unique host in 89 across four targets.
Verified: 853 tests, ruff clean over eleven packages, svelte-check 0 errors,
the count-is-a-promise gate at 131 checks and 0 mismatches, and 143 GET routes
with no 5xx. Four migrations, each with an exercised downgrade.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01DVkdx5ynZ6aof5kLFczmhX
313 lines
10 KiB
Python
313 lines
10 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import time
|
|
from uuid import UUID
|
|
|
|
import httpx
|
|
from fastapi import HTTPException, status
|
|
from sqlalchemy import select, text, update
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.core.crypto import encrypt_secret
|
|
from shared.http import get_async_client
|
|
from shared.models.proxy import (
|
|
PROXY_MODES,
|
|
PROXY_SCHEMES,
|
|
Proxy,
|
|
ProxyCreate,
|
|
ProxyEndpoint,
|
|
ProxyEndpointRead,
|
|
ProxyRead,
|
|
ProxyTestResult,
|
|
ProxyUpdate,
|
|
)
|
|
from shared.services.proxy_resolve import (
|
|
build_proxy_url as _build_url,
|
|
)
|
|
from shared.services.proxy_resolve import (
|
|
load_proxy_endpoints as _load_endpoints,
|
|
)
|
|
from shared.services.proxy_resolve import (
|
|
resolve_proxy_url as _resolve_proxy_url,
|
|
)
|
|
from shared.services.scan_resolve import MASK
|
|
from shared.utils.datetime import utc_now
|
|
from shared.utils.net import host_port
|
|
|
|
_TEST_URL = "https://api.ipify.org"
|
|
_TEST_TIMEOUT = 8.0
|
|
_TCP_TIMEOUT = 6.0
|
|
_MIN_PORT = 1
|
|
_MAX_PORT = 65535
|
|
_DEFAULT_LOCK_KEY = 0x70726F78
|
|
|
|
|
|
def _bad(detail: str) -> HTTPException:
|
|
return HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=detail)
|
|
|
|
|
|
def _validate_mode(mode: str) -> None:
|
|
if mode not in PROXY_MODES:
|
|
msg = f"Invalid mode. Must be one of {', '.join(PROXY_MODES)}."
|
|
raise _bad(msg)
|
|
|
|
|
|
def _validate_endpoints(endpoints: list[ProxyEndpoint]) -> None:
|
|
if not endpoints:
|
|
msg = "At least one endpoint is required."
|
|
raise _bad(msg)
|
|
for ep in endpoints:
|
|
if ep.scheme not in PROXY_SCHEMES:
|
|
msg = f"Invalid scheme '{ep.scheme}'. Must be one of {', '.join(PROXY_SCHEMES)}."
|
|
raise _bad(msg)
|
|
if not ep.host:
|
|
msg = "Endpoint host is required."
|
|
raise _bad(msg)
|
|
if not (_MIN_PORT <= ep.port <= _MAX_PORT):
|
|
msg = (
|
|
f"Invalid port {ep.port}. Must be between {_MIN_PORT} and {_MAX_PORT}."
|
|
)
|
|
raise _bad(msg)
|
|
|
|
|
|
def _endpoint_key(ep: ProxyEndpoint) -> tuple:
|
|
return (ep.scheme, ep.host, ep.port, ep.username)
|
|
|
|
|
|
def _mask_url(ep: ProxyEndpoint) -> str:
|
|
if ep.username:
|
|
cred = f"{ep.username}:{MASK}@" if ep.password else f"{ep.username}@"
|
|
return f"{ep.scheme}://{cred}{ep.host}:{ep.port}"
|
|
return f"{ep.scheme}://{ep.host}:{ep.port}"
|
|
|
|
|
|
def _endpoint_to_read(ep: ProxyEndpoint) -> ProxyEndpointRead:
|
|
return ProxyEndpointRead(
|
|
scheme=ep.scheme,
|
|
host=ep.host,
|
|
port=ep.port,
|
|
username=ep.username,
|
|
has_password=bool(ep.password),
|
|
url_masked=_mask_url(ep),
|
|
)
|
|
|
|
|
|
def _encrypt_endpoints(endpoints: list[ProxyEndpoint]) -> str:
|
|
payload = [e.model_dump() for e in endpoints]
|
|
return encrypt_secret(json.dumps(payload))
|
|
|
|
|
|
class ProxyService:
|
|
def __init__(self, session: AsyncSession):
|
|
self.session = session
|
|
|
|
def to_read(self, proxy: Proxy) -> ProxyRead:
|
|
endpoints = _load_endpoints(proxy)
|
|
return ProxyRead(
|
|
id=proxy.id,
|
|
name=proxy.name,
|
|
description=proxy.description,
|
|
mode=proxy.mode,
|
|
is_active=proxy.is_active,
|
|
is_default=proxy.is_default,
|
|
endpoints=[_endpoint_to_read(e) for e in endpoints],
|
|
endpoint_count=proxy.endpoint_count,
|
|
created_at=proxy.created_at,
|
|
updated_at=proxy.updated_at,
|
|
last_test_at=proxy.last_test_at,
|
|
last_test_ok=proxy.last_test_ok,
|
|
last_test_message=proxy.last_test_message,
|
|
)
|
|
|
|
async def list(self) -> list[ProxyRead]:
|
|
result = await self.session.execute(
|
|
select(Proxy).order_by(Proxy.is_default.desc(), Proxy.updated_at.desc())
|
|
)
|
|
return [self.to_read(p) for p in result.scalars().all()]
|
|
|
|
async def get(self, id: UUID) -> ProxyRead:
|
|
proxy = await self._get_or_404(id)
|
|
return self.to_read(proxy)
|
|
|
|
async def create(self, data: ProxyCreate, created_by: UUID) -> ProxyRead:
|
|
_validate_mode(data.mode)
|
|
_validate_endpoints(data.endpoints)
|
|
|
|
proxy = Proxy(
|
|
name=data.name,
|
|
description=data.description,
|
|
mode=data.mode,
|
|
is_active=data.is_active,
|
|
is_default=data.is_default,
|
|
endpoints_encrypted=_encrypt_endpoints(data.endpoints),
|
|
endpoint_count=len(data.endpoints),
|
|
created_by=created_by,
|
|
)
|
|
self.session.add(proxy)
|
|
if proxy.is_default:
|
|
await self._lock_defaults()
|
|
await self._unset_other_defaults(proxy.id)
|
|
await self.session.commit()
|
|
await self.session.refresh(proxy)
|
|
return self.to_read(proxy)
|
|
|
|
async def update(self, id: UUID, data: ProxyUpdate) -> ProxyRead:
|
|
proxy = await self._get_or_404(id)
|
|
|
|
if data.name is not None:
|
|
proxy.name = data.name
|
|
if data.description is not None:
|
|
proxy.description = data.description
|
|
if data.mode is not None:
|
|
_validate_mode(data.mode)
|
|
proxy.mode = data.mode
|
|
if data.is_active is not None:
|
|
proxy.is_active = data.is_active
|
|
|
|
if data.endpoints is not None:
|
|
_validate_endpoints(data.endpoints)
|
|
merged = self._merge_endpoints(proxy, data.endpoints)
|
|
proxy.endpoints_encrypted = _encrypt_endpoints(merged)
|
|
proxy.endpoint_count = len(merged)
|
|
|
|
if data.is_default is not None:
|
|
proxy.is_default = data.is_default
|
|
|
|
proxy.updated_at = utc_now()
|
|
if data.is_default:
|
|
await self._lock_defaults()
|
|
await self._unset_other_defaults(proxy.id)
|
|
await self.session.commit()
|
|
await self.session.refresh(proxy)
|
|
return self.to_read(proxy)
|
|
|
|
async def delete(self, id: UUID) -> bool:
|
|
proxy = await self._get_or_404(id)
|
|
await self.session.delete(proxy)
|
|
await self.session.commit()
|
|
return True
|
|
|
|
async def set_default(self, id: UUID) -> ProxyRead:
|
|
proxy = await self._get_or_404(id)
|
|
await self._lock_defaults()
|
|
await self._unset_other_defaults(proxy.id)
|
|
proxy.is_default = True
|
|
proxy.updated_at = utc_now()
|
|
await self.session.commit()
|
|
await self.session.refresh(proxy)
|
|
return self.to_read(proxy)
|
|
|
|
async def test(self, id: UUID) -> ProxyTestResult:
|
|
proxy = await self._get_or_404(id)
|
|
endpoints = _load_endpoints(proxy)
|
|
if not endpoints:
|
|
return await self._persist_test(
|
|
proxy,
|
|
ProxyTestResult(success=False, message="No endpoints configured."),
|
|
)
|
|
|
|
ep = endpoints[0]
|
|
url = _build_url(ep)
|
|
result = await self._probe(ep, url)
|
|
return await self._persist_test(proxy, result)
|
|
|
|
async def resolve_proxy_url(self, proxy_id: UUID) -> str | None:
|
|
proxy = await self.session.get(Proxy, proxy_id)
|
|
return _resolve_proxy_url(proxy)
|
|
|
|
def _merge_endpoints(
|
|
self, proxy: Proxy, incoming: list[ProxyEndpoint]
|
|
) -> list[ProxyEndpoint]:
|
|
previous = _load_endpoints(proxy)
|
|
stored = {_endpoint_key(e): e.password for e in previous}
|
|
by_user: dict[str | None, list[str | None]] = {}
|
|
for ep in previous:
|
|
by_user.setdefault(ep.username, []).append(ep.password)
|
|
merged: list[ProxyEndpoint] = []
|
|
for ep in incoming:
|
|
if ep.password != MASK:
|
|
merged.append(ep)
|
|
continue
|
|
key = _endpoint_key(ep)
|
|
if key in stored:
|
|
merged.append(ep.model_copy(update={"password": stored[key]}))
|
|
continue
|
|
same_user = by_user.get(ep.username) or []
|
|
if len(same_user) != 1:
|
|
msg = f"Enter the password again for {host_port(ep.host, ep.port)}."
|
|
raise _bad(msg)
|
|
merged.append(ep.model_copy(update={"password": same_user[0]}))
|
|
return merged
|
|
|
|
async def _probe(self, ep: ProxyEndpoint, url: str) -> ProxyTestResult:
|
|
start = time.monotonic()
|
|
try:
|
|
async with get_async_client(
|
|
proxy=url, timeout=httpx.Timeout(_TEST_TIMEOUT)
|
|
) as client:
|
|
resp = await client.get(_TEST_URL)
|
|
resp.raise_for_status()
|
|
except (httpx.HTTPError, ImportError, ValueError, OSError) as exc:
|
|
return await self._tcp_probe(ep, str(exc) or exc.__class__.__name__)
|
|
latency = int((time.monotonic() - start) * 1000)
|
|
return ProxyTestResult(
|
|
success=True,
|
|
message="Proxy reachable.",
|
|
latency_ms=latency,
|
|
reachable=True,
|
|
)
|
|
|
|
async def _tcp_probe(self, ep: ProxyEndpoint, reason: str) -> ProxyTestResult:
|
|
"""The port's own reachability, after the proxied request failed."""
|
|
try:
|
|
conn = asyncio.open_connection(ep.host, ep.port)
|
|
_, writer = await asyncio.wait_for(conn, timeout=_TCP_TIMEOUT)
|
|
writer.close()
|
|
await writer.wait_closed()
|
|
except (TimeoutError, OSError) as exc:
|
|
return ProxyTestResult(
|
|
success=False, message=f"Connection failed: {exc}", latency_ms=None
|
|
)
|
|
return ProxyTestResult(
|
|
success=False,
|
|
message=(
|
|
f"{host_port(ep.host, ep.port)} accepts connections. "
|
|
f"The request through the proxy failed: {reason[:200]}. "
|
|
"Check the scheme and the credentials."
|
|
),
|
|
latency_ms=None,
|
|
reachable=True,
|
|
)
|
|
|
|
async def _persist_test(
|
|
self, proxy: Proxy, result: ProxyTestResult
|
|
) -> ProxyTestResult:
|
|
proxy.last_test_at = utc_now()
|
|
proxy.last_test_ok = result.success
|
|
proxy.last_test_message = result.message
|
|
await self.session.commit()
|
|
return result
|
|
|
|
async def _lock_defaults(self) -> None:
|
|
await self.session.execute(
|
|
text("SELECT pg_advisory_xact_lock(:k)"), {"k": _DEFAULT_LOCK_KEY}
|
|
)
|
|
|
|
async def _unset_other_defaults(self, keep_id: UUID) -> None:
|
|
await self.session.execute(
|
|
update(Proxy)
|
|
.where(Proxy.id != keep_id, Proxy.is_default.is_(True))
|
|
.values(is_default=False)
|
|
)
|
|
|
|
async def _get_or_404(self, id: UUID) -> Proxy:
|
|
result = await self.session.execute(select(Proxy).where(Proxy.id == id))
|
|
proxy = result.scalar_one_or_none()
|
|
if not proxy:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_404_NOT_FOUND, detail="Proxy not found"
|
|
)
|
|
return proxy
|