mirror of
https://github.com/yogeshojha/rengine.git
synced 2026-10-20 23:31:31 +02:00
Yogesh OjhaandClaude Opus 5
1e2d16eed2
There was no backup path at all, and the connection arithmetic did not add up. scripts/backup.sh dumps the four things an instance actually is: the database, the scan media on disk, the docker volumes holding uploaded wordlists, nuclei templates, report themes and fonts, and the settings a restore needs. SECRET_KEY travels with it deliberately — API keys, proxy credentials and notification secrets are Fernet encrypted with a key derived from it, so an archive without it restores a database full of unreadable secrets. The header says so, and backups/ and *.tar are gitignored, because an archive is a credential. Restore is the half that is usually never tried, so it was: a 305 MB archive of this instance was restored over it and came back at 9 scans, 59,858 subdomains, 535,000 endpoints, 29 findings and 7 targets — identical — with the migration head intact, the API answering and an encrypted API key still decrypting. It stops the api and both workers first so nothing writes underneath it, and asks for the database name before it destroys anything. The pools could ask for more connections than the server would give. The api takes 10 + 20, each celery prefork child takes 2 + 3, and beat takes another pool: at the concurrency of 8 this box runs that is 75 against a max_connections of 100, and at the 12 the compose file now defaults to it is 95 — five short of the ceiling, which fails as a checkout timeout under load and reads like a slow query. max_connections is now set from compose with the arithmetic beside it, and the api checks it at startup: pool_demand() against max_connections less the superuser reserve, warned once rather than discovered during an incident. The inputs moved to shared settings because the sum spans both processes and neither could see the other's numbers. Worth recording: .env on this box pins CELERY_SCAN_CONCURRENCY=8, so raising the compose default to 12 changes new installs and not this one. It also still carries CELERY_DEFAULT_CONCURRENCY, which nothing has read since it was renamed. Three tests: the demand counts every pool, the shipped defaults fit the shipped server, and an oversized pool warns rather than failing. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013dKUV96WkQQETZBMoAG98C
90 lines
2.7 KiB
Python
90 lines
2.7 KiB
Python
from sqlalchemy import text
|
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
|
|
|
from app.config import settings
|
|
from shared.logging import get_logger
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
_server_settings = {"application_name": f"{settings.APP_NAME}-api"}
|
|
|
|
# reclaims any connection a request leaks inside an open transaction
|
|
if settings.DB_IDLE_TX_TIMEOUT > 0:
|
|
_server_settings["idle_in_transaction_session_timeout"] = str(
|
|
settings.DB_IDLE_TX_TIMEOUT * 1000
|
|
)
|
|
|
|
engine = create_async_engine(
|
|
settings.database_url,
|
|
echo=settings.DEBUG,
|
|
future=True,
|
|
pool_size=settings.DB_POOL_SIZE,
|
|
max_overflow=settings.DB_MAX_OVERFLOW,
|
|
pool_timeout=settings.DB_POOL_TIMEOUT,
|
|
pool_recycle=settings.DB_POOL_RECYCLE,
|
|
pool_pre_ping=True,
|
|
connect_args={"server_settings": _server_settings},
|
|
)
|
|
|
|
async_db_session = async_sessionmaker(
|
|
engine,
|
|
class_=AsyncSession,
|
|
expire_on_commit=False,
|
|
)
|
|
|
|
|
|
async def get_session() -> AsyncSession:
|
|
async with async_db_session() as session:
|
|
yield session
|
|
|
|
|
|
def pool_stats() -> dict[str, int]:
|
|
pool = engine.pool
|
|
return {
|
|
"size": pool.size(),
|
|
"checked_out": pool.checkedout(),
|
|
"checked_in": pool.checkedin(),
|
|
"overflow": pool.overflow(),
|
|
"capacity": settings.DB_POOL_SIZE + settings.DB_MAX_OVERFLOW,
|
|
}
|
|
|
|
|
|
def pool_demand() -> int:
|
|
"""Connections every pool can ask for at once: the api, one worker pool per celery
|
|
child, and beat."""
|
|
per_child = settings.WORKER_DB_POOL_SIZE + settings.WORKER_DB_MAX_OVERFLOW
|
|
return (
|
|
settings.DB_POOL_SIZE
|
|
+ settings.DB_MAX_OVERFLOW
|
|
+ settings.CELERY_SCAN_CONCURRENCY * per_child
|
|
+ per_child
|
|
)
|
|
|
|
|
|
async def check_capacity() -> None:
|
|
"""Say out loud when the pools can outgrow the server.
|
|
|
|
Exceeding max_connections does not fail at startup — it fails later, under load,
|
|
as a checkout timeout that reads like a slow query.
|
|
"""
|
|
demand = pool_demand()
|
|
try:
|
|
async with engine.connect() as conn:
|
|
allowed = int(await conn.scalar(text("SHOW max_connections")))
|
|
reserved = int(
|
|
await conn.scalar(text("SHOW superuser_reserved_connections"))
|
|
)
|
|
except Exception:
|
|
logger.debug("could not read the connection ceiling", exc_info=True)
|
|
return
|
|
usable = allowed - reserved
|
|
if demand > usable:
|
|
logger.warning(
|
|
"database pools can outgrow the server",
|
|
demand=demand,
|
|
usable=usable,
|
|
hint="raise POSTGRES_MAX_CONNECTIONS or lower the pool sizes",
|
|
)
|
|
else:
|
|
logger.info("database pool headroom", demand=demand, usable=usable)
|