'use strict'; const assert = require('assert'); const fs = require('fs'); const http = require('http'); const os = require('os'); const path = require('path'); const { startOwnedChild, withCleanup } = require('./helpers/state-store-worker'); const { createStateStore } = require('../../scripts/lib/state-store'); const FIXTURE = path.join(__dirname, 'helpers', 'control-pane-contention-server.js'); function request(url, method = 'GET', body, timeoutMs = 1000, onRequest = () => {}) { return new Promise((resolve, reject) => { let settled = false; let timer; let response; const finish = (failed, error, value) => { if (settled) return; settled = true; clearTimeout(timer); if (failed) { try { if (response) response.destroy(); } catch (_cleanupError) { /* Primary wins. */ } try { req.destroy(); } catch (_cleanupError) { /* Primary wins. */ } reject(error); } else resolve(value); }; const req = http.request(url, { method, agent: false, headers: { 'Content-Type': 'application/json' } }, res => { if (settled) { res.destroy(); return; } response = res; let text = ''; let bytes = 0; res.setEncoding('utf8'); res.on('data', data => { bytes += Buffer.byteLength(data); if (bytes > 128 * 1024) finish(true, new Error('HTTP fixture response exceeded 128 KiB')); else text += data; }); res.on('error', error => finish(true, error)); res.on('close', () => { if (!res.complete) finish(true, new Error('HTTP fixture response closed before completion')); }); res.on('aborted', () => finish(true, new Error('HTTP fixture response aborted'))); res.on('end', () => { try { finish(false, null, { status: res.statusCode, headers: res.headers, body: JSON.parse(text) }); } catch (error) { finish(true, error); } }); }); // A total deadline covers headers and body, even if bytes keep arriving. timer = setTimeout(() => finish(true, new Error(`HTTP ${method} timed out after ${timeoutMs}ms`)), timeoutMs); req.on('error', error => finish(true, error)); try { onRequest(req); req.end(body === undefined ? undefined : JSON.stringify(body)); } catch (error) { finish(true, error); } }); } async function startServer(dbPath, readOnly = false) { const client = startOwnedChild(FIXTURE, [dbPath, readOnly ? 'read-only' : 'editable']); let metrics = { active: 0, peak: 0, started: 0, exited: 0, received: 0, disconnected: 0 }; const observers = new Set(); client.observe(message => { if (message && message.type === 'workers') { metrics = message; for (const notify of observers) notify(); } }); let ready; try { ready = await client.waitFor(message => message && message.type === 'ready'); } catch (error) { return withCleanup(() => { throw error; }, [() => client.stop()]); } return { url: ready.url, get metrics() { return metrics; }, observe(fn) { observers.add(fn); return () => observers.delete(fn); }, mutationReceived() { return client.waitFor(message => message && message.type === 'mutation-received', 5000); }, close: () => client.stop(), }; } function waitForObservation(server, predicate, timeoutMs = 3000) { return new Promise((resolve, reject) => { let unsubscribe; const timer = setTimeout(() => { unsubscribe(); reject(new Error(`Timed out waiting for worker lifecycle: ${JSON.stringify(server.metrics)}`)); }, timeoutMs); function check() { if (!predicate()) return; clearTimeout(timer); unsubscribe(); resolve(); } unsubscribe = server.observe(check); check(); }); } async function holdFixtureLock(dbPath) { const lockPath = `${dbPath}.ecc-state.lock`; const fd = fs.openSync(lockPath, 'wx', 0o600); let released = false; const release = () => { if (released) return; released = true; return withCleanup(() => fs.unlinkSync(lockPath), [() => fs.closeSync(fd)]); }; try { fs.writeFileSync(fd, JSON.stringify({ pid: process.pid, hostname: os.hostname() })); } catch (error) { return withCleanup(() => { throw error; }, [release]); } return release; } async function burstScenario(server, dbPath) { const bytesBefore = fs.readFileSync(dbPath); const releaseLock = await holdFixtureLock(dbPath); const results = []; const requests = []; let resolveRejected; const sixRejected = new Promise(resolve => { resolveRejected = resolve; }); return withCleanup(async () => { for (let index = 0; index < 8; index += 1) { requests.push(request(`${server.url}/api/work-items/burst-${index}/claim`, 'POST', { owner: `owner-${index}` }, 12000) .then(result => { results.push({ index, ...result }); if (results.filter(item => item.status === 503).length >= 6) resolveRejected(); return result; }, error => ({ transportError: error }))); } await waitForObservation(server, () => server.metrics.received >= 8); await waitForObservation(server, () => server.metrics.started >= 2); // Response completion is also observed; no sleep guesses how long Worker // construction or request body parsing needs on the current machine. let timer; try { await Promise.race([ sixRejected, new Promise((_, reject) => { timer = setTimeout(() => reject(new Error(`Excess requests did not promptly return 503; peak workers=${server.metrics.peak}`)), 1500); }) ]); } finally { clearTimeout(timer); } assert.strictEqual(server.metrics.started, 2, 'Overflow must not start additional workers'); assert.strictEqual(server.metrics.peak, 2); assert.strictEqual(results.length, 6, 'Only the six rejected requests complete while lock is held'); for (const result of results) { assert.strictEqual(result.status, 503); assert.strictEqual(result.body.code, 'STATE_STORE_BUSY'); assert.ok(Number(result.headers['retry-after']) > 0); } assert.strictEqual((await request(`${server.url}/api/health`)).status, 200); const snapshot = await request(`${server.url}/api/snapshot`, 'GET', undefined, 3000); assert.strictEqual(snapshot.status, 200); assert.deepStrictEqual(fs.readFileSync(dbPath), bytesBefore); const rejected = results.map(result => result.index); await releaseLock(); await Promise.all(requests); await waitForObservation(server, () => server.metrics.active === 0); assert.strictEqual(results.filter(result => result.status === 200).length, 2); assert.strictEqual(server.metrics.started, 2, 'Rejected requests must never execute later'); const store = await createStateStore({ dbPath }); await withCleanup(() => { for (const index of rejected) assert.strictEqual(store.getWorkItemById(`burst-${index}`).owner, null); }, [() => store.close()]); const retry = await request(`${server.url}/api/work-items/burst-${rejected[0]}/claim`, 'POST', { owner: 'retry-owner' }, 5000); assert.strictEqual(retry.status, 200); assert.strictEqual(retry.body.item.owner, 'retry-owner'); }, [releaseLock, () => Promise.all(requests)]); } async function withServer(readOnly, fn) { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-http-contention-')); const dbPath = path.join(dir, 'state.db'); let server; return withCleanup(async () => { const store = await createStateStore({ dbPath }); await withCleanup(() => { store.upsertWorkItem({ id: 'task', title: 'Synthetic board task', source: 'manual', status: 'open' }); for (let index = 0; index < 8; index += 1) { store.upsertWorkItem({ id: `burst-${index}`, title: `Burst task ${index}`, source: 'manual', status: 'open' }); } }, [() => store.close()]); server = await startServer(dbPath, readOnly); return await fn(server, dbPath); }, [() => { if (server) return server.close(); }, () => fs.rmSync(dir, { recursive: true, force: true })]); } async function disconnectScenario(server, dbPath) { const releaseLock = await holdFixtureLock(dbPath); let disconnectedRequest; const abandoned = request(`${server.url}/api/work-items/burst-0/claim`, 'POST', { owner: 'abandoned-client' }, 12000, req => { disconnectedRequest = req; }).catch(error => ({ error })); const accepted = request(`${server.url}/api/work-items/burst-1/claim`, 'POST', { owner: 'connected-client' }, 12000) .catch(error => ({ error })); await withCleanup(async () => { await waitForObservation(server, () => server.metrics.active === 2); disconnectedRequest.destroy(new Error('Intentional client disconnect')); await waitForObservation(server, () => server.metrics.disconnected === 1); const excess = await request(`${server.url}/api/work-items/burst-2/claim`, 'POST', { owner: 'excess-client' }); assert.strictEqual(excess.status, 503, 'Disconnect must not release a slot while its worker is still alive'); assert.strictEqual(server.metrics.started, 2); assert.strictEqual(server.metrics.active, 2); }, [releaseLock, () => abandoned, () => accepted]); await waitForObservation(server, () => server.metrics.active === 0); const retry = await request(`${server.url}/api/work-items/burst-2/claim`, 'POST', { owner: 'retry-client' }, 5000); assert.strictEqual(retry.status, 200, 'Actual worker exits must restore capacity'); } async function contentionScenario() { return withServer(false, async (server, dbPath) => { const bytesBefore = fs.readFileSync(dbPath); const releaseLock = await holdFixtureLock(dbPath); let outcome; let mutation; await withCleanup(async () => { const received = server.mutationReceived(); let settled = false; mutation = request(`${server.url}/api/work-items/task/claim`, 'POST', { owner: 'alice' }, 12000) .then(result => { settled = true; return result; }, error => { settled = true; return { transportError: error }; }); await received; let healthError = null; let healthChecks = 0; let snapshotChecks = 0; // Probe continuously after actual HTTP receipt, including asynchronous // database initialization. No timing guess about when lock wait begins. while (!settled) { try { const health = await request(`${server.url}/api/health`); assert.strictEqual(health.status, 200); healthChecks += 1; const snapshot = await request(`${server.url}/api/snapshot`, 'GET', undefined, 3000); assert.strictEqual(snapshot.status, 200); assert.ok(snapshot.body.workItems.items.some(item => item.id === 'task')); snapshotChecks += 1; } catch (error) { healthError = error.message; break; } } const result = await mutation; if (result.transportError) throw result.transportError; outcome = { result, healthError, healthChecks, snapshotChecks }; }, [releaseLock, () => mutation]); assert.deepStrictEqual(fs.readFileSync(dbPath), bytesBefore, 'Timed-out mutation must not change database bytes'); const store = await createStateStore({ dbPath }); await withCleanup(() => { const item = store.getWorkItemById('task'); assert.strictEqual(item.status, 'open'); assert.strictEqual(item.owner, null); }, [() => store.close()]); return outcome; }); } async function run() { let passed = 0; let failed = 0; async function test(name, fn) { try { await fn(); console.log(` PASS ${name}`); passed += 1; } catch (error) { console.log(` FAIL ${name}\n ${error && error.message ? error.message : String(error)}`); failed += 1; } } console.log('\n=== Testing control-pane mutation contention ===\n'); const contention = await contentionScenario(); await test('health and snapshots remain responsive while a board mutation waits for another database writer', () => { assert.strictEqual(contention.healthError, null, contention.healthError || undefined); assert.ok(contention.healthChecks > 0); assert.ok(contention.snapshotChecks > 0); }); await test('lock timeout is retryable HTTP 503 with Retry-After', () => { assert.strictEqual(contention.result.status, 503); assert.ok(Number(contention.result.headers['retry-after']) > 0, 'Retry-After must give a positive delay'); assert.strictEqual(contention.result.body.ok, false); assert.strictEqual(contention.result.body.code, 'STATE_STORE_BUSY'); }); await test('claim and move still save their changes, while invalid edits return 400', () => withServer(false, async (server, dbPath) => { const claim = await request(`${server.url}/api/work-items/task/claim`, 'POST', { owner: 'alice', as: 'human' }, 5000); assert.strictEqual(claim.status, 200); assert.strictEqual(claim.body.item.owner, 'alice'); assert.strictEqual(claim.body.item.status, 'running'); const move = await request(`${server.url}/api/work-items/task/move`, 'POST', { lane: 'blocked' }, 5000); assert.strictEqual(move.status, 200); assert.strictEqual(move.body.item.status, 'blocked'); const invalid = await request(`${server.url}/api/work-items/task/move`, 'POST', { lane: 'imaginary' }, 5000); assert.strictEqual(invalid.status, 400); assert.strictEqual(invalid.body.ok, false); const store = await createStateStore({ dbPath }); await withCleanup(() => { const item = store.getWorkItemById('task'); assert.strictEqual(item.owner, 'alice'); assert.strictEqual(item.status, 'blocked'); assert.strictEqual(item.metadata.assigneeKind, 'human'); }, [() => store.close()]); })); await test('read-only mode rejects board edits before database work', () => withServer(true, async server => { for (const action of ['claim', 'move']) { const result = await request(`${server.url}/api/work-items/task/${action}`, 'POST', { owner: 'alice', lane: 'done' }); assert.strictEqual(result.status, 403); } })); await test('an eight-request burst caps live workers at two and rejects excess work without queuing', () => withServer(false, burstScenario)); await test('a disconnected HTTP client retains its worker slot until actual exit', () => withServer(false, disconnectScenario)); console.log(`\nResults: Passed: ${passed}, Failed: ${failed}`); process.exitCode = failed ? 1 : 0; } process.exitCode = 1; run().catch(error => { console.error(error); process.exitCode = 1; });