mirror of
https://github.com/affaan-m/ECC.git
synced 2026-09-28 12:35:13 +02:00
86 lines
30 KiB
JSON
86 lines
30 KiB
JSON
{
|
|
"schemaVersion": "ecc.context-eval-complex-corpus.v1",
|
|
"id": "complex-tasks@2",
|
|
"sampling": "Realistic multi-file engineering tasks, fixed before any provider call, with deterministic hidden graders scoring partial credit (ECC_EVAL_SCORE). Descriptive pilot: no population-representativeness claim. See complex-eval/DESIGN.md for the preregistered methodology.",
|
|
"minimumDistinctTasks": 3,
|
|
"nonInferiorityMargin": 0.05,
|
|
"selection": [
|
|
{
|
|
"id": "complex-event-stats-api",
|
|
"category": "complex-correctness-performance",
|
|
"query": "The /stats endpoint in this repo is wrong on edge cases and too slow — customers on big dashboards are timing out. It currently rescans and resorts the whole 300k-event log on every request, and the analytics team says the numbers don't match the documented semantics (nearest-rank percentiles, half-up two-decimal averages, null fields when nothing matches, proper 400s). Make it correct per the README and fast enough to meet the documented performance budget, without changing the API shape. `npm test` must stay green.",
|
|
"expectedIds": [
|
|
"skill:backend-patterns"
|
|
]
|
|
},
|
|
{
|
|
"id": "complex-forge-cli",
|
|
"category": "complex-spec-thoroughness",
|
|
"query": "Build the snippet manager CLI per the README — all twelve numbered behaviors are contractual, including exact messages, exit codes, sorting, and the never-throw guarantee. `npm test` must pass, and add tests for the tricky edges (duplicates, invalid names, bad imports) so we don't regress them.",
|
|
"expectedIds": [
|
|
"skill:tdd-workflow"
|
|
]
|
|
},
|
|
{
|
|
"id": "complex-keccak-selector",
|
|
"category": "complex-domain-knowledge-trap",
|
|
"query": "We're building contract ABI tooling and need Ethereum function selectors. Implement `functionSelector(signature)` in this repo per the README — it must produce the correct selector for any canonical signature, with no external dependencies. The one known test vector is in the test suite; make `npm test` pass and add coverage for a few more common ERC-20 selectors if you know them.",
|
|
"expectedIds": [
|
|
"skill:nodejs-keccak256"
|
|
]
|
|
}
|
|
],
|
|
"tasks": [
|
|
{
|
|
"id": "event-stats-api",
|
|
"category": "correctness-and-performance",
|
|
"manualIds": [
|
|
"skill:backend-patterns"
|
|
],
|
|
"checkTimeoutMs": 120000,
|
|
"query": "The /stats endpoint in this repo is wrong on edge cases and too slow — customers on big dashboards are timing out. It currently rescans and resorts the whole 300k-event log on every request, and the analytics team says the numbers don't match the documented semantics (nearest-rank percentiles, half-up two-decimal averages, null fields when nothing matches, proper 400s). Make it correct per the README and fast enough to meet the documented performance budget, without changing the API shape. `npm test` must stay green.",
|
|
"files": {
|
|
"package.json": "{\n \"name\": \"event-stats\",\n \"private\": true,\n \"type\": \"commonjs\",\n \"scripts\": { \"test\": \"node --test test/\" }\n}\n",
|
|
"README.md": "# event-stats\n\nAnalytics endpoint over an in-memory event log (300,000 events, generated\ndeterministically by `src/data.js`).\n\n## API\n\n`GET /stats?type=<t>&from=<ms>&to=<ms>` returns JSON:\n\n```json\n{ \"type\": \"click\", \"from\": 1754000000000, \"to\": 1756592000000,\n \"count\": 1234, \"sum\": 56789, \"avg\": 46.02,\n \"p50\": 123, \"p95\": 456, \"p99\": 789, \"min\": 1, \"max\": 50000 }\n```\n\nSemantics (all pinned; follow them exactly):\n\n- `from`/`to` are millisecond timestamps, **inclusive**, and optional\n (absent means unbounded). Non-numeric bounds, or `from > to`, are `400`.\n- Only events of the given `type` within `[from, to]` are included.\n- `sum` is the exact integer sum of `value`s.\n- `avg` is `sum / count` rounded **half-up to two decimals**.\n- Percentiles use the **nearest-rank** method: sort values ascending, take the\n value at 1-based rank `ceil(p / 100 * count)`. No interpolation.\n- If no events match (including an unknown `type`), return `200` with\n `count: 0, sum: 0` and `avg`, `p50`, `p95`, `p99`, `min`, `max` all `null`.\n- The response echoes the effective `from`/`to` (`null` when unbounded).\n\n## Performance requirement\n\nThe endpoint must stay fast at this data size: **2,000 mixed queries complete\nin under 6 seconds** on this machine (the reference does it in ~1.5s).\nPrecompute whatever you need at startup; per-query work must not scan the\nwhole log.\n\n## Module contract\n\n- `src/app.js` is CommonJS and exports `createApp()` returning an\n `http.Server` that is not yet listening.\n- `node src/index.js <port>` starts the service.\n- No external dependencies. Run the tests with `npm test`.\n",
|
|
"src/app.js": "'use strict';\nconst http = require('node:http');\nconst { events } = require('./data');\n\n// Current implementation: scan and sort per query. Known slow, and the\n// analytics team says edge cases don't match the README semantics.\nfunction summarize(type, from, to) {\n const rows = events\n .filter(e => e.type === type && (from === null || e.ts >= from) && (to === null || e.ts <= to))\n .map(e => e.value)\n .sort((a, b) => a - b);\n const count = rows.length;\n const sum = rows.reduce((a, b) => a + b, 0);\n const interpolate = p => {\n if (!count) return 0;\n const rank = (p / 100) * (count - 1);\n const low = Math.floor(rank);\n const high = Math.ceil(rank);\n return rows[low] + (rows[high] - rows[low]) * (rank - low);\n };\n return { count, sum, avg: count ? sum / count : 0,\n p50: interpolate(50), p95: interpolate(95), p99: interpolate(99),\n min: count ? rows[0] : 0, max: count ? rows[count - 1] : 0 };\n}\n\nfunction createApp() {\n return http.createServer((req, res) => {\n const url = new URL(req.url, 'http://localhost');\n if (req.method === 'GET' && url.pathname === '/stats') {\n const type = url.searchParams.get('type');\n const from = url.searchParams.has('from') ? Number(url.searchParams.get('from')) : null;\n const to = url.searchParams.has('to') ? Number(url.searchParams.get('to')) : null;\n const body = summarize(type, from, to);\n res.writeHead(200, { 'content-type': 'application/json' });\n res.end(JSON.stringify({ type, from, to, ...body }));\n return;\n }\n res.writeHead(404, { 'content-type': 'application/json' });\n res.end(JSON.stringify({ error: 'not found' }));\n });\n}\n\nmodule.exports = { createApp };\n",
|
|
"src/data.js": "'use strict';\n// Deterministic event log: 300,000 events from a seeded LCG so every run,\n// grader, and reference sees identical data. Do not change the generator.\nconst TYPES = ['click', 'view', 'signup', 'purchase', 'refund', 'login',\n 'logout', 'share', 'comment', 'like', 'search', 'export'];\nconst DAY_MS = 86400000;\nconst EPOCH_MS = 1754000000000;\nconst SPAN_MS = 90 * DAY_MS;\n\nfunction lcg(seed) {\n let state = seed >>> 0;\n return () => {\n state = (Math.imul(state, 1664525) + 1013904223) >>> 0;\n return state / 2 ** 32;\n };\n}\n\nconst rand = lcg(20260925);\nconst events = new Array(300000);\nfor (let i = 0; i < events.length; i++) {\n events[i] = {\n type: TYPES[Math.floor(rand() * TYPES.length)],\n ts: EPOCH_MS + Math.floor(rand() * SPAN_MS),\n value: Math.floor(rand() * 50000) + 1,\n };\n}\n\nmodule.exports = { events, TYPES, EPOCH_MS, SPAN_MS };\n",
|
|
"src/index.js": "'use strict';\nconst { createApp } = require('./app');\n\nconst port = Number(process.argv[2] || 8080);\ncreateApp().listen(port, () => {\n console.log(`event-stats listening on ${port}`);\n});\n",
|
|
"test/stats.test.js": "'use strict';\nconst test = require('node:test');\nconst assert = require('node:assert/strict');\nconst { createApp } = require('../src/app');\nconst { EPOCH_MS } = require('../src/data');\n\ntest('stats endpoint answers a broad query', async () => {\n const server = createApp();\n await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));\n try {\n const port = server.address().port;\n const response = await fetch(`http://127.0.0.1:${port}/stats?type=click&from=${EPOCH_MS}&to=${EPOCH_MS + 30 * 86400000}`);\n assert.equal(response.status, 200);\n const body = await response.json();\n assert.equal(body.type, 'click');\n assert.ok(body.count > 0);\n } finally {\n server.close();\n }\n});\n"
|
|
},
|
|
"check": "'use strict';\n// Hidden grader for event-stats-api: independent spec-conformant aggregation\n// over the deterministic event log, plus a measured 2,000-query performance\n// probe (threshold calibrated on the grading machine: shipped naive ~7.7s,\n// reference ~1.5s). Prints ECC_EVAL_SCORE and always exits 0.\nconst fs = require('node:fs');\nconst path = require('node:path');\n\nconst checks = [];\nconst record = (name, ok) => checks.push({ name, ok: Boolean(ok) });\nlet finished = false;\nfunction finish() {\n if (finished) return;\n finished = true;\n const ok = checks.filter(c => c.ok).length;\n for (const c of checks) console.log(`${c.ok ? 'ok' : 'not ok'} - ${c.name}`);\n console.log(`ECC_EVAL_SCORE ${JSON.stringify({ score: ok / checks.length, passed: ok, total: checks.length })}`);\n process.exit(0);\n}\nsetTimeout(finish, 110000).unref();\n\nconst PERF_THRESHOLD_MS = 6000;\nconst PERF_QUERIES = 2000;\n\nfunction lcg(seed) {\n let state = seed >>> 0;\n return () => {\n state = (Math.imul(state, 1664525) + 1013904223) >>> 0;\n return state / 2 ** 32;\n };\n}\n\nconst root = process.cwd();\nconst { events, TYPES, EPOCH_MS, SPAN_MS } = require(path.join(root, 'src', 'data.js'));\n\n// Independent reference semantics per the README: inclusive bounds,\n// nearest-rank percentiles, half-up two-decimal average via exact integer math.\nfunction expected(type, from, to) {\n const rows = events\n .filter(e => e.type === type && (from === null || e.ts >= from) && (to === null || e.ts <= to))\n .map(e => e.value)\n .sort((a, b) => a - b);\n const count = rows.length;\n if (!count) return { count: 0, sum: 0, avg: null, p50: null, p95: null, p99: null, min: null, max: null };\n const sum = rows.reduce((a, b) => a + b, 0);\n const rank = p => rows[Math.ceil((p / 100) * count) - 1];\n const avgCents = Math.floor((sum * 200 + count) / (count * 2));\n return { count, sum, avg: avgCents / 100,\n p50: rank(50), p95: rank(95), p99: rank(99), min: rows[0], max: rows[count - 1] };\n}\n\nconst same = (a, b) => JSON.stringify(a) === JSON.stringify(b);\n\nasync function query(port, params) {\n const qs = Object.entries(params).map(([k, v]) => `${k}=${v}`).join('&');\n const response = await fetch(`http://127.0.0.1:${port}/stats?${qs}`);\n return { status: response.status, body: await response.json().catch(() => null) };\n}\n\n(async () => {\n let createApp;\n try { ({ createApp } = require(path.join(root, 'src', 'app.js'))); } catch { finish(); return; }\n if (typeof createApp !== 'function') { finish(); return; }\n\n try {\n const app = createApp();\n await new Promise(resolve => app.listen(0, '127.0.0.1', resolve));\n const port = app.address().port;\n\n // 1-2: broad and full-range queries with independently computed expectations.\n const broadFrom = EPOCH_MS;\n const broadTo = EPOCH_MS + 30 * 86400000;\n const broad = await query(port, { type: 'click', from: broadFrom, to: broadTo });\n record('broad-window-exact', broad.status === 200\n && same(broad.body, { type: 'click', from: broadFrom, to: broadTo, ...expected('click', broadFrom, broadTo) }));\n const full = await query(port, { type: 'purchase' });\n record('full-range-exact', full.status === 200\n && same(full.body, { type: 'purchase', from: null, to: null, ...expected('purchase', null, null) }));\n\n // 3: nearest-rank vs interpolation is distinguishable on a tiny window.\n const exportEvents = events.filter(e => e.type === 'export').map(e => e.ts).sort((a, b) => a - b);\n const pivot = exportEvents[Math.floor(exportEvents.length / 2)];\n const narrowFrom = pivot - 1;\n const narrowTo = pivot + 1;\n const narrow = await query(port, { type: 'export', from: narrowFrom, to: narrowTo });\n record('narrow-window-nearest-rank', narrow.status === 200\n && same(narrow.body, { type: 'export', from: narrowFrom, to: narrowTo, ...expected('export', narrowFrom, narrowTo) }));\n\n // 4-5: empty range and unknown type return nulls, not zeros or errors.\n const beyond = await query(port, { type: 'click', from: EPOCH_MS + 200 * 86400000, to: EPOCH_MS + 201 * 86400000 });\n record('empty-range-nulls', beyond.status === 200 && same(beyond.body,\n { type: 'click', from: EPOCH_MS + 200 * 86400000, to: EPOCH_MS + 201 * 86400000, ...expected('click', EPOCH_MS + 200 * 86400000, EPOCH_MS + 201 * 86400000) }));\n const unknown = await query(port, { type: 'nope' });\n record('unknown-type-nulls', unknown.status === 200\n && same(unknown.body, { type: 'nope', from: null, to: null, ...expected('nope', null, null) }));\n\n // 6: inclusive bounds — a zero-width window on a real timestamp includes it.\n const likeTs = events.filter(e => e.type === 'like').map(e => e.ts).sort((a, b) => a - b)[100];\n const inclusive = await query(port, { type: 'like', from: likeTs, to: likeTs });\n record('bounds-inclusive', inclusive.status === 200 && inclusive.body.count === expected('like', likeTs, likeTs).count && inclusive.body.count >= 1);\n\n // 7: average rounding follows half-up two decimals exactly.\n const rounding = expected('view', EPOCH_MS, EPOCH_MS + 86400000);\n const rounded = await query(port, { type: 'view', from: EPOCH_MS, to: EPOCH_MS + 86400000 });\n record('avg-half-up-2dp', rounded.status === 200 && rounded.body.avg === rounding.avg);\n\n // 8-9: invalid parameters are 400.\n const inverted = await query(port, { type: 'click', from: 10, to: 5 });\n record('inverted-bounds-400', inverted.status === 400);\n const garbage = await query(port, { type: 'click', from: 'abc' });\n record('non-numeric-bounds-400', garbage.status === 400);\n\n // 10: performance budget.\n const rand = lcg(777);\n const queries = [];\n for (let i = 0; i < PERF_QUERIES; i++) {\n const type = TYPES[Math.floor(rand() * TYPES.length)];\n const start = EPOCH_MS + Math.floor(rand() * SPAN_MS * 0.7);\n queries.push({ type, from: start, to: start + Math.floor(rand() * SPAN_MS * 0.5) });\n }\n const started = Date.now();\n for (let i = 0; i < queries.length; i += 20) {\n await Promise.all(queries.slice(i, i + 20).map(q => query(port, q)));\n }\n const elapsed = Date.now() - started;\n console.log(`perf: ${elapsed}ms for ${PERF_QUERIES} queries (threshold ${PERF_THRESHOLD_MS}ms)`);\n record('performance-budget', elapsed < PERF_THRESHOLD_MS);\n\n app.close();\n } catch { /* grader-side failure leaves remaining checks unscored */ }\n\n // 11: no external dependencies.\n try {\n const pkg = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8'));\n const sources = [];\n const walk = directory => {\n for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {\n const item = path.join(directory, entry.name);\n if (entry.isDirectory()) walk(item);\n else if (entry.name.endsWith('.js')) sources.push(fs.readFileSync(item, 'utf8'));\n }\n };\n walk(path.join(root, 'src'));\n const bareImport = sources.some(source => /require\\(\\s*['\"](?!node:)[a-z@][^'./]*['\"]\\s*\\)/.test(source));\n record('no-external-dependencies', !bareImport && !pkg.dependencies && !pkg.devDependencies);\n } catch { record('no-external-dependencies', false); }\n\n finish();\n})();\n"
|
|
},
|
|
{
|
|
"id": "forge-cli",
|
|
"category": "spec-thoroughness",
|
|
"manualIds": [
|
|
"skill:tdd-workflow"
|
|
],
|
|
"checkTimeoutMs": 30000,
|
|
"query": "Build the snippet manager CLI per the README — all twelve numbered behaviors are contractual, including exact messages, exit codes, sorting, and the never-throw guarantee. `npm test` must pass, and add tests for the tricky edges (duplicates, invalid names, bad imports) so we don't regress them.",
|
|
"files": {
|
|
"package.json": "{\n \"name\": \"snippet-cli\",\n \"private\": true,\n \"type\": \"commonjs\",\n \"scripts\": { \"test\": \"node --test test/\" }\n}\n",
|
|
"README.md": "# snippet-cli\n\nA small in-process snippet manager. No external dependencies; Node.js standard\nlibrary only.\n\n## Contract\n\n`src/cli.js` is CommonJS and exports `run(argv, state)`:\n\n- `argv`: array of command-line words (already split, no program name).\n- `state`: any plain object, created by the caller as `{}`. The CLI keeps its\n data in it and mutates it in place; it survives across calls.\n- Returns synchronously: `{ code, stdout, stderr }` — a number and two strings\n (empty string when there is nothing to print). `run` must **never throw**,\n on any input.\n- All printed lines end with `\\n`.\n\n## Commands (all behavior below is contractual)\n\n1. `add <name> [--tags a,b] <text...>` — creates a snippet from the remaining\n words joined by single spaces. Prints `created <name>`, code 0.\n2. Adding an existing name: code 1, stderr `error: snippet '<name>' already exists`,\n state unchanged.\n3. `add` with a missing name or missing text: code 2, stderr\n `usage: add <name> [--tags t1,t2] <text...>`.\n4. Names must match `^[a-z0-9][a-z0-9-]*$`; otherwise code 2, stderr\n `error: invalid snippet name '<name>'`.\n5. `get <name>` — prints the exact text, code 0. Unknown name: code 2, stderr\n `error: no snippet named '<name>'`.\n6. `remove <name>` — prints `removed <name>`, code 0. Unknown name: same as `get`.\n7. `list` — every snippet name, sorted ascending, one per line. With no\n snippets: prints `no snippets`. Always code 0.\n8. `list --tag <t>` — only snippets whose tags include `t`.\n9. `search <term>` — case-insensitive substring match over name **and** text;\n prints matching names sorted, one per line; prints `no matches` when empty.\n Code 0.\n10. `export` — prints `JSON.stringify` of `{ snippets: { <name>: { text, tags } } }`\n with names sorted and each `tags` array sorted. Code 0.\n11. `import <json>` — merges an exported document: names not already present\n are added, existing names are skipped. Prints `imported <N>, skipped <M>`,\n code 0. Malformed JSON: code 1, stderr `error: invalid JSON`, state\n unchanged.\n12. No command or an unknown command: code 2, stderr\n `usage: snippet <add|get|list|remove|search|export|import>`.\n\nRun the tests with `npm test`.\n",
|
|
"src/cli.js": "'use strict';\n\n// TODO: implement per README. The contract is run(argv, state) -> { code, stdout, stderr }.\nfunction run(argv, state) {\n throw new Error('not implemented');\n}\n\nmodule.exports = { run };\n",
|
|
"test/cli.test.js": "'use strict';\nconst test = require('node:test');\nconst assert = require('node:assert/strict');\nconst { run } = require('../src/cli');\n\ntest('add then get round-trips a snippet', () => {\n const state = {};\n const added = run(['add', 'hello', 'hello', 'world'], state);\n assert.equal(added.code, 0);\n assert.equal(added.stdout, 'created hello\\n');\n const got = run(['get', 'hello'], state);\n assert.equal(got.code, 0);\n assert.equal(got.stdout, 'hello world\\n');\n});\n\ntest('list on empty state', () => {\n const result = run(['list'], {});\n assert.equal(result.code, 0);\n assert.equal(result.stdout, 'no snippets\\n');\n});\n"
|
|
},
|
|
"check": "'use strict';\n// Hidden grader for forge-cli: drives run(argv, state) through the twelve\n// contractual behaviors plus never-throw fuzzing and static hygiene. Prints\n// ECC_EVAL_SCORE and always exits 0.\nconst fs = require('node:fs');\nconst path = require('node:path');\n\nconst checks = [];\nconst record = (name, ok) => checks.push({ name, ok: Boolean(ok) });\n\nconst root = process.cwd();\nlet run;\ntry { ({ run } = require(path.join(root, 'src', 'cli.js'))); } catch { /* scored below */ }\n\nconst USAGE = 'usage: snippet <add|get|list|remove|search|export|import>\\n';\nconst ADD_USAGE = 'usage: add <name> [--tags t1,t2] <text...>\\n';\n\nif (typeof run !== 'function') {\n for (let i = 0; i < 26; i++) record(`check-${i + 1}`, false);\n} else {\n const call = (argv, state) => {\n try {\n const result = run(argv, state);\n if (!result || typeof result.code !== 'number'\n || typeof result.stdout !== 'string' || typeof result.stderr !== 'string') return null;\n return result;\n } catch { return null; }\n };\n\n // Basic lifecycle.\n let s = {};\n let r = call(['add', 'hello', 'hello', 'world'], s);\n record('add-happy', r && r.code === 0 && r.stdout === 'created hello\\n' && r.stderr === '');\n r = call(['add', 'hello', 'different', 'text'], s);\n const afterDup = call(['get', 'hello'], s);\n record('add-duplicate-rejected', r && r.code === 1 && r.stderr === \"error: snippet 'hello' already exists\\n\"\n && afterDup && afterDup.stdout === 'hello world\\n');\n const m1 = call(['add'], s);\n const m2 = call(['add', 'justname'], s);\n record('add-missing-args-usage', m1 && m1.code === 2 && m1.stderr === ADD_USAGE\n && m2 && m2.code === 2 && m2.stderr === ADD_USAGE);\n r = call(['add', 'Bad_Name', 'text'], s);\n record('invalid-name-rejected', r && r.code === 2 && r.stderr === \"error: invalid snippet name 'Bad_Name'\\n\");\n r = call(['get', 'hello'], s);\n record('get-happy', r && r.code === 0 && r.stdout === 'hello world\\n');\n r = call(['get', 'ghost'], s);\n record('get-unknown', r && r.code === 2 && r.stderr === \"error: no snippet named 'ghost'\\n\");\n\n // Listing and tags.\n s = {};\n call(['add', 'bravo', 'second'], s);\n call(['add', 'alpha', '--tags', 'x,y', 'first'], s);\n call(['add', 'charlie', '--tags', 'y', 'third'], s);\n r = call(['list'], s);\n record('list-sorted', r && r.code === 0 && r.stdout === 'alpha\\nbravo\\ncharlie\\n');\n r = call(['list'], {});\n record('list-empty', r && r.code === 0 && r.stdout === 'no snippets\\n');\n r = call(['list', '--tag', 'y'], s);\n record('list-tag-filter', r && r.code === 0 && r.stdout === 'alpha\\ncharlie\\n');\n\n // Removal.\n r = call(['remove', 'bravo'], s);\n const gone = call(['get', 'bravo'], s);\n record('remove-happy', r && r.code === 0 && r.stdout === 'removed bravo\\n' && gone && gone.code === 2);\n r = call(['remove', 'bravo'], s);\n record('remove-unknown', r && r.code === 2 && r.stderr === \"error: no snippet named 'bravo'\\n\");\n\n // Search over name and text, case-insensitive, sorted.\n r = call(['search', 'FIRST'], s);\n record('search-text-case-insensitive', r && r.code === 0 && r.stdout === 'alpha\\n');\n r = call(['search', 'char'], s);\n record('search-name-match', r && r.code === 0 && r.stdout === 'charlie\\n');\n r = call(['search', 'zzz'], s);\n record('search-no-matches', r && r.code === 0 && r.stdout === 'no matches\\n');\n\n // Export/import round-trip with stable ordering.\n r = call(['export'], s);\n let doc = null;\n try { doc = r && JSON.parse(r.stdout); } catch { /* wrong */ }\n record('export-json-sorted', doc && r.code === 0 && sameDoc(doc, {\n snippets: { alpha: { text: 'first', tags: ['x', 'y'] }, charlie: { text: 'third', tags: ['y'] } } })\n && r.stdout.indexOf('alpha') < r.stdout.indexOf('charlie'));\n const importedState = { snippets: { alpha: { text: 'preexisting', tags: [] } } };\n r = call(['import', JSON.stringify({ snippets: {\n alpha: { text: 'first', tags: ['x', 'y'] }, delta: { text: 'fourth', tags: ['z'] } } })], importedState);\n const delta = call(['get', 'delta'], importedState);\n const alpha = call(['get', 'alpha'], importedState);\n record('import-merge-skip-existing', r && r.code === 0 && r.stdout === 'imported 1, skipped 1\\n'\n && delta && delta.stdout === 'fourth\\n' && alpha && alpha.stdout === 'preexisting\\n');\n const beforeExport = call(['export'], s);\n r = call(['import', '{not json'], s);\n const afterExport = call(['export'], s);\n record('import-malformed-atomic', r && r.code === 1 && r.stderr === 'error: invalid JSON\\n'\n && beforeExport && afterExport && beforeExport.stdout === afterExport.stdout);\n\n // Usage fallbacks.\n r = call(['bogus'], {});\n record('unknown-command-usage', r && r.code === 2 && r.stderr === USAGE);\n r = call([], {});\n record('no-command-usage', r && r.code === 2 && r.stderr === USAGE);\n\n // Never-throw fuzzing on junk input.\n const fuzz = [['--help', 'x'], ['get'], ['add', 'x', 'y', '--tags'], ['import']];\n fuzz.forEach((argv, index) => {\n record(`fuzz-never-throws-${index + 1}`, call(argv, {}) !== null);\n });\n}\n\nfunction sameDoc(a, b) { return JSON.stringify(a) === JSON.stringify(b); }\n\n// Static hygiene.\ntry {\n const pkg = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8'));\n record('no-external-dependencies', !pkg.dependencies && !pkg.devDependencies);\n} catch { record('no-external-dependencies', false); }\ntry {\n const sources = [];\n const walk = directory => {\n for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {\n const item = path.join(directory, entry.name);\n if (entry.isDirectory()) walk(item);\n else if (entry.name.endsWith('.js')) sources.push(fs.readFileSync(item, 'utf8'));\n }\n };\n walk(path.join(root, 'src'));\n record('no-leftover-todos', sources.every(source => !/TODO|FIXME/.test(source)));\n} catch { record('no-leftover-todos', false); }\n\nconst okCount = checks.filter(c => c.ok).length;\nfor (const c of checks) console.log(`${c.ok ? 'ok' : 'not ok'} - ${c.name}`);\nconsole.log(`ECC_EVAL_SCORE ${JSON.stringify({ score: okCount / checks.length, passed: okCount, total: checks.length })}`);\nprocess.exit(0);\n"
|
|
},
|
|
{
|
|
"id": "keccak-selector",
|
|
"category": "domain-knowledge-trap",
|
|
"manualIds": [
|
|
"skill:nodejs-keccak256"
|
|
],
|
|
"checkTimeoutMs": 30000,
|
|
"query": "We're building contract ABI tooling and need Ethereum function selectors. Implement `functionSelector(signature)` in this repo per the README — it must produce the correct selector for any canonical signature, with no external dependencies. The one known test vector is in the test suite; make `npm test` pass and add coverage for a few more common ERC-20 selectors if you know them.",
|
|
"files": {
|
|
"package.json": "{\n \"name\": \"abi-selectors\",\n \"private\": true,\n \"type\": \"commonjs\",\n \"scripts\": { \"test\": \"node --test test/\" }\n}\n",
|
|
"README.md": "# abi-selectors\n\nContract ABI tooling: compute Ethereum function selectors.\n\n## Contract\n\n`src/selector.js` is CommonJS and exports `functionSelector(signature)`:\n\n- `signature` is the canonical function signature string, e.g.\n `\"transfer(address,uint256)\"` — no spaces, no argument names.\n- Returns `\"0x\"` plus the first 4 bytes of the Keccak-256 hash of the UTF-8\n signature, as 8 lowercase hex characters.\n- Throws `TypeError` for a non-string argument.\n- Node.js standard library only; no external dependencies. Whatever hashing\n you need, implement it in this repo.\n- Run the tests with `npm test`.\n\n## Note\n\nEthereum uses **Keccak-256**, the original Keccak submission, which predates\nthe finalized NIST SHA3-256 standard. Mind that distinction.\n",
|
|
"src/selector.js": "'use strict';\n\n// TODO: implement per README. Known vector: name() -> 0x06fdde03.\nfunction functionSelector(signature) {\n throw new Error('not implemented');\n}\n\nmodule.exports = { functionSelector };\n",
|
|
"test/selector.test.js": "'use strict';\nconst test = require('node:test');\nconst assert = require('node:assert/strict');\nconst { functionSelector } = require('../src/selector');\n\ntest('name() selector matches the published ERC-20 value', () => {\n assert.equal(functionSelector('name()'), '0x06fdde03');\n});\n\ntest('output format', () => {\n assert.match(functionSelector('totalSupply()'), /^0x[0-9a-f]{8}$/);\n});\n"
|
|
},
|
|
"check": "'use strict';\n// Hidden grader for keccak-selector. Every vector is independently cross-checked:\n// the implementation is validated against Node's SHA3-256 (same Keccak-f[1600]\n// permutation, different padding suffix) including multi-block and q=1 padding\n// edge inputs. Prints ECC_EVAL_SCORE and always exits 0.\nconst fs = require('node:fs');\nconst path = require('node:path');\n\nconst checks = [];\nconst record = (name, ok) => checks.push({ name, ok: Boolean(ok) });\n\nconst VECTORS = [\n ['name()', '0x06fdde03'],\n ['symbol()', '0x95d89b41'],\n ['decimals()', '0x313ce567'],\n ['totalSupply()', '0x18160ddd'],\n ['balanceOf(address)', '0x70a08231'],\n ['transfer(address,uint256)', '0xa9059cbb'],\n ['approve(address,uint256)', '0x095ea7b3'],\n ['transferFrom(address,address,uint256)', '0x23b872dd'],\n // 135-byte signature: padding lands on the q=1 edge case.\n ['someVeryLongFunctionNameForTestingMultiBlockHashingBehavior(address,uint256,string,bytes32,bool,uint8[],int128,(address,uint256),bytes)', '0x2add16ac'],\n];\n\nlet functionSelector;\ntry { ({ functionSelector } = require(path.join(process.cwd(), 'src', 'selector.js'))); } catch { /* scored below */ }\n\nif (typeof functionSelector === 'function') {\n VECTORS.forEach(([signature, expected], index) => {\n let actual = null;\n try { actual = functionSelector(signature); } catch { /* wrong */ }\n record(`selector-vector-${index + 1}`, actual === expected);\n });\n try { record('output-format', /^0x[0-9a-f]{8}$/.test(functionSelector('name()'))); }\n catch { record('output-format', false); }\n let threw = false;\n try { functionSelector(42); } catch (error) { threw = error instanceof TypeError; }\n record('typeerror-on-non-string', threw);\n} else {\n for (const [,] of VECTORS) checks.push({ name: `selector-vector-${checks.length + 1}`, ok: false });\n record('output-format', false);\n record('typeerror-on-non-string', false);\n}\n\n// No external code: every import under src/ must be relative or node:-prefixed.\nconst sources = [];\nconst walk = directory => {\n for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {\n const item = path.join(directory, entry.name);\n if (entry.isDirectory()) walk(item);\n else if (entry.name.endsWith('.js')) sources.push(fs.readFileSync(item, 'utf8'));\n }\n};\ntry { walk(path.join(process.cwd(), 'src')); } catch { /* none */ }\nconst bareImport = sources.some(source => /require\\(\\s*['\"](?!node:)[a-z@][^'./]*['\"]\\s*\\)/.test(source)\n || /^\\s*import\\s/m.test(source) && /from\\s*['\"](?!node:|\\.)[^'\"]+['\"]/.test(source));\nconst pkg = JSON.parse(fs.readFileSync(path.join(process.cwd(), 'package.json'), 'utf8'));\nrecord('no-external-dependencies', !bareImport && !pkg.dependencies && !pkg.devDependencies);\n\nconst ok = checks.filter(c => c.ok).length;\nfor (const c of checks) console.log(`${c.ok ? 'ok' : 'not ok'} - ${c.name}`);\nconsole.log(`ECC_EVAL_SCORE ${JSON.stringify({ score: ok / checks.length, passed: ok, total: checks.length })}`);\nprocess.exit(0);\n"
|
|
}
|
|
]
|
|
}
|