{ "schemaVersion": "ecc.context-eval-corpus.v2", "id": "coding-tasks@1", "sampling": "Purposive coding-task corpus fixed before any provider call: 22 small JavaScript repairs paired with one plausibly helpful ECC skill, 8 trivial no-workflow fixes (some with misleading workflow vocabulary), and selection probes for exact names, paraphrases, no-workflow queries and policy blocks; equal weight per distinct task and no population-representativeness claim.", "minimumDistinctTasks": 30, "nonInferiorityMargin": 0.05, "selection": [ { "id":"exact-python", "category":"exact", "query":"Use python-patterns to review typed Python functions.", "expectedIds":["skill:python-patterns"] }, { "id":"exact-api", "category":"exact", "query":"Use api-design for REST pagination.", "expectedIds":["skill:api-design"] }, { "id":"paraphrase-tests", "category":"paraphrase", "query":"Write pytest fixtures and parametrized regression tests for a Python package.", "expectedIds":["skill:python-testing"] }, { "id":"paraphrase-api", "category":"paraphrase", "query":"Design REST endpoints with pagination and status codes.", "expectedIds":["skill:api-design"] }, { "id":"plain-arithmetic", "category":"no-workflow", "query":"What is 17 times 24?", "expectedIds":[] }, { "id":"ambiguous-vocabulary", "category":"no-workflow", "query":"Count words in this literal text: database testing security review. Do not perform any of those activities.", "expectedIds":[] }, { "id":"negative-skill-mention", "category":"no-workflow", "query":"Do not use python-patterns. Return only OK.", "expectedIds":[] }, { "id":"explicit-no-workflow", "category":"no-workflow", "query":"Use python-patterns as plain text only. Add 7 and 5.", "noWorkflow":true,"expectedIds":[] }, { "id":"excluded-name", "category":"policy", "query":"Use python-patterns to simplify Python code.", "exclude":["skill:python-patterns"],"expectedIds":[] }, { "id":"excluded-explicit", "category":"policy", "query":"Use python-patterns.", "exclude":["skill:python-patterns"],"explicitIds":["skill:python-patterns"],"expectedBlock":"excluded" }, { "id":"authority-boundary", "category":"policy", "query":"Use inherit-legacy-style to preserve conventions.", "explicitIds":["skill:inherit-legacy-style"],"expectedBlock":"native-authority" }, { "id":"opt-out-conflict", "category":"policy", "query":"Use python-patterns.", "noWorkflow":true,"explicitIds":["skill:python-patterns"],"expectedBlock":"opt-out-conflict" }, { "id":"unknown-explicit", "category":"policy", "query":"Use an unavailable workflow.", "explicitIds":["skill:ecc-eval-nonexistent"],"expectedBlock":"unknown-id" }, { "id":"exact-security-review", "category":"exact", "query":"Use security-review to check this login handler for SQL injection and leaked secrets.", "expectedIds":["skill:security-review"] }, { "id":"exact-error-handling", "category":"exact", "query":"Use error-handling to add typed error classes to the config loader.", "expectedIds":["skill:error-handling"] }, { "id":"exact-database-migrations", "category":"exact", "query":"Use database-migrations to add a NOT NULL column to a large Postgres table.", "expectedIds":["skill:database-migrations"] }, { "id":"exact-regex-structured-text", "category":"exact", "query":"Use regex-vs-llm-structured-text to decide how to parse vendor invoice lines.", "expectedIds":["skill:regex-vs-llm-structured-text"] }, { "id":"exact-content-hash-cache", "category":"exact", "query":"Use content-hash-cache-pattern to cache PDF text extraction results.", "expectedIds":["skill:content-hash-cache-pattern"] }, { "id":"exact-hexagonal", "category":"exact", "query":"Use hexagonal-architecture to separate the signup use case from its database and email adapters.", "expectedIds":["skill:hexagonal-architecture"] }, { "id":"paraphrase-sql-injection", "category":"paraphrase", "query":"User input is concatenated into SQL strings in our login endpoint; audit the handler for injection and hardcoded credentials before release.", "expectedIds":["skill:security-review"] }, { "id":"paraphrase-retry", "category":"paraphrase", "query":"Wrap a flaky payment provider call with exponential backoff retries and typed error classes so callers get useful failure messages.", "expectedIds":["skill:error-handling"] }, { "id":"paraphrase-zero-downtime-rename", "category":"paraphrase", "query":"Rename a column on a busy PostgreSQL table without downtime, with reversible up and down schema changes.", "expectedIds":["skill:database-migrations"] }, { "id":"paraphrase-redis-cache", "category":"paraphrase", "query":"Add a Redis cache-aside layer with key expiry and a distributed lock for our profile reads.", "expectedIds":["skill:redis-patterns"] }, { "id":"paraphrase-token-decimals", "category":"paraphrase", "query":"Our dashboard shows USDC balances wrong on some EVM chains because token decimals differ; normalize amounts across chains safely.", "expectedIds":["skill:evm-token-decimals"] }, { "id":"paraphrase-keccak", "category":"paraphrase", "query":"Compute Ethereum function selectors in Node without confusing NIST SHA3-256 with Keccak-256.", "expectedIds":["skill:nodejs-keccak256"] }, { "id":"paraphrase-content-hash", "category":"paraphrase", "query":"Cache slow document parsing so results are keyed by the SHA-256 of file content instead of the file path.", "expectedIds":["skill:content-hash-cache-pattern"] }, { "id":"paraphrase-ports-adapters", "category":"paraphrase", "query":"Refactor toward ports and adapters so the domain use case no longer imports the database driver directly.", "expectedIds":["skill:hexagonal-architecture"] }, { "id":"paraphrase-structured-text", "category":"paraphrase", "query":"Should I parse these semi-structured quiz and invoice text lines with regular expressions or an LLM? Start with the cheapest reliable option.", "expectedIds":["skill:regex-vs-llm-structured-text"] }, { "id":"rename-variable", "category":"no-workflow", "query":"Rename the local variable tmp to total in this three-line function.", "expectedIds":[] }, { "id":"misleading-security-typo", "category":"no-workflow", "query":"Fix the spelling of \"recieve\" in the footer text of the security settings page. Nothing else.", "expectedIds":[] }, { "id":"misleading-tests-heading", "category":"no-workflow", "query":"Change the README heading \"Running tests\" to \"Running checks\". Do not write or run any tests.", "expectedIds":[] }, { "id":"explicit-no-workflow-migration", "category":"no-workflow", "query":"Treat database-migrations as plain words. Reverse the string abc.", "noWorkflow":true,"expectedIds":[] }, { "id":"excluded-api-explicit", "category":"policy", "query":"Use api-design.", "exclude":["skill:api-design"],"explicitIds":["skill:api-design"],"expectedBlock":"excluded" }, { "id":"authority-latency", "category":"policy", "query":"Use latency-critical-systems to tune the quote cache.", "explicitIds":["skill:latency-critical-systems"],"expectedBlock":"native-authority" }, { "id":"authority-rust-testing", "category":"policy", "query":"Use rust-testing for property tests.", "explicitIds":["skill:rust-testing"],"expectedBlock":"native-authority" }, { "id":"opt-out-conflict-security", "category":"policy", "query":"Use security-review.", "noWorkflow":true,"explicitIds":["skill:security-review"],"expectedBlock":"opt-out-conflict" }, { "id":"unknown-typo-id", "category":"policy", "query":"Use security-reveiw.", "explicitIds":["skill:security-reveiw"],"expectedBlock":"unknown-id" }, { "id":"explicit-allowed", "category":"policy", "query":"Use error-handling for the retry wrapper.", "explicitIds":["skill:error-handling"],"expectedIds":["skill:error-handling"] } ], "tasks": [ { "id": "sql-injection-query", "category": "security", "manualIds": [ "skill:security-review" ], "query": "src/users.js builds SQL for a node-postgres style driver: each builder returns { text, values } where text uses $1, $2 placeholders. Both buildFindUserQuery(email) and buildSearchUsersQuery(nameFragment, limit) interpolate caller input into the SQL text. Fix them so no caller-supplied string is ever placed in the SQL text; pass it through values instead. The search must still match names containing the fragment case-insensitively. limit must be an integer from 1 to 100; throw a RangeError for anything else (including numeric strings). Keep both exports and the selected columns. Do not add dependencies.", "files": { "src/users.js": "'use strict';\n\n// Query builders used by the /users routes. The db layer calls\n// pool.query(q.text, q.values) with node-postgres.\nfunction buildFindUserQuery(email) {\n return {\n text: \"SELECT id, email, name FROM users WHERE email = '\" + email + \"'\",\n values: [],\n };\n}\n\nfunction buildSearchUsersQuery(nameFragment, limit) {\n return {\n text: \"SELECT id, email, name FROM users WHERE name ILIKE '%\" + nameFragment + \"%' ORDER BY name LIMIT \" + limit,\n values: [],\n };\n}\n\nmodule.exports = { buildFindUserQuery, buildSearchUsersQuery };\n", "README.md": "# users-api\n\nSmall query-builder layer for the users service. Builders return\n{ text, values } objects that are passed directly to node-postgres.\n" }, "check": "'use strict';\nconst assert = require('node:assert/strict');\nconst path = require('node:path');\nconst m = require(path.join(process.cwd(), 'src/users.js'));\nconst evil = \"a' OR '1'='1\";\nconst q = m.buildFindUserQuery(evil);\nassert.ok(!q.text.includes(\"'1'='1\") && !q.text.includes(evil), 'email leaked into SQL text');\nassert.match(q.text, /\\$1/);\nassert.match(q.text, /FROM\\s+users/i);\nassert.ok(q.values.includes(evil));\nconst frag = \"bob'; DROP TABLE users;--\";\nconst s = m.buildSearchUsersQuery(frag, 10);\nassert.ok(!/DROP\\s+TABLE/i.test(s.text), 'fragment leaked into SQL text');\nassert.match(s.text, /ILIKE|lower\\s*\\(/i);\nassert.match(s.text, /LIMIT\\s+(\\$\\d+|10)\\b/i);\nassert.ok(s.values.some(v => typeof v === 'string' && v.includes(frag)));\nfor (const bad of [0, 101, 2.5, '10', '10; DROP TABLE users', NaN, undefined]) {\n assert.throws(() => m.buildSearchUsersQuery('x', bad), RangeError);\n}\nconst max = Math.max(0, ...[...s.text.matchAll(/\\$(\\d+)/g)].map(x => Number(x[1])));\nassert.equal(max, s.values.length, 'placeholders and values disagree');\n" }, { "id": "path-traversal-guard", "category": "security", "manualIds": [ "skill:security-review" ], "query": "src/static.js exports resolvePublicPath(requestPath, root) used by our static file server. requestPath is the raw URL path (for example \"/css/site.css\", possibly percent-encoded). It currently joins it onto root, which allows escaping the public directory. Make it return the absolute file path when the decoded path stays inside root (root itself counts as inside), and return null (never throw) when the path escapes root, contains a NUL byte, or cannot be percent-decoded. Watch out for sibling directories that share root as a string prefix. Keep the export name and signature. Do not add dependencies.", "files": { "src/static.js": "'use strict';\nconst path = require('path');\n\nconst PUBLIC_ROOT = path.resolve(__dirname, '..', 'public');\n\n// Maps a request path such as \"/css/site.css\" to a file on disk.\nfunction resolvePublicPath(requestPath, root = PUBLIC_ROOT) {\n return path.join(root, decodeURIComponent(requestPath));\n}\n\nmodule.exports = { resolvePublicPath, PUBLIC_ROOT };\n", "public/index.html": "
' + body + '