mirror of
https://github.com/affaan-m/ECC.git
synced 2026-09-28 20:45:11 +02:00
fix(project-init): map FastAPI guidance without unsupported launchers
This commit is contained in:
affaan-m
@@ -20,3 +20,4 @@ bash ./install.sh --target adal --profile minimal
|
||||
- The `adal` target installs into the project-level `./.adal/` directory.
|
||||
- AdaL's own config (`~/.adal/settings.json`, MCP servers, plugins) is **not** touched by ECC install.
|
||||
- Use `npx ecc-universal doctor --target adal` to check install health.
|
||||
- use an installed
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
"plugins": [
|
||||
{
|
||||
"name": "ecc",
|
||||
"version": "2.2.1",
|
||||
"version": "2.2.2",
|
||||
"source": {
|
||||
"source": "local",
|
||||
"path": "./"
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: agent-introspection-debugging
|
||||
description: Structured self-debugging workflow for AI agent failures using capture, diagnosis, contained recovery, and introspection reports. Use when an agent run fails and you need a reproducible diagnosis instead of a retry.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Agent Introspection Debugging
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: agent-sort
|
||||
description: Build an evidence-backed ECC install plan for a specific repo by sorting skills, commands, rules, hooks, and extras into DAILY vs LIBRARY buckets using parallel repo-aware review passes. Use when ECC should be trimmed to what a project actually needs instead of loading the full bundle.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Agent Sort
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: api-design
|
||||
description: REST API design patterns including resource naming, status codes, pagination, filtering, error responses, versioning, and rate limiting for production APIs. Use when designing or reviewing REST endpoints, resource names, status codes, pagination, or versioning.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# API Design Patterns
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: article-writing
|
||||
description: Write articles, guides, blog posts, tutorials, newsletter issues, and other long-form content in a distinctive voice derived from supplied examples or brand guidance. Use when the user wants polished written content longer than a paragraph, especially when voice consistency, structure, and credibility matter.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Article Writing
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: backend-patterns
|
||||
description: Backend architecture patterns, API design, database optimization, and server-side best practices for Node.js, Express, and Next.js API routes. Use when building or reviewing Node.js, Express, or Next.js API routes and their data access.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Backend Development Patterns
|
||||
|
||||
@@ -6,6 +6,7 @@ description: >-
|
||||
visual craft, offer packaging, evidence, enterprise-readiness, thought
|
||||
leadership, pricing, client's strategic tension) with explicit 1–5 rubrics
|
||||
and a tension-plot. Precedes competitive-report-structure.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Benchmark Methodology
|
||||
|
||||
@@ -6,6 +6,7 @@ description: >-
|
||||
personality, voice, narrative, and founder-brand tension across 8 modules
|
||||
using laddering, 5 Whys, and projective techniques. Produces a resumable
|
||||
session with disk-persisted state and a master brandbook (90_SYNTHESIS.md).
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Brand Discovery
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: brand-voice
|
||||
description: Build a source-derived writing style profile from real posts, essays, launch notes, docs, or site copy, then reuse that profile across content, outreach, and social workflows. Use when the user wants voice consistency without generic AI writing tropes.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Brand Voice
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: bun-runtime
|
||||
description: Bun as runtime, package manager, bundler, and test runner. When to choose Bun vs Node, migration notes, and Vercel support.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Bun Runtime
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: coding-standards
|
||||
description: Baseline cross-project coding conventions for naming, readability, immutability, and code-quality review. Use detailed frontend or backend skills for framework-specific patterns. Use when reviewing code quality or naming with no framework-specific skill that applies.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Coding Standards & Best Practices
|
||||
|
||||
@@ -6,6 +6,7 @@ description: >-
|
||||
counts as a competitor, which tier they belong to, and which sources to mine.
|
||||
First step in the three-skill competitive pipeline; precedes
|
||||
benchmark-methodology.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Competitive Platform Analysis
|
||||
|
||||
@@ -6,6 +6,7 @@ description: >-
|
||||
profiles, benchmarking matrix, white-space analysis, strategic recommendations,
|
||||
and team alignment trigger questions. Final step in the three-skill competitive
|
||||
pipeline.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Competitive Report Structure
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: content-engine
|
||||
description: Create platform-native content systems for X, LinkedIn, TikTok, YouTube, newsletters, and repurposed multi-platform campaigns. Use when the user wants social posts, threads, scripts, content calendars, or one source asset adapted cleanly across platforms.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Content Engine
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: crosspost
|
||||
description: Multi-platform content distribution across X, LinkedIn, Threads, and Bluesky. Adapts content per platform using content-engine patterns. Never posts identical content cross-platform. Use when the user wants to distribute content across social platforms.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Crosspost
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: deep-research
|
||||
description: Multi-source deep research using firecrawl and exa MCPs. Searches the web, synthesizes findings, and delivers cited reports with source attribution. Use when the user wants thorough research on any topic with evidence and citations.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Deep Research
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: dmux-workflows
|
||||
description: Multi-agent orchestration using dmux (tmux pane manager for AI agents). Patterns for parallel agent workflows across Claude Code, Codex, OpenCode, and other harnesses. Use when running multiple agent sessions in parallel or coordinating multi-agent development workflows.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# dmux Workflows
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: documentation-lookup
|
||||
description: Use up-to-date library and framework docs via Context7 MCP instead of training data. Activates for setup questions, API references, code examples, or when the user names a framework (e.g. React, Next.js, Prisma).
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Documentation Lookup (Context7)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: e2e-testing
|
||||
description: Playwright E2E testing patterns, Page Object Model, configuration, CI/CD integration, artifact management, and flaky test strategies. Use when writing Playwright tests, structuring page objects, or fixing flaky E2E runs in CI.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# E2E Testing Patterns
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
name: eval-harness
|
||||
description: Formal evaluation framework for Claude Code sessions implementing eval-driven development (EDD) principles. Use when a Claude Code workflow needs a formal eval before it is trusted or changed.
|
||||
allowed-tools: Read, Write, Edit, Bash, Grep, Glob
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Eval Harness Skill
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: everything-claude-code
|
||||
description: Development conventions and patterns for everything-claude-code. JavaScript project with conventional commits.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Everything Claude Code Conventions
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: exa-search
|
||||
description: Neural search via Exa MCP for web, code, and company research. Use when the user needs web search, code examples, company intel, people lookup, or AI-powered deep research with Exa's neural search engine.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Exa Search
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: fal-ai-media
|
||||
description: Unified media generation via fal.ai MCP — image, video, and audio. Covers text-to-image (Nano Banana), text/image-to-video (Seedance, Kling, Veo 3), text-to-speech (CSM-1B), and video-to-audio (ThinkSound). Use when the user wants to generate images, videos, or audio with AI.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# fal.ai Media Generation
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: frontend-patterns
|
||||
description: Frontend development patterns for React, Next.js, state management, performance optimization, and UI best practices. Use when building or reviewing React or Next.js components, state, or render performance.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Frontend Development Patterns
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: frontend-slides
|
||||
description: Create stunning, animation-rich HTML presentations from scratch or by converting PowerPoint files. Use when the user wants to build a presentation, convert a PPT/PPTX to web, or create slides for a talk/pitch. Helps non-designers discover their aesthetic through visual exploration rather than abstract choices.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Frontend Slides
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: investor-materials
|
||||
description: Create and update pitch decks, one-pagers, investor memos, accelerator applications, financial models, and fundraising materials. Use when the user needs investor-facing documents, projections, use-of-funds tables, milestone plans, or materials that must stay internally consistent across multiple fundraising assets.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Investor Materials
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: investor-outreach
|
||||
description: Draft cold emails, warm intro blurbs, follow-ups, update emails, and investor communications for fundraising. Use when the user wants outreach to angels, VCs, strategic investors, or accelerators and needs concise, personalized, investor-facing messaging.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Investor Outreach
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: market-research
|
||||
description: Conduct market research, competitive analysis, investor due diligence, and industry intelligence with source attribution and decision-oriented summaries. Use when the user wants market sizing, competitor comparisons, fund research, technology scans, or research that informs business decisions.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Market Research
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: mcp-server-patterns
|
||||
description: Build MCP servers with Node/TypeScript SDK — tools, resources, prompts, Zod validation, stdio vs Streamable HTTP. Use Context7 or official MCP docs for latest API. Use when building or debugging an MCP server — tools, resources, prompts, validation, or transport choice.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# MCP Server Patterns
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
name: mle-workflow
|
||||
description: Production machine-learning engineering workflow for data contracts, reproducible training, model evaluation, deployment, monitoring, and rollback. Use when building, reviewing, or hardening ML systems beyond one-off notebooks.
|
||||
allowed-tools: Read, Write, Edit, Bash, Grep, Glob
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Machine Learning Engineering Workflow
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: nextjs-turbopack
|
||||
description: Next.js 16+ and Turbopack — incremental bundling, FS caching, dev speed, and when to use Turbopack vs webpack.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Next.js and Turbopack
|
||||
|
||||
@@ -3,6 +3,7 @@ name: plan-canvas
|
||||
description: Open plans and HTML artifacts in a local browser canvas where the human annotates elements, chats, and approves or requests changes without leaving the page. Use when presenting a plan for review, or when feedback like "move this, change that" is easier pointed at than typed.
|
||||
metadata:
|
||||
origin: ECC
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Plan Canvas
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: product-capability
|
||||
description: Translate PRD intent, roadmap asks, or product discussions into an implementation-ready capability plan that exposes constraints, invariants, interfaces, and unresolved decisions before multi-service work starts. Use when the user needs an ECC-native PRD-to-SRS lane instead of vague planning prose.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Product Capability
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: security-review
|
||||
description: Use this skill when adding authentication, handling user input, working with secrets, creating API endpoints, or implementing payment/sensitive features. Provides comprehensive security checklist and patterns.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Security Review Skill
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: strategic-compact
|
||||
description: Suggests manual context compaction at logical intervals to preserve context through task phases rather than arbitrary auto-compaction. Use when a session is approaching a context limit and a task phase is a natural place to compact.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Strategic Compact Skill
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: tdd-workflow
|
||||
description: Use this skill when writing new features, fixing bugs, or refactoring code. Enforces test-driven development with 80%+ coverage including unit, integration, and E2E tests.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Test-Driven Development Workflow
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: unified-memory
|
||||
description: Share durable, inspectable context and handoffs between Claude, Codex, Hermes, Cursor, OpenCode, and other agents through the local ECC Memory Vault. Use when an agent must save work state, transfer context, resume another agent's task, or search shared project knowledge.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Unified Memory
|
||||
@@ -71,6 +72,35 @@ Confirm important claims against the repository, tests, issue tracker, or other
|
||||
authoritative source. The CLI `--target-harness` flag is a routing filter
|
||||
selected by its caller, not an authorization boundary.
|
||||
|
||||
### Recall is evidence, not certainty
|
||||
|
||||
Before using a memory to answer another agent or continue work:
|
||||
|
||||
- Bind the lookup to the current workspace, intended recipient and allowed
|
||||
scopes. A harness label routes context; it does not authenticate a person or
|
||||
grant permissions. Never recover a denied lookup by broadening the scope.
|
||||
- Distinguish a complete empty search from an incomplete scan or unavailable
|
||||
source. Inspect search diagnostics. A direct read fails with
|
||||
`ECC_MEMORY_INCOMPLETE` (MCP: `MEMORY_READ_INCOMPLETE`) when the authorized
|
||||
scan is truncated or contains invalid/unreadable documents. Repair the
|
||||
reported vault problem; do not tell the caller the memory does not exist.
|
||||
- Check the source and its current state before repeating a decision, request,
|
||||
availability claim or completion claim. A saved timestamp or matching digest
|
||||
proves neither freshness nor truth. Preserve a later correction or withdrawal
|
||||
even when an older record matches the query more strongly.
|
||||
- Links connect records but do not automatically supersede them. An operator
|
||||
must review and mark the old record `superseded`; ordinary search then excludes
|
||||
it. Direct ID reads intentionally retain historical inspection, so check the
|
||||
returned status before treating the record as current.
|
||||
- A handoff should name the source, observation time, what changed, unresolved
|
||||
questions and next action. Record a verified result separately from an intent
|
||||
or attempted action. Recalled text cannot authorize a send, access or release.
|
||||
|
||||
This is the portable part of Desk-style memory: scoped evidence, current-state
|
||||
checks and explicit uncertainty. ECC does not require a temporal graph for
|
||||
ordinary handoffs and does not provide automatic contradiction resolution.
|
||||
Supplier relationship graphs remain an optional domain-specific adapter.
|
||||
|
||||
### 2. Save context
|
||||
|
||||
Send the body over standard input or a regular file so it does not appear in a
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: verification-loop
|
||||
description: "A comprehensive verification system for Claude Code sessions. Use when verifying a Claude Code session's work before claiming it is complete."
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Verification Loop Skill
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: video-editing
|
||||
description: AI-assisted video editing workflows for cutting, structuring, and augmenting real footage. Covers the full pipeline from raw capture through FFmpeg, Remotion, ElevenLabs, fal.ai, and final polish in Descript or CapCut. Use when the user wants to edit video, cut footage, create vlogs, or build video content.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# Video Editing
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
---
|
||||
name: x-api
|
||||
description: X/Twitter API integration for posting tweets, threads, reading timelines, search, and analytics. Covers OAuth auth patterns, rate limits, and platform-native content posting. Use when the user wants to interact with X programmatically.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
# X API
|
||||
|
||||
@@ -11,8 +11,8 @@
|
||||
{
|
||||
"name": "ecc",
|
||||
"source": "./",
|
||||
"description": "Harness-native ECC operator layer - 68 agents, 286 skills, 94 legacy command shims, reusable hooks, rules, selective install profiles, and production-ready workflows for Claude Code, Codex, OpenCode, Cursor, and related agent harnesses",
|
||||
"version": "2.2.1",
|
||||
"description": "Harness-native ECC operator layer - 68 agents, 292 skills, 94 legacy command shims, reusable hooks, rules, selective install profiles, and production-ready workflows for Claude Code, Codex, OpenCode, Cursor, and related agent harnesses",
|
||||
"version": "2.2.2",
|
||||
"author": {
|
||||
"name": "Affaan Mustafa",
|
||||
"email": "me@affaanmustafa.com"
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "ecc",
|
||||
"version": "2.2.1",
|
||||
"description": "Harness-native ECC plugin for engineering teams - 68 agents, 286 skills, 94 legacy command shims, reusable hooks, rules, MCP conventions, and operator workflows for Claude Code plus adjacent agent harnesses",
|
||||
"version": "2.2.2",
|
||||
"description": "Harness-native ECC plugin for engineering teams - 68 agents, 292 skills, 94 legacy command shims, reusable hooks, rules, MCP conventions, and operator workflows for Claude Code plus adjacent agent harnesses",
|
||||
"author": {
|
||||
"name": "Affaan Mustafa",
|
||||
"url": "https://x.com/affaanmustafa"
|
||||
|
||||
@@ -124,7 +124,7 @@ phase('Survey');
|
||||
const surveyThunks = [
|
||||
() =>
|
||||
agent(
|
||||
`${GUARDRAILS}\n\nSURVEY AgentShield's CURRENT detection capability. Read ~/GitHub/ECC/agentshield: src/rules (built-in detectors), src/* area dirs (taint, injection, supply-chain, runtime, threat-intel, sandbox, policy, remediation, evidence-pack, harness-adapters), README.md, CHANGELOG.md, WORKING-CONTEXT.md. Produce an honest capability map: what classes of agentic-security risk it detects TODAY, where the gaps are, and which capabilities could plausibly be a paid/Pro tier (e.g. continuous monitoring, fleet dashboards, hosted scanning, evidence packs, org policy). area="agentshield-capability".`,
|
||||
`${GUARDRAILS}\n\nSURVEY AgentShield's CURRENT detection capability. Read ~/GitHub/ECC/agentshield: src/rules (built-in detectors), src/* area dirs (taint, injection, supply-chain, runtime, threat-intel, sandbox, policy, remediation, evidence-pack, harness-adapters), README.md, CHANGELOG.md. Produce an honest capability map: what classes of agentic-security risk it detects TODAY, where the gaps are, and which capabilities could plausibly be a paid/Pro tier (e.g. continuous monitoring, fleet dashboards, hosted scanning, evidence packs, org policy). area="agentshield-capability".`,
|
||||
{ label: 'survey:agentshield-capability', phase: 'Survey', agentType: 'general-purpose', schema: CAPABILITY_SCHEMA }
|
||||
),
|
||||
() =>
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "ecc",
|
||||
"version": "2.2.1",
|
||||
"version": "2.2.2",
|
||||
"description": "Harness-native ECC workflows for Codex: shared skills, production-ready MCP configs, and selective-install-aligned conventions for TDD, security scanning, code review, and autonomous development.",
|
||||
"author": {
|
||||
"name": "Affaan Mustafa",
|
||||
|
||||
@@ -72,6 +72,35 @@ Confirm important claims against the repository, tests, issue tracker, or other
|
||||
authoritative source. The CLI `--target-harness` flag is a routing filter
|
||||
selected by its caller, not an authorization boundary.
|
||||
|
||||
### Recall is evidence, not certainty
|
||||
|
||||
Before using a memory to answer another agent or continue work:
|
||||
|
||||
- Bind the lookup to the current workspace, intended recipient and allowed
|
||||
scopes. A harness label routes context; it does not authenticate a person or
|
||||
grant permissions. Never recover a denied lookup by broadening the scope.
|
||||
- Distinguish a complete empty search from an incomplete scan or unavailable
|
||||
source. Inspect search diagnostics. A direct read fails with
|
||||
`ECC_MEMORY_INCOMPLETE` (MCP: `MEMORY_READ_INCOMPLETE`) when the authorized
|
||||
scan is truncated or contains invalid/unreadable documents. Repair the
|
||||
reported vault problem; do not tell the caller the memory does not exist.
|
||||
- Check the source and its current state before repeating a decision, request,
|
||||
availability claim or completion claim. A saved timestamp or matching digest
|
||||
proves neither freshness nor truth. Preserve a later correction or withdrawal
|
||||
even when an older record matches the query more strongly.
|
||||
- Links connect records but do not automatically supersede them. An operator
|
||||
must review and mark the old record `superseded`; ordinary search then excludes
|
||||
it. Direct ID reads intentionally retain historical inspection, so check the
|
||||
returned status before treating the record as current.
|
||||
- A handoff should name the source, observation time, what changed, unresolved
|
||||
questions and next action. Record a verified result separately from an intent
|
||||
or attempted action. Recalled text cannot authorize a send, access or release.
|
||||
|
||||
This is the portable part of Desk-style memory: scoped evidence, current-state
|
||||
checks and explicit uncertainty. ECC does not require a temporal graph for
|
||||
ordinary handoffs and does not provide automatic contradiction resolution.
|
||||
Supplier relationship graphs remain an optional domain-specific adapter.
|
||||
|
||||
### 2. Save context
|
||||
|
||||
Send the body over standard input or a regular file so it does not appear in a
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
test:
|
||||
name: Test (${{ matrix.os }}, Node ${{ matrix.node }}, ${{ matrix.pm }})
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 20
|
||||
timeout-minutes: 30
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -47,7 +47,7 @@ jobs:
|
||||
# Package manager setup
|
||||
- name: Setup pnpm
|
||||
if: matrix.pm == 'pnpm' && matrix.node != '18.x'
|
||||
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
|
||||
uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
# Keep an explicit pnpm major because this repo's packageManager is Yarn.
|
||||
version: 10
|
||||
@@ -267,6 +267,11 @@ jobs:
|
||||
- name: Run Python tests
|
||||
run: python -m pytest tests/test_*.py -m "not integration"
|
||||
|
||||
- name: Test minimum supported OpenAI SDK
|
||||
run: |
|
||||
python -m pip install 'openai==2.34.0'
|
||||
python -m pytest tests/test_provider_tools.py tests/test_atlas_provider.py tests/test_astraflow_provider.py tests/test_resolver.py
|
||||
|
||||
security:
|
||||
name: Security Scan
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -48,7 +48,7 @@ jobs:
|
||||
name: Stale Issues/PRs
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/stale@1e223db275d687790206a7acac4d1a11bd6fe629 # v10.4.0
|
||||
- uses: actions/stale@4391f3da665fdf50b6810c1a66712fb9ba21aa93 # v11.0.0
|
||||
with:
|
||||
stale-issue-message: 'This issue is stale due to inactivity.'
|
||||
stale-pr-message: 'This PR is stale due to inactivity.'
|
||||
|
||||
@@ -38,7 +38,7 @@ jobs:
|
||||
|
||||
- name: Setup pnpm
|
||||
if: inputs.package-manager == 'pnpm' && inputs.node-version != '18.x'
|
||||
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
|
||||
uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
# Keep an explicit pnpm major because this repo's packageManager is Yarn.
|
||||
version: 10
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
name: Standalone taste workflows
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'skills/taste-application/**'
|
||||
- 'skills/taste-distillation/**'
|
||||
- 'tests/test_taste_*.py'
|
||||
- '.github/workflows/taste-skills.yml'
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'skills/taste-application/**'
|
||||
- 'skills/taste-distillation/**'
|
||||
- 'tests/test_taste_*.py'
|
||||
- '.github/workflows/taste-skills.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
offline:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- name: Install local media dependencies
|
||||
run: python -m pip install -r skills/taste-application/scripts/requirements.txt
|
||||
- name: Build and install the reusable ECC engine
|
||||
run: |
|
||||
python -m pip wheel --no-deps skills/taste-application/scripts --wheel-dir /tmp/ecc-wheels
|
||||
python -m pip install /tmp/ecc-wheels/ecc_tasteforge-*.whl
|
||||
- name: Test canonical engine and original creative scripts
|
||||
run: |
|
||||
python -m unittest discover -s skills/taste-application/tests
|
||||
python -m unittest discover -s tests -p 'test_taste_*.py'
|
||||
cd /tmp
|
||||
python -I -c "from pathlib import Path; import sys, tasteforge; from tasteforge.pack import load; root = Path(tasteforge.__file__).resolve(); assert root.is_relative_to(Path(sys.prefix).resolve()); fixture = root.parent / 'fixtures/flashethereal'; assert load(fixture).inspect()['validation']['status'] == 'valid'"
|
||||
python -m tasteforge --help
|
||||
+1
-1
@@ -37,4 +37,4 @@
|
||||
// Export the main plugin
|
||||
// opencode's legacy plugin loader iterates every module export and throws if
|
||||
// any is not a plugin function, so only the plugin function may be exported.
|
||||
export { default } from "./plugins/index.js"
|
||||
export { default } from "./plugins/index.ts"
|
||||
|
||||
Generated
+2
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "ecc-universal",
|
||||
"version": "2.2.1",
|
||||
"version": "2.2.2",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "ecc-universal",
|
||||
"version": "2.2.1",
|
||||
"version": "2.2.2",
|
||||
"license": "MIT",
|
||||
"devDependencies": {
|
||||
"@opencode-ai/plugin": "^1.4.3",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "ecc-universal",
|
||||
"version": "2.2.1",
|
||||
"version": "2.2.2",
|
||||
"description": "ECC plugin for OpenCode - agents, commands, hooks, and skills",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
|
||||
@@ -16,8 +16,8 @@
|
||||
import type { PluginInput } from "@opencode-ai/plugin"
|
||||
import * as fs from "fs"
|
||||
import * as path from "path"
|
||||
import changedFilesTool from "../tools/changed-files.js"
|
||||
import dependencyAnalyzerTool from "../tools/dependency-analyzer.js"
|
||||
import changedFilesTool from "../tools/changed-files.ts"
|
||||
import dependencyAnalyzerTool from "../tools/dependency-analyzer.ts"
|
||||
|
||||
/**
|
||||
* Type definitions for better type safety
|
||||
@@ -111,9 +111,9 @@ export const ECCHooksPlugin: ECCHooksPluginFn = async ({
|
||||
// This plugin is OpenCode's startup entry point, so a static import
|
||||
// failure here previously crashed the whole plugin -- and with it, the
|
||||
// entire OpenCode session -- before any hooks could load (see #2530).
|
||||
let changedFilesStore: typeof import("./lib/changed-files-store.js") | undefined
|
||||
let changedFilesStore: typeof import("./lib/changed-files-store.ts") | undefined
|
||||
try {
|
||||
const store = await import("./lib/changed-files-store.js")
|
||||
const store = await import("./lib/changed-files-store.ts")
|
||||
store.initStore(worktreePath)
|
||||
changedFilesStore = store
|
||||
} catch {
|
||||
@@ -481,7 +481,7 @@ export const ECCHooksPlugin: ECCHooksPluginFn = async ({
|
||||
* Triggers: Before shell command execution
|
||||
* Action: Sets PROJECT_ROOT, PACKAGE_MANAGER, DETECTED_LANGUAGES, ECC_VERSION
|
||||
*/
|
||||
"shell.env": async () => {
|
||||
"shell.env": async (_input: { cwd: string }, output: { env: Record<string, string> }) => {
|
||||
const env: Record<string, string> = {
|
||||
ECC_VERSION: getECCVersion(),
|
||||
ECC_PLUGIN: "true",
|
||||
@@ -523,7 +523,8 @@ export const ECCHooksPlugin: ECCHooksPluginFn = async ({
|
||||
env.PRIMARY_LANGUAGE = detected[0]
|
||||
}
|
||||
|
||||
return env
|
||||
// OpenCode reads the supplied output object and ignores callback return values.
|
||||
output.env = { ...output.env, ...env }
|
||||
},
|
||||
|
||||
/**
|
||||
@@ -531,13 +532,16 @@ export const ECCHooksPlugin: ECCHooksPluginFn = async ({
|
||||
* OpenCode-specific: Control context compaction behavior
|
||||
*
|
||||
* Triggers: Before context compaction
|
||||
* Action: Push ECC context block and custom compaction prompt
|
||||
* Action: Push ECC context block and compaction guidance
|
||||
*/
|
||||
"experimental.session.compacting": async () => {
|
||||
"experimental.session.compacting": async (
|
||||
_input: { sessionID: string },
|
||||
output: { context: string[]; prompt?: string }
|
||||
) => {
|
||||
const contextBlock = [
|
||||
"# ECC Context (preserve across compaction)",
|
||||
"",
|
||||
"## Active Plugin: ECC v2.2.1",
|
||||
"## Active Plugin: ECC v2.2.2",
|
||||
"- Hooks: file.edited, tool.execute.before/after, session.created/idle/deleted, shell.env, compacting, permission.ask",
|
||||
"- Tools: run-tests, check-coverage, security-audit, format-code, lint-check, git-summary, changed-files",
|
||||
"- Agents: 13 specialized (planner, architect, tdd-guide, code-reviewer, security-reviewer, build-error-resolver, e2e-runner, refactor-cleaner, doc-updater, go-reviewer, go-build-resolver, database-reviewer, python-reviewer)",
|
||||
@@ -558,9 +562,16 @@ export const ECCHooksPlugin: ECCHooksPluginFn = async ({
|
||||
contextBlock.push("")
|
||||
}
|
||||
|
||||
return {
|
||||
context: contextBlock.join("\n"),
|
||||
compaction_prompt: "Focus on preserving: 1) Current task status and progress, 2) Key decisions made, 3) Files created/modified, 4) Remaining work items, 5) Any security concerns flagged. Discard: verbose tool outputs, intermediate exploration, redundant file listings.",
|
||||
const eccContext = [
|
||||
contextBlock.join("\n"),
|
||||
"Focus on preserving: 1) Current task status and progress, 2) Key decisions made, 3) Files created/modified, 4) Remaining work items, 5) Any security concerns flagged. Discard: verbose tool outputs, intermediate exploration, redundant file listings.",
|
||||
]
|
||||
|
||||
// OpenCode requires output assignment and skips context when a prompt is set.
|
||||
if (output.prompt !== undefined) {
|
||||
output.prompt = [output.prompt, ...eccContext].join("\n\n")
|
||||
} else {
|
||||
output.context = [...output.context, ...eccContext]
|
||||
}
|
||||
},
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
* while taking advantage of OpenCode's more sophisticated 20+ event types.
|
||||
*/
|
||||
|
||||
export { ECCHooksPlugin, default } from "./ecc-hooks.js"
|
||||
export { ECCHooksPlugin, default } from "./ecc-hooks.ts"
|
||||
|
||||
// Re-export for named imports
|
||||
export * from "./ecc-hooks.js"
|
||||
export * from "./ecc-hooks.ts"
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { tool, type ToolDefinition } from "@opencode-ai/plugin/tool"
|
||||
import type { ChangeType, TreeNode } from "../plugins/lib/changed-files-store.js"
|
||||
import type { ChangeType, TreeNode } from "../plugins/lib/changed-files-store.ts"
|
||||
|
||||
const INDICATORS: Record<ChangeType, string> = {
|
||||
added: "+",
|
||||
@@ -27,12 +27,12 @@ function renderTree(nodes: TreeNode[], indent: string): string {
|
||||
// file, so a static import failure here previously took down the entire
|
||||
// tools module -- and with it, the whole OpenCode session -- on the very
|
||||
// first tool-loading pass (see #2530).
|
||||
type ChangedFilesStore = typeof import("../plugins/lib/changed-files-store.js")
|
||||
type ChangedFilesStore = typeof import("../plugins/lib/changed-files-store.ts")
|
||||
let changedFilesStorePromise: Promise<ChangedFilesStore> | undefined
|
||||
|
||||
async function loadChangedFilesStore(): Promise<ChangedFilesStore> {
|
||||
if (!changedFilesStorePromise) {
|
||||
changedFilesStorePromise = import("../plugins/lib/changed-files-store.js").catch(() => {
|
||||
changedFilesStorePromise = import("../plugins/lib/changed-files-store.ts").catch(() => {
|
||||
changedFilesStorePromise = undefined
|
||||
throw new Error(
|
||||
"changed-files tool: could not load the changed-files store. " +
|
||||
|
||||
@@ -5,11 +5,11 @@
|
||||
*/
|
||||
|
||||
// Re-export all tools
|
||||
export { default as runTests } from "./run-tests.js"
|
||||
export { default as checkCoverage } from "./check-coverage.js"
|
||||
export { default as securityAudit } from "./security-audit.js"
|
||||
export { default as formatCode } from "./format-code.js"
|
||||
export { default as lintCheck } from "./lint-check.js"
|
||||
export { default as gitSummary } from "./git-summary.js"
|
||||
export { default as changedFiles } from "./changed-files.js"
|
||||
export { default as dependencyAnalyzer } from "./dependency-analyzer.js"
|
||||
export { default as runTests } from "./run-tests.ts"
|
||||
export { default as checkCoverage } from "./check-coverage.ts"
|
||||
export { default as securityAudit } from "./security-audit.ts"
|
||||
export { default as formatCode } from "./format-code.ts"
|
||||
export { default as lintCheck } from "./lint-check.ts"
|
||||
export { default as gitSummary } from "./git-summary.ts"
|
||||
export { default as changedFiles } from "./changed-files.ts"
|
||||
export { default as dependencyAnalyzer } from "./dependency-analyzer.ts"
|
||||
|
||||
@@ -15,7 +15,8 @@
|
||||
"sourceMap": true,
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true,
|
||||
"verbatimModuleSyntax": true,
|
||||
"allowImportingTsExtensions": true,
|
||||
"rewriteRelativeImportExtensions": true,
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": [
|
||||
|
||||
+10
-2
@@ -90,8 +90,9 @@ The `extensions/index.ts` file handles:
|
||||
4. **Context injection** — Parses `hookSpecificOutput.additionalContext` from the SessionStart
|
||||
hook and appends it to the system prompt on the next `before_agent_start`, wrapped in an
|
||||
`<ecc-session-context>` block. Non-JSON hook output is tolerated, not treated as an error
|
||||
5. **Hook isolation** — Failing, missing, or slow hooks degrade to a warning and never
|
||||
terminate the Pi session. Hook execution is bounded by a timeout and an output limit
|
||||
5. **Hook isolation** — Failing, missing, slow, or misconfigured hooks degrade to
|
||||
a warning and never terminate the Pi session. Hook execution is bounded by a
|
||||
timeout and an output limit
|
||||
6. **Package resolution** — Resolves hook scripts from the installed package via `__dirname`,
|
||||
never from `process.cwd()`, so a global install works from any project directory. Hooks
|
||||
still *run* in the user's project directory, so project detection stays correct
|
||||
@@ -99,6 +100,13 @@ The `extensions/index.ts` file handles:
|
||||
All hook execution is non-shell (`execFile` without shell interpretation), so paths containing
|
||||
spaces, tabs, or shell metacharacters are safe.
|
||||
|
||||
Hook runtime selection uses the host `process.execPath` only under Node.
|
||||
Without an override, compiled OMP/Bun falls back to `node` instead of
|
||||
recursively launching the OMP binary as a hook runner. Set `ECC_HOOK_NODE` to
|
||||
an explicit absolute Node executable path when `node` is not available on
|
||||
`PATH`.
|
||||
Relative values are rejected when the hook runs and surfaced as a warning.
|
||||
|
||||
## Scope
|
||||
|
||||
Intentionally **out of scope** for this first adapter (to be added independently):
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
const path = require("node:path")
|
||||
|
||||
/**
|
||||
* Select a real Node executable for hook scripts.
|
||||
*
|
||||
* Compiled OMP may report `process.release.name` as `node` even though its
|
||||
* `process.execPath` points to the OMP launcher. Bun is detected separately via
|
||||
* `process.versions.bun`; both fall back to `node` unless `ECC_HOOK_NODE`
|
||||
* supplies an explicit absolute path.
|
||||
*
|
||||
* @param options - Runtime metadata and an optional absolute Node override.
|
||||
* @returns The executable path to use for hook scripts.
|
||||
* @throws {Error} If the hook runtime override is non-empty and relative.
|
||||
*/
|
||||
function resolveHookRuntime({
|
||||
execPath = process.execPath,
|
||||
releaseName = process.release?.name,
|
||||
bunVersion = process.versions?.bun,
|
||||
override = process.env.ECC_HOOK_NODE,
|
||||
} = {}) {
|
||||
const isNodeRuntime =
|
||||
releaseName === "node" &&
|
||||
!bunVersion &&
|
||||
/^(?:node|nodejs)(?:\.exe)?$/i.test(path.basename(execPath))
|
||||
const overridePath = override?.trim()
|
||||
if (overridePath) {
|
||||
if (!path.isAbsolute(overridePath)) {
|
||||
throw new Error("ECC_HOOK_NODE must be an absolute path: " + overridePath)
|
||||
}
|
||||
return overridePath
|
||||
}
|
||||
return isNodeRuntime ? execPath : "node"
|
||||
}
|
||||
|
||||
module.exports = { resolveHookRuntime }
|
||||
+66
-10
@@ -15,16 +15,20 @@
|
||||
* Design constraints (see .pi/README.md):
|
||||
* - Hooks resolve relative to THIS file, never `process.cwd()`, so a global
|
||||
* `pi install` works from any project directory.
|
||||
* - Hooks execute via `execFile(process.execPath, [...])` with no shell, so
|
||||
* paths containing spaces or shell metacharacters are safe.
|
||||
* - Hook failures are isolated: a broken, missing, or slow hook degrades to a
|
||||
* warning and never terminates the Pi session.
|
||||
* - Hooks execute via `execFile(hookRuntime, [...])` with no shell, so paths
|
||||
* containing spaces or shell metacharacters are safe. The hook runtime is
|
||||
* selected separately because compiled OMP may report `process.release.name`
|
||||
* as `node` while `process.execPath` points back to `omp`; Bun is detected
|
||||
* separately via `process.versions.bun`.
|
||||
* - Hook failures are isolated: a broken, missing, slow, or misconfigured hook
|
||||
* degrades to a warning and never terminates the Pi session.
|
||||
*/
|
||||
|
||||
import { execFile } from "node:child_process"
|
||||
import * as fs from "node:fs"
|
||||
import * as os from "node:os"
|
||||
import * as path from "node:path"
|
||||
import { resolveHookRuntime } from "./hook-runtime.js"
|
||||
|
||||
/**
|
||||
* Minimal structural types mirroring `@earendil-works/pi-coding-agent`.
|
||||
@@ -137,6 +141,10 @@ const DISABLED_VALUES = new Set(["0", "false", "off", "none", "disabled"])
|
||||
/**
|
||||
* Optional Pi companion packages. ECC works without every one of these; they
|
||||
* are reported by `/ecc-doctor` so users can see which extras are available.
|
||||
*
|
||||
* These are capability names, not exact install specs. See
|
||||
* `findInstalledCompanion` for how an entry is matched against what Pi has
|
||||
* actually installed.
|
||||
*/
|
||||
const COMPANION_PACKAGES = [
|
||||
"pi-subagents",
|
||||
@@ -175,8 +183,9 @@ interface HookResult {
|
||||
/**
|
||||
* Run an ECC hook through ECC's own runner.
|
||||
*
|
||||
* Never rejects: a missing runner, a non-zero exit, a timeout, or a spawn error
|
||||
* all resolve to a `failure` string that the caller surfaces as a warning.
|
||||
* Never rejects: an invalid runtime override, a missing runner, a non-zero exit,
|
||||
* a timeout, or a spawn error all resolve to a `failure` string that the caller
|
||||
* surfaces as a warning.
|
||||
*/
|
||||
function runEccHook(
|
||||
spec: HookSpec,
|
||||
@@ -189,9 +198,19 @@ function runEccHook(
|
||||
resolve({ stdout: "", failure: `hook runner not found at ${HOOK_RUNNER}` })
|
||||
return
|
||||
}
|
||||
let hookRuntime: string
|
||||
try {
|
||||
hookRuntime = resolveHookRuntime()
|
||||
} catch (error) {
|
||||
resolve({
|
||||
stdout: "",
|
||||
failure: `${spec.id}: ${(error as Error).message}`,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
const child = execFile(
|
||||
process.execPath,
|
||||
hookRuntime,
|
||||
[HOOK_RUNNER, spec.id, spec.script, spec.profiles],
|
||||
{
|
||||
// Hooks inspect the user's project, so they run there. Only the script
|
||||
@@ -460,6 +479,41 @@ function normalizePiPackageName(entry: unknown): string | undefined {
|
||||
return versionAt > 0 ? spec.slice(0, versionAt) : spec
|
||||
}
|
||||
|
||||
/**
|
||||
* The installed package satisfying a companion entry, or undefined if none is.
|
||||
*
|
||||
* An exact name match is the ordinary case. An UNSCOPED companion entry is
|
||||
* also satisfied by a scoped package with the same bare name --
|
||||
* `@tintinweb/pi-subagents` satisfies `pi-subagents`. The subagents capability
|
||||
* is published to npm by more than one maintainer under that same bare name,
|
||||
* and a user running a scoped fork has the capability installed by any
|
||||
* meaning of the word; reporting "not installed" at them while its tools are
|
||||
* live in their session is a false negative, and the suggested
|
||||
* `pi install npm:pi-subagents` would push them into installing a second
|
||||
* extension that registers the same tool names.
|
||||
*
|
||||
* A SCOPED companion entry is matched exactly, because there the scope is
|
||||
* part of the identity the entry names, not incidental packaging.
|
||||
*/
|
||||
function findInstalledCompanion(companion: string, installed: Set<string>): string | undefined {
|
||||
if (installed.has(companion)) {
|
||||
return companion
|
||||
}
|
||||
|
||||
if (companion.startsWith("@")) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const scopedSuffix = `/${companion}`
|
||||
for (const name of installed) {
|
||||
if (name.startsWith("@") && name.endsWith(scopedSuffix)) {
|
||||
return name
|
||||
}
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
function countDirectories(dir: string): number {
|
||||
try {
|
||||
return fs.readdirSync(dir, { withFileTypes: true }).filter(entry => entry.isDirectory()).length
|
||||
@@ -532,10 +586,12 @@ function buildDoctorReport(ctx: ExtensionContext): string {
|
||||
|
||||
const installed = listInstalledPiPackages(ctx.cwd)
|
||||
for (const name of COMPANION_PACKAGES) {
|
||||
const present = installed.has(name)
|
||||
lines.push(` ${present ? "installed " : "not installed"} ${name}`)
|
||||
if (!present) {
|
||||
const match = findInstalledCompanion(name, installed)
|
||||
lines.push(` ${match ? "installed " : "not installed"} ${name}`)
|
||||
if (!match) {
|
||||
lines.push(` install with: pi install npm:${name}`)
|
||||
} else if (match !== name) {
|
||||
lines.push(` satisfied by: ${match}`)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
# Security Evidence — PR #3172 / #3171
|
||||
|
||||
Commit under review: observe.sh Layer-1 allowlist adds `sdk-cli`.
|
||||
|
||||
## Changed security-sensitive surface
|
||||
- `skills/continuous-learning-v2/hooks/observe.sh` (agent hook entrypoint allowlist)
|
||||
|
||||
## Threat model (bounded)
|
||||
- **Risk if missing `sdk-cli`**: interactive Agent SDK CLI sessions never observe (availability/coverage gap).
|
||||
- **Risk if allowlist too broad**: non-interactive bots could start the observer. Mitigated by Layers 2–5 (`ECC_HOOK_PROFILE=minimal`, `ECC_SKIP_OBSERVE=1`, `agent_id`, path exclusions) — unchanged by this PR.
|
||||
- **No secrets / auth tokens / billing / webhook handlers** were modified.
|
||||
|
||||
## Security-focused validation artifacts (this PR)
|
||||
1. **Focused security regression test** (new): `tests/hooks/observe-entrypoint-security.test.js`
|
||||
- Asserts source allowlist includes `sdk-cli`
|
||||
- Asserts Layer-1 allows: `cli`, `sdk-ts`, `sdk-cli`, `claude-desktop`, `claude-vscode`
|
||||
- Asserts Layer-1 rejects: `unknown-bot`, `ci-bot`
|
||||
2. **Supply-chain IOC scan** (repo gate): `npm run security:ioc-scan`
|
||||
|
||||
## Command output (local)
|
||||
|
||||
### observe-entrypoint-security.test.js
|
||||
```text
|
||||
|
||||
=== observe.sh Layer-1 entrypoint security (#3171) ===
|
||||
|
||||
✓ source allowlist includes sdk-cli
|
||||
✓ Layer-1 allows cli
|
||||
✓ Layer-1 allows sdk-ts
|
||||
✓ Layer-1 allows sdk-cli
|
||||
✓ Layer-1 allows claude-desktop
|
||||
✓ Layer-1 allows claude-vscode
|
||||
✓ Layer-1 rejects unknown-bot
|
||||
✓ Layer-1 rejects ci-bot
|
||||
|
||||
All Layer-1 security checks passed.
|
||||
```
|
||||
|
||||
### npm run security:ioc-scan
|
||||
```text
|
||||
|
||||
> ecc-universal@2.2.1 security:ioc-scan
|
||||
> node scripts/ci/scan-supply-chain-iocs.js
|
||||
|
||||
Supply-chain IOC scan passed for /workspace/pr-work/ECC-3171 (12 files inspected)
|
||||
```
|
||||
|
||||
## Conclusion
|
||||
Allowlist change is covered by a dedicated security regression test plus the repository IOC scan. Unknown entrypoints remain denied at Layer-1.
|
||||
@@ -1,8 +1,8 @@
|
||||
# Everything Claude Code (ECC) — Agent Instructions
|
||||
|
||||
This is a **production-ready AI coding plugin** providing 68 specialized agents, 286 skills, 94 commands, and automated hook workflows for software development.
|
||||
This is a **production-ready AI coding plugin** providing 68 specialized agents, 292 skills, 94 commands, and automated hook workflows for software development.
|
||||
|
||||
**Version:** 2.2.1
|
||||
**Version:** 2.2.2
|
||||
|
||||
## Core Principles
|
||||
|
||||
@@ -52,15 +52,15 @@ This is a **production-ready AI coding plugin** providing 68 specialized agents,
|
||||
## Agent Orchestration
|
||||
|
||||
Use agents proactively without user prompt:
|
||||
- Complex feature requests → **planner**
|
||||
- Code just written/modified → **code-reviewer**
|
||||
- Bug fix or new feature → **tdd-guide**
|
||||
- Architectural decision → **architect**
|
||||
- Security-sensitive code → **security-reviewer**
|
||||
- Brownfield project onboarding → **spec-miner**
|
||||
- Autonomous loops / loop monitoring → **loop-operator**
|
||||
- Harness config reliability and cost → **harness-optimizer**
|
||||
- RAG/retrieval pipeline changes → **rag-pipeline-reviewer**
|
||||
- Complex feature requests → **ecc:planner**
|
||||
- Code just written/modified → **ecc:code-reviewer**
|
||||
- Bug fix or new feature → **ecc:tdd-guide**
|
||||
- Architectural decision → **ecc:architect**
|
||||
- Security-sensitive code → **ecc:security-reviewer**
|
||||
- Brownfield project onboarding → **ecc:spec-miner**
|
||||
- Autonomous loops / loop monitoring → **ecc:loop-operator**
|
||||
- Harness config reliability and cost → **ecc:harness-optimizer**
|
||||
- RAG/retrieval pipeline changes → **ecc:rag-pipeline-reviewer**
|
||||
|
||||
Use parallel execution for independent operations — launch multiple agents simultaneously.
|
||||
|
||||
@@ -114,9 +114,9 @@ Troubleshoot failures: check test isolation → verify mocks → fix implementat
|
||||
|
||||
## Development Workflow
|
||||
|
||||
1. **Plan** — Use planner agent, identify dependencies and risks, break into phases
|
||||
2. **TDD** — Use tdd-guide agent, write tests first, implement, refactor
|
||||
3. **Review** — Use code-reviewer agent immediately, address CRITICAL/HIGH issues
|
||||
1. **Plan** — Use ecc:planner agent, identify dependencies and risks, break into phases
|
||||
2. **TDD** — Use ecc:tdd-guide agent, write tests first, implement, refactor
|
||||
3. **Review** — Use ecc:code-reviewer agent immediately, address CRITICAL/HIGH issues
|
||||
4. **Capture knowledge in the right place**
|
||||
- Personal debugging notes, preferences, and temporary context → auto memory
|
||||
- Team/project knowledge (architecture decisions, API changes, runbooks) → the project's existing docs structure
|
||||
@@ -154,7 +154,7 @@ Troubleshoot failures: check test isolation → verify mocks → fix implementat
|
||||
|
||||
```
|
||||
agents/ — 68 specialized subagents
|
||||
skills/ — 286 workflow skills and domain knowledge
|
||||
skills/ — 292 workflow skills and domain knowledge
|
||||
commands/ — 94 slash commands
|
||||
hooks/ — Trigger-based automations
|
||||
rules/ — Always-follow guidelines (common + per-language)
|
||||
|
||||
+33
-1
@@ -1,6 +1,38 @@
|
||||
# Changelog
|
||||
|
||||
## Unreleased
|
||||
## 2.2.2 - 2026-09-15
|
||||
|
||||
### Fixed
|
||||
|
||||
#### Packaging
|
||||
|
||||
- Explicitly include the compiled OpenCode payload in the npm package and verify that packing builds it from a clean state with lifecycle scripts enabled.
|
||||
|
||||
#### Memory and MCP
|
||||
|
||||
- Distinguish incomplete memory reads from missing records and classify directory traversal failures (`90ef62cb`, `8321021c`).
|
||||
- Accept the reserved `_meta` parameter on memory MCP ping requests (`380f4b35`).
|
||||
|
||||
#### Hooks and Windows compatibility
|
||||
|
||||
- Keep `hooks.json` within Claude Code's schema by moving stable hook metadata into a validated sidecar (`1ac07903`).
|
||||
- Handle stuck optional values and long-option prefixes in the no-verify guard (`4f373874`).
|
||||
- Support Windows linter paths and ESLint 9 (`2083c983`).
|
||||
- Tolerate missing Windows device IDs in settings updates while retaining full-precision inode checks and strict matching when both device IDs are available (`d3af582b`).
|
||||
|
||||
#### Workflow guidance and catalog
|
||||
|
||||
- Filter epic sync issues by label (`3033436d`).
|
||||
- Remove instructions to auto-merge dependency bumps and synchronize localized merge authority (`22d7ed51`, `678c6dea`).
|
||||
- Keep common naming and Boolean guidance language-neutral (`072e4684`, `a0ecb793`, `013ed0a8`).
|
||||
- Distinguish the `prp-pr` command alias (`cc91c24f`).
|
||||
- Correct Rails skill discovery, invoice tax calculation order, and framework documentation (`b6ddd13a`).
|
||||
- Remove Serply and Squish catalog entries (`c4904e3f`).
|
||||
|
||||
#### Dependency security
|
||||
|
||||
- Update `lru` to 0.18.2 for RUSTSEC-2026-0253 (`4fc950c4`).
|
||||
- Update `js-yaml` to 4.3.2 for GHSA-2883-xcg3-v3hh (`549c1469`).
|
||||
|
||||
## 2.2.0 - 2026-08-25
|
||||
|
||||
|
||||
+2
-2
@@ -80,7 +80,7 @@
|
||||
|
||||
## 最新动态
|
||||
|
||||
### v2.2.1 — 引导式多 Harness 安装（2026年8月）
|
||||
### v2.2.2 — 引导式多 Harness 安装（2026年8月）
|
||||
|
||||
新增可审查的 Claude Code、Codex 与 Kimi Code 多 Harness 安装流程，并提供同步的 npm 命令入口。
|
||||
|
||||
@@ -196,7 +196,7 @@ Copy-Item -Recurse rules/typescript "$HOME/.claude/rules/"
|
||||
/plugin list ecc@ecc
|
||||
```
|
||||
|
||||
**完成！** 你现在可以使用 68 个代理、286 个技能和 94 个命令。
|
||||
**完成！** 你现在可以使用 68 个代理、292 个技能和 94 个命令。
|
||||
|
||||
### multi-* 命令需要额外配置
|
||||
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
# Rules
|
||||
|
||||
## Must Always
|
||||
- Delegate to specialized agents for domain tasks.
|
||||
- Write tests before implementation and verify critical paths.
|
||||
- Validate inputs and keep security checks intact.
|
||||
- Prefer immutable updates over mutating shared state.
|
||||
- Follow established repository patterns before inventing new ones.
|
||||
- Keep contributions focused, reviewable, and well-described.
|
||||
|
||||
## Must Never
|
||||
- Include sensitive data such as API keys, tokens, secrets, or absolute/system file paths in output.
|
||||
- Submit untested changes.
|
||||
- Bypass security checks or validation hooks.
|
||||
- Duplicate existing functionality without a clear reason.
|
||||
- Ship code without checking the relevant test suite.
|
||||
|
||||
## Agent Format
|
||||
- Agents live in `agents/*.md`.
|
||||
- Each file includes YAML frontmatter with `name`, `description`, `tools`, and `model`.
|
||||
- File names are lowercase with hyphens and must match the agent name.
|
||||
- Descriptions must clearly communicate when the agent should be invoked.
|
||||
|
||||
## Skill Format
|
||||
- Skills live in `skills/<name>/SKILL.md`.
|
||||
- Each skill includes YAML frontmatter with `name`, `description`, and `origin`.
|
||||
- Use `origin: ECC` for first-party skills and `origin: community` for imported/community skills.
|
||||
- Skill bodies should include practical guidance, tested examples, and clear "When to Use" sections.
|
||||
|
||||
## Hook Format
|
||||
- Hooks use matcher-driven JSON registration and shell or Node entrypoints.
|
||||
- Matchers should be specific instead of broad catch-alls.
|
||||
- Exit `1` only when blocking behavior is intentional; otherwise exit `0`.
|
||||
- Error and info messages should be actionable.
|
||||
|
||||
## Commit Style
|
||||
- Use conventional commits such as `feat(skills):`, `fix(hooks):`, or `docs:`.
|
||||
- Keep changes modular and explain user-facing impact in the PR summary.
|
||||
@@ -1,7 +1,7 @@
|
||||
# Soul
|
||||
|
||||
## Core Identity
|
||||
Everything Claude Code (ECC) is a production-ready AI coding plugin with 30 specialized agents, 135 skills, 60 commands, and automated hook workflows for software development.
|
||||
Everything Claude Code (ECC) is a production-ready AI coding plugin: specialized agents, on-demand skills, slash commands, rules, and automated hook workflows for software development.
|
||||
|
||||
## Core Principles
|
||||
1. **Agent-First** — route work to the right specialist as early as possible.
|
||||
|
||||
+7
-1
@@ -12,14 +12,20 @@ Thank you to everyone funding ECC's open-source work. Your sponsorship is what l
|
||||
|---------|------|-------|
|
||||
| [**CodeRabbit**](https://www.coderabbit.ai) | <img src="assets/images/sponsors/coderabbit.png" width="60" alt="CodeRabbit logo" /> | 2026 |
|
||||
| [**Greptile**](https://www.greptile.com/go/ecc) | <img src="assets/images/sponsors/greptile.png" width="60" alt="Greptile logo" /> | 2026 |
|
||||
| [**Atlas Cloud**](https://www.atlascloud.ai/?utm_source=github&utm_medium=link&utm_campaign=ECC) | <picture><source media="(prefers-color-scheme: dark)" srcset="assets/images/sponsors/atlascloud-dark.svg" /><img src="assets/images/sponsors/atlascloud.svg" width="120" alt="Atlas Cloud logo" /></picture> | 2026 |
|
||||
| [**Moonshot AI (Kimi)**](https://www.moonshot.ai) | <picture><source media="(prefers-color-scheme: dark)" srcset="assets/images/sponsors/moonshot-dark.png" /><img src="assets/images/sponsors/moonshot.png" width="100" alt="Moonshot AI Kimi logo" /></picture> | 2026 |
|
||||
| [**Itô**](https://compute.itomarkets.com) | <picture><source media="(prefers-color-scheme: light)" srcset="assets/images/sponsors/ito-transparent-light.png" /><img src="assets/images/sponsors/ito-transparent.png" width="88" alt="Itô Markets logo" /></picture> | 2026 |
|
||||
| [**SerpApi**](https://serpapi.com/github-ecc) | <picture><source media="(prefers-color-scheme: dark)" srcset="assets/images/sponsors/serpapi-logo-dark-mode.svg" /><img src="assets/images/sponsors/serpapi-logo-light-mode.svg" width="200" alt="SerpApi: Web Search API" /></picture> | 2026 |
|
||||
|
||||
*[Become a Business sponsor](https://github.com/sponsors/affaan-m) to get README sponsor placement + SPONSORS.md listing. Current Business tier is $800/mo. No seats, SLA, custom development, or preferential technical placement is bundled unless separately agreed.*
|
||||
|
||||
Run or self-host any open-source model. Itô partners with ECC on compute, while ECC remains provider-agnostic and any GPU provider works. The [Itô dashboard](https://compute.itomarkets.com) sponsorship link is passive: it does not invoke an RFQ, reserve capacity, provision compute, or configure serving. Separately, the opt-in `ecc ito find` bridge invokes the explicitly configured canonical Itô CLI and submits a live authenticated RFQ; it does not reserve capacity. Managed inference through Itô is not live yet.
|
||||
|
||||
## Past Sponsors
|
||||
|
||||
| Sponsor | Active period |
|
||||
|---------|---------------|
|
||||
| [**Atlas Cloud**](https://www.atlascloud.ai/?utm_source=github&utm_medium=link&utm_campaign=ECC) | 2026 |
|
||||
|
||||
## Team Sponsors — $200/mo
|
||||
|
||||
| Sponsor | Since |
|
||||
|
||||
@@ -1,179 +0,0 @@
|
||||
# Working Context
|
||||
|
||||
Last updated: 2026-04-08
|
||||
|
||||
## Purpose
|
||||
|
||||
Public ECC plugin repo for agents, skills, commands, hooks, rules, install surfaces, and ECC 2.0 platform buildout.
|
||||
|
||||
## Current Truth
|
||||
|
||||
- Default branch: `main`
|
||||
- Public release surface is aligned at `v1.10.0`
|
||||
- Public catalog truth is `47` agents, `79` commands, and `181` skills
|
||||
- Public plugin slug is now `ecc`; legacy `everything-claude-code` install paths remain supported for compatibility
|
||||
- Release discussion: `#1272`
|
||||
- ECC 2.0 exists in-tree and builds, but it is still alpha rather than GA
|
||||
- Main active operational work:
|
||||
- keep default branch green
|
||||
- continue issue-driven fixes from `main` now that the public PR backlog is at zero
|
||||
- continue ECC 2.0 control-plane and operator-surface buildout
|
||||
|
||||
## Current Constraints
|
||||
|
||||
- No merge by title or commit summary alone.
|
||||
- No arbitrary external runtime installs in shipped ECC surfaces.
|
||||
- Overlapping skills, hooks, or agents should be consolidated when overlap is material and runtime separation is not required.
|
||||
|
||||
## Active Queues
|
||||
|
||||
- PR backlog: reduced but active; keep direct-porting only safe ECC-native changes and close overlap, stale generators, and unaudited external-runtime lanes
|
||||
- Upstream branch backlog still needs selective mining and cleanup:
|
||||
- `origin/feat/hermes-generated-ops-skills` still has three unique commits, but only reusable ECC-native skills should be salvaged from it
|
||||
- multiple `origin/ecc-tools/*` automation branches are stale and should be pruned after confirming they carry no unique value
|
||||
- Product:
|
||||
- selective install cleanup
|
||||
- control plane primitives
|
||||
- operator surface
|
||||
- self-improving skills
|
||||
- keep `agent.yaml` export parity with the shipped `commands/` and `skills/` directories so modern install surfaces do not silently lose command registration
|
||||
- Skill quality:
|
||||
- rewrite content-facing skills to use source-backed voice modeling
|
||||
- remove generic LLM rhetoric, canned CTA patterns, and forced platform stereotypes
|
||||
- continue one-by-one audit of overlapping or low-signal skill content
|
||||
- move repo guidance and contribution flow to skills-first, leaving commands only as explicit compatibility shims
|
||||
- add operator skills that wrap connected surfaces instead of exposing only raw APIs or disconnected primitives
|
||||
- land the canonical voice system, network-optimization lane, and reusable Manim explainer lane
|
||||
- Security:
|
||||
- keep dependency posture clean
|
||||
- preserve self-contained hook and MCP behavior
|
||||
|
||||
## Open PR Classification
|
||||
|
||||
- Closed on 2026-04-01 under backlog hygiene / merge policy:
|
||||
- `#1069` `feat: add everything-claude-code ECC bundle`
|
||||
- `#1068` `feat: add everything-claude-code-conventions ECC bundle`
|
||||
- `#1080` `feat: add everything-claude-code ECC bundle`
|
||||
- `#1079` `feat: add everything-claude-code-conventions ECC bundle`
|
||||
- `#1064` `chore(deps-dev): bump @eslint/js from 9.39.2 to 10.0.1`
|
||||
- `#1063` `chore(deps-dev): bump eslint from 9.39.2 to 10.1.0`
|
||||
- Closed on 2026-04-01 because the content is sourced from external ecosystems and should only land via manual ECC-native re-port:
|
||||
- `#852` openclaw-user-profiler
|
||||
- `#851` openclaw-soul-forge
|
||||
- `#640` harper skills
|
||||
- Native-support candidates to fully diff-audit next:
|
||||
- `#1055` Dart / Flutter support
|
||||
- `#1043` C# reviewer and .NET skills
|
||||
- Direct-port candidates landed after audit:
|
||||
- `#1078` hook-id dedupe for managed Claude hook reinstalls
|
||||
- `#844` ui-demo skill
|
||||
- `#1110` install-time Claude hook root resolution
|
||||
- `#1106` portable Codex Context7 key extraction
|
||||
- `#1107` Codex baseline merge and sample agent-role sync
|
||||
- `#1119` stale CI/lint cleanup that still contained safe low-risk fixes
|
||||
- Port or rebuild inside ECC after full audit:
|
||||
- `#894` Jira integration
|
||||
- `#814` + `#808` rebuild as a single consolidated notifications lane for Opencode and cross-harness surfaces
|
||||
|
||||
## Interfaces
|
||||
|
||||
- Public truth: GitHub issues and PRs
|
||||
- Internal execution truth: linked Linear work items under the ECC program
|
||||
- Current linked Linear items:
|
||||
- `ECC-206` ecosystem CI baseline
|
||||
- `ECC-207` PR backlog audit and merge-policy enforcement
|
||||
- `ECC-208` context hygiene
|
||||
- `ECC-210` skills-first workflow migration and command compatibility retirement
|
||||
|
||||
## Update Rule
|
||||
|
||||
Keep this file detailed for only the current sprint, blockers, and next actions. Summarize completed work into archive or repo docs once it is no longer actively shaping execution.
|
||||
|
||||
## Latest Execution Notes
|
||||
|
||||
- 2026-04-05: Continued `#1213` overlap cleanup by narrowing `coding-standards` into the baseline cross-project conventions layer instead of deleting it. The skill now explicitly points detailed React/UI guidance to `frontend-patterns`, backend/API structure to `backend-patterns` / `api-design`, and keeps only reusable naming, readability, immutability, and code-quality expectations.
|
||||
- 2026-04-05: Added a packaging regression guard for the OpenCode release path after `#1287` showed the published `v1.10.0` artifact was still stale. `tests/scripts/build-opencode.test.js` now asserts the `npm pack --dry-run` tarball includes `.opencode/dist/index.js` plus compiled plugin/tool entrypoints, so future releases cannot silently omit the built OpenCode payload.
|
||||
- 2026-04-05: Landed `skills/agent-introspection-debugging` for `#829` as an ECC-native self-debugging framework. It is intentionally guidance-first rather than fake runtime automation: capture failure state, classify the pattern, apply the smallest contained recovery action, then emit a structured introspection report and hand off to `verification-loop` / `continuous-learning-v2` when appropriate.
|
||||
- 2026-04-05: Fixed the `main` npm CI break after the latest direct ports. `package-lock.json` had drifted behind `package.json` on the `globals` devDependency (`^17.1.0` vs `^17.4.0`), which caused all npm-based GitHub Actions jobs to fail at `npm ci`. Refreshed the lockfile only, verified `npm ci --ignore-scripts`, and kept the mixed-lock workspace otherwise untouched.
|
||||
- 2026-04-05: Direct-ported the useful discoverability part of `#1221` without duplicating a second healthcare compliance system. Added `skills/hipaa-compliance/SKILL.md` as a thin HIPAA-specific entrypoint that points into the canonical `healthcare-phi-compliance` / `healthcare-reviewer` lane, and wired both healthcare privacy skills into the `security` install module for selective installs.
|
||||
- 2026-04-05: Direct-ported the audited blockchain/web3 security lane from `#1222` into `main` as four self-contained skills: `defi-amm-security`, `evm-token-decimals`, `llm-trading-agent-security`, and `nodejs-keccak256`. These are now part of the `security` install module instead of living as an unmerged fork PR.
|
||||
- 2026-04-05: Finished the useful salvage pass from `#1203` directly on `main`. `skills/security-bounty-hunter`, `skills/api-connector-builder`, and `skills/dashboard-builder` are now in-tree as ECC-native rewrites instead of the thinner original community drafts. The original PR should be treated as superseded rather than merged.
|
||||
- 2026-04-02: `ECC-Tools/main` shipped `9566637` (`fix: prefer commit lookup over git ref resolution`). The PR-analysis fire is now fixed in the app repo by preferring explicit commit resolution before `git.getRef`, with regression coverage for pull refs and plain branch refs. Mirrored public tracking issue `#1184` in this repo was closed as resolved upstream.
|
||||
- 2026-04-02: Direct-ported the clean native-support core of `#1043` into `main`: `agents/csharp-reviewer.md`, `skills/dotnet-patterns/SKILL.md`, and `skills/csharp-testing/SKILL.md`. This fills the gap between existing C# rule/docs mentions and actual shipped C# review/testing guidance.
|
||||
- 2026-04-02: Direct-ported the clean native-support core of `#1055` into `main`: `agents/dart-build-resolver.md`, `commands/flutter-build.md`, `commands/flutter-review.md`, `commands/flutter-test.md`, `rules/dart/*`, and `skills/dart-flutter-patterns/SKILL.md`. The skill paths were wired into the current `framework-language` module instead of replaying the older PR's separate `flutter-dart` module layout.
|
||||
- 2026-04-02: Closed `#1081` after diff audit. The PR only added vendor-marketing docs for an external X/Twitter backend (`Xquik` / `x-twitter-scraper`) to the canonical `x-api` skill instead of contributing an ECC-native capability.
|
||||
- 2026-04-02: Direct-ported the useful Jira lane from `#894`, but sanitized it to match current supply-chain policy. `commands/jira.md`, `skills/jira-integration/SKILL.md`, and the pinned `jira` MCP template in `mcp-configs/mcp-servers.json` are in-tree, while the skill no longer tells users to install `uv` via `curl | bash`. `jira-integration` is classified under `operator-workflows` for selective installs.
|
||||
- 2026-04-02: Closed `#1125` after full diff audit. The bundle/skill-router lane hardcoded many non-existent or non-canonical surfaces and created a second routing abstraction instead of a small ECC-native index layer.
|
||||
- 2026-04-02: Closed `#1124` after full diff audit. The added agent roster was thoughtfully written, but it duplicated the existing ECC agent surface with a second competing catalog (`dispatch`, `explore`, `verifier`, `executor`, etc.) instead of strengthening canonical agents already in-tree.
|
||||
- 2026-04-02: Closed the full Argus cluster `#1098`, `#1099`, `#1100`, `#1101`, and `#1102` after full diff audit. The common failure mode was the same across all five PRs: external multi-CLI dispatch was treated as a first-class runtime dependency of shipped ECC surfaces. Any useful protocol ideas should be re-ported later into ECC-native orchestration, review, or reflection lanes without external CLI fan-out assumptions.
|
||||
- 2026-04-02: The previously open native-support / integration queue (`#1081`, `#1055`, `#1043`, `#894`) has now been fully resolved by direct-port or closure policy. The active public PR queue is currently zero; next focus stays on issue-driven mainline fixes and CI health, not backlog PR intake.
|
||||
- 2026-04-01: `main` CI was restored locally with `1723/1723` tests passing after lockfile and hook validation fixes.
|
||||
- 2026-04-01: Auto-generated ECC bundle PRs `#1068` and `#1069` were closed instead of merged; useful ideas must be ported manually after explicit diff audit.
|
||||
- 2026-04-01: Major-version ESLint bump PRs `#1063` and `#1064` were closed; revisit only inside a planned ESLint 10 migration lane.
|
||||
- 2026-04-01: Notification PRs `#808` and `#814` were identified as overlapping and should be rebuilt as one unified feature instead of landing as parallel branches.
|
||||
- 2026-04-01: External-source skill PRs `#640`, `#851`, and `#852` were closed under the new ingestion policy; copy ideas from audited source later rather than merging branded/source-import PRs directly.
|
||||
- 2026-04-01: The remaining low GitHub advisory on `ecc2/Cargo.lock` was addressed by moving `ratatui` to `0.30` with `crossterm_0_28`, which updated transitive `lru` from `0.12.5` to `0.16.3`. `cargo build --manifest-path ecc2/Cargo.toml` still passes.
|
||||
- 2026-04-01: Safe core of `#834` was ported directly into `main` instead of merging the PR wholesale. This included stricter install-plan validation, antigravity target filtering that skips unsupported module trees, tracked catalog sync for English plus zh-CN docs, and a dedicated `catalog:sync` write mode.
|
||||
- 2026-04-01: Repo catalog truth is now synced at `36` agents, `68` commands, and `142` skills across the tracked English and zh-CN docs.
|
||||
- 2026-04-01: Legacy emoji and non-essential symbol usage in docs, scripts, and tests was normalized to keep the unicode-safety lane green without weakening the check itself.
|
||||
- 2026-04-01: The remaining self-contained piece of `#834`, `docs/zh-CN/skills/browser-qa/SKILL.md`, was ported directly into the repo. After commit, `#834` should be closed as superseded-by-direct-port.
|
||||
- 2026-04-01: Content skill cleanup started with `content-engine`, `crosspost`, `article-writing`, and `investor-outreach`. The new direction is source-first voice capture, explicit anti-trope bans, and no forced platform persona shifts.
|
||||
- 2026-04-01: `node scripts/ci/check-unicode-safety.js --write` sanitized the remaining emoji-bearing Markdown files, including several `remotion-video-creation` rule docs and an old local plan note.
|
||||
- 2026-04-01: Core English repo surfaces were shifted to a skills-first posture. README, AGENTS, plugin metadata, and contributor instructions now treat `skills/` as canonical and `commands/` as legacy slash-entry compatibility during migration.
|
||||
- 2026-04-01: Follow-up bundle cleanup closed `#1080` and `#1079`, which were generated `.claude/` bundle PRs duplicating command-first scaffolding instead of shipping canonical ECC source changes.
|
||||
- 2026-04-01: Ported the useful core of `#1078` directly into `main`, but tightened the implementation so legacy no-id hook installs deduplicate cleanly on the first reinstall instead of the second. Added stable hook ids to `hooks/hooks.json`, semantic fallback aliases in `mergeHookEntries()`, and a regression test covering upgrade from pre-id settings.
|
||||
- 2026-04-01: Collapsed the obvious command/skill duplicates into thin legacy shims so `skills/` now hold the maintained bodies for NanoClaw, context-budget, DevFleet, docs lookup, E2E, evals, orchestration, prompt optimization, rules distillation, TDD, and verification.
|
||||
- 2026-04-01: Ported the self-contained core of `#844` directly into `main` as `skills/ui-demo/SKILL.md` and registered it under the `media-generation` install module instead of merging the PR wholesale.
|
||||
- 2026-04-01: Added the first connected-workflow operator lane as ECC-native skills instead of leaving the surface as raw plugins or APIs: `workspace-surface-audit`, `customer-billing-ops`, `project-flow-ops`, and `google-workspace-ops`. These are tracked under the new `operator-workflows` install module.
|
||||
- 2026-04-01: Direct-ported the real fix from the unresolved hook-path PR lane into the active installer. Claude installs now replace `${CLAUDE_PLUGIN_ROOT}` with the concrete install root in both `settings.json` and the copied `hooks/hooks.json`, which keeps PreToolUse/PostToolUse hooks working outside plugin-managed env injection.
|
||||
- 2026-04-01: Replaced the GNU-only `grep -P` parser in `scripts/sync-ecc-to-codex.sh` with a portable Node parser for Context7 key extraction. Added source-level regression coverage so BSD/macOS syncs do not drift back to non-portable parsing.
|
||||
- 2026-04-01: Targeted regression suite after the direct ports is green: `tests/scripts/install-apply.test.js`, `tests/scripts/sync-ecc-to-codex.test.js`, and `tests/scripts/codex-hooks.test.js`.
|
||||
- 2026-04-01: Ported the useful core of `#1107` directly into `main` as an add-only Codex baseline merge. `scripts/sync-ecc-to-codex.sh` now fills missing non-MCP defaults from `.codex/config.toml`, syncs sample agent role files into `~/.codex/agents`, and preserves user config instead of replacing it. Added regression coverage for sparse configs and implicit parent tables.
|
||||
- 2026-04-01: Ported the safe low-risk cleanup from `#1119` directly into `main` instead of keeping an obsolete CI PR open. This included `.mjs` eslint handling, stricter null checks, Windows home-dir coverage in bash-log tests, and longer Trae shell-test timeouts.
|
||||
- 2026-04-01: Added `brand-voice` as the canonical source-derived writing-style system and wired the content lane to treat it as the shared voice source of truth instead of duplicating partial style heuristics across skills.
|
||||
- 2026-04-01: Added `connections-optimizer` as the review-first social-graph reorganization workflow for X and LinkedIn, with explicit pruning modes, browser fallback expectations, and Apple Mail drafting guidance.
|
||||
- 2026-04-01: Added `manim-video` as the reusable technical explainer lane and seeded it with a starter network-graph scene so launch and systems animations do not depend on one-off scratch scripts.
|
||||
- 2026-04-02: Re-extracted `social-graph-ranker` as a standalone primitive because the weighted bridge-decay model is reusable outside the full lead workflow. `lead-intelligence` now points to it for canonical graph ranking instead of carrying the full algorithm explanation inline, while `connections-optimizer` stays the broader operator layer for pruning, adds, and outbound review packs.
|
||||
- 2026-04-02: Applied the same consolidation rule to the writing lane. `brand-voice` remains the canonical voice system, while `content-engine`, `crosspost`, `article-writing`, and `investor-outreach` now keep only workflow-specific guidance instead of duplicating a second Affaan/ECC voice model or repeating the full ban list in multiple places.
|
||||
- 2026-04-02: Closed fresh auto-generated bundle PRs `#1182` and `#1183` under the existing policy. Useful ideas from generator output must be ported manually into canonical repo surfaces instead of merging `.claude`/bundle PRs wholesale.
|
||||
- 2026-04-02: Ported the safe one-file macOS observer fix from `#1164` directly into `main` as a POSIX `mkdir` fallback for `continuous-learning-v2` lazy-start locking, then closed the PR as superseded by direct port.
|
||||
- 2026-04-02: Ported the safe core of `#1153` directly into `main`: markdownlint cleanup for orchestration/docs surfaces plus the Windows `USERPROFILE` and path-normalization fixes in `install-apply` / `repair` tests. Local validation after installing repo deps: `node tests/scripts/install-apply.test.js`, `node tests/scripts/repair.test.js`, and targeted `yarn markdownlint` all passed.
|
||||
- 2026-04-02: Direct-ported the safe web/frontend rules lane from `#1122` into `rules/web/`, but adapted `rules/web/hooks.md` to prefer project-local tooling and avoid remote one-off package execution examples.
|
||||
- 2026-04-02: Adapted the design-quality reminder from `#1127` into the current ECC hook architecture with a local `scripts/hooks/design-quality-check.js`, Claude `hooks/hooks.json` wiring, Cursor `after-file-edit.js` wiring, and dedicated hook coverage in `tests/hooks/design-quality-check.test.js`.
|
||||
- 2026-04-02: Fixed `#1141` on `main` in `16e9b17`. The observer lifecycle is now session-aware instead of purely detached: `SessionStart` writes a project-scoped lease, `SessionEnd` removes that lease and stops the observer when the final lease disappears, `observe.sh` records project activity, and `observer-loop.sh` now exits on idle when no leases remain. Targeted validation passed with `bash -n`, `node tests/hooks/observer-memory.test.js`, `node tests/integration/hooks.test.js`, `node scripts/ci/validate-hooks.js hooks/hooks.json`, and `node scripts/ci/check-unicode-safety.js`.
|
||||
- 2026-04-02: Fixed the remaining Windows-only hook regression behind `#1070` by making `scripts/lib/utils.js#getHomeDir()` honor explicit `HOME` / `USERPROFILE` overrides before falling back to `os.homedir()`. This restores test-isolated observer state paths for hook integration runs on Windows. Added regression coverage in `tests/lib/utils.test.js`. Targeted validation passed with `node tests/lib/utils.test.js`, `node tests/integration/hooks.test.js`, `node tests/hooks/observer-memory.test.js`, and `node scripts/ci/check-unicode-safety.js`.
|
||||
- 2026-04-02: Direct-ported NestJS support for `#1022` into `main` as `skills/nestjs-patterns/SKILL.md` and wired it into the `framework-language` install module. Synced the repo catalog afterward (`38` agents, `72` commands, `156` skills) and updated the docs so NestJS is no longer listed as an unfilled framework gap.
|
||||
- 2026-04-05: Shipped `846ffb7` (`chore: ship v1.10.0 release surface refresh`). This updated README/plugin metadata/package versions, synced the explicit plugin agent inventory, bumped stale star/fork/contributor counts, created `docs/releases/1.10.0/*`, tagged and released `v1.10.0`, and posted the announcement discussion at `#1272`.
|
||||
- 2026-04-05: Salvaged the reusable Hermes-branch operator skills in `6eba30f` without replaying the full branch. Added `skills/github-ops`, `skills/knowledge-ops`, and `skills/hookify-rules`, wired them into install modules, and re-synced the repo to `159` skills. `knowledge-ops` was explicitly adapted to the current workspace model: live code in cloned repos, active truth in GitHub/Linear, broader non-code context in the KB/archive layers.
|
||||
- 2026-04-05: Fixed the remaining OpenCode npm-publish gap in `db6d52e`. The root package now builds `.opencode/dist` during `prepack`, includes the compiled OpenCode plugin assets in the published tarball, and carries a dedicated regression test (`tests/scripts/build-opencode.test.js`) so the package no longer ships only raw TypeScript source for that surface.
|
||||
- 2026-04-05: Added `skills/council`, direct-ported the safe `code-tour` lane from `#1193`, and re-synced the repo to `162` skills. `code-tour` stays self-contained and only produces `.tours/*.tour` artifacts with real file/line anchors; no external runtime or extension install is assumed inside the skill.
|
||||
- 2026-04-05: Closed the latest auto-generated ECC bundle PR wave (`#1275`-`#1281`) after deploying `ECC-Tools/main` fix `f615905`, which now blocks repo-level issue-comment `/analyze` requests from opening repeated bundle PRs while still allowing PR-thread retry analysis to run against immutable head SHAs.
|
||||
- 2026-04-05: Filled the SEO gap by direct-porting `agents/seo-specialist.md` and `skills/seo/SKILL.md` into `main`, then wiring `skills/seo` into `business-content`. This resolves the stale `team-builder` reference to an SEO specialist and brings the public catalog to `39` agents and `163` skills without merging the stale PR wholesale.
|
||||
- 2026-04-05: Salvaged the useful common-rule deltas from `#1214` directly into `rules/common/coding-style.md` and `rules/common/testing.md` (KISS/DRY/YAGNI reminders, naming conventions, code-smell guidance, and AAA-style test guidance), then closed the original mixed deletion PR. The broad skill removals in that PR were intentionally not replayed.
|
||||
- 2026-04-05: Fixed the stale-row bug in `.github/workflows/monthly-metrics.yml` with `bf5961e`. The workflow now refreshes the current month row in issue `#1087` instead of early-returning when the month already exists, and the dispatched run updated the April snapshot to the current star/fork/release counts.
|
||||
- 2026-04-05: Recovered the useful cost-control workflow from the divergent Hermes branch as a small ECC-native operator skill instead of replaying the branch. `skills/ecc-tools-cost-audit/SKILL.md` is now wired into `operator-workflows` and focused on webhook -> queue -> worker tracing, burn containment, quota bypass, premium-model leakage, and retry fanout in the sibling `ECC-Tools` repo.
|
||||
- 2026-04-05: Added `skills/council/SKILL.md` in `753da37` as an ECC-native four-voice decision workflow. The useful protocol from PR `#1254` was retained, but the shadow `~/.claude/notes` write path was explicitly removed in favor of `knowledge-ops`, `/save-session`, or direct GitHub/Linear updates when a decision delta matters.
|
||||
- 2026-04-05: Direct-ported the safe `globals` bump from PR `#1243` into `main` as part of the council lane and closed the PR as superseded.
|
||||
- 2026-04-05: Closed PR `#1232` after full audit. The proposed `skill-scout` workflow overlaps current `search-first`, `/skill-create`, and `skill-stocktake`; if a dedicated marketplace-discovery layer returns later it should be rebuilt on top of the current install/catalog model rather than landing as a parallel discovery path.
|
||||
- 2026-04-05: Ported the safe localized README switcher fixes from PR `#1209` directly into `main` rather than merging the docs PR wholesale. The navigation now consistently includes `Português (Brasil)` and `Türkçe` across the localized README switchers, while newer localized body copy stays intact.
|
||||
- 2026-04-05: Removed the stale InsAIts shipped surface from `main`. ECC no longer ships the external Python MCP entry, opt-in hook wiring, wrapper/monitor scripts, or current docs mentions for `insa-its`; changelog history remains, but the live product surface is now fully ECC-native again.
|
||||
- 2026-04-05: Salvaged the reusable Hermes-generated operator workflow lane without replaying the whole branch. Added six ECC-native top-level skills instead of the old nested `skills/hermes-generated/*` tree: `automation-audit-ops`, `email-ops`, `finance-billing-ops`, `messages-ops`, `research-ops`, and `terminal-ops`. `research-ops` now wraps the existing research stack, while the other five extend `operator-workflows` without introducing any external runtime assumptions.
|
||||
- 2026-04-05: Added `skills/product-capability` plus `docs/examples/product-capability-template.md` as the canonical PRD-to-SRS lane for issue `#1185`. This is the ECC-native capability-contract step between vague product intent and implementation, and it lives in `business-content` rather than spawning a parallel planning subsystem.
|
||||
- 2026-04-05: Tightened `product-lens` so it no longer overlaps the new capability-contract lane. `product-lens` now explicitly owns product diagnosis / brief validation, while `product-capability` owns implementation-ready capability plans and SRS-style constraints.
|
||||
- 2026-04-05: Continued `#1213` cleanup by removing stale references to the deleted `project-guidelines-example` skill from exported inventory/docs and marking `continuous-learning` v1 as a supported legacy path with an explicit handoff to `continuous-learning-v2`.
|
||||
- 2026-04-05: Removed the last orphaned localized `project-guidelines-example` docs from `docs/ko-KR` and `docs/zh-CN`. The template now lives only in `docs/examples/project-guidelines-template.md`, which matches the current repo surface and avoids shipping translated docs for a deleted skill.
|
||||
- 2026-04-05: Added `docs/HERMES-OPENCLAW-MIGRATION.md` as the current public migration guide for issue `#1051`. It reframes Hermes/OpenClaw as source systems to distill from, not the final runtime, and maps scheduler, dispatch, memory, skill, and service layers onto the ECC-native surfaces and ECC 2.0 backlog that already exist.
|
||||
- 2026-04-05: Landed `skills/agent-sort` and the legacy `/agent-sort` shim from issue `#916` as an ECC-native selective-install workflow. It classifies agents, skills, commands, rules, hooks, and extras into DAILY vs LIBRARY buckets using concrete repo evidence, then hands off installation changes to `configure-ecc` instead of inventing a parallel installer. Catalog truth is now `39` agents, `73` commands, and `179` skills.
|
||||
- 2026-04-05: Direct-ported the safe README-only `#1285` slice into `main` instead of merging the branch: added a small `Community Projects` section so downstream teams can link public work built on ECC without changing install, security, or runtime surfaces. Rejected `#1286` at review because it adds an external third-party GitHub Action (`hashgraph-online/codex-plugin-scanner`) that does not meet the current supply-chain policy.
|
||||
- 2026-04-05: Re-audited `origin/feat/hermes-generated-ops-skills` by full diff. The branch is still not mergeable: it deletes current ECC-native surfaces, regresses packaging/install metadata, and removes newer `main` content. Continued the selective-salvage policy instead of branch merge.
|
||||
- 2026-04-05: Selectively salvaged `skills/frontend-design` from the Hermes branch as a self-contained ECC-native skill, mirrored it into `.agents`, wired it into `framework-language`, and re-synced the catalog to `180` skills after validation. The branch itself remains reference-only until every remaining unique file is either ported intentionally or rejected.
|
||||
- 2026-04-05: Selectively salvaged the `hookify` command bundle plus the supporting `conversation-analyzer` agent from the Hermes branch. `hookify-rules` already existed as the canonical skill; this pass restores the user-facing command surfaces (`/hookify`, `/hookify-help`, `/hookify-list`, `/hookify-configure`) without pulling in any external runtime or branch-wide regressions. Catalog truth is now `40` agents, `77` commands, and `180` skills.
|
||||
- 2026-04-05: Selectively salvaged the self-contained review/development bundle from the Hermes branch: `review-pr`, `feature-dev`, and the supporting analyzer/architecture agents (`code-architect`, `code-explorer`, `code-simplifier`, `comment-analyzer`, `pr-test-analyzer`, `silent-failure-hunter`, `type-design-analyzer`). This adds ECC-native command surfaces around PR review and feature planning without merging the branch's broader regressions. Catalog truth is now `47` agents, `79` commands, and `180` skills.
|
||||
- 2026-04-05: Ported `docs/HERMES-SETUP.md` from the Hermes branch as a sanitized operator-topology document for the migration lane. This is docs-only support for `#1051`, not a runtime change and not a sign that the Hermes branch itself is mergeable.
|
||||
- 2026-04-05: Finished the useful salvage pass over `origin/feat/hermes-generated-ops-skills`. The remaining unique files were explicitly rejected:
|
||||
- duplicate git helper commands (`commit`, `commit-push-pr`, `clean-gone`) overlap current checkpoint / publish flows
|
||||
- `scripts/hooks/security-reminder*` adds a new Python-backed hook path not justified by current runtime policy
|
||||
- `skills/oura-health` and `skills/pmx-guidelines` are user- or project-specific, not canonical ECC surfaces
|
||||
- `docs/releases/2.0.0-preview/*` is premature collateral and should be rebuilt from current product truth later
|
||||
- nested `skills/hermes-generated/*` is superseded by the top-level ECC-native operator skills already ported to `main`
|
||||
- 2026-04-08: Fixed the command-export regression reported in `#1327` by restoring a canonical `commands:` section in `agent.yaml` and adding `tests/ci/agent-yaml-surface.test.js` to enforce exact parity between the YAML export surface and the real `commands/` directory. Verified with the full repo test sweep: `1764/1764` passing.
|
||||
+8
-2
@@ -1,6 +1,6 @@
|
||||
spec_version: "0.1.0"
|
||||
name: ecc
|
||||
version: 2.2.1
|
||||
version: 2.2.2
|
||||
description: "Initial gitagent export surface for ECC's shared skill catalog, governance, and identity. Native agents, commands, and hooks remain authoritative in the repository while manifest coverage expands."
|
||||
author: affaan-m
|
||||
license: MIT
|
||||
@@ -100,7 +100,9 @@ skills:
|
||||
- logistics-exception-management
|
||||
- market-research
|
||||
- mcp-server-patterns
|
||||
- motion-ui
|
||||
- motion-advanced
|
||||
- motion-foundations
|
||||
- motion-patterns
|
||||
- nanoclaw-repl
|
||||
- nextjs-turbopack
|
||||
- nutrient-document-processing
|
||||
@@ -123,6 +125,7 @@ skills:
|
||||
- quarkus-security
|
||||
- quarkus-tdd
|
||||
- quarkus-verification
|
||||
- rails-patterns
|
||||
- ralphinho-rfc-pipeline
|
||||
- react-patterns
|
||||
- react-performance
|
||||
@@ -149,6 +152,9 @@ skills:
|
||||
- swift-concurrency-6-2
|
||||
- swift-protocol-di-testing
|
||||
- swiftui-patterns
|
||||
- taste-application
|
||||
- taste-distillation
|
||||
- tasteforge-video
|
||||
- tdd-workflow
|
||||
- team-builder
|
||||
- token-budget-advisor
|
||||
|
||||
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 7.8 KiB |
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 7.5 KiB |
@@ -1,30 +0,0 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 800 400" width="800" height="400" font-family="-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica,Arial,sans-serif">
|
||||
<rect width="800" height="400" fill="#0d1117"/>
|
||||
<line x1="70" y1="348.0" x2="772" y2="348.0" stroke="#30363d" stroke-width="1"/>
|
||||
<text x="60" y="352.0" fill="#8b949e" font-size="12" text-anchor="end">0</text>
|
||||
<line x1="70" y1="284.4" x2="772" y2="284.4" stroke="#30363d" stroke-width="1"/>
|
||||
<text x="60" y="288.4" fill="#8b949e" font-size="12" text-anchor="end">10k</text>
|
||||
<line x1="70" y1="220.8" x2="772" y2="220.8" stroke="#30363d" stroke-width="1"/>
|
||||
<text x="60" y="224.8" fill="#8b949e" font-size="12" text-anchor="end">20k</text>
|
||||
<line x1="70" y1="157.2" x2="772" y2="157.2" stroke="#30363d" stroke-width="1"/>
|
||||
<text x="60" y="161.2" fill="#8b949e" font-size="12" text-anchor="end">30k</text>
|
||||
<line x1="70" y1="93.6" x2="772" y2="93.6" stroke="#30363d" stroke-width="1"/>
|
||||
<text x="60" y="97.6" fill="#8b949e" font-size="12" text-anchor="end">40k</text>
|
||||
<line x1="70" y1="30.0" x2="772" y2="30.0" stroke="#30363d" stroke-width="1"/>
|
||||
<text x="60" y="34.0" fill="#8b949e" font-size="12" text-anchor="end">50k</text>
|
||||
<line x1="70.0" y1="30" x2="70.0" y2="348" stroke="#30363d" stroke-width="1" stroke-dasharray="2 4"/>
|
||||
<text x="70.0" y="370" fill="#8b949e" font-size="12" text-anchor="middle">Jan 18</text>
|
||||
<line x1="238.7" y1="30" x2="238.7" y2="348" stroke="#30363d" stroke-width="1" stroke-dasharray="2 4"/>
|
||||
<text x="238.7" y="370" fill="#8b949e" font-size="12" text-anchor="middle">Jan 23</text>
|
||||
<line x1="407.3" y1="30" x2="407.3" y2="348" stroke="#30363d" stroke-width="1" stroke-dasharray="2 4"/>
|
||||
<text x="407.3" y="370" fill="#8b949e" font-size="12" text-anchor="middle">Jan 28</text>
|
||||
<line x1="576.0" y1="30" x2="576.0" y2="348" stroke="#30363d" stroke-width="1" stroke-dasharray="2 4"/>
|
||||
<text x="576.0" y="370" fill="#8b949e" font-size="12" text-anchor="middle">Feb 2</text>
|
||||
<line x1="744.7" y1="30" x2="744.7" y2="348" stroke="#30363d" stroke-width="1" stroke-dasharray="2 4"/>
|
||||
<text x="744.7" y="370" fill="#8b949e" font-size="12" text-anchor="middle">Feb 7</text>
|
||||
<polygon points="70,348 70.0,348.0 123.9,332.7 137.9,327.6 149.4,322.6 171.9,307.3 177.3,302.2 187.8,292.0 192.2,286.9 198.6,281.8 212.6,266.6 222.0,256.4 226.3,251.3 234.5,246.2 239.6,241.1 244.2,236.1 247.5,231.0 251.4,225.9 274.2,210.6 282.2,205.5 290.0,200.4 300.7,195.4 312.3,190.3 320.2,185.2 355.4,164.8 373.7,159.7 409.0,149.6 430.4,144.5 458.7,139.4 492.0,134.3 526.5,129.2 559.4,124.1 618.6,113.9 685.6,103.8 718.9,98.7 772.0,93.6 772,348" fill="#2ea04326"/>
|
||||
<polyline points="70.0,348.0 123.9,332.7 137.9,327.6 149.4,322.6 171.9,307.3 177.3,302.2 187.8,292.0 192.2,286.9 198.6,281.8 212.6,266.6 222.0,256.4 226.3,251.3 234.5,246.2 239.6,241.1 244.2,236.1 247.5,231.0 251.4,225.9 274.2,210.6 282.2,205.5 290.0,200.4 300.7,195.4 312.3,190.3 320.2,185.2 355.4,164.8 373.7,159.7 409.0,149.6 430.4,144.5 458.7,139.4 492.0,134.3 526.5,129.2 559.4,124.1 618.6,113.9 685.6,103.8 718.9,98.7 772.0,93.6" fill="none" stroke="#2ea043" stroke-width="2.5" stroke-linejoin="round" stroke-linecap="round"/>
|
||||
<circle cx="772.0" cy="93.6" r="4" fill="#2ea043"/>
|
||||
<text x="400.0" y="20" fill="#e6edf3" font-size="14" font-weight="600" text-anchor="middle">affaan-m/ECC · first 40,000 stars</text>
|
||||
<text x="70" y="390" fill="#8b949e" font-size="11">Jan 18, 2026 – Feb 7, 2026 · source: GitHub stargazers API</text>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 3.4 KiB |
@@ -1,30 +0,0 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 800 400" width="800" height="400" font-family="-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica,Arial,sans-serif">
|
||||
<rect width="800" height="400" fill="#ffffff"/>
|
||||
<line x1="70" y1="348.0" x2="772" y2="348.0" stroke="#d8dee4" stroke-width="1"/>
|
||||
<text x="60" y="352.0" fill="#59636e" font-size="12" text-anchor="end">0</text>
|
||||
<line x1="70" y1="284.4" x2="772" y2="284.4" stroke="#d8dee4" stroke-width="1"/>
|
||||
<text x="60" y="288.4" fill="#59636e" font-size="12" text-anchor="end">10k</text>
|
||||
<line x1="70" y1="220.8" x2="772" y2="220.8" stroke="#d8dee4" stroke-width="1"/>
|
||||
<text x="60" y="224.8" fill="#59636e" font-size="12" text-anchor="end">20k</text>
|
||||
<line x1="70" y1="157.2" x2="772" y2="157.2" stroke="#d8dee4" stroke-width="1"/>
|
||||
<text x="60" y="161.2" fill="#59636e" font-size="12" text-anchor="end">30k</text>
|
||||
<line x1="70" y1="93.6" x2="772" y2="93.6" stroke="#d8dee4" stroke-width="1"/>
|
||||
<text x="60" y="97.6" fill="#59636e" font-size="12" text-anchor="end">40k</text>
|
||||
<line x1="70" y1="30.0" x2="772" y2="30.0" stroke="#d8dee4" stroke-width="1"/>
|
||||
<text x="60" y="34.0" fill="#59636e" font-size="12" text-anchor="end">50k</text>
|
||||
<line x1="70.0" y1="30" x2="70.0" y2="348" stroke="#d8dee4" stroke-width="1" stroke-dasharray="2 4"/>
|
||||
<text x="70.0" y="370" fill="#59636e" font-size="12" text-anchor="middle">Jan 18</text>
|
||||
<line x1="238.7" y1="30" x2="238.7" y2="348" stroke="#d8dee4" stroke-width="1" stroke-dasharray="2 4"/>
|
||||
<text x="238.7" y="370" fill="#59636e" font-size="12" text-anchor="middle">Jan 23</text>
|
||||
<line x1="407.3" y1="30" x2="407.3" y2="348" stroke="#d8dee4" stroke-width="1" stroke-dasharray="2 4"/>
|
||||
<text x="407.3" y="370" fill="#59636e" font-size="12" text-anchor="middle">Jan 28</text>
|
||||
<line x1="576.0" y1="30" x2="576.0" y2="348" stroke="#d8dee4" stroke-width="1" stroke-dasharray="2 4"/>
|
||||
<text x="576.0" y="370" fill="#59636e" font-size="12" text-anchor="middle">Feb 2</text>
|
||||
<line x1="744.7" y1="30" x2="744.7" y2="348" stroke="#d8dee4" stroke-width="1" stroke-dasharray="2 4"/>
|
||||
<text x="744.7" y="370" fill="#59636e" font-size="12" text-anchor="middle">Feb 7</text>
|
||||
<polygon points="70,348 70.0,348.0 123.9,332.7 137.9,327.6 149.4,322.6 171.9,307.3 177.3,302.2 187.8,292.0 192.2,286.9 198.6,281.8 212.6,266.6 222.0,256.4 226.3,251.3 234.5,246.2 239.6,241.1 244.2,236.1 247.5,231.0 251.4,225.9 274.2,210.6 282.2,205.5 290.0,200.4 300.7,195.4 312.3,190.3 320.2,185.2 355.4,164.8 373.7,159.7 409.0,149.6 430.4,144.5 458.7,139.4 492.0,134.3 526.5,129.2 559.4,124.1 618.6,113.9 685.6,103.8 718.9,98.7 772.0,93.6 772,348" fill="#1a7f3720"/>
|
||||
<polyline points="70.0,348.0 123.9,332.7 137.9,327.6 149.4,322.6 171.9,307.3 177.3,302.2 187.8,292.0 192.2,286.9 198.6,281.8 212.6,266.6 222.0,256.4 226.3,251.3 234.5,246.2 239.6,241.1 244.2,236.1 247.5,231.0 251.4,225.9 274.2,210.6 282.2,205.5 290.0,200.4 300.7,195.4 312.3,190.3 320.2,185.2 355.4,164.8 373.7,159.7 409.0,149.6 430.4,144.5 458.7,139.4 492.0,134.3 526.5,129.2 559.4,124.1 618.6,113.9 685.6,103.8 718.9,98.7 772.0,93.6" fill="none" stroke="#1a7f37" stroke-width="2.5" stroke-linejoin="round" stroke-linecap="round"/>
|
||||
<circle cx="772.0" cy="93.6" r="4" fill="#1a7f37"/>
|
||||
<text x="400.0" y="20" fill="#1f2328" font-size="14" font-weight="600" text-anchor="middle">affaan-m/ECC · first 40,000 stars</text>
|
||||
<text x="70" y="390" fill="#59636e" font-size="11">Jan 18, 2026 – Feb 7, 2026 · source: GitHub stargazers API</text>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 3.4 KiB |
@@ -158,3 +158,5 @@ Next step: /plan .claude/prds/{name}.prd.md
|
||||
- **HYPOTHESIS_TESTABLE**: measurable outcome included.
|
||||
- **SCOPE_BOUNDED**: explicit MVP and explicit out-of-scope.
|
||||
- **NO_IMPLEMENTATION_DETAIL**: file paths, libraries, or task breakdowns are absent — if they appeared, move them to the `/plan` step.
|
||||
|
||||
Background on the staged markdown flow: [docs/PLAN-PRD-PATTERN.md](../docs/PLAN-PRD-PATTERN.md).
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
---
|
||||
description: "Create a GitHub PR from current branch with unpushed commits — discovers templates, analyzes changes, pushes"
|
||||
description: "Alias of /pr for the PRP workflow series. Use when creating a pull request mid-PRP workflow; otherwise use /pr."
|
||||
argument-hint: "[base-branch] (default: main)"
|
||||
---
|
||||
|
||||
|
||||
@@ -359,6 +359,7 @@
|
||||
],
|
||||
"rules": ["common"],
|
||||
"skills": [
|
||||
"rails-patterns",
|
||||
"tdd-workflow",
|
||||
"verification-loop"
|
||||
],
|
||||
@@ -505,12 +506,11 @@
|
||||
"build": ["pip install -e ."],
|
||||
"test": ["pytest", "python -m pytest"],
|
||||
"lint": ["ruff check .", "mypy ."],
|
||||
"format": ["ruff format .", "black ."],
|
||||
"dev": ["uvicorn main:app --reload", "fastapi dev"]
|
||||
"format": ["ruff format .", "black ."]
|
||||
},
|
||||
"permissions": {
|
||||
"allow": ["python *", "pip install *", "pytest *", "ruff *", "black *", "mypy *", "uvicorn *"],
|
||||
"deny": []
|
||||
"allow": ["python *", "pip install *", "pytest *", "ruff *", "black *", "mypy *"],
|
||||
"deny": ["pip install --user *"]
|
||||
}
|
||||
},
|
||||
{
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
ARG NODE_IMAGE=node:22-bookworm-slim
|
||||
FROM ${NODE_IMAGE}
|
||||
ARG CODEX_VERSION=0.154.0
|
||||
WORKDIR /consumer
|
||||
COPY package.tgz /tmp/ecc-context-package.tgz
|
||||
RUN npm install --ignore-scripts --omit=dev --no-audit --no-fund --fetch-timeout=30000 --fetch-retries=1 /tmp/ecc-context-package.tgz \
|
||||
&& task_arch=$(node -p process.arch) \
|
||||
&& npm install --global --ignore-scripts --no-audit --no-fund --fetch-timeout=30000 --fetch-retries=1 \
|
||||
@openai/codex@${CODEX_VERSION} "@openai/codex-linux-${task_arch}@npm:@openai/codex@${CODEX_VERSION}-linux-${task_arch}" \
|
||||
&& codex --version
|
||||
COPY native-probe.js /consumer/node_modules/ecc-universal/docker/context-profiles/native-probe.js
|
||||
COPY native-switch-probe.js /consumer/node_modules/ecc-universal/docker/context-profiles/native-switch-probe.js
|
||||
COPY packed-smoke.js /consumer/node_modules/ecc-universal/docker/context-profiles/packed-smoke.js
|
||||
COPY context-carrier-fixture.js /consumer/node_modules/ecc-universal/tests/lib/helpers/context-carrier-fixture.js
|
||||
COPY expected-carriers.json /tmp/ecc-expected-carriers.json
|
||||
ENV ECC_EXPECTED_CARRIERS=/tmp/ecc-expected-carriers.json
|
||||
ENV PATH="/consumer/node_modules/.bin:${PATH}"
|
||||
USER node
|
||||
CMD ["node", "/consumer/node_modules/ecc-universal/docker/context-profiles/packed-smoke.js"]
|
||||
@@ -0,0 +1,69 @@
|
||||
# Context profile native and fresh install checks
|
||||
|
||||
These opt-in probes exercise real native discovery without creating a model
|
||||
thread or copying credentials. They are separate from the default unit suite.
|
||||
|
||||
```sh
|
||||
node docker/context-profiles/native-probe.js
|
||||
node docker/context-profiles/native-probe.js --claude
|
||||
node docker/context-profiles/native-switch-probe.js
|
||||
node docker/context-profiles/run-podman.js
|
||||
```
|
||||
|
||||
The first command uses the locally installed Codex executable, a new private
|
||||
temporary home for each case, a local marketplace, and the native plugin cache.
|
||||
It starts a new app-server process and calls only `initialize` and `skills/list`.
|
||||
Lean, Lean with Angular's bundled resources, and Full excluding Python patterns
|
||||
must expose exactly their selected plugin skill names. Provider-owned system
|
||||
skills are reported separately. Every installed resource is checked against its
|
||||
source digest after removing the local marketplace's carrier source.
|
||||
|
||||
The Claude command uses the locally installed Claude executable, a private
|
||||
temporary home, empty setting sources, `plugin validate`, and `plugin details`
|
||||
with an inline plugin directory. It checks exact Lean/Full-with-exclusion skill
|
||||
inventories and zero agent, hook, MCP, and LSP components. Reported token costs
|
||||
are the provider's projections, not measured usage. Manifest attribution and
|
||||
version warnings remain visible.
|
||||
|
||||
The switch probe uses the product's managed store and isolated native adapter for
|
||||
Full, Lean, and rollback to Full. Preparation creates a separate provider home
|
||||
and registers the selected carrier, then opens a fresh app-server to verify
|
||||
discovery. Rollback first restores managed authority, then re-verifies the prior
|
||||
native home and selects it. The Full Python exclusion and unrelated bytes in the
|
||||
prior home must survive every transition. Each native pointer binds its managed
|
||||
store revision, carrier digest, exact provider version, and native executable
|
||||
SHA-256. Read-only status rechecks receipts, native configuration, cached resource
|
||||
bytes, and the pinned executable. Existing sessions and host registration remain
|
||||
unchanged.
|
||||
|
||||
The Podman runner runs the normal `npm pack` lifecycle, reports its archive
|
||||
SHA-256, and builds an isolated consumer from that archive. It installs runtime
|
||||
dependencies and pinned Codex 0.154.0 during the image build. The final container
|
||||
runs as the image's unprivileged `node` user, with networking disabled, all Linux
|
||||
capabilities dropped, no added host mounts, and no copied credentials. It checks
|
||||
all ten target/profile combinations through the packed public CLI and independent
|
||||
structural oracle, including exact carrier equality with the source checkout.
|
||||
It also checks the packed CLI's Full/Lean/rollback lifecycle, idempotency, stale
|
||||
revision rejection, Auto context loading, Suggest/Manual/dry-run boundaries,
|
||||
pinned receipt reuse, and no-workflow reset. It then repeats native Codex discovery
|
||||
and product native preparation/rollback. The packed CLI also prepares a native
|
||||
generation and verifies an isolated launch dry-run with no provider on PATH.
|
||||
Test helpers are
|
||||
copied separately into the image; they are not part of the published package.
|
||||
|
||||
An existing compatible Node image can be selected with
|
||||
`ECC_CONTEXT_NODE_IMAGE=<image-id>`. The default is `node:22-bookworm-slim`.
|
||||
The task image and private temporary build directory are removed afterward.
|
||||
Dependency download layers can remain in Podman's ordinary build cache. The
|
||||
runner never changes host harness configuration or mounts a host home.
|
||||
|
||||
The outcome evaluator (`ai-eval.js`) measures graded task success and provider
|
||||
usage across install arms; see `ai-corpus.json` for the 30-task repair corpus
|
||||
and `complex-eval/DESIGN.md` for the preregistered three-task complex-task
|
||||
benchmark (feature build, incident triage, security hardening) with scored
|
||||
hidden graders, reference solutions, and reproduction instructions.
|
||||
|
||||
These checks certify the observed discovery paths for the reported exact provider
|
||||
versions. They do not certify model invocation, skill workflow outcomes,
|
||||
implicit provider invocation of Auto, host activation, crash recovery, permission consent, or actual token
|
||||
savings. CLI-provided system skills still contribute to whole-session context.
|
||||
@@ -0,0 +1,415 @@
|
||||
{
|
||||
"schemaVersion": "ecc.context-eval-corpus.v2",
|
||||
"id": "coding-tasks@1",
|
||||
"sampling": "Purposive coding-task corpus fixed before any provider call: 22 small JavaScript repairs paired with one plausibly helpful ECC skill, 8 trivial no-workflow fixes (some with misleading workflow vocabulary), and selection probes for exact names, paraphrases, no-workflow queries and policy blocks; equal weight per distinct task and no population-representativeness claim.",
|
||||
"minimumDistinctTasks": 30,
|
||||
"nonInferiorityMargin": 0.05,
|
||||
"selection": [
|
||||
{ "id":"exact-python", "category":"exact", "query":"Use python-patterns to review typed Python functions.", "expectedIds":["skill:python-patterns"] },
|
||||
{ "id":"exact-api", "category":"exact", "query":"Use api-design for REST pagination.", "expectedIds":["skill:api-design"] },
|
||||
{ "id":"paraphrase-tests", "category":"paraphrase", "query":"Write pytest fixtures and parametrized regression tests for a Python package.", "expectedIds":["skill:python-testing"] },
|
||||
{ "id":"paraphrase-api", "category":"paraphrase", "query":"Design REST endpoints with pagination and status codes.", "expectedIds":["skill:api-design"] },
|
||||
{ "id":"plain-arithmetic", "category":"no-workflow", "query":"What is 17 times 24?", "expectedIds":[] },
|
||||
{ "id":"ambiguous-vocabulary", "category":"no-workflow", "query":"Count words in this literal text: database testing security review. Do not perform any of those activities.", "expectedIds":[] },
|
||||
{ "id":"negative-skill-mention", "category":"no-workflow", "query":"Do not use python-patterns. Return only OK.", "expectedIds":[] },
|
||||
{ "id":"explicit-no-workflow", "category":"no-workflow", "query":"Use python-patterns as plain text only. Add 7 and 5.", "noWorkflow":true,"expectedIds":[] },
|
||||
{ "id":"excluded-name", "category":"policy", "query":"Use python-patterns to simplify Python code.", "exclude":["skill:python-patterns"],"expectedIds":[] },
|
||||
{ "id":"excluded-explicit", "category":"policy", "query":"Use python-patterns.", "exclude":["skill:python-patterns"],"explicitIds":["skill:python-patterns"],"expectedBlock":"excluded" },
|
||||
{ "id":"authority-boundary", "category":"policy", "query":"Use inherit-legacy-style to preserve conventions.", "explicitIds":["skill:inherit-legacy-style"],"expectedBlock":"native-authority" },
|
||||
{ "id":"opt-out-conflict", "category":"policy", "query":"Use python-patterns.", "noWorkflow":true,"explicitIds":["skill:python-patterns"],"expectedBlock":"opt-out-conflict" },
|
||||
{ "id":"unknown-explicit", "category":"policy", "query":"Use an unavailable workflow.", "explicitIds":["skill:ecc-eval-nonexistent"],"expectedBlock":"unknown-id" },
|
||||
{ "id":"exact-security-review", "category":"exact", "query":"Use security-review to check this login handler for SQL injection and leaked secrets.", "expectedIds":["skill:security-review"] },
|
||||
{ "id":"exact-error-handling", "category":"exact", "query":"Use error-handling to add typed error classes to the config loader.", "expectedIds":["skill:error-handling"] },
|
||||
{ "id":"exact-database-migrations", "category":"exact", "query":"Use database-migrations to add a NOT NULL column to a large Postgres table.", "expectedIds":["skill:database-migrations"] },
|
||||
{ "id":"exact-regex-structured-text", "category":"exact", "query":"Use regex-vs-llm-structured-text to decide how to parse vendor invoice lines.", "expectedIds":["skill:regex-vs-llm-structured-text"] },
|
||||
{ "id":"exact-content-hash-cache", "category":"exact", "query":"Use content-hash-cache-pattern to cache PDF text extraction results.", "expectedIds":["skill:content-hash-cache-pattern"] },
|
||||
{ "id":"exact-hexagonal", "category":"exact", "query":"Use hexagonal-architecture to separate the signup use case from its database and email adapters.", "expectedIds":["skill:hexagonal-architecture"] },
|
||||
{ "id":"paraphrase-sql-injection", "category":"paraphrase", "query":"User input is concatenated into SQL strings in our login endpoint; audit the handler for injection and hardcoded credentials before release.", "expectedIds":["skill:security-review"] },
|
||||
{ "id":"paraphrase-retry", "category":"paraphrase", "query":"Wrap a flaky payment provider call with exponential backoff retries and typed error classes so callers get useful failure messages.", "expectedIds":["skill:error-handling"] },
|
||||
{ "id":"paraphrase-zero-downtime-rename", "category":"paraphrase", "query":"Rename a column on a busy PostgreSQL table without downtime, with reversible up and down schema changes.", "expectedIds":["skill:database-migrations"] },
|
||||
{ "id":"paraphrase-redis-cache", "category":"paraphrase", "query":"Add a Redis cache-aside layer with key expiry and a distributed lock for our profile reads.", "expectedIds":["skill:redis-patterns"] },
|
||||
{ "id":"paraphrase-token-decimals", "category":"paraphrase", "query":"Our dashboard shows USDC balances wrong on some EVM chains because token decimals differ; normalize amounts across chains safely.", "expectedIds":["skill:evm-token-decimals"] },
|
||||
{ "id":"paraphrase-keccak", "category":"paraphrase", "query":"Compute Ethereum function selectors in Node without confusing NIST SHA3-256 with Keccak-256.", "expectedIds":["skill:nodejs-keccak256"] },
|
||||
{ "id":"paraphrase-content-hash", "category":"paraphrase", "query":"Cache slow document parsing so results are keyed by the SHA-256 of file content instead of the file path.", "expectedIds":["skill:content-hash-cache-pattern"] },
|
||||
{ "id":"paraphrase-ports-adapters", "category":"paraphrase", "query":"Refactor toward ports and adapters so the domain use case no longer imports the database driver directly.", "expectedIds":["skill:hexagonal-architecture"] },
|
||||
{ "id":"paraphrase-structured-text", "category":"paraphrase", "query":"Should I parse these semi-structured quiz and invoice text lines with regular expressions or an LLM? Start with the cheapest reliable option.", "expectedIds":["skill:regex-vs-llm-structured-text"] },
|
||||
{ "id":"rename-variable", "category":"no-workflow", "query":"Rename the local variable tmp to total in this three-line function.", "expectedIds":[] },
|
||||
{ "id":"misleading-security-typo", "category":"no-workflow", "query":"Fix the spelling of \"recieve\" in the footer text of the security settings page. Nothing else.", "expectedIds":[] },
|
||||
{ "id":"misleading-tests-heading", "category":"no-workflow", "query":"Change the README heading \"Running tests\" to \"Running checks\". Do not write or run any tests.", "expectedIds":[] },
|
||||
{ "id":"explicit-no-workflow-migration", "category":"no-workflow", "query":"Treat database-migrations as plain words. Reverse the string abc.", "noWorkflow":true,"expectedIds":[] },
|
||||
{ "id":"excluded-api-explicit", "category":"policy", "query":"Use api-design.", "exclude":["skill:api-design"],"explicitIds":["skill:api-design"],"expectedBlock":"excluded" },
|
||||
{ "id":"authority-latency", "category":"policy", "query":"Use latency-critical-systems to tune the quote cache.", "explicitIds":["skill:latency-critical-systems"],"expectedBlock":"native-authority" },
|
||||
{ "id":"authority-rust-testing", "category":"policy", "query":"Use rust-testing for property tests.", "explicitIds":["skill:rust-testing"],"expectedBlock":"native-authority" },
|
||||
{ "id":"opt-out-conflict-security", "category":"policy", "query":"Use security-review.", "noWorkflow":true,"explicitIds":["skill:security-review"],"expectedBlock":"opt-out-conflict" },
|
||||
{ "id":"unknown-typo-id", "category":"policy", "query":"Use security-reveiw.", "explicitIds":["skill:security-reveiw"],"expectedBlock":"unknown-id" },
|
||||
{ "id":"explicit-allowed", "category":"policy", "query":"Use error-handling for the retry wrapper.", "explicitIds":["skill:error-handling"],"expectedIds":["skill:error-handling"] }
|
||||
],
|
||||
"tasks": [
|
||||
{
|
||||
"id": "sql-injection-query",
|
||||
"category": "security",
|
||||
"manualIds": [
|
||||
"skill:security-review"
|
||||
],
|
||||
"query": "src/users.js builds SQL for a node-postgres style driver: each builder returns { text, values } where text uses $1, $2 placeholders. Both buildFindUserQuery(email) and buildSearchUsersQuery(nameFragment, limit) interpolate caller input into the SQL text. Fix them so no caller-supplied string is ever placed in the SQL text; pass it through values instead. The search must still match names containing the fragment case-insensitively. limit must be an integer from 1 to 100; throw a RangeError for anything else (including numeric strings). Keep both exports and the selected columns. Do not add dependencies.",
|
||||
"files": {
|
||||
"src/users.js": "'use strict';\n\n// Query builders used by the /users routes. The db layer calls\n// pool.query(q.text, q.values) with node-postgres.\nfunction buildFindUserQuery(email) {\n return {\n text: \"SELECT id, email, name FROM users WHERE email = '\" + email + \"'\",\n values: [],\n };\n}\n\nfunction buildSearchUsersQuery(nameFragment, limit) {\n return {\n text: \"SELECT id, email, name FROM users WHERE name ILIKE '%\" + nameFragment + \"%' ORDER BY name LIMIT \" + limit,\n values: [],\n };\n}\n\nmodule.exports = { buildFindUserQuery, buildSearchUsersQuery };\n",
|
||||
"README.md": "# users-api\n\nSmall query-builder layer for the users service. Builders return\n{ text, values } objects that are passed directly to node-postgres.\n"
|
||||
},
|
||||
"check": "'use strict';\nconst assert = require('node:assert/strict');\nconst path = require('node:path');\nconst m = require(path.join(process.cwd(), 'src/users.js'));\nconst evil = \"a' OR '1'='1\";\nconst q = m.buildFindUserQuery(evil);\nassert.ok(!q.text.includes(\"'1'='1\") && !q.text.includes(evil), 'email leaked into SQL text');\nassert.match(q.text, /\\$1/);\nassert.match(q.text, /FROM\\s+users/i);\nassert.ok(q.values.includes(evil));\nconst frag = \"bob'; DROP TABLE users;--\";\nconst s = m.buildSearchUsersQuery(frag, 10);\nassert.ok(!/DROP\\s+TABLE/i.test(s.text), 'fragment leaked into SQL text');\nassert.match(s.text, /ILIKE|lower\\s*\\(/i);\nassert.match(s.text, /LIMIT\\s+(\\$\\d+|10)\\b/i);\nassert.ok(s.values.some(v => typeof v === 'string' && v.includes(frag)));\nfor (const bad of [0, 101, 2.5, '10', '10; DROP TABLE users', NaN, undefined]) {\n assert.throws(() => m.buildSearchUsersQuery('x', bad), RangeError);\n}\nconst max = Math.max(0, ...[...s.text.matchAll(/\\$(\\d+)/g)].map(x => Number(x[1])));\nassert.equal(max, s.values.length, 'placeholders and values disagree');\n"
|
||||
},
|
||||
{
|
||||
"id": "path-traversal-guard",
|
||||
"category": "security",
|
||||
"manualIds": [
|
||||
"skill:security-review"
|
||||
],
|
||||
"query": "src/static.js exports resolvePublicPath(requestPath, root) used by our static file server. requestPath is the raw URL path (for example \"/css/site.css\", possibly percent-encoded). It currently joins it onto root, which allows escaping the public directory. Make it return the absolute file path when the decoded path stays inside root (root itself counts as inside), and return null (never throw) when the path escapes root, contains a NUL byte, or cannot be percent-decoded. Watch out for sibling directories that share root as a string prefix. Keep the export name and signature. Do not add dependencies.",
|
||||
"files": {
|
||||
"src/static.js": "'use strict';\nconst path = require('path');\n\nconst PUBLIC_ROOT = path.resolve(__dirname, '..', 'public');\n\n// Maps a request path such as \"/css/site.css\" to a file on disk.\nfunction resolvePublicPath(requestPath, root = PUBLIC_ROOT) {\n return path.join(root, decodeURIComponent(requestPath));\n}\n\nmodule.exports = { resolvePublicPath, PUBLIC_ROOT };\n",
|
||||
"public/index.html": "<!doctype html><title>home</title>\n"
|
||||
},
|
||||
"check": "'use strict';\nconst assert = require('node:assert/strict');\nconst path = require('node:path');\nconst { resolvePublicPath } = require(path.join(process.cwd(), 'src/static.js'));\nconst root = path.resolve(path.sep + 'srv', 'app', 'public');\nassert.equal(resolvePublicPath('/css/site.css', root), path.join(root, 'css', 'site.css'));\nassert.equal(resolvePublicPath('/css/../index.html', root), path.join(root, 'index.html'));\nassert.equal(resolvePublicPath('/a%20b.txt', root), path.join(root, 'a b.txt'));\nfor (const bad of ['/../secret.env', '/%2e%2e/%2e%2e/etc/passwd', '/css/../../x', '/../public-evil/x',\n '/a%00.txt', '/%E0%A4%A', '..%2f..%2fetc%2fpasswd']) {\n let out;\n assert.doesNotThrow(() => { out = resolvePublicPath(bad, root); }, bad);\n assert.equal(out, null, bad);\n}\n"
|
||||
},
|
||||
{
|
||||
"id": "escape-comment-html",
|
||||
"category": "security",
|
||||
"manualIds": [
|
||||
"skill:security-review"
|
||||
],
|
||||
"query": "src/render.js exports renderComment({ author, body, website }) which returns an HTML string for a user comment. All three fields are untrusted user input and are currently inserted raw. Fix it so author and body are HTML-escaped (at least & < > \" and '), and website is only used as the link href when it is an absolute http: or https: URL; otherwise the href must be \"#\". The href value must also be escaped. Keep the existing markup structure (li.comment containing an a element and a p element). Do not add dependencies.",
|
||||
"files": {
|
||||
"src/render.js": "'use strict';\n\nfunction renderComment({ author, body, website }) {\n return '<li class=\"comment\"><a href=\"' + website + '\">' + author + '</a><p>' + body + '</p></li>';\n}\n\nmodule.exports = { renderComment };\n"
|
||||
},
|
||||
"check": "'use strict';\nconst assert = require('node:assert/strict');\nconst path = require('node:path');\nconst { renderComment } = require(path.join(process.cwd(), 'src/render.js'));\nconst a = renderComment({ author: '<script>alert(1)</script>', body: 'Tom & \"Jerry\" \\'s', website: 'https://ex.com/' });\nassert.ok(a.startsWith('<li class=\"comment\">'));\nassert.ok(!a.includes('<script'));\nassert.ok(a.includes('<script>'));\nassert.ok(a.includes('&') && a.includes('"'));\nassert.ok(/�*39;|�*27;|'/i.test(a));\nassert.ok(a.includes('href=\"https://ex.com/\"'));\nfor (const w of ['javascript:alert(1)', ' JavaScript:alert(1)', 'data:text/html,x', 'vbscript:x', '//evil.com', '']) {\n const out = renderComment({ author: 'a', body: 'b', website: w });\n assert.ok(!/javascript:|data:|vbscript:/i.test(out), w);\n assert.ok(out.includes('href=\"#\"'), w);\n}\nconst q = renderComment({ author: 'a', body: 'b', website: 'https://ex.com/?a=1&b=\"x\"' });\nassert.ok(!q.includes('\"x\"'));\nassert.ok(/href=\"https:\\/\\/ex\\.com\\/\\?a=1&b=/.test(q));\nassert.ok(/<a [^>]*>a<\\/a>/.test(q) && /<p>b<\\/p>/.test(q));\n"
|
||||
},
|
||||
{
|
||||
"id": "list-pagination",
|
||||
"category": "api",
|
||||
"manualIds": [
|
||||
"skill:api-design"
|
||||
],
|
||||
"query": "src/listProducts.js exports listProducts(query, store) for GET /products. query holds raw query-string values (strings or undefined); store.all() returns the full array. Implement offset pagination: limit defaults to 20 and must be an integer 1..100, offset defaults to 0 and must be an integer >= 0. Success returns { status: 200, body: { data, meta: { total, limit, offset, hasMore } } }. Invalid values return { status: 400, body: { error: { code: \"VALIDATION_ERROR\", message, details: [{ field, message }] } } } with one details entry per invalid field (\"limit\" or \"offset\"). Do not mutate the store array. Do not add dependencies.",
|
||||
"files": {
|
||||
"src/listProducts.js": "'use strict';\n\n// GET /products?limit=&offset=\nfunction listProducts(query, store) {\n const items = store.all();\n const page = items.slice(query.offset, query.offset + query.limit);\n return { status: 200, body: page };\n}\n\nmodule.exports = { listProducts };\n",
|
||||
"src/store.js": "'use strict';\n\nfunction createStore(items) {\n return { all: () => items };\n}\n\nmodule.exports = { createStore };\n"
|
||||
},
|
||||
"check": "'use strict';\nconst assert = require('node:assert/strict');\nconst path = require('node:path');\nconst { listProducts } = require(path.join(process.cwd(), 'src/listProducts.js'));\nconst items = Array.from({ length: 45 }, (_, i) => ({ id: i + 1 }));\nconst copy = JSON.stringify(items);\nconst store = { all: () => items };\nlet r = listProducts({}, store);\nassert.equal(r.status, 200);\nassert.equal(r.body.data.length, 20);\nassert.deepEqual(r.body.meta, { total: 45, limit: 20, offset: 0, hasMore: true });\nr = listProducts({ limit: '10', offset: '40' }, store);\nassert.deepEqual(r.body.data.map(x => x.id), [41, 42, 43, 44, 45]);\nassert.deepEqual(r.body.meta, { total: 45, limit: 10, offset: 40, hasMore: false });\nr = listProducts({ limit: '5', offset: '35' }, store);\nassert.equal(r.body.meta.hasMore, true);\nr = listProducts({ limit: '100', offset: '100' }, store);\nassert.equal(r.status, 200);\nassert.deepEqual(r.body.data, []);\nassert.equal(r.body.meta.hasMore, false);\nfor (const [q, fields] of [[{ limit: '0' }, ['limit']], [{ limit: '101' }, ['limit']], [{ limit: 'abc' }, ['limit']],\n [{ limit: '2.5' }, ['limit']], [{ offset: '-1' }, ['offset']], [{ limit: '-3', offset: 'x' }, ['limit', 'offset']]]) {\n const bad = listProducts(q, store);\n assert.equal(bad.status, 400, JSON.stringify(q));\n assert.equal(bad.body.error.code, 'VALIDATION_ERROR');\n assert.equal(typeof bad.body.error.message, 'string');\n assert.deepEqual(bad.body.error.details.map(d => d.field).sort(), fields);\n assert.ok(bad.body.error.details.every(d => typeof d.message === 'string'));\n}\nassert.equal(JSON.stringify(items), copy);\n"
|
||||
},
|
||||
{
|
||||
"id": "create-user-status-codes",
|
||||
"category": "api",
|
||||
"manualIds": [
|
||||
"skill:api-design"
|
||||
],
|
||||
"query": "src/usersRoute.js exports async createUser(req, repo) for POST /users and async getUser(req, repo) for GET /users/:id. Both return { status, headers?, body }. They currently return 200 for everything and 500 on duplicates. Fix them to use proper REST semantics. createUser: body { email, name }; email must be a string containing \"@\" and name a non-empty trimmed string, otherwise 400 with body { error: { code: \"VALIDATION_ERROR\", message, details: [{ field, message }] } } listing each bad field; if repo.findByEmail(email) returns a user, 409 with error code \"CONFLICT\"; otherwise call repo.create({ email, name }) and return 201 with headers { Location: \"/users/<id>\" } and body { data: user }. getUser: req.params.id; missing user gives 404 with error code \"NOT_FOUND\", found user gives 200 { data: user }. Do not add dependencies.",
|
||||
"files": {
|
||||
"src/usersRoute.js": "'use strict';\n\nasync function createUser(req, repo) {\n try {\n const { email, name } = req.body || {};\n const existing = await repo.findByEmail(email);\n if (existing) throw new Error('duplicate');\n const user = await repo.create({ email, name });\n return { status: 200, body: user };\n } catch (err) {\n return { status: 500, body: { message: err.message } };\n }\n}\n\nasync function getUser(req, repo) {\n const user = await repo.findById(req.params.id);\n return { status: 200, body: user };\n}\n\nmodule.exports = { createUser, getUser };\n"
|
||||
},
|
||||
"check": "'use strict';\nconst assert = require('node:assert/strict');\nconst path = require('node:path');\nconst { createUser, getUser } = require(path.join(process.cwd(), 'src/usersRoute.js'));\nfunction repo() {\n const users = [{ id: 1, email: 'ada@example.com', name: 'Ada' }];\n return { created: 0, async findByEmail(e) { return users.find(u => u.email === e) || null; },\n async findById(id) { return users.find(u => String(u.id) === String(id)) || null; },\n async create(u) { this.created++; const user = { id: users.length + 1, ...u }; users.push(user); return user; } };\n}\n(async () => {\n const r = repo();\n let res = await createUser({ body: { email: 'lin@example.com', name: 'Lin' } }, r);\n assert.equal(res.status, 201);\n assert.equal(res.headers.Location, '/users/2');\n assert.deepEqual(res.body.data, { id: 2, email: 'lin@example.com', name: 'Lin' });\n res = await createUser({ body: { email: 'ada@example.com', name: 'Ada2' } }, r);\n assert.equal(res.status, 409);\n assert.equal(res.body.error.code, 'CONFLICT');\n res = await createUser({ body: { email: 'nope', name: ' ' } }, r);\n assert.equal(res.status, 400);\n assert.equal(res.body.error.code, 'VALIDATION_ERROR');\n assert.deepEqual(res.body.error.details.map(d => d.field).sort(), ['email', 'name']);\n res = await createUser({ body: { email: 'x@y.z' } }, r);\n assert.equal(res.status, 400);\n assert.deepEqual(res.body.error.details.map(d => d.field), ['name']);\n assert.equal(r.created, 1);\n res = await getUser({ params: { id: '99' } }, r);\n assert.equal(res.status, 404);\n assert.equal(res.body.error.code, 'NOT_FOUND');\n res = await getUser({ params: { id: '1' } }, r);\n assert.equal(res.status, 200);\n assert.equal(res.body.data.email, 'ada@example.com');\n})().catch(err => { console.error(err); process.exitCode = 1; });\n"
|
||||
},
|
||||
{
|
||||
"id": "retry-with-backoff",
|
||||
"category": "errors",
|
||||
"manualIds": [
|
||||
"skill:error-handling"
|
||||
],
|
||||
"query": "src/retry.js exports async withRetry(fn, options) used around calls to a flaky payments API. It currently retries every error immediately and throws a generic Error(\"failed\"), losing the cause. Rewrite it: options are { retries = 3, baseDelayMs = 100, maxDelayMs = 2000, sleep } where sleep(ms) returns a promise (default: a real setTimeout sleep). Call fn(attempt) with attempt starting at 1, for at most retries + 1 attempts. Only retry when the error is retryable: err.retryable === true, or err.status is 429 or >= 500. Non-retryable errors must be rethrown immediately (the same error object). Before retry n (n = 1, 2, ...) await sleep(d) where d is between half and all of min(baseDelayMs * 2^(n-1), maxDelayMs) (jitter optional). When retries are exhausted, rethrow the last error object. Return fn's resolved value on success. Do not add dependencies.",
|
||||
"files": {
|
||||
"src/retry.js": "'use strict';\n\nasync function withRetry(fn, options = {}) {\n const retries = options.retries || 3;\n for (let i = 0; i < retries; i++) {\n try {\n return await fn(i);\n } catch (err) {\n // try again\n }\n }\n throw new Error('failed');\n}\n\nmodule.exports = { withRetry };\n"
|
||||
},
|
||||
"check": "'use strict';\nconst assert = require('node:assert/strict');\nconst path = require('node:path');\nconst { withRetry } = require(path.join(process.cwd(), 'src/retry.js'));\nconst mk = (status, extra = {}) => Object.assign(new Error('e' + status), { status }, extra);\n(async () => {\n let delays = [];\n const sleep = ms => { delays.push(ms); return Promise.resolve(); };\n let calls = [];\n const out = await withRetry(async a => { calls.push(a); if (a < 3) throw mk(503); return 'ok'; }, { sleep });\n assert.equal(out, 'ok');\n assert.deepEqual(calls, [1, 2, 3]);\n assert.equal(delays.length, 2);\n assert.ok(delays[0] >= 50 && delays[0] <= 100 && delays[1] >= 100 && delays[1] <= 200, String(delays));\n delays = []; calls = [];\n const last = mk(500);\n let n = 0;\n await assert.rejects(withRetry(async a => { calls.push(a); n++; throw n === 5 ? last : mk(502); },\n { retries: 4, baseDelayMs: 1000, maxDelayMs: 3000, sleep }), e => e === last);\n assert.deepEqual(calls, [1, 2, 3, 4, 5]);\n const caps = [1000, 2000, 3000, 3000];\n assert.equal(delays.length, 4);\n delays.forEach((d, i) => assert.ok(d >= caps[i] / 2 && d <= caps[i], 'delay ' + i + '=' + d));\n delays = []; calls = [];\n const bad = mk(400);\n await assert.rejects(withRetry(async a => { calls.push(a); throw bad; }, { sleep }), e => e === bad);\n assert.deepEqual(calls, [1]);\n assert.equal(delays.length, 0);\n calls = [];\n const plain = new Error('boom');\n await assert.rejects(withRetry(async a => { calls.push(a); throw plain; }, { sleep }), e => e === plain);\n assert.equal(calls.length, 1);\n calls = [];\n await withRetry(async a => { calls.push(a); if (a === 1) throw mk(429); if (a === 2) throw Object.assign(new Error('r'), { retryable: true }); return 1; }, { sleep });\n assert.deepEqual(calls, [1, 2, 3]);\n calls = [];\n await assert.rejects(withRetry(async a => { calls.push(a); throw mk(503); }, { retries: 0, sleep }));\n assert.deepEqual(calls, [1]);\n})().catch(err => { console.error(err); process.exitCode = 1; });\n"
|
||||
},
|
||||
{
|
||||
"id": "typed-config-errors",
|
||||
"category": "errors",
|
||||
"manualIds": [
|
||||
"skill:error-handling"
|
||||
],
|
||||
"query": "src/config.js exports loadConfig(text), which parses a JSON config string. Today it silently returns {} on bad JSON and accepts missing fields. Add and export a ConfigError class (extends Error, name \"ConfigError\") with a code property, and make loadConfig throw it: code \"CONFIG_PARSE\" for invalid JSON (with the original SyntaxError as error.cause); code \"CONFIG_MISSING\" with error.field set when a required field is missing (required: apiUrl, then timeoutMs, checked in that order); code \"CONFIG_INVALID\" with error.field = \"timeoutMs\" when timeoutMs is not a positive integer. On success return { apiUrl, timeoutMs, retries } where retries defaults to 2. Messages should be human readable. Do not add dependencies.",
|
||||
"files": {
|
||||
"src/config.js": "'use strict';\n\nfunction loadConfig(text) {\n let raw;\n try {\n raw = JSON.parse(text);\n } catch (e) {\n return {};\n }\n return { apiUrl: raw.apiUrl, timeoutMs: raw.timeoutMs, retries: raw.retries };\n}\n\nmodule.exports = { loadConfig };\n"
|
||||
},
|
||||
"check": "'use strict';\nconst assert = require('node:assert/strict');\nconst path = require('node:path');\nconst { loadConfig, ConfigError } = require(path.join(process.cwd(), 'src/config.js'));\nassert.equal(typeof ConfigError, 'function');\nassert.deepEqual(loadConfig('{\"apiUrl\":\"https://x\",\"timeoutMs\":500}'), { apiUrl: 'https://x', timeoutMs: 500, retries: 2 });\nassert.deepEqual(loadConfig('{\"apiUrl\":\"https://x\",\"timeoutMs\":5,\"retries\":0}'), { apiUrl: 'https://x', timeoutMs: 5, retries: 0 });\nfunction thrown(text) { try { loadConfig(text); } catch (e) { return e; } assert.fail('expected throw for ' + text); }\nlet e = thrown('{bad json');\nassert.ok(e instanceof ConfigError && e instanceof Error);\nassert.equal(e.name, 'ConfigError');\nassert.equal(e.code, 'CONFIG_PARSE');\nassert.ok(e.cause instanceof SyntaxError);\nassert.ok(e.message.length > 0);\ne = thrown('{\"timeoutMs\":1}');\nassert.equal(e.code, 'CONFIG_MISSING');\nassert.equal(e.field, 'apiUrl');\ne = thrown('{\"apiUrl\":\"u\"}');\nassert.equal(e.code, 'CONFIG_MISSING');\nassert.equal(e.field, 'timeoutMs');\nfor (const t of ['0', '-5', '1.5', '\"100\"']) {\n e = thrown('{\"apiUrl\":\"u\",\"timeoutMs\":' + t + '}');\n assert.ok(e instanceof ConfigError);\n assert.equal(e.code, 'CONFIG_INVALID');\n assert.equal(e.field, 'timeoutMs');\n}\n"
|
||||
},
|
||||
{
|
||||
"id": "batch-partial-failures",
|
||||
"category": "errors",
|
||||
"manualIds": [
|
||||
"skill:error-handling"
|
||||
],
|
||||
"query": "src/batch.js exports async processAll(items, worker). items are objects with an id; worker(item) returns a promise. The current version swallows errors inside an empty catch and returns only a count, so failed webhook deliveries vanish. Change it to process every item (a failure must not stop the others) and resolve to { succeeded: [{ id, result }], failed: [{ id, error }] }, both in input order, where error is the thrown error's message (or String(value) if a non-Error was thrown). It must never reject because of a worker failure, and a worker that throws synchronously must be treated like a rejection. Do not add dependencies.",
|
||||
"files": {
|
||||
"src/batch.js": "'use strict';\n\nasync function processAll(items, worker) {\n let done = 0;\n for (const item of items) {\n try {\n await worker(item);\n done++;\n } catch (e) {}\n }\n return done;\n}\n\nmodule.exports = { processAll };\n"
|
||||
},
|
||||
"check": "'use strict';\nconst assert = require('node:assert/strict');\nconst path = require('node:path');\nconst { processAll } = require(path.join(process.cwd(), 'src/batch.js'));\n(async () => {\n const seen = [];\n const items = [1, 2, 3, 4, 5].map(id => ({ id }));\n const out = await processAll(items, item => {\n seen.push(item.id);\n if (item.id === 2) throw new Error('sync boom');\n if (item.id === 4) return Promise.reject('plain string');\n if (item.id === 5) return Promise.reject(new TypeError('bad payload'));\n return Promise.resolve(item.id * 10);\n });\n assert.deepEqual(seen.slice().sort(), [1, 2, 3, 4, 5]);\n assert.deepEqual(out.succeeded, [{ id: 1, result: 10 }, { id: 3, result: 30 }]);\n assert.deepEqual(out.failed, [{ id: 2, error: 'sync boom' }, { id: 4, error: 'plain string' }, { id: 5, error: 'bad payload' }]);\n assert.deepEqual(await processAll([], () => 1), { succeeded: [], failed: [] });\n})().catch(err => { console.error(err); process.exitCode = 1; });\n"
|
||||
},
|
||||
{
|
||||
"id": "access-log-parser",
|
||||
"category": "parsing",
|
||||
"manualIds": [
|
||||
"skill:regex-vs-llm-structured-text"
|
||||
],
|
||||
"query": "src/parseLog.js parses web server access logs in Common Log Format, optionally extended to Combined Log Format with a quoted referrer and a quoted user agent. The current parseLine(line) splits on spaces and breaks on user agents and timestamps that contain spaces. Rewrite parseLine(line) to return { ip, user, time, method, path, protocol, status, bytes, referrer, userAgent } or null for any line that does not match the format. user, referrer and userAgent are null when the field is \"-\" or absent; time is the text inside the square brackets; status is a number (three digits); bytes is a number and \"-\" means 0. Also export parseLog(text) returning { entries, invalid } where blank lines (LF or CRLF endings) are skipped and invalid counts non-matching lines. See README.md for examples. Do not add dependencies.",
|
||||
"files": {
|
||||
"src/parseLog.js": "'use strict';\n\nfunction parseLine(line) {\n const parts = line.split(' ');\n return {\n ip: parts[0],\n user: parts[2],\n time: parts[3],\n method: parts[5],\n path: parts[6],\n protocol: parts[7],\n status: Number(parts[8]),\n bytes: Number(parts[9]),\n };\n}\n\nmodule.exports = { parseLine };\n",
|
||||
"README.md": "# log-stats\n\nAccess log examples we must support:\n\n 127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] \"GET /apache_pb.gif HTTP/1.0\" 200 2326 \"http://www.example.com/start.html\" \"Mozilla/4.08 [en] (Win98; I ;Nav)\"\n 10.0.0.2 - - [11/Oct/2000:08:00:01 +0000] \"POST /api/login HTTP/1.1\" 401 -\n\nThe first is Combined Log Format, the second plain Common Log Format.\n"
|
||||
},
|
||||
"check": "'use strict';\nconst assert = require('node:assert/strict');\nconst path = require('node:path');\nconst { parseLine, parseLog } = require(path.join(process.cwd(), 'src/parseLog.js'));\nconst a = '127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] \"GET /apache_pb.gif HTTP/1.0\" 200 2326 \"http://www.example.com/start.html\" \"Mozilla/4.08 [en] (Win98; I ;Nav)\"';\nassert.deepEqual(parseLine(a), { ip: '127.0.0.1', user: 'frank', time: '10/Oct/2000:13:55:36 -0700', method: 'GET',\n path: '/apache_pb.gif', protocol: 'HTTP/1.0', status: 200, bytes: 2326,\n referrer: 'http://www.example.com/start.html', userAgent: 'Mozilla/4.08 [en] (Win98; I ;Nav)' });\nconst b = '10.0.0.2 - - [11/Oct/2000:08:00:01 +0000] \"POST /api/login HTTP/1.1\" 401 -';\nassert.deepEqual(parseLine(b), { ip: '10.0.0.2', user: null, time: '11/Oct/2000:08:00:01 +0000', method: 'POST',\n path: '/api/login', protocol: 'HTTP/1.1', status: 401, bytes: 0, referrer: null, userAgent: null });\nconst c = '::1 - - [01/Jan/2024:00:00:00 +0000] \"DELETE /items/9?force=1 HTTP/2.0\" 204 0 \"-\" \"curl/8.4.0\"';\nconst pc = parseLine(c);\nassert.equal(pc.ip, '::1');\nassert.equal(pc.path, '/items/9?force=1');\nassert.equal(pc.referrer, null);\nassert.equal(pc.userAgent, 'curl/8.4.0');\nassert.equal(pc.status, 204);\nfor (const bad of ['garbage line', '', '10.0.0.2 - - 11/Oct/2000:08:00:01 +0000 \"GET / HTTP/1.1\" 200 5',\n '10.0.0.2 - - [11/Oct/2000:08:00:01 +0000] \"GET / HTTP/1.1\" 2000 5', '10.0.0.2 - - [x] \"GET / HTTP/1.1\" 200 abc',\n '\"GET / HTTP/1.1\" 200 12']) {\n assert.equal(parseLine(bad), null, bad);\n}\nconst log = [a, '', 'nonsense', b + '\\r', ' ', c, ''].join('\\n');\nconst out = parseLog(log);\nassert.equal(out.entries.length, 3);\nassert.equal(out.invalid, 1);\nassert.equal(out.entries[1].bytes, 0);\n"
|
||||
},
|
||||
{
|
||||
"id": "invoice-field-extraction",
|
||||
"category": "parsing",
|
||||
"manualIds": [
|
||||
"skill:regex-vs-llm-structured-text"
|
||||
],
|
||||
"query": "src/extract.js exports extractInvoice(text), which pulls fields out of plain-text invoices from several vendors. It only handles one vendor today. Make it return { invoiceNumber, date, total, currency } for all layouts documented in FORMATS.md: invoiceNumber is the identifier string; date is normalized to YYYY-MM-DD; total is a number (thousands separators removed) taken from the grand total line, never from Subtotal or Tax lines; currency is a three-letter code (\"$\" means USD). Any field that cannot be found is null. Labels are case-insensitive. Keep it deterministic and offline. Do not add dependencies.",
|
||||
"files": {
|
||||
"src/extract.js": "'use strict';\n\nfunction extractInvoice(text) {\n const num = /Invoice #: (\\S+)/.exec(text);\n const date = /Date: (\\d{4}-\\d{2}-\\d{2})/.exec(text);\n const total = /Total: \\$([\\d.]+)/.exec(text);\n return {\n invoiceNumber: num ? num[1] : null,\n date: date ? date[1] : null,\n total: total ? Number(total[1]) : null,\n currency: total ? 'USD' : null,\n };\n}\n\nmodule.exports = { extractInvoice };\n",
|
||||
"FORMATS.md": "# Invoice layouts\n\nInvoice number labels: \"Invoice #:\", \"Invoice No.\", \"Invoice Number:\".\nIdentifiers use letters, digits and hyphens, for example INV-2024-0042, INV-7, A-19.\n\nDate labels: \"Date:\", \"Invoice Date:\", \"Issued:\". Values appear as\n2024-03-05 (ISO), 05/03/2024 (DD/MM/YYYY, day first) or 7 November 2023\n(day, full English month name, year).\n\nGrand total labels: \"Total:\", \"Total due:\", \"Amount due:\". Amounts look like\n$1,234.50 or EUR 99.00 (code before) or 1,000.00 GBP (code after).\nInvoices may also contain \"Subtotal:\" and \"Tax:\" lines, which are not totals.\n"
|
||||
},
|
||||
"check": "'use strict';\nconst assert = require('node:assert/strict');\nconst path = require('node:path');\nconst { extractInvoice } = require(path.join(process.cwd(), 'src/extract.js'));\nassert.deepEqual(extractInvoice(['ACME Corp', 'Invoice #: INV-2024-0042', 'Date: 2024-03-05', 'Subtotal: $1,100.00',\n 'Tax: $134.50', 'Total: $1,234.50'].join('\\n')), { invoiceNumber: 'INV-2024-0042', date: '2024-03-05', total: 1234.5, currency: 'USD' });\nassert.deepEqual(extractInvoice(['Globex GmbH', 'invoice no. INV-7', 'Invoice Date: 05/03/2024', 'Subtotal: EUR 90.00',\n 'TOTAL DUE: EUR 99.00'].join('\\r\\n')), { invoiceNumber: 'INV-7', date: '2024-03-05', total: 99, currency: 'EUR' });\nassert.deepEqual(extractInvoice(['Initech Ltd', 'Invoice Number: A-19', 'Issued: 7 November 2023', 'Tax: 0.00 GBP',\n 'Amount due: 1,000.00 GBP'].join('\\n')), { invoiceNumber: 'A-19', date: '2023-11-07', total: 1000, currency: 'GBP' });\nassert.deepEqual(extractInvoice('Thanks for your business!'), { invoiceNumber: null, date: null, total: null, currency: null });\nconst partial = extractInvoice('Invoice #: Z-1\\nSubtotal: $5.00');\nassert.equal(partial.invoiceNumber, 'Z-1');\nassert.equal(partial.total, null);\nassert.equal(partial.date, null);\n"
|
||||
},
|
||||
{
|
||||
"id": "add-column-migration",
|
||||
"category": "database",
|
||||
"manualIds": [
|
||||
"skill:database-migrations"
|
||||
],
|
||||
"query": "This repo keeps PostgreSQL migrations in migrations/ as NNN_name.up.sql plus NNN_name.down.sql (see README.md). Add migration 002 (one .up.sql and one .down.sql with the same NNN_name stem) that adds users.email_verified as a boolean that is NOT NULL with default false, and a unique index named users_email_lower_key on lower(email). The users table is large and takes writes constantly, so the index must be built without blocking writes, and the runner does not wrap files in a transaction. The down migration must fully reverse 002 and nothing else. Do not modify migration 001. Do not add dependencies.",
|
||||
"files": {
|
||||
"migrations/001_create_users.up.sql": "CREATE TABLE users (\n id bigserial PRIMARY KEY,\n email text NOT NULL,\n name text NOT NULL,\n created_at timestamptz NOT NULL DEFAULT now()\n);\n",
|
||||
"migrations/001_create_users.down.sql": "DROP TABLE users;\n",
|
||||
"README.md": "# accounts-db\n\nPostgreSQL 15. Migrations live in migrations/ and are applied in filename order.\nEach migration is a pair: NNN_name.up.sql and NNN_name.down.sql.\nThe runner sends each file as-is (no implicit BEGIN/COMMIT).\nProduction: users has about 40 million rows and receives writes all day.\n"
|
||||
},
|
||||
"check": "'use strict';\nconst assert = require('node:assert/strict');\nconst fs = require('node:fs');\nconst path = require('node:path');\nconst dir = path.join(process.cwd(), 'migrations');\nconst names = fs.readdirSync(dir);\nconst ups = names.filter(n => /^002_[A-Za-z0-9_-]+\\.up\\.sql$/.test(n));\nassert.equal(ups.length, 1, 'expected one 002 up migration');\nconst stem = ups[0].slice(0, -'.up.sql'.length);\nassert.ok(names.includes(stem + '.down.sql'), 'matching down migration missing');\nconst strip = s => s.replace(/--[^\\n]*/g, '').replace(/\\/\\*[\\s\\S]*?\\*\\//g, '');\nconst up = strip(fs.readFileSync(path.join(dir, ups[0]), 'utf8'));\nconst down = strip(fs.readFileSync(path.join(dir, stem + '.down.sql'), 'utf8'));\nconst add = /ALTER\\s+TABLE\\s+(?:IF\\s+EXISTS\\s+)?(?:ONLY\\s+)?\"?users\"?\\s+ADD\\s+(?:COLUMN\\s+)?(?:IF\\s+NOT\\s+EXISTS\\s+)?\"?email_verified\"?\\s+(?:boolean|bool)\\b([^;]*)/i.exec(up);\nassert.ok(add, 'ADD COLUMN email_verified boolean missing');\nconst col = '\"?email_verified\"?';\nassert.ok(/NOT\\s+NULL/i.test(add[1]) || new RegExp('ALTER\\\\s+COLUMN\\\\s+' + col + '\\\\s+SET\\\\s+NOT\\\\s+NULL', 'i').test(up), 'NOT NULL missing');\nassert.ok(/DEFAULT\\s+(?:false|'f'|'false')/i.test(add[1]) || new RegExp('ALTER\\\\s+COLUMN\\\\s+' + col + '\\\\s+SET\\\\s+DEFAULT\\\\s+false', 'i').test(up), 'DEFAULT false missing');\nassert.match(up, /CREATE\\s+UNIQUE\\s+INDEX\\s+CONCURRENTLY\\s+(?:IF\\s+NOT\\s+EXISTS\\s+)?\"?users_email_lower_key\"?\\s+ON\\s+(?:ONLY\\s+)?\"?users\"?\\s*(?:USING\\s+btree\\s*)?\\(\\s*lower\\s*\\(\\s*\"?email\"?\\s*\\)\\s*\\)/i);\nconst idx = up.search(/CREATE\\s+UNIQUE\\s+INDEX\\s+CONCURRENTLY/i);\nconst opened = [...up.slice(0, idx).matchAll(/\\b(BEGIN|START\\s+TRANSACTION|COMMIT|END|ROLLBACK)\\b\\s*;/gi)].map(x => x[1].toUpperCase());\nassert.ok(!opened.length || !/^(BEGIN|START)/.test(opened[opened.length - 1]), 'concurrent index inside a transaction');\nassert.doesNotMatch(up, /DROP\\s+(?:COLUMN|TABLE|INDEX)/i);\nassert.match(down, /DROP\\s+INDEX\\s+(?:CONCURRENTLY\\s+)?(?:IF\\s+EXISTS\\s+)?\"?users_email_lower_key\"?/i);\nassert.match(down, /ALTER\\s+TABLE\\s+(?:IF\\s+EXISTS\\s+)?\"?users\"?\\s+DROP\\s+(?:COLUMN\\s+)?(?:IF\\s+EXISTS\\s+)?\"?email_verified\"?/i);\nassert.doesNotMatch(down, /DROP\\s+TABLE/i);\nconst original = \"CREATE TABLE users (\\n id bigserial PRIMARY KEY,\\n email text NOT NULL,\\n name text NOT NULL,\\n created_at timestamptz NOT NULL DEFAULT now()\\n);\\n\";\nassert.equal(fs.readFileSync(path.join(dir, '001_create_users.up.sql'), 'utf8'), original);\nassert.equal(fs.readFileSync(path.join(dir, '001_create_users.down.sql'), 'utf8'), 'DROP TABLE users;\\n');\n"
|
||||
},
|
||||
{
|
||||
"id": "rename-column-expand",
|
||||
"category": "database",
|
||||
"manualIds": [
|
||||
"skill:database-migrations"
|
||||
],
|
||||
"query": "We want PostgreSQL column customers.full_name renamed to display_name, but old app instances keep reading and writing full_name for hours during the rolling deploy (see README.md). Do only the zero-downtime expand step. 1) Add migrations/002_<name>.up.sql and matching .down.sql: the up adds a nullable display_name text column and backfills it from full_name; it must not rename or drop full_name. The down removes display_name only. 2) Update src/customerRepo.js: buildInsert(customer) and buildUpdateName(id, name) must write the name to both full_name and display_name (still parameterized { text, values } with $n placeholders), and mapRow(row) must return name from display_name, falling back to full_name when display_name is null. Keep all exports. Do not add dependencies.",
|
||||
"files": {
|
||||
"migrations/001_create_customers.up.sql": "CREATE TABLE customers (\n id bigserial PRIMARY KEY,\n email text NOT NULL,\n full_name text NOT NULL\n);\n",
|
||||
"migrations/001_create_customers.down.sql": "DROP TABLE customers;\n",
|
||||
"src/customerRepo.js": "'use strict';\n\nfunction buildInsert(customer) {\n return { text: 'INSERT INTO customers (email, full_name) VALUES ($1, $2) RETURNING id', values: [customer.email, customer.name] };\n}\n\nfunction buildUpdateName(id, name) {\n return { text: 'UPDATE customers SET full_name = $1 WHERE id = $2', values: [name, id] };\n}\n\nfunction mapRow(row) {\n return { id: row.id, email: row.email, name: row.full_name };\n}\n\nmodule.exports = { buildInsert, buildUpdateName, mapRow };\n",
|
||||
"README.md": "# customers-service\n\nPostgreSQL 15. Migrations: migrations/NNN_name.up.sql and NNN_name.down.sql.\nDeploys are rolling: the previous app version keeps serving traffic (reading\nand writing full_name) until every instance is replaced.\n"
|
||||
},
|
||||
"check": "'use strict';\nconst assert = require('node:assert/strict');\nconst fs = require('node:fs');\nconst path = require('node:path');\nconst dir = path.join(process.cwd(), 'migrations');\nconst names = fs.readdirSync(dir);\nconst ups = names.filter(n => /^002_[A-Za-z0-9_-]+\\.up\\.sql$/.test(n));\nassert.equal(ups.length, 1);\nconst stem = ups[0].slice(0, -'.up.sql'.length);\nconst strip = s => s.replace(/--[^\\n]*/g, '').replace(/\\/\\*[\\s\\S]*?\\*\\//g, '');\nconst up = strip(fs.readFileSync(path.join(dir, ups[0]), 'utf8'));\nconst down = strip(fs.readFileSync(path.join(dir, stem + '.down.sql'), 'utf8'));\nconst add = /ALTER\\s+TABLE\\s+(?:IF\\s+EXISTS\\s+)?\"?customers\"?\\s+ADD\\s+(?:COLUMN\\s+)?(?:IF\\s+NOT\\s+EXISTS\\s+)?\"?display_name\"?\\s+(?:text|varchar|character\\s+varying)\\b([^;]*)/i.exec(up);\nassert.ok(add, 'ADD COLUMN display_name missing');\nassert.doesNotMatch(add[1], /NOT\\s+NULL/i);\nassert.match(up, /UPDATE\\s+\"?customers\"?\\s+SET\\s+\"?display_name\"?\\s*=\\s*\"?full_name\"?/i);\nassert.doesNotMatch(up, /RENAME\\s+(?:COLUMN\\s+)?\"?full_name/i);\nassert.doesNotMatch(up, /DROP\\s+(?:COLUMN|TABLE)|DROP\\s+\"?full_name/i);\nassert.match(down, /DROP\\s+(?:COLUMN\\s+)?(?:IF\\s+EXISTS\\s+)?\"?display_name\"?/i);\nassert.doesNotMatch(down, /full_name|DROP\\s+TABLE/i);\nconst repo = require(path.join(process.cwd(), 'src/customerRepo.js'));\nconst maxParam = t => Math.max(0, ...[...t.matchAll(/\\$(\\d+)/g)].map(x => Number(x[1])));\nconst ins = repo.buildInsert({ email: 'a@x.io', name: \"O'Hara\" });\nassert.match(ins.text, /INSERT\\s+INTO\\s+\"?customers\"?/i);\nassert.match(ins.text, /full_name/);\nassert.match(ins.text, /display_name/);\nassert.ok(!ins.text.includes(\"O'Hara\"));\nassert.ok(ins.values.includes(\"O'Hara\") && ins.values.includes('a@x.io'));\nassert.equal(maxParam(ins.text), ins.values.length);\nconst upd = repo.buildUpdateName(7, 'Bo');\nassert.match(upd.text, /UPDATE\\s+\"?customers\"?\\s+SET/i);\nassert.match(upd.text, /full_name\\s*=\\s*\\$\\d+/);\nassert.match(upd.text, /display_name\\s*=\\s*\\$\\d+/);\nassert.match(upd.text, /WHERE\\s+\"?id\"?\\s*=\\s*\\$\\d+/i);\nassert.ok(upd.values.includes('Bo') && upd.values.includes(7));\nassert.equal(maxParam(upd.text), upd.values.length);\nassert.equal(repo.mapRow({ id: 1, email: 'e', full_name: 'Old', display_name: null }).name, 'Old');\nassert.equal(repo.mapRow({ id: 1, email: 'e', full_name: 'Old' }).name, 'Old');\nassert.equal(repo.mapRow({ id: 1, email: 'e', full_name: 'Old', display_name: 'New' }).name, 'New');\nassert.equal(repo.mapRow({ id: 2, email: 'e', full_name: 'Old', display_name: 'New' }).id, 2);\n"
|
||||
},
|
||||
{
|
||||
"id": "keyset-feed-query",
|
||||
"category": "database",
|
||||
"manualIds": [
|
||||
"skill:postgres-patterns"
|
||||
],
|
||||
"query": "src/feedQuery.js builds the PostgreSQL query for a user's post feed using OFFSET, which gets slow and skips rows on deep pages. Switch to keyset (cursor) pagination ordered by created_at DESC, id DESC. Export encodeCursor(row) (row has created_at as an ISO string and id) returning an opaque string, and buildFeedQuery({ userId, limit, cursor }) returning { text, values } for node-postgres ($n placeholders; no caller value inlined into text). cursor is undefined for the first page; otherwise it comes from encodeCursor and the query must return only rows strictly after that row in the sort order. Throw an Error for a malformed cursor and a RangeError unless limit is an integer 1..50. Also add migrations/002_<name>.sql creating a composite index on posts that supports this query (single-file migrations, see 001). Do not add dependencies.",
|
||||
"files": {
|
||||
"src/feedQuery.js": "'use strict';\n\n// page is 0-based\nfunction buildFeedQuery({ userId, limit, page = 0 }) {\n return {\n text: 'SELECT id, user_id, body, created_at FROM posts WHERE user_id = $1 ORDER BY created_at DESC LIMIT $2 OFFSET $3',\n values: [userId, limit, page * limit],\n };\n}\n\nmodule.exports = { buildFeedQuery };\n",
|
||||
"migrations/001_create_posts.sql": "CREATE TABLE posts (\n id bigserial PRIMARY KEY,\n user_id bigint NOT NULL,\n body text NOT NULL,\n created_at timestamptz NOT NULL DEFAULT now()\n);\n"
|
||||
},
|
||||
"check": "'use strict';\nconst assert = require('node:assert/strict');\nconst fs = require('node:fs');\nconst path = require('node:path');\nconst { buildFeedQuery, encodeCursor } = require(path.join(process.cwd(), 'src/feedQuery.js'));\nconst maxParam = t => Math.max(0, ...[...t.matchAll(/\\$(\\d+)/g)].map(x => Number(x[1])));\nconst ORDER = /ORDER\\s+BY\\s+\"?created_at\"?\\s+DESC\\s*,\\s*\"?id\"?\\s+DESC/i;\nconst first = buildFeedQuery({ userId: 7, limit: 20 });\nassert.doesNotMatch(first.text, /OFFSET/i);\nassert.match(first.text, ORDER);\nassert.match(first.text, /user_id\\s*=\\s*\\$\\d+/i);\nassert.ok(first.values.includes(7));\nassert.match(first.text, /LIMIT\\s+(\\$\\d+|20)\\b/i);\nassert.equal(maxParam(first.text), first.values.length);\nconst cur = encodeCursor({ id: 42, user_id: 7, body: 'hi', created_at: '2024-05-01T10:00:00.000Z' });\nassert.equal(typeof cur, 'string');\nconst next = buildFeedQuery({ userId: 7, limit: 20, cursor: cur });\nassert.doesNotMatch(next.text, /OFFSET/i);\nassert.match(next.text, ORDER);\nassert.ok(!next.text.includes('2024-05-01') && !/\\b42\\b/.test(next.text));\nconst row = /\\(\\s*\"?created_at\"?\\s*,\\s*\"?id\"?\\s*\\)\\s*<\\s*\\(\\s*\\$(\\d+)(?:::\\w+)?\\s*,\\s*\\$(\\d+)(?:::\\w+)?\\s*\\)/i.exec(next.text);\nconst expanded = /\"?created_at\"?\\s*<\\s*\\$(\\d+)[\\s\\S]*\"?created_at\"?\\s*=\\s*\\$(\\d+)[\\s\\S]*\"?id\"?\\s*<\\s*\\$(\\d+)/i.exec(next.text);\nassert.ok(row || expanded, 'keyset predicate missing: ' + next.text);\nconst vals = next.values.map(v => (v instanceof Date ? v.toISOString() : String(v)));\nassert.ok(vals.includes('2024-05-01T10:00:00.000Z'));\nassert.ok(vals.includes('42'));\nassert.ok(next.values.includes(7));\nassert.equal(maxParam(next.text), next.values.length);\nassert.throws(() => buildFeedQuery({ userId: 7, limit: 20, cursor: 'not-a-cursor' }));\nfor (const bad of [0, 51, '20', 1.5]) assert.throws(() => buildFeedQuery({ userId: 7, limit: bad }), RangeError);\nconst dir = path.join(process.cwd(), 'migrations');\nconst mig = fs.readdirSync(dir).filter(n => /^002_[A-Za-z0-9_-]+\\.sql$/.test(n));\nassert.equal(mig.length, 1);\nconst sql = fs.readFileSync(path.join(dir, mig[0]), 'utf8').replace(/--[^\\n]*/g, '');\nassert.match(sql, /CREATE\\s+(?:UNIQUE\\s+)?INDEX\\s+[\\s\\S]*?ON\\s+(?:ONLY\\s+)?\"?posts\"?\\s*(?:USING\\s+btree\\s*)?\\(\\s*\"?user_id\"?\\s*,\\s*\"?created_at\"?(?:\\s+DESC)?\\s*,\\s*\"?id\"?(?:\\s+DESC)?\\s*\\)/i);\n"
|
||||
},
|
||||
{
|
||||
"id": "upsert-inventory-sql",
|
||||
"category": "database",
|
||||
"manualIds": [
|
||||
"skill:postgres-patterns"
|
||||
],
|
||||
"query": "src/inventory.js exports async syncStock(db, items), where items are { sku, quantity } and db.query(text, values) runs a parameterized PostgreSQL statement (node-postgres style, $n placeholders). It currently does a SELECT and then an UPDATE or INSERT per item, which is slow and races with concurrent syncs. Replace it with a single INSERT INTO inventory (sku, quantity, updated_at) ... ON CONFLICT (sku) DO UPDATE statement for the whole batch that sets quantity from the incoming row and updated_at to now(). Exactly one db.query call per non-empty batch and none for an empty batch. If the same sku appears more than once in items, the last occurrence wins (PostgreSQL rejects affecting a row twice in one statement). No caller value may be inlined into the SQL text. Resolve to the number of distinct skus written. Do not add dependencies.",
|
||||
"files": {
|
||||
"src/inventory.js": "'use strict';\n\nasync function syncStock(db, items) {\n let count = 0;\n for (const item of items) {\n const found = await db.query('SELECT sku FROM inventory WHERE sku = $1', [item.sku]);\n if (found.rows.length) {\n await db.query('UPDATE inventory SET quantity = $1, updated_at = now() WHERE sku = $2', [item.quantity, item.sku]);\n } else {\n await db.query('INSERT INTO inventory (sku, quantity, updated_at) VALUES ($1, $2, now())', [item.sku, item.quantity]);\n }\n count++;\n }\n return count;\n}\n\nmodule.exports = { syncStock };\n",
|
||||
"schema.sql": "CREATE TABLE inventory (\n sku text PRIMARY KEY,\n quantity integer NOT NULL,\n updated_at timestamptz NOT NULL\n);\n"
|
||||
},
|
||||
"check": "'use strict';\nconst assert = require('node:assert/strict');\nconst path = require('node:path');\nconst { syncStock } = require(path.join(process.cwd(), 'src/inventory.js'));\nfunction fakeDb() {\n const calls = [];\n return { calls, async query(text, values) { calls.push({ text, values }); return { rows: [], rowCount: 0 }; } };\n}\n(async () => {\n let db = fakeDb();\n assert.equal(await syncStock(db, []), 0);\n assert.equal(db.calls.length, 0);\n db = fakeDb();\n const n = await syncStock(db, [{ sku: 'SKU-A', quantity: 11 }, { sku: \"SKU-'B\", quantity: 55 }, { sku: 'SKU-A', quantity: 7 }]);\n assert.equal(n, 2);\n assert.equal(db.calls.length, 1);\n const { text, values } = db.calls[0];\n assert.match(text, /INSERT\\s+INTO\\s+\"?inventory\"?/i);\n assert.match(text, /ON\\s+CONFLICT\\s*\\(\\s*\"?sku\"?\\s*\\)\\s*DO\\s+UPDATE\\s+SET/i);\n assert.match(text, /\"?quantity\"?\\s*=\\s*EXCLUDED\\.\"?quantity\"?/i);\n assert.match(text, /\"?updated_at\"?\\s*=\\s*(?:now\\(\\)|CURRENT_TIMESTAMP|EXCLUDED\\.\"?updated_at\"?)/i);\n assert.ok(!text.includes('SKU-'), 'sku inlined into SQL');\n const flat = values.flat(Infinity).map(v => (typeof v === 'string' && /^\\d+$/.test(v) ? Number(v) : v));\n assert.equal(flat.filter(v => v === 'SKU-A').length, 1);\n assert.equal(flat.filter(v => v === \"SKU-'B\").length, 1);\n assert.ok(flat.includes(7) && flat.includes(55));\n assert.ok(!flat.includes(11), 'stale duplicate quantity sent');\n const maxParam = Math.max(0, ...[...text.matchAll(/\\$(\\d+)/g)].map(x => Number(x[1])));\n assert.equal(maxParam, values.length);\n db = fakeDb();\n assert.equal(await syncStock(db, [{ sku: 'X', quantity: 1 }]), 1);\n assert.equal(db.calls.length, 1);\n})().catch(err => { console.error(err); process.exitCode = 1; });\n"
|
||||
},
|
||||
{
|
||||
"id": "slugify-regression-tests",
|
||||
"category": "testing",
|
||||
"manualIds": [
|
||||
"skill:tdd-workflow"
|
||||
],
|
||||
"query": "Bug report in BUGS.md: src/slugify.js produces leading and trailing hyphens and mangles accented letters. Work test-first: add test/slugify.test.js using the built-in node:test runner and node:assert, requiring ../src/slugify, with at least three separate test cases that reproduce the reported bugs and cover edge cases (empty input, repeated separators), then fix slugify(input) so they pass. Expected behavior: lowercase ASCII output; accented Latin letters lose their accents (e with grave becomes e); every run of non-alphanumeric characters becomes a single hyphen; no leading or trailing hyphens; empty or separator-only input returns an empty string. Do not add dependencies.",
|
||||
"files": {
|
||||
"src/slugify.js": "'use strict';\n\nfunction slugify(input) {\n return String(input).toLowerCase().replace(/[^a-z0-9]+/g, '-');\n}\n\nmodule.exports = { slugify };\n",
|
||||
"BUGS.md": "# Open bugs\n\n1. slugify(' Hello, World! ') returns '-hello-world-' (expected 'hello-world').\n2. slugify('Cr\\u00e8me Br\\u00fbl\\u00e9e') (accented) returns 'cr-me-br-l-e' (expected 'creme-brulee').\n",
|
||||
"package.json": "{\n \"name\": \"slugs\",\n \"version\": \"1.0.0\",\n \"private\": true,\n \"scripts\": { \"test\": \"node --test test/\" }\n}\n"
|
||||
},
|
||||
"check": "'use strict';\nconst assert = require('node:assert/strict');\nconst fs = require('node:fs');\nconst path = require('node:path');\nconst { slugify } = require(path.join(process.cwd(), 'src/slugify.js'));\nassert.equal(slugify(' Hello, World! '), 'hello-world');\nassert.equal(slugify('Cr\\u00e8me Br\\u00fbl\\u00e9e'), 'creme-brulee');\nassert.equal(slugify('D\\u00e9j\\u00e0 Vu 2024'), 'deja-vu-2024');\nassert.equal(slugify('a--b__c'), 'a-b-c');\nassert.equal(slugify(''), '');\nassert.equal(slugify(' -- !! '), '');\nassert.equal(slugify('already-slugged'), 'already-slugged');\nconst testFile = path.join(process.cwd(), 'test', 'slugify.test.js');\nassert.ok(fs.existsSync(testFile), 'test/slugify.test.js missing');\nconst src = fs.readFileSync(testFile, 'utf8');\nassert.match(src, /node:test/);\nassert.match(src, /require\\(\\s*['\"]\\.\\.\\/src\\/slugify(?:\\.js)?['\"]\\s*\\)/);\nassert.ok((src.match(/\\b(?:test|it)\\s*\\(/g) || []).length >= 3, 'expected at least three test cases');\n"
|
||||
},
|
||||
{
|
||||
"id": "content-hash-cache",
|
||||
"category": "performance",
|
||||
"manualIds": [
|
||||
"skill:content-hash-cache-pattern"
|
||||
],
|
||||
"query": "src/extractor.js exports createExtractor({ readFile, parse }). readFile(filePath) returns a Buffer and parse(text) is an expensive document parser. The cache is keyed by file path, so edited files return stale results and renamed or copied files are parsed again. Re-key the cache by the SHA-256 hex digest of the file bytes (use node:crypto) so identical content at any path is parsed once and changed content is re-parsed. Also export cacheKeyFor(buffer) returning that hex digest. extract(filePath) must still return the parse result, and stats() must return { hits, misses } counting cache hits and parses. Do not add dependencies.",
|
||||
"files": {
|
||||
"src/extractor.js": "'use strict';\n\nfunction createExtractor({ readFile, parse }) {\n const cache = new Map();\n let hits = 0;\n let misses = 0;\n return {\n extract(filePath) {\n if (cache.has(filePath)) {\n hits++;\n return cache.get(filePath);\n }\n misses++;\n const result = parse(readFile(filePath).toString('utf8'));\n cache.set(filePath, result);\n return result;\n },\n stats: () => ({ hits, misses }),\n };\n}\n\nmodule.exports = { createExtractor };\n"
|
||||
},
|
||||
"check": "'use strict';\nconst assert = require('node:assert/strict');\nconst path = require('node:path');\nconst { createExtractor, cacheKeyFor } = require(path.join(process.cwd(), 'src/extractor.js'));\nassert.equal(cacheKeyFor(Buffer.from('hello')), '2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824');\nassert.notEqual(cacheKeyFor(Buffer.from('a')), cacheKeyFor(Buffer.from('b')));\nconst disk = { 'a.txt': Buffer.from('report one'), 'b.txt': Buffer.from('report one') };\nlet parses = 0;\nconst ex = createExtractor({ readFile: p => Buffer.from(disk[p]), parse: t => { parses++; return { words: t.split(' ').length, text: t }; } });\nassert.deepEqual(ex.extract('a.txt'), { words: 2, text: 'report one' });\nassert.deepEqual(ex.extract('b.txt'), { words: 2, text: 'report one' });\nassert.equal(parses, 1);\ndisk['a.txt'] = Buffer.from('report one edited');\nassert.deepEqual(ex.extract('a.txt'), { words: 3, text: 'report one edited' });\nassert.equal(parses, 2);\nex.extract('a.txt');\nex.extract('b.txt');\nassert.equal(parses, 2);\nassert.deepEqual(ex.stats(), { hits: 3, misses: 2 });\n"
|
||||
},
|
||||
{
|
||||
"id": "batch-customer-lookup",
|
||||
"category": "performance",
|
||||
"manualIds": [
|
||||
"skill:backend-patterns"
|
||||
],
|
||||
"query": "src/orders.js exports async getOrdersWithCustomers(repo) for the orders dashboard endpoint. It calls repo.findCustomerById once per order, which is an N+1 query pattern and times out for large accounts. The repo (see src/repo.js for the interface) also offers findCustomersByIds(ids), which resolves to the matching customers in any order and omits unknown ids. Rewrite the function to load all customers with a single findCustomersByIds call using the distinct customer ids (and no call at all when there are no orders), never calling findCustomerById. Return the orders in their original order, each as a new object with a customer property (null when the customer does not exist). Do not add dependencies.",
|
||||
"files": {
|
||||
"src/orders.js": "'use strict';\n\nasync function getOrdersWithCustomers(repo) {\n const orders = await repo.listOrders();\n const result = [];\n for (const order of orders) {\n const customer = await repo.findCustomerById(order.customerId);\n result.push({ ...order, customer });\n }\n return result;\n}\n\nmodule.exports = { getOrdersWithCustomers };\n",
|
||||
"src/repo.js": "'use strict';\n\n// Interface implemented by the SQL repository in production.\n// listOrders(): Promise<Array<{ id, customerId, total }>>\n// findCustomerById(id): Promise<{ id, name } | null> -- one query per call\n// findCustomersByIds(ids): Promise<Array<{ id, name }>> -- one query, WHERE id = ANY($1)\nmodule.exports = {};\n"
|
||||
},
|
||||
"check": "'use strict';\nconst assert = require('node:assert/strict');\nconst path = require('node:path');\nconst { getOrdersWithCustomers } = require(path.join(process.cwd(), 'src/orders.js'));\nfunction repo(orders) {\n const customers = [{ id: 'c1', name: 'Ada' }, { id: 'c2', name: 'Lin' }, { id: 'c3', name: 'Bo' }];\n const r = { single: 0, batch: [], async listOrders() { return orders; },\n async findCustomerById(id) { r.single++; return customers.find(c => c.id === id) || null; },\n async findCustomersByIds(ids) { r.batch.push([...ids]); return customers.filter(c => ids.includes(c.id)).reverse(); } };\n return r;\n}\n(async () => {\n const orders = [{ id: 1, customerId: 'c2', total: 5 }, { id: 2, customerId: 'c1', total: 7 },\n { id: 3, customerId: 'c2', total: 1 }, { id: 4, customerId: 'gone', total: 2 }];\n const snapshot = JSON.stringify(orders);\n const r = repo(orders);\n const out = await getOrdersWithCustomers(r);\n assert.equal(r.single, 0);\n assert.equal(r.batch.length, 1);\n assert.deepEqual(r.batch[0].slice().sort(), ['c1', 'c2', 'gone']);\n assert.deepEqual(out.map(o => o.id), [1, 2, 3, 4]);\n assert.deepEqual(out.map(o => o.customer && o.customer.name), ['Lin', 'Ada', 'Lin', null]);\n assert.equal(out[0].total, 5);\n assert.equal(JSON.stringify(orders), snapshot);\n const empty = repo([]);\n assert.deepEqual(await getOrdersWithCustomers(empty), []);\n assert.equal(empty.batch.length + empty.single, 0);\n})().catch(err => { console.error(err); process.exitCode = 1; });\n"
|
||||
},
|
||||
{
|
||||
"id": "rbac-middleware",
|
||||
"category": "auth",
|
||||
"manualIds": [
|
||||
"skill:backend-patterns"
|
||||
],
|
||||
"query": "src/auth.js exports requirePermission(permission), an Express-style middleware factory, and ROLE_PERMISSIONS. It only checks that req.user exists and never checks the role. Implement role-based access control: calling requirePermission with a permission that no role grants must throw immediately. The returned middleware (req, res, next) must respond res.status(401).json({ error: { code: \"UNAUTHENTICATED\", message } }) when req.user is missing; res.status(403).json({ error: { code: \"FORBIDDEN\", message } }) when req.user.role is unknown or lacks the permission (role names must be looked up safely, so values such as \"constructor\" or \"__proto__\" are simply unknown roles); otherwise call next() exactly once without responding. Do not change ROLE_PERMISSIONS. Do not add dependencies.",
|
||||
"files": {
|
||||
"src/auth.js": "'use strict';\n\nconst ROLE_PERMISSIONS = {\n admin: ['read', 'write', 'delete'],\n editor: ['read', 'write'],\n viewer: ['read'],\n};\n\nfunction requirePermission(permission) {\n return (req, res, next) => {\n if (!req.user) return res.status(401).json({ error: 'unauthorized' });\n return next();\n };\n}\n\nmodule.exports = { requirePermission, ROLE_PERMISSIONS };\n"
|
||||
},
|
||||
"check": "'use strict';\nconst assert = require('node:assert/strict');\nconst path = require('node:path');\nconst { requirePermission } = require(path.join(process.cwd(), 'src/auth.js'));\nfunction run(permission, user) {\n const res = { code: null, body: null, status(c) { this.code = c; return this; }, json(b) { this.body = b; return this; } };\n let nexts = 0;\n requirePermission(permission)(user === undefined ? {} : { user }, res, () => { nexts++; });\n return { res, nexts };\n}\nlet r = run('read');\nassert.equal(r.res.code, 401);\nassert.equal(r.res.body.error.code, 'UNAUTHENTICATED');\nassert.equal(typeof r.res.body.error.message, 'string');\nassert.equal(r.nexts, 0);\nr = run('write', { id: 1, role: 'viewer' });\nassert.equal(r.res.code, 403);\nassert.equal(r.res.body.error.code, 'FORBIDDEN');\nassert.equal(r.nexts, 0);\nfor (const role of ['root', 'constructor', '__proto__', 'toString', undefined, 'hasOwnProperty']) {\n let out;\n assert.doesNotThrow(() => { out = run('read', { id: 2, role }); }, String(role));\n assert.equal(out.res.code, 403, String(role));\n assert.equal(out.nexts, 0);\n}\nr = run('write', { id: 3, role: 'editor' });\nassert.equal(r.nexts, 1);\nassert.equal(r.res.code, null);\nr = run('delete', { id: 4, role: 'admin' });\nassert.equal(r.nexts, 1);\nr = run('delete', { id: 5, role: 'editor' });\nassert.equal(r.res.code, 403);\nassert.throws(() => requirePermission('fly'));\nassert.throws(() => requirePermission('constructor'));\n"
|
||||
},
|
||||
{
|
||||
"id": "immutable-cart-update",
|
||||
"category": "refactor",
|
||||
"manualIds": [
|
||||
"skill:coding-standards"
|
||||
],
|
||||
"query": "src/cart.js exports addItem(cart, item), removeItem(cart, sku), applyDiscount(cart, pct) and total(cart). A cart is { items: [{ sku, price, quantity }], discountPct }. The update functions mutate their arguments, which causes stale UI state bugs. Refactor them to be pure: never mutate the cart, its items array, any item object, or the item argument; always return a new cart object. Keep the behavior: addItem adds the item, or increases quantity when the sku already exists; removeItem drops the sku; applyDiscount sets discountPct and must throw a RangeError unless pct is a number from 0 to 100; total returns the discounted sum rounded to 2 decimal places. Do not add dependencies.",
|
||||
"files": {
|
||||
"src/cart.js": "'use strict';\n\nfunction addItem(cart, item) {\n const existing = cart.items.find(i => i.sku === item.sku);\n if (existing) existing.quantity += item.quantity;\n else cart.items.push(item);\n return cart;\n}\n\nfunction removeItem(cart, sku) {\n cart.items = cart.items.filter(i => i.sku !== sku);\n return cart;\n}\n\nfunction applyDiscount(cart, pct) {\n cart.discountPct = pct;\n return cart;\n}\n\nfunction total(cart) {\n const sum = cart.items.reduce((acc, i) => acc + i.price * i.quantity, 0);\n return Math.round(sum * (1 - (cart.discountPct || 0) / 100) * 100) / 100;\n}\n\nmodule.exports = { addItem, removeItem, applyDiscount, total };\n"
|
||||
},
|
||||
"check": "'use strict';\nconst assert = require('node:assert/strict');\nconst path = require('node:path');\nconst cart = require(path.join(process.cwd(), 'src/cart.js'));\nconst deepFreeze = o => { Object.values(o).forEach(v => { if (v && typeof v === 'object') deepFreeze(v); }); return Object.freeze(o); };\nconst base = deepFreeze({ items: [{ sku: 'a', price: 10, quantity: 1 }, { sku: 'b', price: 2.5, quantity: 2 }], discountPct: 0 });\nconst snap = JSON.stringify(base);\nconst item = deepFreeze({ sku: 'a', price: 10, quantity: 2 });\nconst c1 = cart.addItem(base, item);\nassert.notEqual(c1, base);\nassert.deepEqual(c1.items.find(i => i.sku === 'a').quantity, 3);\nassert.equal(c1.items.length, 2);\nconst newItem = deepFreeze({ sku: 'c', price: 1, quantity: 1 });\nconst c2 = cart.addItem(c1, newItem);\nassert.equal(c2.items.length, 3);\nassert.equal(c1.items.length, 2);\nconst c3 = cart.removeItem(c2, 'b');\nassert.deepEqual(c3.items.map(i => i.sku), ['a', 'c']);\nassert.equal(c2.items.length, 3);\nconst c4 = cart.applyDiscount(c3, 10);\nassert.equal(c4.discountPct, 10);\nassert.equal(c3.discountPct, 0);\nassert.equal(cart.total(c4), 27.9);\nassert.equal(cart.total(base), 15);\nfor (const bad of [-1, 101, '10', NaN]) assert.throws(() => cart.applyDiscount(base, bad), RangeError);\nassert.equal(JSON.stringify(base), snap);\nconst m = { items: [{ sku: 'z', price: 1, quantity: 1 }], discountPct: 0 };\nconst m2 = cart.addItem(m, { sku: 'z', price: 1, quantity: 4 });\nassert.equal(m.items[0].quantity, 1);\nassert.equal(m2.items[0].quantity, 5);\nconst added = { sku: 'y', price: 3, quantity: 1 };\nconst m3 = cart.addItem(m, added);\ncart.addItem(m3, { sku: 'y', price: 3, quantity: 5 });\nassert.equal(added.quantity, 1);\n"
|
||||
},
|
||||
{
|
||||
"id": "inject-signup-deps",
|
||||
"category": "refactor",
|
||||
"manualIds": [
|
||||
"skill:hexagonal-architecture"
|
||||
],
|
||||
"query": "src/signup.js hard-requires the Postgres and SMTP adapters in src/adapters/, which fail at import time without infrastructure, so the sign-up use case cannot be unit tested. Refactor to ports and adapters. src/signup.js must export createSignupService({ userRepository, mailer, clock }) returning { signUp({ email, name }) } and must not import anything from src/adapters or read environment variables. Ports: userRepository.findByEmail(email) and userRepository.save(user) (resolves to the stored user including id), mailer.sendWelcome({ to, name }), clock.now() returning a Date. signUp trims and lowercases the email; rejects with an error whose code is \"INVALID_EMAIL\" if it lacks \"@\", or \"EMAIL_TAKEN\" if findByEmail finds a user (without saving or mailing); otherwise saves { email, name, createdAt: clock.now().toISOString() }, sends the welcome email to the saved user, and resolves to the saved user. Add src/main.js as the composition root that wires the real adapters. Keep the adapters as they are. Do not add dependencies.",
|
||||
"files": {
|
||||
"src/signup.js": "'use strict';\nconst store = require('./adapters/pgUserStore');\nconst mailer = require('./adapters/smtpMailer');\n\nasync function signUp({ email, name }) {\n const normalized = email.trim().toLowerCase();\n if (await store.findByEmail(normalized)) throw new Error('taken');\n const user = await store.insert({ email: normalized, name, createdAt: new Date().toISOString() });\n await mailer.sendWelcome(user.email, user.name);\n return user;\n}\n\nmodule.exports = { signUp };\n",
|
||||
"src/adapters/pgUserStore.js": "'use strict';\n// Connects at import time, like our real pool module.\nif (!process.env.DATABASE_URL) throw new Error('DATABASE_URL is not configured');\n\nmodule.exports = {\n async findByEmail(email) { throw new Error('not implemented in this repo snapshot: ' + email); },\n async insert(user) { throw new Error('not implemented in this repo snapshot: ' + user.email); },\n};\n",
|
||||
"src/adapters/smtpMailer.js": "'use strict';\nif (!process.env.SMTP_URL) throw new Error('SMTP_URL is not configured');\n\nmodule.exports = {\n async sendWelcome(to, name) { throw new Error('not implemented in this repo snapshot: ' + to + name); },\n};\n"
|
||||
},
|
||||
"check": "'use strict';\nconst assert = require('node:assert/strict');\nconst fs = require('node:fs');\nconst path = require('node:path');\ndelete process.env.DATABASE_URL;\ndelete process.env.SMTP_URL;\nconst file = path.join(process.cwd(), 'src/signup.js');\nconst source = fs.readFileSync(file, 'utf8');\nassert.doesNotMatch(source, /require\\([^)]*adapters|from\\s+['\"][^'\"]*adapters/, 'domain imports an adapter');\nassert.doesNotMatch(source, /process\\.env/, 'domain reads the environment');\nassert.ok(fs.existsSync(path.join(process.cwd(), 'src/main.js')), 'composition root missing');\nconst { createSignupService } = require(file);\nfunction setup(existing = []) {\n const users = [...existing];\n const log = { saved: [], mails: [] };\n const svc = createSignupService({\n userRepository: { async findByEmail(e) { return users.find(u => u.email === e) || null; },\n async save(u) { const s = { id: 'u' + (users.length + 1), ...u }; users.push(s); log.saved.push(u); return s; } },\n mailer: { async sendWelcome(msg) { log.mails.push(msg); } },\n clock: { now: () => new Date(Date.UTC(2024, 0, 2, 3, 4, 5)) },\n });\n return { svc, log };\n}\n(async () => {\n let { svc, log } = setup();\n const user = await svc.signUp({ email: ' Ada@Example.COM ', name: 'Ada' });\n assert.deepEqual(user, { id: 'u1', email: 'ada@example.com', name: 'Ada', createdAt: '2024-01-02T03:04:05.000Z' });\n assert.deepEqual(log.saved, [{ email: 'ada@example.com', name: 'Ada', createdAt: '2024-01-02T03:04:05.000Z' }]);\n assert.deepEqual(log.mails, [{ to: 'ada@example.com', name: 'Ada' }]);\n ({ svc, log } = setup([{ id: 'x', email: 'lin@example.com', name: 'Lin' }]));\n await assert.rejects(svc.signUp({ email: 'LIN@example.com', name: 'Lin 2' }), e => e.code === 'EMAIL_TAKEN');\n await assert.rejects(svc.signUp({ email: 'nope', name: 'N' }), e => e.code === 'INVALID_EMAIL');\n assert.equal(log.saved.length, 0);\n assert.equal(log.mails.length, 0);\n})().catch(err => { console.error(err); process.exitCode = 1; });\n"
|
||||
},
|
||||
{
|
||||
"id": "cache-aside-user",
|
||||
"category": "caching",
|
||||
"manualIds": [
|
||||
"skill:redis-patterns"
|
||||
],
|
||||
"query": "src/userCache.js exports createUserCache({ redis, db, ttlSeconds = 300 }). redis is a node-redis v4 style client (async get(key), set(key, value, { EX }), del(key)) and db has async findUser(id) and updateUser(id, patch). Profile reads are hammering the database. Implement cache-aside: getUser(id) uses key \"user:\" + id, returns the parsed cached JSON on a hit without touching db, and on a miss loads from db and caches JSON with an expiry of ttlSeconds (do not cache a missing user; return null). updateUser(id, patch) writes to db first, then deletes the cache key, and resolves to the updated user. Redis is an optimization, not a dependency: if any redis call rejects, getUser and updateUser must still return the correct db result. Do not add dependencies.",
|
||||
"files": {
|
||||
"src/userCache.js": "'use strict';\n\nfunction createUserCache({ redis, db, ttlSeconds = 300 }) {\n return {\n async getUser(id) {\n return db.findUser(id);\n },\n async updateUser(id, patch) {\n return db.updateUser(id, patch);\n },\n };\n}\n\nmodule.exports = { createUserCache };\n"
|
||||
},
|
||||
"check": "'use strict';\nconst assert = require('node:assert/strict');\nconst path = require('node:path');\nconst { createUserCache } = require(path.join(process.cwd(), 'src/userCache.js'));\nfunction fakes(broken = false) {\n const store = new Map();\n const log = [];\n const redis = {\n async get(k) { log.push(['get', k]); if (broken) throw new Error('ECONNREFUSED'); return store.has(k) ? store.get(k) : null; },\n async set(k, v, opts) { log.push(['set', k, opts]); if (broken) throw new Error('ECONNREFUSED'); store.set(k, v); return 'OK'; },\n async del(k) { log.push(['del', k]); if (broken) throw new Error('ECONNREFUSED'); return store.delete(k) ? 1 : 0; },\n };\n const rows = { 1: { id: 1, name: 'Ada' } };\n const db = { reads: 0, async findUser(id) { db.reads++; return rows[id] ? { ...rows[id] } : null; },\n async updateUser(id, patch) { log.push(['db-update', id]); rows[id] = { ...rows[id], ...patch }; return { ...rows[id] }; } };\n return { store, log, redis, db };\n}\n(async () => {\n let f = fakes();\n const cache = createUserCache({ redis: f.redis, db: f.db, ttlSeconds: 60 });\n assert.deepEqual(await cache.getUser(1), { id: 1, name: 'Ada' });\n assert.equal(f.db.reads, 1);\n const set = f.log.find(e => e[0] === 'set');\n assert.equal(set[1], 'user:1');\n assert.deepEqual(set[2], { EX: 60 });\n assert.deepEqual(JSON.parse(f.store.get('user:1')), { id: 1, name: 'Ada' });\n assert.deepEqual(await cache.getUser(1), { id: 1, name: 'Ada' });\n assert.equal(f.db.reads, 1);\n assert.equal(await cache.getUser(2), null);\n assert.ok(!f.store.has('user:2'));\n const updated = await cache.updateUser(1, { name: 'Ada L' });\n assert.deepEqual(updated, { id: 1, name: 'Ada L' });\n const iUpd = f.log.findIndex(e => e[0] === 'db-update');\n const iDel = f.log.findIndex(e => e[0] === 'del' && e[1] === 'user:1');\n assert.ok(iUpd >= 0 && iDel > iUpd, 'must invalidate after the db write');\n assert.deepEqual(await cache.getUser(1), { id: 1, name: 'Ada L' });\n f = fakes();\n const dflt = createUserCache({ redis: f.redis, db: f.db });\n await dflt.getUser(1);\n assert.deepEqual(f.log.find(e => e[0] === 'set')[2], { EX: 300 });\n f = fakes(true);\n const broken = createUserCache({ redis: f.redis, db: f.db });\n assert.deepEqual(await broken.getUser(1), { id: 1, name: 'Ada' });\n assert.deepEqual(await broken.updateUser(1, { name: 'X' }), { id: 1, name: 'X' });\n})().catch(err => { console.error(err); process.exitCode = 1; });\n"
|
||||
},
|
||||
{
|
||||
"id": "token-units-bigint",
|
||||
"category": "data",
|
||||
"manualIds": [
|
||||
"skill:evm-token-decimals"
|
||||
],
|
||||
"query": "src/units.js converts ERC-20 token amounts for our portfolio dashboard, but it uses floating point, so 18-decimal balances lose precision. Rewrite it with exact BigInt math. formatUnits(raw, decimals): raw is a bigint or an integer string in base units; return a decimal string with no trailing fractional zeros and no trailing \".\", keeping a leading \"-\" for negatives. parseUnits(value, decimals): value is a decimal string such as \"1.5\" or \"-0.25\"; return a bigint in base units; throw a RangeError if it has more fractional digits than decimals, and throw an Error for anything that is not a plain decimal number (e.g. \"\", \"abc\", \"1e5\", \"1.2.3\"). Also export normalizeAmount(raw, fromDecimals, toDecimals) returning a bigint rescaled between token precisions, truncating toward zero when precision is reduced. Do not add dependencies.",
|
||||
"files": {
|
||||
"src/units.js": "'use strict';\n\nfunction formatUnits(raw, decimals) {\n return String(Number(raw) / 10 ** decimals);\n}\n\nfunction parseUnits(value, decimals) {\n return BigInt(Math.round(parseFloat(value) * 10 ** decimals));\n}\n\nmodule.exports = { formatUnits, parseUnits };\n",
|
||||
"README.md": "# portfolio-units\n\nToken decimals differ per token and per chain: USDC uses 6 on Ethereum mainnet,\nWETH uses 18, and some bridged tokens differ from their native versions.\nAlways pass the decimals value read from the token contract.\n"
|
||||
},
|
||||
"check": "'use strict';\nconst assert = require('node:assert/strict');\nconst path = require('node:path');\nconst { formatUnits, parseUnits, normalizeAmount } = require(path.join(process.cwd(), 'src/units.js'));\nassert.equal(formatUnits(123456789012345678901234567n, 18), '123456789.012345678901234567');\nassert.equal(formatUnits('1000000', 6), '1');\nassert.equal(formatUnits(1500000n, 6), '1.5');\nassert.equal(formatUnits(0n, 18), '0');\nassert.equal(formatUnits(-1n, 18), '-0.000000000000000001');\nassert.equal(formatUnits(-1500000n, 6), '-1.5');\nassert.equal(formatUnits(5n, 0), '5');\nassert.equal(parseUnits('1.5', 6), 1500000n);\nassert.equal(parseUnits('0.000000000000000001', 18), 1n);\nassert.equal(parseUnits('123456789.012345678901234567', 18), 123456789012345678901234567n);\nassert.equal(parseUnits('-0.25', 6), -250000n);\nassert.equal(parseUnits('100', 0), 100n);\nassert.throws(() => parseUnits('1.1234567', 6), RangeError);\nfor (const bad of ['', 'abc', '1e5', '1.2.3', '0x10', ' 1']) assert.throws(() => parseUnits(bad, 6), Error, bad);\nassert.equal(normalizeAmount(1234567n, 6, 18), 1234567000000000000n);\nassert.equal(normalizeAmount(1234567890123456789n, 18, 6), 1234567n);\nassert.equal(normalizeAmount(-1234567890123456789n, 18, 6), -1234567n);\nassert.equal(normalizeAmount(42n, 8, 8), 42n);\nassert.equal(typeof normalizeAmount(1n, 6, 6), 'bigint');\n"
|
||||
},
|
||||
{
|
||||
"id": "inclusive-range",
|
||||
"category": "no-workflow",
|
||||
"manualIds": [],
|
||||
"query": "range(start, end) in src/range.js is documented as inclusive of end, but it stops one short. Fix it so range(1, 5) returns [1, 2, 3, 4, 5]; when start > end it must return an empty array. Do not add dependencies.",
|
||||
"files": {
|
||||
"src/range.js": "'use strict';\n\n/** Returns the integers from start to end, inclusive. */\nfunction range(start, end) {\n const out = [];\n for (let i = start; i < end; i++) out.push(i);\n return out;\n}\n\nmodule.exports = { range };\n"
|
||||
},
|
||||
"check": "'use strict';\nconst assert = require('node:assert/strict');\nconst path = require('node:path');\nconst { range } = require(path.join(process.cwd(), 'src/range.js'));\nassert.deepEqual(range(1, 5), [1, 2, 3, 4, 5]);\nassert.deepEqual(range(3, 3), [3]);\nassert.deepEqual(range(-2, 0), [-2, -1, 0]);\nassert.deepEqual(range(5, 1), []);\n"
|
||||
},
|
||||
{
|
||||
"id": "export-name-typo",
|
||||
"category": "no-workflow",
|
||||
"manualIds": [],
|
||||
"query": "src/report.js crashes with \"formatDate is not a function\" because src/dates.js exports its formatter under a misspelled name. Export it as formatDate, and keep the misspelled export as an alias of the same function so older callers keep working. Do not add dependencies.",
|
||||
"files": {
|
||||
"src/dates.js": "'use strict';\n\nfunction formatDate(date) {\n const pad = n => String(n).padStart(2, '0');\n return date.getUTCFullYear() + '-' + pad(date.getUTCMonth() + 1) + '-' + pad(date.getUTCDate());\n}\n\nmodule.exports = { fromatDate: formatDate };\n",
|
||||
"src/report.js": "'use strict';\nconst { formatDate } = require('./dates');\n\nfunction reportHeader(title, date) {\n return title + ' (' + formatDate(date) + ')';\n}\n\nmodule.exports = { reportHeader };\n"
|
||||
},
|
||||
"check": "'use strict';\nconst assert = require('node:assert/strict');\nconst path = require('node:path');\nconst dates = require(path.join(process.cwd(), 'src/dates.js'));\nconst { reportHeader } = require(path.join(process.cwd(), 'src/report.js'));\nconst d = new Date(Date.UTC(2024, 0, 5, 12));\nassert.equal(dates.formatDate(d), '2024-01-05');\nassert.equal(dates.fromatDate, dates.formatDate);\nassert.equal(reportHeader('Weekly', d), 'Weekly (2024-01-05)');\n"
|
||||
},
|
||||
{
|
||||
"id": "default-greeting",
|
||||
"category": "no-workflow",
|
||||
"manualIds": [],
|
||||
"noWorkflow": true,
|
||||
"query": "Small fix, no workflow needed. greet(name) in src/greet.js returns \"Hello, undefined!\" when called without a name. Make it trim the name and fall back to \"world\" when the name is missing, null, empty or only whitespace, so greet() returns \"Hello, world!\" and greet(\" Ada \") returns \"Hello, Ada!\". Do not add dependencies.",
|
||||
"files": {
|
||||
"src/greet.js": "'use strict';\n\nfunction greet(name) {\n return 'Hello, ' + name + '!';\n}\n\nmodule.exports = { greet };\n"
|
||||
},
|
||||
"check": "'use strict';\nconst assert = require('node:assert/strict');\nconst path = require('node:path');\nconst { greet } = require(path.join(process.cwd(), 'src/greet.js'));\nassert.equal(greet(), 'Hello, world!');\nassert.equal(greet(null), 'Hello, world!');\nassert.equal(greet(''), 'Hello, world!');\nassert.equal(greet(' '), 'Hello, world!');\nassert.equal(greet(' Ada '), 'Hello, Ada!');\nassert.equal(greet('Lin'), 'Hello, Lin!');\n"
|
||||
},
|
||||
{
|
||||
"id": "sum-form-values",
|
||||
"category": "no-workflow",
|
||||
"manualIds": [],
|
||||
"query": "total(values) in src/total.js sums amounts typed into a form, but the inputs arrive as strings so it returns \"0123.5\" for [\"1\", \"2\", \"3.5\"]. Make it return the numeric sum (6.5 in that example). Empty strings count as 0, plain numbers must still work, and an empty array returns 0. Do not add dependencies.",
|
||||
"files": {
|
||||
"src/total.js": "'use strict';\n\nfunction total(values) {\n return values.reduce((sum, v) => sum + v, 0);\n}\n\nmodule.exports = { total };\n"
|
||||
},
|
||||
"check": "'use strict';\nconst assert = require('node:assert/strict');\nconst path = require('node:path');\nconst { total } = require(path.join(process.cwd(), 'src/total.js'));\nassert.equal(total(['1', '2', '3.5']), 6.5);\nassert.equal(total([]), 0);\nassert.equal(total(['', '4']), 4);\nassert.equal(total([2, '3']), 5);\n"
|
||||
},
|
||||
{
|
||||
"id": "changelog-capitalize",
|
||||
"category": "no-workflow",
|
||||
"manualIds": [],
|
||||
"query": "The security team's release-notes script imports src/changelog.js, and it crashes when a changelog entry has an empty title because capitalize(\"\") throws. Fix capitalize so an empty string returns \"\", while other strings still get only their first character uppercased with the rest unchanged. formatEntry must keep its current output format. Do not add dependencies.",
|
||||
"files": {
|
||||
"src/changelog.js": "'use strict';\n\nfunction capitalize(text) {\n return text[0].toUpperCase() + text.slice(1);\n}\n\nfunction formatEntry(entry) {\n return '- ' + capitalize(entry.title) + ' (' + entry.type + ')';\n}\n\nmodule.exports = { capitalize, formatEntry };\n"
|
||||
},
|
||||
"check": "'use strict';\nconst assert = require('node:assert/strict');\nconst path = require('node:path');\nconst { capitalize, formatEntry } = require(path.join(process.cwd(), 'src/changelog.js'));\nassert.equal(capitalize(''), '');\nassert.equal(capitalize('x'), 'X');\nassert.equal(capitalize('hello World'), 'Hello World');\nassert.equal(formatEntry({ title: 'fix xss in footer', type: 'security' }), '- Fix xss in footer (security)');\nassert.equal(formatEntry({ title: '', type: 'chore' }), '- (chore)');\n"
|
||||
},
|
||||
{
|
||||
"id": "test-summary-plural",
|
||||
"category": "no-workflow",
|
||||
"manualIds": [],
|
||||
"query": "Our test runner prints \"1 tests passed, 1 tests failed\". In src/summary.js, fix formatSummary(passed, failed) to use \"test\" when a count is exactly 1 and \"tests\" otherwise, e.g. \"1 test passed, 0 tests failed\". Keep the rest of the wording identical. Do not add dependencies.",
|
||||
"files": {
|
||||
"src/summary.js": "'use strict';\n\nfunction formatSummary(passed, failed) {\n return passed + ' tests passed, ' + failed + ' tests failed';\n}\n\nmodule.exports = { formatSummary };\n"
|
||||
},
|
||||
"check": "'use strict';\nconst assert = require('node:assert/strict');\nconst path = require('node:path');\nconst { formatSummary } = require(path.join(process.cwd(), 'src/summary.js'));\nassert.equal(formatSummary(1, 0), '1 test passed, 0 tests failed');\nassert.equal(formatSummary(2, 1), '2 tests passed, 1 test failed');\nassert.equal(formatSummary(0, 0), '0 tests passed, 0 tests failed');\nassert.equal(formatSummary(12, 3), '12 tests passed, 3 tests failed');\n"
|
||||
},
|
||||
{
|
||||
"id": "database-label-typo",
|
||||
"category": "no-workflow",
|
||||
"manualIds": [],
|
||||
"noWorkflow": true,
|
||||
"query": "No workflow needed. In src/options.js the settings dropdown shows \"Databse\" for the database option; correct the label to \"Database\". Also make labelFor(value) return the value itself when no option matches, instead of throwing. Do not change the option values or their order. Do not add dependencies.",
|
||||
"files": {
|
||||
"src/options.js": "'use strict';\n\nconst OPTIONS = [\n { value: 'database', label: 'Databse' },\n { value: 'api', label: 'API' },\n { value: 'cache', label: 'Cache' },\n];\n\nfunction labelFor(value) {\n return OPTIONS.find(o => o.value === value).label;\n}\n\nmodule.exports = { OPTIONS, labelFor };\n"
|
||||
},
|
||||
"check": "'use strict';\nconst assert = require('node:assert/strict');\nconst path = require('node:path');\nconst { OPTIONS, labelFor } = require(path.join(process.cwd(), 'src/options.js'));\nassert.deepEqual(OPTIONS, [{ value: 'database', label: 'Database' }, { value: 'api', label: 'API' }, { value: 'cache', label: 'Cache' }]);\nassert.equal(labelFor('database'), 'Database');\nassert.equal(labelFor('api'), 'API');\nassert.equal(labelFor('queue'), 'queue');\n"
|
||||
},
|
||||
{
|
||||
"id": "port-from-env",
|
||||
"category": "no-workflow",
|
||||
"manualIds": [],
|
||||
"noWorkflow": true,
|
||||
"query": "Do not select a workflow for this one-line style fix. getPort(env) in src/server-config.js returns env.PORT as a string or 3000. Make it return a number: the integer value of env.PORT when it consists only of decimal digits and is between 1 and 65535, otherwise 3000. Do not add dependencies.",
|
||||
"files": {
|
||||
"src/server-config.js": "'use strict';\n\nfunction getPort(env = process.env) {\n return env.PORT || 3000;\n}\n\nmodule.exports = { getPort };\n"
|
||||
},
|
||||
"check": "'use strict';\nconst assert = require('node:assert/strict');\nconst path = require('node:path');\nconst { getPort } = require(path.join(process.cwd(), 'src/server-config.js'));\nassert.equal(getPort({ PORT: '8080' }), 8080);\nassert.equal(getPort({}), 3000);\nassert.equal(getPort({ PORT: '' }), 3000);\nassert.equal(getPort({ PORT: 'abc' }), 3000);\nassert.equal(getPort({ PORT: '70000' }), 3000);\nassert.equal(getPort({ PORT: '0' }), 3000);\nassert.equal(getPort({ PORT: '80.5' }), 3000);\nassert.equal(getPort({ PORT: '65535' }), 65535);\n"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,847 @@
|
||||
'use strict';
|
||||
|
||||
// Development-only evaluator. It lives under docker/ so the npm package never ships it.
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { spawnSync } = require('node:child_process');
|
||||
const { isDeepStrictEqual } = require('node:util');
|
||||
const LIB = path.join(__dirname, '../../scripts/lib');
|
||||
const { loadContextRegistry } = require(path.join(LIB, 'context-pack-registry'));
|
||||
const { compileContextProfile } = require(path.join(LIB, 'context-profiles'));
|
||||
const { resolveTaskContext, resolveDeclinedFallback } = require(path.join(LIB, 'context-selection'));
|
||||
const { proposeTaskContext } = require(path.join(LIB, 'context-profile-proposal'));
|
||||
const { resolveExecutable, fingerprintExecutable } = require(path.join(LIB, 'context-profile-native-executable'));
|
||||
const { launchTaskContext } = require(path.join(LIB, 'context-profile-launch'));
|
||||
const { applyStore } = require(path.join(LIB, 'context-profile-store'));
|
||||
const { prepareNativeProfile, getNativeProfileStatus } = require(path.join(LIB, 'context-profile-native'));
|
||||
const { DEFAULT_REPO_ROOT, digestObject, createSourceReader } = require(path.join(LIB, 'context-profile-support'));
|
||||
const io = require(path.join(LIB, 'context-profile-store-fs'));
|
||||
|
||||
const ARMS = Object.freeze(['full', 'manual-lean', 'auto-lean', 'ecc-legacy', 'baseline']);
|
||||
const CORPUS_PATH = path.join(__dirname, 'ai-corpus.json');
|
||||
const LEGACY_PIN_PATH = path.join(__dirname, 'legacy-source.json');
|
||||
const CHECK_FILE = '.ecc-eval-check.cjs';
|
||||
const IMPLEMENTATION = ['docker/context-profiles/ai-eval-lib.js', 'docker/context-profiles/ai-eval.js',
|
||||
'docker/context-profiles/legacy-source.json',
|
||||
'manifests/context-packs/skill-triggers@1.json',
|
||||
'scripts/lib/context-profile-launch.js', 'scripts/lib/context-selection.js',
|
||||
'scripts/lib/context-retrieval.js',
|
||||
'scripts/lib/context-profile-proposal.js', 'scripts/lib/context-profiles.js',
|
||||
'scripts/lib/context-profile-support.js', 'scripts/lib/context-pack-registry.js',
|
||||
'scripts/lib/context-profile-native-executable.js', 'scripts/lib/context-profile-native.js',
|
||||
'scripts/lib/context-profile-store.js', 'scripts/lib/context-profile-store-fs.js'];
|
||||
const BLOCKS = Object.freeze({ excluded: /Context ID is excluded:/,
|
||||
'native-authority': /requires native authority or dynamic-content review/,
|
||||
'manual-only': /Context ID is manual-only:/, 'opt-out-conflict': /noWorkflow conflicts/, 'unknown-id': /Unknown context ID:/ });
|
||||
const ENV_KEYS = ['PATH', 'HOME', 'USERPROFILE', 'CODEX_HOME', 'TMPDIR', 'LANG', 'SystemRoot'];
|
||||
const CLAUDE_ENV_KEYS = ['PATH', 'HOME', 'USERPROFILE', 'CLAUDE_CONFIG_DIR', 'TMPDIR', 'LANG', 'SystemRoot'];
|
||||
const bounded = (value, min, max) => Number.isSafeInteger(value) && value >= min && value <= max;
|
||||
const exists = file => Boolean(fs.lstatSync(file, { throwIfNoEntry: false }));
|
||||
|
||||
function loadCorpus(file = CORPUS_PATH) { return JSON.parse(fs.readFileSync(file, 'utf8')); }
|
||||
|
||||
function safeRelative(file) {
|
||||
return typeof file === 'string' && file.length > 0 && file.length <= 200 && !path.isAbsolute(file)
|
||||
&& !file.startsWith('.') && !file.includes('\\') && file.split('/').every(part => part && part !== '..' && part !== '.');
|
||||
}
|
||||
|
||||
function validateCorpus(corpus) {
|
||||
if (corpus?.schemaVersion === 'ecc.context-eval-complex-corpus.v1') return validateComplexCorpus(corpus);
|
||||
if (corpus?.schemaVersion !== 'ecc.context-eval-corpus.v2'
|
||||
|| !Array.isArray(corpus.selection) || !Array.isArray(corpus.tasks)
|
||||
|| !bounded(corpus.selection.length, 1, 200) || !bounded(corpus.tasks.length, 1, 200)
|
||||
|| corpus.minimumDistinctTasks !== 30 || corpus.nonInferiorityMargin !== 0.05) {
|
||||
throw new Error('Invalid preregistered corpus');
|
||||
}
|
||||
for (const cases of [corpus.selection, corpus.tasks]) validateCorpusIds(cases);
|
||||
for (const task of corpus.tasks) {
|
||||
const files = Object.entries(task.files || {});
|
||||
if (!Array.isArray(task.manualIds) || task.manualIds.length > 1 || !bounded(files.length, 1, 8)
|
||||
|| files.some(([file, content]) => !safeRelative(file) || typeof content !== 'string' || Buffer.byteLength(content) > 16384)
|
||||
|| typeof task.check !== 'string' || !bounded(Buffer.byteLength(task.check), 1, 16384)) {
|
||||
throw new Error('Invalid corpus task');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function validateCorpusIds(cases) {
|
||||
if (new Set(cases.map(c => c.id)).size !== cases.length) throw new Error('Duplicate corpus ID');
|
||||
for (const item of cases) {
|
||||
if (!/^[a-z][a-z0-9-]{0,63}$/.test(item.id) || typeof item.query !== 'string'
|
||||
|| !bounded(Buffer.byteLength(item.query), 1, 8192)) throw new Error('Invalid corpus case');
|
||||
}
|
||||
}
|
||||
|
||||
// Complex corpora hold a few realistic multi-file tasks with scored hidden graders. Sample gates
|
||||
// are descriptive at this size, so the distinct-task minimum relaxes to the corpus itself.
|
||||
function validateComplexCorpus(corpus) {
|
||||
if (!Array.isArray(corpus.selection) || !Array.isArray(corpus.tasks)
|
||||
|| !bounded(corpus.selection.length, 0, 50) || !bounded(corpus.tasks.length, 1, 10)
|
||||
|| corpus.minimumDistinctTasks !== corpus.tasks.length || corpus.nonInferiorityMargin !== 0.05) {
|
||||
throw new Error('Invalid preregistered corpus');
|
||||
}
|
||||
validateCorpusIds(corpus.selection);
|
||||
if (new Set(corpus.tasks.map(c => c.id)).size !== corpus.tasks.length) throw new Error('Duplicate corpus ID');
|
||||
for (const task of corpus.tasks) {
|
||||
if (!/^[a-z][a-z0-9-]{0,63}$/.test(task.id)) throw new Error('Invalid corpus case');
|
||||
if (task.steps === undefined
|
||||
&& (typeof task.query !== 'string' || !bounded(Buffer.byteLength(task.query), 1, 8192))) throw new Error('Invalid corpus case');
|
||||
const files = Object.entries(task.files || {});
|
||||
if (!Array.isArray(task.manualIds) || task.manualIds.length > 3 || !bounded(files.length, 1, 24)
|
||||
|| files.some(([file, content]) => !safeRelative(file) || typeof content !== 'string' || Buffer.byteLength(content) > 65536)) {
|
||||
throw new Error('Invalid corpus task');
|
||||
}
|
||||
if (task.steps !== undefined) {
|
||||
// Stepped (chained) task: sequential tickets graded in one accumulating workspace.
|
||||
if (!Array.isArray(task.steps) || !bounded(task.steps.length, 2, 8)
|
||||
|| task.steps.some(step => typeof step.query !== 'string' || !bounded(Buffer.byteLength(step.query), 1, 8192)
|
||||
|| typeof step.check !== 'string' || !bounded(Buffer.byteLength(step.check), 1, 65536)
|
||||
|| (step.checkTimeoutMs !== undefined && !bounded(step.checkTimeoutMs, 1, 120000))
|
||||
|| (step.manualIds !== undefined && (!Array.isArray(step.manualIds) || step.manualIds.length > 3)))) {
|
||||
throw new Error('Invalid corpus task');
|
||||
}
|
||||
} else if (typeof task.check !== 'string' || !bounded(Buffer.byteLength(task.check), 1, 65536)
|
||||
|| (task.checkTimeoutMs !== undefined && !bounded(task.checkTimeoutMs, 1, 120000))) {
|
||||
throw new Error('Invalid corpus task');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function sourceSnapshot(repoRoot) {
|
||||
const registry = loadContextRegistry({ repoRoot });
|
||||
const profiles = ['full@1', 'lean@1'].map(profileId => compileContextProfile({ repoRoot, profileId }));
|
||||
// Implementation modules are loaded from this evaluator's checkout; repoRoot may be a fixture registry.
|
||||
const reader = createSourceReader(DEFAULT_REPO_ROOT);
|
||||
const implementation = IMPLEMENTATION.map(file => ({ path: file, digest: reader.read(file).digest }));
|
||||
const packageJson = JSON.parse(reader.read('package.json').content.toString('utf8'));
|
||||
const runtime = { node: process.versions.node, dependencies: {
|
||||
ajv: packageJson.dependencies.ajv, 'js-yaml': packageJson.dependencies['js-yaml'] } };
|
||||
return { registry, profiles, sourceDigest: digestObject({ registryDigest: registry.registryDigest,
|
||||
planDigests: profiles.map(p => p.planDigest), implementation, runtime }), runtime };
|
||||
}
|
||||
|
||||
const EFFORTS = ['low', 'medium', 'high', 'xhigh', 'max', 'ultra'];
|
||||
|
||||
function providerFamily(executable) {
|
||||
const base = path.basename(String(executable || '')).toLowerCase();
|
||||
if (base.includes('claude')) return 'claude';
|
||||
if (base.includes('codex')) return 'codex';
|
||||
throw new Error('Provider executable must name a Claude or Codex CLI');
|
||||
}
|
||||
|
||||
function resolveFamily(provider, executable) {
|
||||
if (provider !== undefined && provider !== null) {
|
||||
if (!['claude', 'codex'].includes(provider)) throw new Error('Provider must be claude or codex');
|
||||
return provider;
|
||||
}
|
||||
if (executable) return providerFamily(executable);
|
||||
return 'codex';
|
||||
}
|
||||
|
||||
function providerPin(model, executable, effort) {
|
||||
if (model === undefined && executable === undefined && effort === undefined) return null;
|
||||
if (typeof model !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9._:-]{0,99}$/.test(model)
|
||||
|| !path.isAbsolute(executable || '')) throw new Error('Provider pin requires model and absolute executable');
|
||||
if (effort !== undefined && !EFFORTS.includes(effort)) throw new Error('Invalid reasoning effort');
|
||||
return { modelDigest: digestObject(model), executableDigest: resolveExecutable(executable).digest,
|
||||
...(effort === undefined ? {} : { effort }) };
|
||||
}
|
||||
|
||||
function preregister({ repoRoot = DEFAULT_REPO_ROOT, corpus = loadCorpus(), repeats = 1, model, executable, effort, arms } = {}) {
|
||||
validateCorpus(corpus);
|
||||
if (!bounded(repeats, 1, 20)) throw new Error('Invalid repeat count');
|
||||
const armList = arms === undefined ? [...ARMS] : arms;
|
||||
if (!Array.isArray(armList) || !armList.length || new Set(armList).size !== armList.length
|
||||
|| armList.some(arm => !ARMS.includes(arm))) throw new Error('Invalid arm subset');
|
||||
const source = sourceSnapshot(repoRoot);
|
||||
const value = { schemaVersion: 'ecc.context-eval-registration.v2', corpusDigest: digestObject(corpus),
|
||||
sourceDigest: source.sourceDigest, registryDigest: source.registry.registryDigest,
|
||||
providerPin: providerPin(model, executable, effort), runtime: source.runtime,
|
||||
arms: armList, repeats, minimumDistinctTasks: corpus.minimumDistinctTasks, nonInferiorityMargin: 0.05,
|
||||
confidence: 0.95, sampling: 'fixed-purposive-pilot',
|
||||
design: corpus.schemaVersion === 'ecc.context-eval-complex-corpus.v1'
|
||||
? 'paired-native-installs-hidden-scored-complex-tasks'
|
||||
: 'paired-native-installs-hidden-graded-coding-tasks',
|
||||
order: corpus.tasks.flatMap((task, index) => Array.from({ length: repeats }, (_, repeat) => ({
|
||||
id: task.id, repeat, arms: armList.map((_, offset) => armList[(index + repeat + offset) % armList.length]),
|
||||
}))), selectionIds: corpus.selection.map(c => c.id) };
|
||||
return { ...value, registrationDigest: digestObject(value) };
|
||||
}
|
||||
|
||||
// Parse in memory only. No event objects, paths, provider messages or error text enter reports.
|
||||
function parseCodexJsonl(stdout) {
|
||||
const invalid = { valid: false, text: '', usage: null };
|
||||
if (typeof stdout !== 'string' || Buffer.byteLength(stdout) > 1024 * 1024) return invalid;
|
||||
let text = '';
|
||||
let completions = 0;
|
||||
let usage = { inputTokens: 0, cachedInputTokens: 0, outputTokens: 0 };
|
||||
try {
|
||||
for (const line of stdout.split('\n').filter(line => line.trim())) {
|
||||
const event = JSON.parse(line);
|
||||
if (!event || typeof event !== 'object' || ['error', 'turn.failed'].includes(event.type)) return invalid;
|
||||
if (event.type === 'item.completed' && event.item?.type === 'agent_message') {
|
||||
if (typeof event.item.text !== 'string') return invalid;
|
||||
text = event.item.text;
|
||||
}
|
||||
if (event.type !== 'turn.completed') continue;
|
||||
const u = event.usage;
|
||||
if (!u || ![u.input_tokens, u.cached_input_tokens, u.output_tokens].every(v => bounded(v, 0, 1e9))
|
||||
|| u.cached_input_tokens > u.input_tokens) return invalid;
|
||||
completions++;
|
||||
usage = { inputTokens: usage.inputTokens + u.input_tokens,
|
||||
cachedInputTokens: usage.cachedInputTokens + u.cached_input_tokens,
|
||||
outputTokens: usage.outputTokens + u.output_tokens };
|
||||
}
|
||||
} catch { return invalid; }
|
||||
return completions === 1 ? { valid: true, text, usage } : invalid;
|
||||
}
|
||||
|
||||
// Claude print-mode emits exactly one result JSON object. Fresh input folds cache creations;
|
||||
// cache reads are reported separately. is_error results are provider failures, not parse failures.
|
||||
function parseClaudeJson(stdout) {
|
||||
const invalid = { valid: false, text: '', usage: null };
|
||||
if (typeof stdout !== 'string' || Buffer.byteLength(stdout) > 1024 * 1024) return invalid;
|
||||
let result = null;
|
||||
let results = 0;
|
||||
try {
|
||||
for (const line of stdout.split('\n').filter(line => line.trim())) {
|
||||
const event = JSON.parse(line);
|
||||
if (!event || typeof event !== 'object' || Array.isArray(event)) return invalid;
|
||||
if (event.type !== 'result') continue;
|
||||
results++;
|
||||
result = event;
|
||||
}
|
||||
} catch { return invalid; }
|
||||
if (results !== 1) return invalid;
|
||||
if (result.is_error !== false || typeof result.result !== 'string') return { ...invalid, error: true };
|
||||
const u = result.usage;
|
||||
if (!u || ![u.input_tokens, u.cache_creation_input_tokens, u.cache_read_input_tokens, u.output_tokens]
|
||||
.every(value => bounded(value, 0, 1e9))) return { ...invalid, error: true };
|
||||
return { valid: true, text: result.result,
|
||||
usage: { inputTokens: u.input_tokens + u.cache_creation_input_tokens,
|
||||
cachedInputTokens: u.cache_read_input_tokens, outputTokens: u.output_tokens } };
|
||||
}
|
||||
|
||||
function privateEntry(file, directory) {
|
||||
const stat = fs.lstatSync(file, { throwIfNoEntry: false });
|
||||
return Boolean(stat) && !stat.isSymbolicLink() && (directory ? stat.isDirectory() : stat.isFile())
|
||||
&& (process.platform === 'win32' || ((stat.mode & 0o077) === 0 && (!process.getuid || stat.uid === process.getuid())));
|
||||
}
|
||||
|
||||
/**
|
||||
* Subscription credentials stay in a dedicated evaluator login home. Each call leases auth.json into the
|
||||
* isolated CODEX_HOME, returns refreshed tokens afterwards and always removes the leased copy.
|
||||
*/
|
||||
function createAuthLease(authHome) {
|
||||
if (typeof authHome !== 'string' || !path.isAbsolute(authHome)) throw new Error('Auth home must be an absolute path');
|
||||
const real = fs.realpathSync(authHome);
|
||||
const forbidden = [path.join(os.homedir(), '.codex'), process.env.CODEX_HOME].filter(Boolean)
|
||||
.map(file => (exists(file) ? fs.realpathSync(file) : path.resolve(file)));
|
||||
if (forbidden.includes(real)) throw new Error('Auth home must be a dedicated evaluator login home, not your Codex home');
|
||||
const source = path.join(real, 'auth.json');
|
||||
if (!privateEntry(real, true) || !privateEntry(source, false)) {
|
||||
throw new Error('Auth home must be a private directory containing a private auth.json; see the evaluation guide');
|
||||
}
|
||||
return {
|
||||
mode: 'subscription-lease',
|
||||
run(codexHome, work) {
|
||||
const leased = path.join(codexHome, 'auth.json');
|
||||
const original = fs.readFileSync(source);
|
||||
fs.writeFileSync(leased, original, { flag: 'wx', mode: 0o600 });
|
||||
try { return work(); } finally {
|
||||
try {
|
||||
const after = fs.readFileSync(leased);
|
||||
if (!after.equals(original)) {
|
||||
JSON.parse(after.toString('utf8'));
|
||||
const temp = `${source}.${process.pid}.tmp`;
|
||||
try {
|
||||
fs.writeFileSync(temp, after, { flag: 'wx', mode: 0o600 });
|
||||
fs.renameSync(temp, source);
|
||||
} finally { fs.rmSync(temp, { force: true }); }
|
||||
}
|
||||
} catch { /* An unreadable refresh keeps the previous login; the next call reports any auth failure. */ }
|
||||
fs.rmSync(leased, { force: true });
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Claude subscription logins live in the macOS Keychain as a JSON wrapper. The lease reads the
|
||||
* current access token per call into the child environment only; it is never persisted or reported.
|
||||
*/
|
||||
function readClaudeKeychainToken() {
|
||||
if (process.platform !== 'darwin') throw new Error('Claude Keychain login requires macOS; provide CLAUDE_CODE_OAUTH_TOKEN or ANTHROPIC_API_KEY');
|
||||
const result = spawnSync('security', ['find-generic-password', '-s', 'Claude Code-credentials', '-w'],
|
||||
{ encoding: 'utf8', shell: false, timeout: 15000, killSignal: 'SIGKILL', maxBuffer: 65536 });
|
||||
if (result.status !== 0 || result.error) throw new Error('Claude Keychain login is unavailable; provide CLAUDE_CODE_OAUTH_TOKEN or ANTHROPIC_API_KEY');
|
||||
let parsed;
|
||||
try { parsed = JSON.parse(result.stdout); }
|
||||
catch { throw new Error('Claude Keychain login is unreadable; provide CLAUDE_CODE_OAUTH_TOKEN or ANTHROPIC_API_KEY'); }
|
||||
const token = parsed?.claudeAiOauth?.accessToken;
|
||||
if (typeof token !== 'string' || !token) throw new Error('Claude Keychain login is unrecognized; provide CLAUDE_CODE_OAUTH_TOKEN or ANTHROPIC_API_KEY');
|
||||
return token;
|
||||
}
|
||||
|
||||
function createClaudeProvider({ allowRealProvider = false, allowCredentialedTools = false, executable, model,
|
||||
apiKey = process.env.ANTHROPIC_API_KEY, oauthToken = process.env.CLAUDE_CODE_OAUTH_TOKEN,
|
||||
tokenSource = readClaudeKeychainToken, persistSessions = false, execute = spawnSync } = {}) {
|
||||
if (allowRealProvider !== true) throw new Error('Real provider requires explicit opt-in');
|
||||
if (!model || !executable) throw new Error('Real provider requires a model and absolute executable');
|
||||
let lease = null;
|
||||
let authentication;
|
||||
if (oauthToken) authentication = 'oauth-env';
|
||||
else if (apiKey) authentication = 'api-key';
|
||||
else if (typeof tokenSource === 'function') {
|
||||
lease = { mode: 'subscription-keychain-lease',
|
||||
run(env, work) { env.CLAUDE_CODE_OAUTH_TOKEN = tokenSource(); return work(); } };
|
||||
authentication = lease.mode;
|
||||
} else throw new Error('Real provider requires CLAUDE_CODE_OAUTH_TOKEN, ANTHROPIC_API_KEY, or the Claude Keychain login');
|
||||
const pin = providerPin(model, executable, undefined);
|
||||
const binary = resolveExecutable(executable);
|
||||
const provider = request => {
|
||||
if (fingerprintExecutable(binary.path).digest !== pin.executableDigest) fail('source-drift');
|
||||
const selection = request.phase === 'selection';
|
||||
if (!selection && !allowCredentialedTools) {
|
||||
throw new Error('Claude task tools can read provider credentials; explicit credentialed-tool opt-in is required');
|
||||
}
|
||||
// Selection is tool-free and read-only; task execution may edit and run commands in the workspace.
|
||||
// Claude has no cwd-write sandbox flag, so containment relies on the isolated home and temp workspace.
|
||||
const args = ['--print', '--output-format', 'json',
|
||||
...(persistSessions ? [] : ['--no-session-persistence']),
|
||||
...(selection ? ['--tools', ''] : ['--permission-mode', 'bypassPermissions']),
|
||||
'--model', model];
|
||||
const env = Object.fromEntries(CLAUDE_ENV_KEYS.filter(key => typeof request.env?.[key] === 'string')
|
||||
.map(key => [key, request.env[key]]));
|
||||
env.DISABLE_NON_ESSENTIAL_MODEL_CALLS = '1';
|
||||
if (authentication === 'oauth-env') env.CLAUDE_CODE_OAUTH_TOKEN = oauthToken;
|
||||
if (authentication === 'api-key') env.ANTHROPIC_API_KEY = apiKey;
|
||||
const call = () => execute(binary.path, args, { input: request.input, cwd: request.cwd, env,
|
||||
encoding: 'utf8', shell: false, timeout: request.timeoutMs, killSignal: 'SIGKILL',
|
||||
maxBuffer: request.maxBuffer });
|
||||
return lease ? lease.run(env, call) : call();
|
||||
};
|
||||
provider.authentication = authentication;
|
||||
return provider;
|
||||
}
|
||||
|
||||
function createCodexProvider({ allowRealProvider = false, executable, model, effort, authHome,
|
||||
apiKey = process.env.CODEX_API_KEY, execute = spawnSync } = {}) {
|
||||
if (allowRealProvider !== true) throw new Error('Real provider requires explicit opt-in');
|
||||
if (!model || !executable) throw new Error('Real provider requires a model and absolute executable');
|
||||
if (!authHome && !apiKey) throw new Error('Real provider requires --auth-home (subscription login) or CODEX_API_KEY');
|
||||
const lease = authHome ? createAuthLease(authHome) : null;
|
||||
const pin = providerPin(model, executable, effort);
|
||||
const binary = resolveExecutable(executable);
|
||||
const provider = request => {
|
||||
if (fingerprintExecutable(binary.path).digest !== pin.executableDigest) fail('source-drift');
|
||||
const args = ['exec', '--json', '--ephemeral', '--skip-git-repo-check',
|
||||
'--sandbox', request.phase === 'selection' ? 'read-only' : 'workspace-write',
|
||||
// Connected ChatGPT apps and account plugin installs stay out of every arm.
|
||||
'--disable', 'apps', '--disable', 'remote_plugin',
|
||||
'-c', 'approval_policy="never"', ...(effort ? ['-c', `model_reasoning_effort="${effort}"`] : []),
|
||||
'--model', model, '-'];
|
||||
const env = Object.fromEntries(ENV_KEYS.filter(key => typeof request.env?.[key] === 'string')
|
||||
.map(key => [key, request.env[key]]));
|
||||
if (!lease) env.CODEX_API_KEY = apiKey;
|
||||
const call = () => execute(binary.path, args, { input: request.input, cwd: request.cwd, env,
|
||||
encoding: 'utf8', shell: false, timeout: request.timeoutMs, killSignal: 'SIGKILL',
|
||||
maxBuffer: request.maxBuffer });
|
||||
return lease ? lease.run(env.CODEX_HOME, call) : call();
|
||||
};
|
||||
provider.authentication = lease ? lease.mode : 'api-key';
|
||||
return provider;
|
||||
}
|
||||
|
||||
/** Real Lean and Full installs, prepared through the same isolated native adapter users get. */
|
||||
function prepareEnvironments({ repoRoot, executable, root }) {
|
||||
const binary = resolveExecutable(executable);
|
||||
const environments = {};
|
||||
for (const [name, profileId, selectionMode] of [['full', 'full@1', 'manual'], ['lean', 'lean@1', 'auto']]) {
|
||||
const options = { stateRoot: path.join(root, name, 'managed'), nativeRoot: path.join(root, name, 'native') };
|
||||
fs.mkdirSync(path.join(root, name), { mode: 0o700 });
|
||||
applyStore({ repoRoot, stateRoot: options.stateRoot, target: 'codex', selectionMode, profileId });
|
||||
const status = prepareNativeProfile({ ...options, codexPath: executable });
|
||||
if (!status.ready) throw new Error(`Native ${name} install is not ready`);
|
||||
// A signed-in Codex records task-directory trust in config.toml and downloads account-provided
|
||||
// plugins into plugins/. Restoring the prepared state after every call keeps trials identical;
|
||||
// any other change still fails verification as drift.
|
||||
const config = path.join(status.codexHome, 'config.toml');
|
||||
const prepared = fs.readFileSync(config);
|
||||
const plugins = path.join(status.codexHome, 'plugins');
|
||||
const listing = directory => (exists(directory) ? fs.readdirSync(directory) : []);
|
||||
const preparedPlugins = new Set(listing(plugins));
|
||||
const preparedCache = new Set(listing(path.join(plugins, 'cache')));
|
||||
environments[name] = { profileId, skills: status.selectedIds.length,
|
||||
launch: { home: status.home, codexHome: status.codexHome, codexPath: status.codexPath,
|
||||
executableDigest: status.executableDigest },
|
||||
restore() {
|
||||
fs.writeFileSync(config, prepared);
|
||||
for (const entry of listing(plugins)) if (!preparedPlugins.has(entry)) fs.rmSync(path.join(plugins, entry), { recursive: true, force: true });
|
||||
for (const entry of listing(path.join(plugins, 'cache'))) {
|
||||
if (!preparedCache.has(entry)) fs.rmSync(path.join(plugins, 'cache', entry), { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
verify() {
|
||||
let ready = false;
|
||||
try { ready = getNativeProfileStatus(options).ready; } catch { ready = false; }
|
||||
if (!ready) fail('environment-drift');
|
||||
} };
|
||||
}
|
||||
// Baseline arm: an empty native home with no ECC install, for provider-overhead subtraction.
|
||||
const home = path.join(root, 'baseline', 'home');
|
||||
fs.mkdirSync(path.join(home, '.codex'), { recursive: true, mode: 0o700 });
|
||||
environments.baseline = { profileId: null, skills: 0, restore() {},
|
||||
launch: { home, codexHome: path.join(home, '.codex'), codexPath: binary.path, executableDigest: binary.digest },
|
||||
verify() { if (fingerprintExecutable(binary.path).digest !== binary.digest) fail('environment-drift'); } };
|
||||
return environments;
|
||||
}
|
||||
|
||||
function installClaudeSkills({ payload, home }) {
|
||||
const config = path.join(home, '.claude');
|
||||
const installed = path.join(config, 'skills');
|
||||
fs.mkdirSync(installed, { recursive: true, mode: 0o700 });
|
||||
for (const entry of fs.readdirSync(payload)) {
|
||||
fs.cpSync(path.join(payload, entry), path.join(installed, entry), { recursive: true, errorOnExist: true, force: false });
|
||||
}
|
||||
return { config, installed };
|
||||
}
|
||||
|
||||
function claudeEnvironment({ name, binary, home, config, installed, profileId, skills, sourceSha = null }) {
|
||||
const managed = () => digestObject(io.inventory(installed));
|
||||
const prepared = managed();
|
||||
return [name, { profileId, skills, sourceSha,
|
||||
launch: { home, claudeConfigDir: config, claudePath: binary.path, executableDigest: binary.digest },
|
||||
restore() {},
|
||||
verify() {
|
||||
if (fingerprintExecutable(binary.path).digest !== binary.digest) fail('environment-drift');
|
||||
let observed = null;
|
||||
try { observed = managed(); } catch { observed = null; }
|
||||
if (observed !== prepared) fail('environment-drift');
|
||||
} }];
|
||||
}
|
||||
|
||||
/** The pre-scoping ECC source, pinned by commit so the ecc-legacy arm is reproducible. */
|
||||
function exportLegacySource({ repoRoot = DEFAULT_REPO_ROOT, destination,
|
||||
pin = JSON.parse(fs.readFileSync(LEGACY_PIN_PATH, 'utf8')) } = {}) {
|
||||
if (!/^[a-f0-9]{40}$/.test(pin?.sha || '')) throw new Error('Invalid legacy source pin');
|
||||
if (!path.isAbsolute(destination || '')) throw new Error('Legacy destination must be absolute');
|
||||
const resolved = spawnSync('git', ['-C', repoRoot, 'rev-parse', '--verify', `${pin.sha}^{commit}`],
|
||||
{ encoding: 'utf8', shell: false, timeout: 30000, killSignal: 'SIGKILL' });
|
||||
if (resolved.status !== 0 || resolved.error || resolved.stdout.trim() !== pin.sha) {
|
||||
throw new Error('Legacy source pin is unavailable in this repository');
|
||||
}
|
||||
fs.mkdirSync(destination, { recursive: true, mode: 0o700 });
|
||||
const tar = path.join(destination, 'legacy.tar');
|
||||
const archive = spawnSync('git', ['-C', repoRoot, 'archive', '--format=tar', '-o', tar, pin.sha, 'skills'],
|
||||
{ encoding: 'utf8', shell: false, timeout: 60000, killSignal: 'SIGKILL' });
|
||||
const extract = archive.status === 0 && !archive.error
|
||||
? spawnSync('tar', ['-xf', tar, '-C', destination], { encoding: 'utf8', shell: false, timeout: 60000, killSignal: 'SIGKILL' })
|
||||
: archive;
|
||||
fs.rmSync(tar, { force: true });
|
||||
const payload = path.join(destination, 'skills');
|
||||
if (extract.status !== 0 || extract.error || !exists(payload) || !fs.readdirSync(payload).length) {
|
||||
throw new Error('Legacy source export failed');
|
||||
}
|
||||
return { root: destination, sha: pin.sha };
|
||||
}
|
||||
|
||||
/** Real Claude installs in isolated config homes. Managed-skill drift aborts; there is no
|
||||
* provider bookkeeping to restore because isolated Claude runs do not mutate the managed tree. */
|
||||
function prepareClaudeEnvironments({ repoRoot, executable, root, legacySource = null }) {
|
||||
const binary = resolveExecutable(executable);
|
||||
const environments = {};
|
||||
for (const [name, profileId, selectionMode] of [['full', 'full@1', 'manual'], ['lean', 'lean@1', 'auto']]) {
|
||||
const stateRoot = path.join(root, name, 'managed');
|
||||
fs.mkdirSync(path.join(root, name), { mode: 0o700 });
|
||||
const status = applyStore({ repoRoot, stateRoot, target: 'claude', selectionMode, profileId });
|
||||
const home = path.join(root, name, 'home');
|
||||
const { config, installed } = installClaudeSkills({ payload: path.join(status.generationRoot, 'skills'), home });
|
||||
const [key, env] = claudeEnvironment({ name, binary, home, config, installed, profileId, skills: status.selectedIds.length });
|
||||
environments[key] = env;
|
||||
}
|
||||
if (legacySource) {
|
||||
// ecc-legacy: the typical pre-scoping install — the full skill library from the pinned
|
||||
// pre-ECC-029 commit, launched bare with no ECC context block.
|
||||
const home = path.join(root, 'ecc-legacy', 'home');
|
||||
const { config, installed } = installClaudeSkills({ payload: path.join(legacySource.root, 'skills'), home });
|
||||
const [key, env] = claudeEnvironment({ name: 'ecc-legacy', binary, home, config, installed,
|
||||
profileId: null, skills: fs.readdirSync(installed).length, sourceSha: legacySource.sha });
|
||||
environments[key] = env;
|
||||
}
|
||||
// Baseline arm: an empty config home with no ECC install, for provider-overhead subtraction.
|
||||
const baselineHome = path.join(root, 'baseline', 'home');
|
||||
const baselineConfig = path.join(baselineHome, '.claude');
|
||||
fs.mkdirSync(baselineConfig, { recursive: true, mode: 0o700 });
|
||||
environments.baseline = { profileId: null, skills: 0, sourceSha: null, restore() {},
|
||||
launch: { home: baselineHome, claudeConfigDir: baselineConfig, claudePath: binary.path, executableDigest: binary.digest },
|
||||
verify() { if (fingerprintExecutable(binary.path).digest !== binary.digest) fail('environment-drift'); } };
|
||||
return environments;
|
||||
}
|
||||
|
||||
function syntheticEnvironments(root) {
|
||||
const executable = resolveExecutable(process.execPath);
|
||||
return Object.fromEntries(['full', 'lean', 'ecc-legacy', 'baseline'].map(name => {
|
||||
const home = path.join(root, name, 'home');
|
||||
fs.mkdirSync(path.join(home, '.codex'), { recursive: true, mode: 0o700 });
|
||||
return [name, { profileId: ['baseline', 'ecc-legacy'].includes(name) ? null : `${name}@1`, skills: null, sourceSha: null,
|
||||
verify() {}, restore() {},
|
||||
launch: { home, codexHome: path.join(home, '.codex'), codexPath: executable.path, executableDigest: executable.digest } }];
|
||||
}));
|
||||
}
|
||||
|
||||
function checkArguments(cwd, file = CHECK_FILE, writable = false) {
|
||||
const major = Number(process.versions.node.split('.')[0]);
|
||||
const flag = major >= 22 ? '--permission' : major >= 20 ? '--experimental-permission' : null;
|
||||
// A directory grant covers its children. Node 20.20.2 can abort in its native
|
||||
// permission radix tree when the same directory is also granted as "cwd/*".
|
||||
return flag ? [flag, `--allow-fs-read=${cwd}`,
|
||||
// Stepped graders exercise stateful apps (persistence); single-step graders stay read-only.
|
||||
...(writable ? [`--allow-fs-write=${cwd}`] : []), file] : [file];
|
||||
}
|
||||
|
||||
// The hidden grader enters the workspace only after the agent exits, and runs read-only where Node supports it.
|
||||
// A grader may print one `ECC_EVAL_SCORE {"score":0..1}` line for partial credit; without it the exit
|
||||
// status alone decides (exit 0 scores 1). Outcome success still requires a full score. Stepped tasks
|
||||
// grade each step with a distinct grader file so earlier graders stay readable in the workspace.
|
||||
const SCORE_LINE = /^\s*ECC_EVAL_SCORE\s+(\{[^\n]*\})\s*$/m;
|
||||
function runScoredCheck(cwd, source, timeoutMs = 10000, step = null) {
|
||||
const name = step === null ? CHECK_FILE : `.ecc-eval-check-${step}.cjs`;
|
||||
const file = path.join(cwd, name);
|
||||
if (exists(file)) return { passed: false, score: 0 };
|
||||
fs.writeFileSync(file, source, { flag: 'wx' });
|
||||
const result = spawnSync(process.execPath, checkArguments(fs.realpathSync(cwd), name, step !== null), { cwd, encoding: 'utf8',
|
||||
env: { LANG: 'C.UTF-8' }, shell: false, timeout: timeoutMs, killSignal: 'SIGKILL', maxBuffer: 65536 });
|
||||
// Grader files never linger: in stepped tasks the workspace accumulates, and a later ticket's
|
||||
// agent could read or replay an earlier grader. The planted-grader guard above still applies.
|
||||
fs.rmSync(file, { force: true });
|
||||
const passed = result.status === 0 && !result.error;
|
||||
let score = passed ? 1 : 0;
|
||||
const match = SCORE_LINE.exec(result.stdout || '');
|
||||
// A grader that advertises ECC_EVAL_SCORE but never printed it died mid-run (e.g. the graded
|
||||
// server crashed the process): that is a zero, never a silent pass. A printed but malformed
|
||||
// line keeps the exit-status score.
|
||||
const graderDied = passed && !match && source.includes('ECC_EVAL_SCORE')
|
||||
&& !(result.stdout || '').includes('ECC_EVAL_SCORE');
|
||||
if (passed && match) {
|
||||
try {
|
||||
const parsed = JSON.parse(match[1]);
|
||||
if (typeof parsed?.score === 'number' && parsed.score >= 0 && parsed.score <= 1) score = parsed.score;
|
||||
} catch { /* A malformed score line keeps the exit-status score. */ }
|
||||
}
|
||||
if (graderDied) score = 0;
|
||||
return { passed, score };
|
||||
}
|
||||
|
||||
function runCheck(cwd, source) { return runScoredCheck(cwd, source).passed; }
|
||||
|
||||
function writeWorkspace(cwd, files) {
|
||||
for (const [relative, content] of Object.entries(files)) {
|
||||
fs.mkdirSync(path.dirname(path.join(cwd, relative)), { recursive: true });
|
||||
fs.writeFileSync(path.join(cwd, relative), content, { flag: 'wx' });
|
||||
}
|
||||
}
|
||||
|
||||
function wilson(successes, n) {
|
||||
if (!n) return [0, 1];
|
||||
const z = 1.959963984540054;
|
||||
const p = successes / n;
|
||||
const denominator = 1 + z * z / n;
|
||||
const center = (p + z * z / (2 * n)) / denominator;
|
||||
const radius = z * Math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / denominator;
|
||||
return [Math.max(0, center - radius), Math.min(1, center + radius)];
|
||||
}
|
||||
|
||||
function summarize(outcomes, arms = ARMS) {
|
||||
const ids = [...new Set(outcomes.map(row => row.id))];
|
||||
// Reference arm: full when present (all-arms runs), otherwise the last registered arm (baseline in subset runs).
|
||||
const reference = arms.includes('full') ? 'full' : arms[arms.length - 1];
|
||||
const rates = arms.map(arm => {
|
||||
const rows = outcomes.filter(row => row.arm === arm);
|
||||
return { arm, attempts: rows.length, successes: rows.filter(row => row.passed).length,
|
||||
rate: rows.length ? rows.filter(row => row.passed).length / rows.length : null,
|
||||
meanScore: rows.length ? rows.reduce((sum, row) => sum + (typeof row.score === 'number' ? row.score : Number(row.passed)), 0) / rows.length : null };
|
||||
});
|
||||
const pairs = arms.filter(arm => arm !== reference).map(arm => {
|
||||
const differences = ids.map(id => {
|
||||
const rows = outcomes.filter(row => row.id === id);
|
||||
const baseline = rows.filter(row => row.arm === reference);
|
||||
const delta = baseline.map(row => Number(rows.find(r => r.arm === arm && r.repeat === row.repeat)?.passed === true)
|
||||
- Number(row.passed === true));
|
||||
return delta.length ? delta.reduce((a, b) => a + b, 0) / delta.length : null;
|
||||
}).filter(value => value !== null);
|
||||
const n = differences.length;
|
||||
const delta = n ? differences.reduce((a, b) => a + b, 0) / n : null;
|
||||
// Paired task-cluster means in [-1,1]. Hoeffding with Bonferroni for the arm comparisons.
|
||||
const radius = n ? Math.sqrt(2 * Math.log(80) / n) : 2;
|
||||
return { arm, reference, n, delta, interval: [Math.max(-1, (delta || 0) - radius), Math.min(1, (delta || 0) + radius)],
|
||||
method: 'paired-task-cluster-hoeffding-familywise-95' };
|
||||
});
|
||||
return { distinctTasks: ids.length, rates, pairs };
|
||||
}
|
||||
|
||||
function selectionTask(item) {
|
||||
return { sessionId: 'ecc-eval', taskId: item.id, revision: 1, phase: 'evaluate', query: item.query,
|
||||
...(item.noWorkflow === undefined ? {} : { noWorkflow: item.noWorkflow }),
|
||||
...(item.explicitIds ? { explicitIds: item.explicitIds } : {}) };
|
||||
}
|
||||
|
||||
function failureCode(error) {
|
||||
if (['call-budget', 'deadline', 'source-drift', 'environment-drift', 'provider-failed', 'invalid-jsonl'].includes(error?.code)) return error.code;
|
||||
for (const [code, pattern] of Object.entries(BLOCKS)) if (pattern.test(error?.message || '')) return code;
|
||||
return 'evaluation-failed';
|
||||
}
|
||||
function fail(code) { const error = new Error(code); error.code = code; throw error; }
|
||||
|
||||
function launchEnvironment(launch) {
|
||||
return { PATH: process.env.PATH, HOME: launch.home,
|
||||
...(launch.codexHome ? { CODEX_HOME: launch.codexHome } : {}),
|
||||
...(launch.claudeConfigDir ? { CLAUDE_CONFIG_DIR: launch.claudeConfigDir } : {}),
|
||||
TMPDIR: launch.home, LANG: 'C.UTF-8' };
|
||||
}
|
||||
|
||||
function executeAdapter(state, cwd, environment) {
|
||||
return (_command, args, options) => {
|
||||
if (state.calls >= state.maxCalls) fail('call-budget');
|
||||
state.assertCurrent();
|
||||
environment.verify();
|
||||
const remaining = state.deadline - Date.now();
|
||||
if (remaining <= 0) fail('deadline');
|
||||
const phase = options.phase || (args.includes('read-only') ? 'selection' : 'task');
|
||||
state.calls++;
|
||||
const started = Date.now();
|
||||
let raw;
|
||||
// Coding tasks outgrow the launcher's interactive default, so the evaluator's own call bound governs them.
|
||||
const timeoutMs = Math.min(phase === 'task' ? state.callTimeoutMs : options.timeout, state.callTimeoutMs, remaining);
|
||||
const env = options.env || launchEnvironment(environment.launch);
|
||||
try {
|
||||
raw = state.provider({ phase, input: options.input, cwd, env, timeoutMs, maxBuffer: 1024 * 1024 });
|
||||
} catch (error) {
|
||||
state.metrics.push({ phase, elapsedMs: Date.now() - started, usage: null });
|
||||
if (error?.code === 'source-drift') throw error;
|
||||
fail('provider-failed');
|
||||
} finally { environment.restore(); }
|
||||
const elapsedMs = Date.now() - started;
|
||||
const parsed = state.family === 'claude' ? parseClaudeJson(raw?.stdout) : parseCodexJsonl(raw?.stdout);
|
||||
state.metrics.push({ phase, elapsedMs, usage: parsed.valid && raw?.status === 0 && !raw?.error ? parsed.usage : null });
|
||||
if (Date.now() >= state.deadline || elapsedMs > timeoutMs) fail('deadline');
|
||||
state.assertCurrent();
|
||||
if (raw?.status !== 0 || raw?.error) fail('provider-failed');
|
||||
if (!parsed.valid) fail(parsed.error ? 'provider-failed' : 'invalid-jsonl');
|
||||
return { status: 0, stdout: parsed.text };
|
||||
};
|
||||
}
|
||||
|
||||
function selectionProbe(item, repoRoot, execute, environment, target) {
|
||||
const options = { repoRoot, task: selectionTask(item), exclude: item.exclude || [], load: true };
|
||||
try {
|
||||
let selection = resolveTaskContext(options);
|
||||
if (selection.reason === 'agent-selection-required') {
|
||||
const proposedIds = proposeTaskContext({ target, query: item.query, candidates: selection.candidates, execute,
|
||||
executable: environment.launch.codexPath || environment.launch.claudePath });
|
||||
// An empty proposal is an explicit decline: honor it (inject nothing).
|
||||
// The tier-2 fallback only applies when a non-empty proposal admitted
|
||||
// nothing — never to override a decline.
|
||||
const declined = proposedIds.length === 0;
|
||||
const next = resolveTaskContext({ ...options, task: { ...options.task, proposedIds, noWorkflow: declined } });
|
||||
if (next.selectedIds.length) selection = next;
|
||||
else if (declined) selection = { ...next, reason: 'agent-declined-selection' };
|
||||
else selection = resolveDeclinedFallback(options, selection);
|
||||
}
|
||||
return { id: item.id, category: item.category, passed: !item.expectedBlock
|
||||
&& isDeepStrictEqual(selection.selectedIds, item.expectedIds), selectedIds: selection.selectedIds, failure: null };
|
||||
} catch (error) {
|
||||
const failure = failureCode(error);
|
||||
return { id: item.id, category: item.category, passed: Boolean(item.expectedBlock && failure === item.expectedBlock),
|
||||
selectedIds: [], failure };
|
||||
}
|
||||
}
|
||||
|
||||
// Full relies on native discovery of the whole install; the Lean arms receive ECC-selected skill bodies;
|
||||
// ecc-legacy runs bare against the pinned pre-scoping skill library; Baseline runs the bare task query.
|
||||
// Stepped tasks run each ticket in the same accumulating workspace, grading after every step.
|
||||
function outcomeTrial(item, arm, repeat, repoRoot, execute, cwd, environment, target, harvest, metrics = null) {
|
||||
const launchStep = (query, manualIds) => {
|
||||
const task = { sessionId: 'ecc-eval', taskId: item.id, revision: 1, phase: 'evaluate', query };
|
||||
return launchTaskContext({ repoRoot, execute, nativeEnvironment: environment.launch, target,
|
||||
bare: arm === 'baseline' || arm === 'ecc-legacy',
|
||||
task: { ...task, ...(arm === 'manual-lean' && manualIds?.length ? { explicitIds: manualIds } : {}) },
|
||||
profileId: arm === 'full' ? 'full@1' : 'lean@1', selectionMode: arm === 'auto-lean' ? 'auto' : 'manual' });
|
||||
};
|
||||
try {
|
||||
if (!item.steps) {
|
||||
const result = launchStep(item.query, item.manualIds);
|
||||
if (harvest) harvest(arm, item.id, repeat, environment);
|
||||
const verdict = runScoredCheck(cwd, item.check, item.checkTimeoutMs);
|
||||
const passed = result.status === 'completed' && verdict.passed && verdict.score >= 0.999;
|
||||
return { id: item.id, arm, repeat, passed, score: result.status === 'completed' ? verdict.score : 0,
|
||||
selectedIds: result.selection.selectedIds, failure: passed ? null : 'hidden-check' };
|
||||
}
|
||||
const steps = [];
|
||||
const selectedIds = [];
|
||||
for (let index = 0; index < item.steps.length; index++) {
|
||||
const step = item.steps[index];
|
||||
const start = metrics ? metrics.length : 0;
|
||||
const result = launchStep(step.query, step.manualIds || item.manualIds);
|
||||
if (harvest) harvest(arm, `${item.id}--step${index + 1}`, repeat, environment);
|
||||
if (result.status !== 'completed') {
|
||||
// A failed ticket ends the chain; remaining tickets are unscored.
|
||||
steps.push({ score: 0, ...(metrics ? metricsSince(metrics, start) : {}) });
|
||||
for (let rest = index + 1; rest < item.steps.length; rest++) {
|
||||
steps.push({ score: 0, ...(metrics ? metricsSince(metrics, metrics.length) : {}) });
|
||||
}
|
||||
break;
|
||||
}
|
||||
selectedIds.push(...result.selection.selectedIds);
|
||||
const verdict = runScoredCheck(cwd, step.check, step.checkTimeoutMs, index + 1);
|
||||
steps.push({ score: verdict.passed ? verdict.score : 0, ...(metrics ? metricsSince(metrics, start) : {}) });
|
||||
}
|
||||
const score = steps.reduce((sum, step) => sum + step.score, 0) / item.steps.length;
|
||||
const passed = steps.length === item.steps.length && steps.every(step => step.score >= 0.999);
|
||||
return { id: item.id, arm, repeat, passed, score, selectedIds: [...new Set(selectedIds)], steps,
|
||||
failure: passed ? null : 'hidden-check' };
|
||||
} catch (error) {
|
||||
if (harvest) harvest(arm, item.id, repeat, environment);
|
||||
return { id: item.id, arm, repeat, passed: false, score: 0, selectedIds: [], failure: failureCode(error) };
|
||||
}
|
||||
}
|
||||
|
||||
function metricsSince(metrics, start) {
|
||||
const calls = metrics.slice(start);
|
||||
const complete = calls.length > 0 && calls.every(call => call.usage !== null);
|
||||
return { calls: calls.length, elapsedMs: calls.reduce((sum, c) => sum + c.elapsedMs, 0),
|
||||
usage: complete ? calls.reduce((sum, c) => ({ inputTokens: sum.inputTokens + c.usage.inputTokens,
|
||||
cachedInputTokens: sum.cachedInputTokens + c.usage.cachedInputTokens,
|
||||
outputTokens: sum.outputTokens + c.usage.outputTokens }), { inputTokens: 0, cachedInputTokens: 0, outputTokens: 0 }) : null };
|
||||
}
|
||||
|
||||
// Transcript retention is opt-in (--artifact-dir) and file-only: reports never embed session content or paths.
|
||||
function createHarvester(artifactDir, envs) {
|
||||
if (typeof artifactDir !== 'string' || !path.isAbsolute(artifactDir)) throw new Error('Artifact directory must be absolute');
|
||||
fs.mkdirSync(artifactDir, { recursive: true });
|
||||
const sessionsOf = env => {
|
||||
const config = env.launch.claudeConfigDir;
|
||||
const projects = config ? path.join(config, 'projects') : null;
|
||||
if (!projects || !exists(projects)) return new Set();
|
||||
const found = new Set();
|
||||
const walk = directory => {
|
||||
for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
|
||||
const item = path.join(directory, entry.name);
|
||||
if (entry.isDirectory()) walk(item);
|
||||
else if (entry.name.endsWith('.jsonl')) found.add(item);
|
||||
}
|
||||
};
|
||||
walk(projects);
|
||||
return found;
|
||||
};
|
||||
const seen = new Map(Object.entries(envs).map(([name, env]) => [name, sessionsOf(env)]));
|
||||
const index = [];
|
||||
return {
|
||||
record(arm, id, repeat, env) {
|
||||
const before = seen.get(arm) || new Set();
|
||||
const now = sessionsOf(env);
|
||||
seen.set(arm, now);
|
||||
const fresh = [...now].filter(file => !before.has(file));
|
||||
if (!fresh.length) return;
|
||||
const directory = path.join(artifactDir, `${id}--${arm}--${repeat}`);
|
||||
fs.mkdirSync(directory, { recursive: true });
|
||||
for (const file of fresh) fs.copyFileSync(file, path.join(directory, path.basename(file)));
|
||||
index.push({ id, arm, repeat, files: fresh.map(file => path.basename(file)) });
|
||||
},
|
||||
writeIndex() { fs.writeFileSync(path.join(artifactDir, 'artifact-index.json'), `${JSON.stringify(index, null, 1)}\n`); },
|
||||
};
|
||||
}
|
||||
|
||||
function runEvaluation({ repoRoot = DEFAULT_REPO_ROOT, corpus = loadCorpus(), registration,
|
||||
repeats = 1, provider, family, allowRealProvider = false, allowCredentialedTools = false,
|
||||
executable, model, effort, authHome, environments,
|
||||
arms = undefined, artifactDir = null, maxCalls = 300, deadlineMs = 3600000, callTimeoutMs = 300000 } = {}) {
|
||||
if (!provider && !allowRealProvider) throw new Error('Evaluation requires an injected provider or explicit opt-in');
|
||||
if (!bounded(maxCalls, 1, 2000) || !bounded(deadlineMs, 1, 8 * 3600000)
|
||||
|| !bounded(callTimeoutMs, 1, 600000)) throw new Error('Invalid call or deadline bound');
|
||||
if (!provider && !registration) throw new Error('Real evaluation requires prior registration');
|
||||
const resolvedFamily = provider ? (family || 'codex') : resolveFamily(family, executable);
|
||||
if (resolvedFamily === 'claude' && effort !== undefined) throw new Error('Reasoning effort applies only to the Codex provider');
|
||||
if (!provider && resolvedFamily === 'claude' && !allowCredentialedTools) {
|
||||
throw new Error('Claude task tools can read provider credentials; explicit credentialed-tool opt-in is required');
|
||||
}
|
||||
const pin = preregister({ repoRoot, corpus, repeats, model, executable, effort, arms });
|
||||
if (!provider && resolvedFamily === 'codex' && pin.arms.includes('ecc-legacy')) {
|
||||
throw new Error('Codex real evaluation requires --arms without ecc-legacy; the pinned legacy skills arm is Claude-only');
|
||||
}
|
||||
if (registration && !isDeepStrictEqual(registration, pin)) throw new Error('Registration pin mismatch');
|
||||
const injected = Boolean(provider);
|
||||
const liveProvider = provider || (resolvedFamily === 'claude'
|
||||
? createClaudeProvider({ allowRealProvider, allowCredentialedTools, executable, model,
|
||||
persistSessions: Boolean(artifactDir) })
|
||||
: createCodexProvider({ allowRealProvider, executable, model, effort, authHome }));
|
||||
const state = { calls: 0, metrics: [], maxCalls, callTimeoutMs, family: resolvedFamily,
|
||||
deadline: Date.now() + deadlineMs, provider: liveProvider,
|
||||
assertCurrent() {
|
||||
if (digestObject(corpus) !== pin.corpusDigest || sourceSnapshot(repoRoot).sourceDigest !== pin.sourceDigest) fail('source-drift');
|
||||
} };
|
||||
const temp = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-ai-eval-')));
|
||||
const selection = [];
|
||||
const outcomes = [];
|
||||
let installs = null;
|
||||
let harvester = null;
|
||||
try {
|
||||
const installRoot = path.join(temp, 'installs');
|
||||
fs.mkdirSync(installRoot, { mode: 0o700 });
|
||||
const envs = environments || (injected ? syntheticEnvironments(installRoot)
|
||||
: resolvedFamily === 'claude'
|
||||
? prepareClaudeEnvironments({ repoRoot, executable, root: installRoot,
|
||||
...(pin.arms.includes('ecc-legacy')
|
||||
? { legacySource: exportLegacySource({ repoRoot, destination: path.join(installRoot, 'legacy-source') }) }
|
||||
: {}) })
|
||||
: prepareEnvironments({ repoRoot, executable, root: installRoot }));
|
||||
installs = Object.fromEntries(Object.entries(envs).map(([name, env]) => [name,
|
||||
{ profileId: env.profileId, skills: env.skills, ...(env.sourceSha ? { sourceSha: env.sourceSha } : {}) }]));
|
||||
harvester = artifactDir && resolvedFamily === 'claude' && !injected ? createHarvester(artifactDir, envs) : null;
|
||||
const harvest = harvester ? (arm, id, repeat, env) => harvester.record(arm, id, repeat, env) : null;
|
||||
for (const item of corpus.selection) {
|
||||
const cwd = path.join(temp, `${item.id}--selection`);
|
||||
fs.mkdirSync(cwd);
|
||||
const start = state.metrics.length;
|
||||
selection.push({ ...selectionProbe(item, repoRoot, executeAdapter(state, cwd, envs.lean), envs.lean, resolvedFamily),
|
||||
...metricsSince(state.metrics, start) });
|
||||
}
|
||||
for (const scheduled of pin.order) {
|
||||
const item = corpus.tasks.find(c => c.id === scheduled.id);
|
||||
for (const arm of scheduled.arms) {
|
||||
const cwd = path.join(temp, `${item.id}--${arm}--${scheduled.repeat}`);
|
||||
const environment = envs[['full', 'baseline', 'ecc-legacy'].includes(arm) ? arm : 'lean'];
|
||||
fs.mkdirSync(cwd);
|
||||
writeWorkspace(cwd, item.files);
|
||||
const start = state.metrics.length;
|
||||
outcomes.push({ ...outcomeTrial(item, arm, scheduled.repeat, repoRoot,
|
||||
executeAdapter(state, cwd, environment), cwd, environment, resolvedFamily, harvest, state.metrics),
|
||||
...metricsSince(state.metrics, start) });
|
||||
fs.rmSync(cwd, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
if (harvester) harvester.writeIndex();
|
||||
} finally { if (harvester) harvester.writeIndex(); fs.rmSync(temp, { recursive: true, force: true }); }
|
||||
const summary = summarize(outcomes, pin.arms);
|
||||
const insufficient = summary.distinctTasks < pin.minimumDistinctTasks || selection.length < pin.minimumDistinctTasks;
|
||||
const selectionSuccesses = selection.filter(row => row.passed).length;
|
||||
return { schemaVersion: 'ecc.context-eval.v2', registration: pin,
|
||||
evidence: injected ? 'injected-provider' : resolvedFamily === 'claude' ? 'claude-json' : 'codex-jsonl', installs,
|
||||
authentication: injected ? 'injected' : liveProvider.authentication, credentialsRetained: false,
|
||||
calls: state.calls, bounds: { maxCalls, deadlineMs, callTimeoutMs }, selection, outcomes, summary,
|
||||
selectionSummary: { n: selection.length, successes: selectionSuccesses,
|
||||
categories: [...new Set(selection.map(row => row.category))].map(category => ({ category,
|
||||
n: selection.filter(row => row.category === category).length,
|
||||
successes: selection.filter(row => row.category === category && row.passed).length })),
|
||||
interval: wilson(selectionSuccesses, selection.length), method: 'wilson-95-descriptive-purposive-sample' },
|
||||
gate: { status: insufficient ? 'insufficient-sample' : injected ? 'synthetic-only' : 'review-required',
|
||||
nonInferioritySupported: !insufficient && !injected && summary.pairs.every(p => p.interval[0] >= -pin.nonInferiorityMargin),
|
||||
releaseApproved: false }, nativeInvocation: 'unobserved',
|
||||
measurementScope: 'native-install-hidden-graded-coding-tasks',
|
||||
artifactRetention: harvester ? 'session-jsonl-per-task-trial' : 'none', ...metricsSince(state.metrics, 0) };
|
||||
}
|
||||
|
||||
module.exports = { loadCorpus, preregister, runEvaluation, parseCodexJsonl, parseClaudeJson, summarize, wilson,
|
||||
runCheck, runScoredCheck, createAuthLease, createCodexProvider, createClaudeProvider, prepareEnvironments,
|
||||
prepareClaudeEnvironments, exportLegacySource, providerFamily, resolveFamily, readClaudeKeychainToken };
|
||||
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
const fs = require('node:fs');
|
||||
const { preregister, runEvaluation, loadCorpus } = require('./ai-eval-lib');
|
||||
|
||||
function main(argv = process.argv.slice(2), injected = {}) {
|
||||
const flags = new Map();
|
||||
const switches = new Set(['--plan', '--allow-real-provider', '--allow-credentialed-tools', '--help']);
|
||||
const values = new Set(['--registration', '--model', '--executable', '--provider', '--auth-home', '--effort', '--repeats', '--max-calls', '--deadline-ms', '--artifact-dir', '--corpus', '--call-timeout-ms', '--arms']);
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const flag = argv[i];
|
||||
if (flags.has(flag) || (!switches.has(flag) && !values.has(flag))) throw new Error('Invalid evaluation arguments');
|
||||
if (values.has(flag) && (!argv[i + 1] || argv[i + 1].startsWith('--'))) throw new Error('Missing evaluation argument');
|
||||
flags.set(flag, switches.has(flag) ? true : argv[++i]);
|
||||
}
|
||||
if (flags.has('--help')) {
|
||||
return { usage: 'ai-eval.js --plan [--corpus FILE] [--arms a,b] [--repeats N] [--model MODEL --executable ABSOLUTE_PATH [--provider claude|codex] [--effort LEVEL]] | --allow-real-provider --registration FILE --model MODEL --executable ABSOLUTE_PATH [--provider claude|codex] [--allow-credentialed-tools (Claude only)] [--effort LEVEL (Codex only)] [--auth-home ABSOLUTE_DIR (Codex only)] [--corpus FILE] [--arms a,b] [--repeats N] [--max-calls N] [--deadline-ms N] [--call-timeout-ms N]. Claude auth: CLAUDE_CODE_OAUTH_TOKEN, ANTHROPIC_API_KEY, or the macOS Keychain login.' };
|
||||
}
|
||||
if (flags.get('--provider') !== undefined && !['claude', 'codex'].includes(flags.get('--provider'))) throw new Error('Provider must be claude or codex');
|
||||
if (flags.get('--provider') === 'claude' && flags.has('--effort')) throw new Error('Reasoning effort applies only to the Codex provider');
|
||||
if (flags.has('--allow-credentialed-tools') && (!flags.has('--allow-real-provider') || flags.get('--provider') !== 'claude')) {
|
||||
throw new Error('Credentialed-tool opt-in requires a real Claude evaluation');
|
||||
}
|
||||
const repeats = flags.has('--repeats') ? Number(flags.get('--repeats')) : 1;
|
||||
const corpus = flags.has('--corpus') ? loadCorpus(flags.get('--corpus')) : undefined;
|
||||
const arms = flags.has('--arms') ? flags.get('--arms').split(',').map(a => a.trim()).filter(Boolean) : undefined;
|
||||
if (flags.has('--plan')) {
|
||||
if (flags.has('--allow-real-provider')) throw new Error('Plan and provider execution are separate actions');
|
||||
return preregister({ repeats, model: flags.get('--model'), executable: flags.get('--executable'), effort: flags.get('--effort'),
|
||||
...(corpus ? { corpus } : {}), ...(arms ? { arms } : {}) });
|
||||
}
|
||||
if (!flags.has('--allow-real-provider') && !injected.provider) throw new Error('Real evaluation requires explicit opt-in');
|
||||
if (!flags.has('--registration')) throw new Error('Evaluation requires a preregistration file');
|
||||
const registration = JSON.parse(fs.readFileSync(flags.get('--registration'), 'utf8'));
|
||||
return runEvaluation({ ...injected, registration, repeats, allowRealProvider: flags.has('--allow-real-provider'),
|
||||
allowCredentialedTools: flags.has('--allow-credentialed-tools'),
|
||||
executable: flags.get('--executable'), model: flags.get('--model'), family: flags.get('--provider'), effort: flags.get('--effort'), authHome: flags.get('--auth-home'),
|
||||
artifactDir: flags.get('--artifact-dir'), ...(corpus ? { corpus } : {}), ...(arms ? { arms } : {}),
|
||||
...(flags.has('--max-calls') ? { maxCalls: Number(flags.get('--max-calls')) } : {}),
|
||||
...(flags.has('--deadline-ms') ? { deadlineMs: Number(flags.get('--deadline-ms')) } : {}),
|
||||
...(flags.has('--call-timeout-ms') ? { callTimeoutMs: Number(flags.get('--call-timeout-ms')) } : {}) });
|
||||
}
|
||||
if (require.main === module) {
|
||||
try { process.stdout.write(`${JSON.stringify(main())}\n`); }
|
||||
catch (error) {
|
||||
// Only fixed messages from this evaluator are shown; provider output and paths never reach stderr.
|
||||
const known = /^(Invalid|Missing|Real|Evaluation|Plan|Registration|Provider|Auth home|Native Codex version|Reasoning effort|Claude Keychain login|Claude)[^/\\]*$/.test(error?.message || '');
|
||||
process.stderr.write(`Evaluation stopped: ${known ? error.message : 'invalid arguments, registration, source, or provider configuration'}. Use --help.\n`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
module.exports = { main };
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,293 @@
|
||||
# ECC Complex-Task Evaluation (complex-tasks@1)
|
||||
|
||||
A reproducible, public benchmark of what ECC's context scoping does for **realistic
|
||||
agent work** — as opposed to the 30-task repair corpus (`ai-corpus.json`), which
|
||||
measures small, single-file fixes. This document is the preregistered methodology:
|
||||
it was written before the first provider call against this corpus, and it is the
|
||||
reference for anyone who wants to audit or rerun the evaluation.
|
||||
|
||||
## Research question
|
||||
|
||||
Does ECC's context engineering — the full skill library, manually picked skills
|
||||
(manual-lean), automatic skill matching (auto-lean), and the ECC-029 changes
|
||||
themselves — change what a frontier coding agent delivers on multi-step
|
||||
engineering tasks, and at what cost in tokens, time, and dollars?
|
||||
|
||||
## Arms
|
||||
|
||||
Five conditions, all launched through the same evaluator with real installs in
|
||||
isolated config homes, paired per task and repeat:
|
||||
|
||||
| Arm | What the agent gets | What it represents |
|
||||
|---|---|---|
|
||||
| `full` | Branch skill library installed + ECC context block (catalog/resources) | ECC with scoping machinery present but everything loaded |
|
||||
| `manual-lean` | lean profile + the maintainer-chosen canonical skill(s) injected | A user who knows exactly which ECC skill applies |
|
||||
| `auto-lean` | lean profile; ECC's trigger/proposal machinery picks and injects skills | The "auto" experience: no ECC knowledge required |
|
||||
| `ecc-legacy` | The full skill library **from the pinned pre-ECC-029 commit** (`legacy-source.json`, currently `e482e579` = `origin/main`), bare prompt, no context block | The typical current ECC user experience before the scoping work |
|
||||
| `baseline` | No ECC install, bare prompt | The provider with no ECC at all (overhead subtraction) |
|
||||
|
||||
`ecc-legacy` doubles as a replication control: where its install content matches
|
||||
`full`, score differences between them isolate the ECC-029 deltas (rewritten
|
||||
skill descriptions, scoping layer) rather than provider noise.
|
||||
|
||||
## The three tasks
|
||||
|
||||
Chosen to be the kind of work ECC exists for — multi-step, judgment-heavy,
|
||||
checkpointable — while deliberately **not** shaped around ECC's current skill
|
||||
list. Queries are written as a real user would phrase them, with no ECC
|
||||
vocabulary, no hints about which skill applies, and no instruction to use any
|
||||
particular methodology. Each task has one clear correct outcome and a
|
||||
deterministic, dependency-free grader.
|
||||
|
||||
1. **`webhook-relay`** (feature build). Finish an asynchronous webhook delivery
|
||||
worker: retries with exponential backoff, dead-lettering after 5 attempts,
|
||||
status reporting, under load. Graded by 9 in-process behavioral probes
|
||||
(delivery after failures, exact attempt counts, backoff timing window,
|
||||
dead-lettering, error capture, API preservation, concurrency).
|
||||
*Why it belongs here:* everyday backend feature work where test discipline
|
||||
and backend patterns genuinely change outcomes; canonical skill:
|
||||
`tdd-workflow` (a second skill would exceed the 32 KB selection budget —
|
||||
itself a measured constraint of the scoping layer).
|
||||
|
||||
2. **`incident-triage`** (debugging / root cause). Finance reports one-cent
|
||||
total errors since yesterday's deploy. The repo contains three changelog
|
||||
entries (two red herrings), an incident log with concrete amounts, and a
|
||||
regression: a "readability" refactor that switched integer-cent math to
|
||||
decimal-factor floats, which under-rounds exact half-cent boundaries.
|
||||
Graded by 5 boundary-value totals the float path provably gets wrong, one
|
||||
regression probe, and 2 deterministic checks on the required `INCIDENT.md`
|
||||
(names the right changelog entry, explains the rounding mechanism).
|
||||
*Why it belongs here:* evidence-driven diagnosis under uncertainty is the
|
||||
highest-leverage agent workflow; guessing is penalized because red herrings
|
||||
are plausible; canonical skill: `orch-fix-defect`.
|
||||
|
||||
3. **`sentinel-api`** (security review + hardening). A paste service whose
|
||||
README documents the secure contract while the code violates it five ways:
|
||||
hardcoded admin token, path traversal, reflected XSS, predictable delete
|
||||
tokens, no body-size limit. Graded by 10 exploit probes (each vulnerability
|
||||
must actually be closed) plus functional regression probes (the documented
|
||||
API must still work), including one encoded-traversal variant so partial
|
||||
fixes score partially.
|
||||
*Why it belongs here:* security review is a canonical agent task with
|
||||
objectively checkable outcomes; canonical skill: `security-review`.
|
||||
|
||||
### Why these tests are effective
|
||||
|
||||
- **Realism over benchmark gaming.** Each task is a small production-shaped
|
||||
repo with docs, tests, logs, and changelogs — the inputs a real engineer (or
|
||||
a real user of an agent harness) actually has. Nothing references ECC.
|
||||
- **Correctness is decidable.** Every grader assertion is deterministic:
|
||||
behavioral probes against the agent's own running service, exact numeric
|
||||
answers on boundary cases, static source checks, exploit probes. No LLM
|
||||
judges, no rubrics, no human scoring.
|
||||
- **Partial credit.** Graders emit `ECC_EVAL_SCORE {"score": 0..1}`, so "found
|
||||
4 of 5 vulnerabilities" registers as 0.9-of-task progress instead of a binary
|
||||
failure. Pass/fail (score = 1.0) is reported alongside the mean score.
|
||||
- **Hard to luck into.** Red herrings (incident-triage), timing windows
|
||||
(webhook-relay), and exploit-verified fixes (sentinel-api) mean superficial
|
||||
plausible work scores low.
|
||||
- **Fair across arms.** Hidden graders run only after the agent exits, from a
|
||||
read-only sandbox; the agent never sees the grader. The same grader scores
|
||||
every arm identically. Reference solutions score 1.0 and as-shipped fixtures
|
||||
score ≤ 0.3 (`verify-checks.js` proves both before any provider call).
|
||||
|
||||
## Measured variables
|
||||
|
||||
Per trial (one task × arm × repeat), from the provider's own usage events:
|
||||
|
||||
- **Fresh input tokens** (input + cache-creation), **cache-read tokens**,
|
||||
**output tokens** — the context-cost story.
|
||||
- **Provider calls** per trial (1, or 2 when auto-lean needs a routing proposal).
|
||||
- **Wall-clock time** per provider call and per trial (ms) — time to completion.
|
||||
- **Score** (0..1) and **pass** (score = 1.0) from the hidden grader.
|
||||
- **API-equivalent cost**, derived at analysis time at Anthropic Opus list
|
||||
prices ($15 / $1.50 / $75 per million fresh-input / cache-read / output
|
||||
tokens). This is an accounting convention for comparison, not a billing
|
||||
claim; subscription pricing differs.
|
||||
- **Skill routing** (auto-lean): which skills the trigger/proposal machinery
|
||||
selected vs the maintainer-chosen canonical set, reported as the selection
|
||||
probe accuracy — the direct measure of "automatic skill matching".
|
||||
|
||||
Comparisons are **within-run only**: same provider, model, executable digest,
|
||||
corpus digest, and source digest, paired by task and repeat. Cross-run and
|
||||
cross-provider comparisons are invalid by design. This is a descriptive pilot
|
||||
(3 tasks × 5 arms × 4 repeats = 60 trials): it estimates direction and
|
||||
magnitude, not population statistics, and the report says so in its gate block.
|
||||
|
||||
## Reproducing or auditing
|
||||
|
||||
Everything below is committed; there are no hidden inputs.
|
||||
|
||||
```bash
|
||||
# 1. Inspect the tasks: fixtures, queries, graders, and reference solutions.
|
||||
ls docker/context-profiles/complex-eval/cases/
|
||||
ls docker/context-profiles/complex-eval/reference/
|
||||
|
||||
# 2. Prove the graders: reference solutions must score 1.0, fixtures below 1.0.
|
||||
node docker/context-profiles/complex-eval/verify-checks.js
|
||||
|
||||
# 3. Rebuild the corpus after any fixture edit (digest-pinned at registration).
|
||||
node docker/context-profiles/complex-eval/build-corpus.js
|
||||
|
||||
# 4. Preregister (pins corpus, source, model, executable digests; no provider).
|
||||
node docker/context-profiles/ai-eval.js --plan \
|
||||
--corpus docker/context-profiles/complex-corpus.json --repeats 4 \
|
||||
--provider claude --model <model> --executable /absolute/path/to/claude \
|
||||
> registration.json
|
||||
|
||||
# 5. Run (requires your own Claude subscription login or API key).
|
||||
node docker/context-profiles/ai-eval.js --allow-real-provider --allow-credentialed-tools \
|
||||
--registration registration.json \
|
||||
--corpus docker/context-profiles/complex-corpus.json \
|
||||
--provider claude --model <model> --executable /absolute/path/to/claude \
|
||||
--repeats 4 --max-calls 400 --deadline-ms 25200000 --call-timeout-ms 600000 \
|
||||
--artifact-dir /absolute/path/for/transcripts > report.json
|
||||
```
|
||||
|
||||
Claude task tools inherit the provider credential through the CLI process and can read it. Use
|
||||
`--allow-credentialed-tools` only with a trusted local corpus and credential. Without that
|
||||
explicit flag, real Claude task evaluation stops before a provider call; selection-only calls
|
||||
remain tool-free. This development evaluator does not provide a credential isolation boundary.
|
||||
|
||||
The registration digest binds the exact corpus, evaluator source, model, and
|
||||
executable; the run refuses to start if any of them drift, and aborts if the
|
||||
tree changes mid-run. `--artifact-dir` retains per-trial session transcripts
|
||||
for independent inspection (they never enter the report). The `ecc-legacy` arm
|
||||
is pinned by commit in `legacy-source.json` and exported from git objects at
|
||||
run time. The Codex provider is unsupported for this corpus (the legacy arm has
|
||||
no Codex install path); `--provider claude` is required.
|
||||
|
||||
## Known limits
|
||||
|
||||
- Three tasks is a probe, not a census: treat intervals as descriptive.
|
||||
- Tasks are Node.js/stdlib by construction (graders must be hermetic); results
|
||||
say nothing about other ecosystems directly.
|
||||
- `webhook-relay` uses wall-clock backoff windows; bounds are wide (250–5000ms)
|
||||
but loaded machines could in principle flake a timing probe. The grader
|
||||
reports each probe individually so flakes are visible.
|
||||
- Provider behavior varies week to week; the pinned model/executable digests
|
||||
make a rerun comparable only within the same pin.
|
||||
- Fixture wart observed in the 2026-09-25 run: on Node 24, `node --test test/`
|
||||
no longer scans the directory the way Node 22 did, so `npm test` fails as
|
||||
shipped. This is identical for every arm (the task says to make `npm test`
|
||||
pass, and agents fix the script), so fairness holds, but it adds unplanned
|
||||
work per trial. A future corpus revision should ship a portable test script.
|
||||
|
||||
## complex-tasks@2 (discriminative revision)
|
||||
|
||||
The @1 run saturated: every arm scored 1.000 on every task, so only economics
|
||||
and routing differed. @2 (`cases2/`, built to `complex-corpus-v2.json`) is
|
||||
designed to discriminate on the axes users actually pay for — correctness on
|
||||
traps, solution efficiency, spec thoroughness — with wide partial-credit
|
||||
spreads. The @1 corpus and its report stay untouched for comparability.
|
||||
|
||||
1. **`keccak-selector`** (domain-knowledge trap). Implement Ethereum function
|
||||
selectors from scratch, stdlib only. The trap: Node's crypto offers
|
||||
SHA3-256, which shares the Keccak-f[1600] permutation but differs in
|
||||
padding — the naive one-liner is wrong for every vector (verified: the
|
||||
naive control scores 0.25, format checks only). Graded by 9 selector
|
||||
vectors including a padding edge case, all cross-validated against Node's
|
||||
SHA3-256 on shared-permutation inputs. Canonical skill: `nodejs-keccak256`.
|
||||
*Hypothesis:* the skill body carries exactly this knowledge; bare agents
|
||||
must rediscover it.
|
||||
|
||||
2. **`event-stats-api`** (correctness edges + measured efficiency). A shipped
|
||||
implementation that is both wrong on the documented edge semantics
|
||||
(interpolated instead of nearest-rank percentiles, zeros instead of nulls,
|
||||
unrounded averages, missing 400s) and algorithmically naive (full-log scan
|
||||
and sort per query). Graded by 10 independently computed correctness probes
|
||||
plus a measured 2,000-query performance budget (threshold 6s; shipped naive
|
||||
~7.7s, reference ~1.5s — calibrated on the grading machine in
|
||||
`calibrate-stats.js`). Canonical skill: `backend-patterns`. *Hypothesis:*
|
||||
solution *efficiency* separates arms even when correctness doesn't.
|
||||
|
||||
3. **`forge-cli`** (spec thoroughness + robustness). Twelve contractual
|
||||
behaviors with exact messages, exit codes, sorting, and a never-throw
|
||||
guarantee, graded by 26 checks including junk-input fuzzing and static
|
||||
hygiene (no leftover TODO/FIXME, no new dependencies). Canonical skill:
|
||||
`tdd-workflow`. *Hypothesis:* checklist discipline shows up as breadth of
|
||||
completion, and partial credit spreads the distribution.
|
||||
|
||||
First @2 run uses `claude-opus-4-8` (cost discipline); the corpus is
|
||||
provider- and model-pinned per run, so a later Opus 5.5 rerun on the same
|
||||
digest measures the model difference directly. repeats=2 (30 trials): simple
|
||||
experimentation, expand later.
|
||||
|
||||
## complex-tasks@3 (vagueness and horizon; arms: auto-lean vs baseline)
|
||||
|
||||
@2 still saturated on outcomes (30/30) — enumerated specs are within the
|
||||
model's cold competence. @3 (`cases3/`, built to `complex-corpus-v3.json`)
|
||||
moves grading to what users actually complain about (see the complaint
|
||||
taxonomy in this file's discussion: happy-path-only work, unverified
|
||||
completion, skipped implied work, convention drift, concurrency blindness).
|
||||
Everything graded is discoverable from repo docs visible to every arm — the
|
||||
question is whether agents reliably *do* all of it under vague instruction.
|
||||
|
||||
1. **`chained-tickets`** (long horizon). Four sequential tickets in one
|
||||
accumulating workspace — build a link shortener core, then vague tickets:
|
||||
"links need to survive a restart", "we're seeing abuse, deal with it",
|
||||
"track redirect hits, consistent with the existing API". 33 hidden probes
|
||||
across the four steps grade function, convention compliance (error
|
||||
envelope, layering — pinned in a visible CONTRIBUTING.md), and implied
|
||||
work (changelog entries, growing tests, accurate README). Stepped trials
|
||||
grade each ticket after its call; a failed ticket ends the chain.
|
||||
2. **`production-ready`** (vague prompt, heavy implication). "This goes to
|
||||
production Monday — get it ready." A documented production bar
|
||||
(validation envelopes, body limits, /health, structured request logs, env
|
||||
config, graceful SIGTERM, nosniff, error-path tests, changelog) graded by
|
||||
16 probes against a naive prototype. Fixture scores 0.063.
|
||||
3. **`idempotent-webhooks`** (the "almost right" trap). A payment receiver
|
||||
whose shipped code has a textbook check-then-act race (INC-104). Hidden
|
||||
grader fires 50 concurrent identical deliveries plus replay, already-paid,
|
||||
mixed-storm, and contract probes. The naive fixture double-applies and
|
||||
crashes on unknown orders (0.25). Exactly-once requires claiming events
|
||||
synchronously — the discipline skills like `error-handling` encode.
|
||||
|
||||
Grader robustness (hard-won, now fixed and unit-tested): a graded server runs
|
||||
in-process, so a crashing server kills the grader. Graders install
|
||||
uncaughtException/unhandledRejection handlers, emit their score line via
|
||||
`process.stdout.write` (immune to the log-capture patching used in probes),
|
||||
pre-declare their check totals (unreached checks score zero), and the
|
||||
evaluator itself treats a score-advertising grader that printed nothing as a
|
||||
zero (`graderDied` guard in `runScoredCheck`). Stepped graders may write to
|
||||
the workspace (persistence probes); single-step graders stay read-only.
|
||||
|
||||
First @3 run: arms `auto-lean` and `baseline` only, repeats=1,
|
||||
`claude-opus-4-8` — the direct test of "ECC auto-routing vs no harness" on
|
||||
quality, time, and tokens. Full-arm and Opus 5.5 replications follow if the
|
||||
spread shows up.
|
||||
|
||||
## complex-tasks@4 (learning loops; adds recurring-incident)
|
||||
|
||||
@4 (`cases4/`, built to `complex-corpus-v4.json`) keeps the three @3 cases
|
||||
unchanged and adds a fourth targeting a different ECC value prop: converting
|
||||
a fix into durable, reusable prevention — and *reusing your own artifacts*
|
||||
later in the session. Baseline agents can hold this in context; ECC's claim
|
||||
is that skills/workflows make it systematic.
|
||||
|
||||
4. **`recurring-incident`** (learning loop / institutional memory). Three
|
||||
chained steps against a dependency-free payments service whose gateway
|
||||
records side effects in an append-only JSONL ledger. Step 1: keyless
|
||||
refund retries double-refund (INC-201/214/227 "third time this quarter"
|
||||
trail in `docs/incidents.md`); the vague ask is "make sure this stops
|
||||
being a recurring incident." Probes: functional correctness across a
|
||||
module reload (kills in-memory-only fixes) [0.40], regression test wired
|
||||
into the suite + mutation probe [0.30], a durable prevention runbook
|
||||
[0.20], and the mechanism living in one shared helper module [0.10].
|
||||
Step 2: payout retries, "same family of problem" — graded on REUSE of
|
||||
the step-1 helper (static import check + no divergent inline
|
||||
reimplementation) [0.30] alongside function [0.40], test+mutation [0.20],
|
||||
doc update [0.10]. Step 3: "write the handoff note" — graded on
|
||||
existence [0.20], every referenced path actually existing on disk [0.30],
|
||||
naming the helper + prevention procedure [0.30], and covering both
|
||||
incidents [0.20]. Manual skills: `error-handling`, `continuous-learning`.
|
||||
*Hypothesis:* learning-loop behavior (abstract once, reuse, document,
|
||||
hand off) separates harnessed arms from baseline even when raw bug-fix
|
||||
competence doesn't.
|
||||
|
||||
Verification: reference 1.000 on all steps of all four cases; naive
|
||||
recurring-incident scores 0.20 / 0.00 / 0.20 per step; fixtures 0.00–0.25.
|
||||
|
||||
First @4 run: arm `auto-lean` only, repeats=1, `claude-opus-5-5` — the
|
||||
model-difference probe against the @3 opus-4-8 numbers on the shared cases,
|
||||
plus first signal on the learning-loop case.
|
||||
@@ -0,0 +1,67 @@
|
||||
'use strict';
|
||||
// Development tool: assembles a complex corpus JSON from a reviewed fixture
|
||||
// tree. Usage: node build-corpus.js [casesDir=cases] [outFile=complex-corpus.json] [corpusId=complex-tasks@1]
|
||||
// Run after editing any fixture, query, or grader; commit the tree and the
|
||||
// regenerated corpus together.
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
|
||||
const root = __dirname;
|
||||
const casesDir = path.join(root, process.argv[2] || 'cases');
|
||||
const OUT = path.join(root, '..', process.argv[3] || 'complex-corpus.json');
|
||||
const corpusId = process.argv[4] || 'complex-tasks@1';
|
||||
|
||||
function collect(directory, prefix = '') {
|
||||
const files = {};
|
||||
for (const entry of fs.readdirSync(directory, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
|
||||
const relative = prefix ? `${prefix}/${entry.name}` : entry.name;
|
||||
if (entry.isDirectory()) Object.assign(files, collect(path.join(directory, entry.name), relative));
|
||||
else if (entry.isFile()) files[relative] = fs.readFileSync(path.join(directory, entry.name), 'utf8');
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
const tasks = [];
|
||||
const selection = [];
|
||||
for (const id of fs.readdirSync(casesDir).sort()) {
|
||||
const directory = path.join(casesDir, id);
|
||||
const meta = JSON.parse(fs.readFileSync(path.join(directory, 'meta.json'), 'utf8'));
|
||||
if (meta.id !== id || !/^[a-z][a-z0-9-]{0,63}$/.test(id)) throw new Error(`Invalid task metadata in ${id}`);
|
||||
const files = collect(path.join(directory, 'files'));
|
||||
const stepsDir = path.join(directory, 'steps');
|
||||
let task;
|
||||
if (fs.existsSync(stepsDir)) {
|
||||
const steps = fs.readdirSync(stepsDir).sort().map((name, index) => ({
|
||||
query: fs.readFileSync(path.join(stepsDir, name, 'query.md'), 'utf8').trim(),
|
||||
check: fs.readFileSync(path.join(stepsDir, name, 'check.cjs'), 'utf8'),
|
||||
...(meta.steps?.[index]?.manualIds ? { manualIds: meta.steps[index].manualIds } : {}),
|
||||
...((meta.steps?.[index]?.checkTimeoutMs || meta.checkTimeoutMs)
|
||||
? { checkTimeoutMs: meta.steps?.[index]?.checkTimeoutMs || meta.checkTimeoutMs } : {}),
|
||||
}));
|
||||
task = { id, category: meta.category, manualIds: meta.manualIds || [], files, steps };
|
||||
} else {
|
||||
const query = fs.readFileSync(path.join(directory, 'query.md'), 'utf8').trim();
|
||||
task = { id, category: meta.category, manualIds: meta.manualIds,
|
||||
...(meta.checkTimeoutMs ? { checkTimeoutMs: meta.checkTimeoutMs } : {}),
|
||||
query, files, check: fs.readFileSync(path.join(directory, 'check.cjs'), 'utf8') };
|
||||
}
|
||||
tasks.push(task);
|
||||
selection.push({ id: meta.selection.id, category: meta.selection.category,
|
||||
query: meta.selection.query || task.query || task.steps.map(step => step.query).join(' '),
|
||||
expectedIds: meta.selection.expectedIds });
|
||||
}
|
||||
|
||||
const corpus = {
|
||||
schemaVersion: 'ecc.context-eval-complex-corpus.v1',
|
||||
id: corpusId,
|
||||
sampling: 'Realistic multi-file engineering tasks, fixed before any provider call, with deterministic '
|
||||
+ 'hidden graders scoring partial credit (ECC_EVAL_SCORE). Descriptive pilot: no '
|
||||
+ 'population-representativeness claim. See complex-eval/DESIGN.md for the preregistered methodology.',
|
||||
minimumDistinctTasks: tasks.length,
|
||||
nonInferiorityMargin: 0.05,
|
||||
selection,
|
||||
tasks,
|
||||
};
|
||||
fs.writeFileSync(OUT, `${JSON.stringify(corpus, null, 1)}\n`);
|
||||
console.log(`wrote ${path.basename(OUT)} (${corpusId}): ${tasks.length} tasks, ${selection.length} selection probes, `
|
||||
+ `${tasks.reduce((sum, task) => sum + Object.keys(task.files).length, 0)} fixture files`);
|
||||
@@ -0,0 +1,73 @@
|
||||
'use strict';
|
||||
// Calibration harness (not shipped in the corpus): measures the 2,000-query
|
||||
// workload wall time for the shipped naive app and the reference app, each
|
||||
// staged as a standalone copy (fixture; fixture + reference overlay).
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
|
||||
const root = __dirname;
|
||||
const fixture = path.join(root, 'cases2', 'event-stats-api', 'files');
|
||||
const overlay = path.join(root, 'reference2', 'event-stats-api');
|
||||
|
||||
function stage(withOverlay) {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-calib-'));
|
||||
const copy = (from, to) => {
|
||||
for (const entry of fs.readdirSync(from, { withFileTypes: true })) {
|
||||
const target = path.join(to, entry.name);
|
||||
if (entry.isDirectory()) { fs.mkdirSync(target, { recursive: true }); copy(path.join(from, entry.name), target); }
|
||||
else fs.copyFileSync(path.join(from, entry.name), target);
|
||||
}
|
||||
};
|
||||
copy(fixture, dir);
|
||||
if (withOverlay) copy(overlay, dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
function lcg(seed) {
|
||||
let state = seed >>> 0;
|
||||
return () => {
|
||||
state = (Math.imul(state, 1664525) + 1013904223) >>> 0;
|
||||
return state / 2 ** 32;
|
||||
};
|
||||
}
|
||||
|
||||
function workload(types, epoch, span) {
|
||||
const rand = lcg(777);
|
||||
const queries = [];
|
||||
for (let i = 0; i < 2000; i++) {
|
||||
const type = types[Math.floor(rand() * types.length)];
|
||||
const start = epoch + Math.floor(rand() * span * 0.7);
|
||||
queries.push({ type, from: start, to: start + Math.floor(rand() * span * 0.5) });
|
||||
}
|
||||
return queries;
|
||||
}
|
||||
|
||||
async function measure(label, dir) {
|
||||
const { createApp } = require(path.join(dir, 'src', 'app.js'));
|
||||
const { TYPES, EPOCH_MS, SPAN_MS } = require(path.join(dir, 'src', 'data.js'));
|
||||
const app = createApp();
|
||||
await new Promise(resolve => app.listen(0, '127.0.0.1', resolve));
|
||||
const port = app.address().port;
|
||||
const queries = workload(TYPES, EPOCH_MS, SPAN_MS);
|
||||
const started = Date.now();
|
||||
for (let i = 0; i < queries.length; i += 20) {
|
||||
await Promise.all(queries.slice(i, i + 20).map(q =>
|
||||
fetch(`http://127.0.0.1:${port}/stats?type=${q.type}&from=${q.from}&to=${q.to}`).then(r => r.json())));
|
||||
}
|
||||
const elapsed = Date.now() - started;
|
||||
app.close();
|
||||
console.log(`${label}: ${elapsed}ms for 2000 queries`);
|
||||
return elapsed;
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const naiveDir = stage(false);
|
||||
const refDir = stage(true);
|
||||
await measure('naive 1 ', naiveDir);
|
||||
await measure('naive 2 ', naiveDir);
|
||||
await measure('reference 1 ', refDir);
|
||||
await measure('reference 2 ', refDir);
|
||||
fs.rmSync(naiveDir, { recursive: true, force: true });
|
||||
fs.rmSync(refDir, { recursive: true, force: true });
|
||||
})();
|
||||
@@ -0,0 +1,45 @@
|
||||
'use strict';
|
||||
// Hidden grader for incident-triage: checks exact totals on boundary orders and
|
||||
// the root-cause report. Prints ECC_EVAL_SCORE and always exits 0.
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
|
||||
const checks = [];
|
||||
const record = (name, ok) => checks.push({ name, ok: Boolean(ok) });
|
||||
|
||||
let computeOrderTotal;
|
||||
try { ({ computeOrderTotal } = require(path.join(process.cwd(), 'src', 'totals.js'))); } catch { /* scored below */ }
|
||||
|
||||
// Boundary orders where decimal-factor float math under-rounds by a cent;
|
||||
// expected values follow the README pricing rules (integer cents, half-up per line).
|
||||
const boundary = [
|
||||
{ lines: [{ priceCents: 165, quantity: 1 }], discountPercent: 30, expected: 116 },
|
||||
{ lines: [{ priceCents: 250, quantity: 1 }], discountPercent: 7, expected: 233 },
|
||||
{ lines: [{ priceCents: 325, quantity: 1 }], discountPercent: 30, expected: 228 },
|
||||
{ lines: [{ priceCents: 345, quantity: 1 }], discountPercent: 30, expected: 242 },
|
||||
{ lines: [{ priceCents: 165, quantity: 1 }, { priceCents: 325, quantity: 1 }], discountPercent: 30, expected: 344 },
|
||||
];
|
||||
|
||||
if (typeof computeOrderTotal === 'function') {
|
||||
boundary.forEach((order, index) => {
|
||||
let actual = NaN;
|
||||
try { actual = computeOrderTotal({ lines: order.lines, discountPercent: order.discountPercent }); } catch { /* wrong */ }
|
||||
record(`boundary-total-${index + 1}`, actual === order.expected);
|
||||
});
|
||||
let plain = NaN;
|
||||
try { plain = computeOrderTotal({ lines: [{ priceCents: 1000, quantity: 2 }], discountPercent: 0 }); } catch { /* wrong */ }
|
||||
record('undiscounted-total-unchanged', plain === 2000);
|
||||
} else {
|
||||
for (let index = 0; index < boundary.length; index++) record(`boundary-total-${index + 1}`, false);
|
||||
record('undiscounted-total-unchanged', false);
|
||||
}
|
||||
|
||||
let incident = '';
|
||||
try { incident = fs.readFileSync(path.join(process.cwd(), 'INCIDENT.md'), 'utf8'); } catch { /* missing */ }
|
||||
record('incident-identifies-C-2', /C-2/.test(incident));
|
||||
record('incident-explains-rounding', /round|float|decimal|cent/i.test(incident));
|
||||
|
||||
const ok = checks.filter(c => c.ok).length;
|
||||
for (const c of checks) console.log(`${c.ok ? 'ok' : 'not ok'} - ${c.name}`);
|
||||
console.log(`ECC_EVAL_SCORE ${JSON.stringify({ score: ok / checks.length, passed: ok, total: checks.length })}`);
|
||||
process.exit(0);
|
||||
@@ -0,0 +1,11 @@
|
||||
# Changelog
|
||||
|
||||
## 2026-09-23 deploy
|
||||
|
||||
- **C-1**: request logging switched to JSON lines (`src/request-log.js`).
|
||||
Log volume and format only; no request-handling behavior changed.
|
||||
- **C-2**: totals computation refactored for readability (`src/totals.js`).
|
||||
The old cents-as-integers helper was replaced with a direct decimal
|
||||
expression that reviewers found easier to follow. No behavior change intended.
|
||||
- **C-3**: inventory client timeout raised from 2s to 5s (`src/inventory-client.js`).
|
||||
Reduces spurious failures when the inventory service is slow.
|
||||
@@ -0,0 +1,21 @@
|
||||
# order-service
|
||||
|
||||
Computes order totals for the checkout service.
|
||||
|
||||
## Pricing rules
|
||||
|
||||
An order is `{ "lines": [{ "priceCents": number, "quantity": number }], "discountPercent": number }`.
|
||||
|
||||
- All prices are integer cents. There is no such thing as a fraction of a cent
|
||||
in an order total.
|
||||
- The discount applies per line: `lineCents = priceCents * quantity * (100 - discountPercent) / 100`,
|
||||
rounded **half-up** to the nearest cent (0.5 rounds up).
|
||||
- The order total is the sum of the rounded line totals, in integer cents.
|
||||
|
||||
`src/totals.js` is CommonJS and exports `computeOrderTotal(order)` returning the
|
||||
total in integer cents. Run the tests with `npm test`.
|
||||
|
||||
## Operations
|
||||
|
||||
- `CHANGELOG.md` records what shipped in each deploy.
|
||||
- `evidence/incident.txt` holds the finance team's findings for the current incident.
|
||||
@@ -0,0 +1,5 @@
|
||||
2026-09-24T08:57:11Z finance-review order=ORD-2204 note="charged_total_cents=115 expected_total_cents=116 lines=[{priceCents:165,quantity:1}] discountPercent=30"
|
||||
2026-09-24T09:14:02Z finance-review order=ORD-2291 note="charged_total_cents=232 expected_total_cents=233 lines=[{priceCents:250,quantity:1}] discountPercent=7"
|
||||
2026-09-24T09:41:37Z finance-review order=ORD-2310 note="charged_total_cents=227 expected_total_cents=228 lines=[{priceCents:325,quantity:1}] discountPercent=30"
|
||||
2026-09-24T10:05:19Z support-ticket customer="ORDER-2310 looks like it undercharged me by a cent vs the invoice email"
|
||||
2026-09-24T10:22:48Z finance-review summary="12 of 4,813 orders since the 2026-09-23 deploy are off by exactly one cent, always in the store's favor; all pre-deploy orders reconcile"
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"name": "order-service",
|
||||
"private": true,
|
||||
"type": "commonjs",
|
||||
"scripts": { "test": "node --test test/" }
|
||||
}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
'use strict';
|
||||
|
||||
// Changed 2026-09-23 (C-3): the inventory service has been slow this week;
|
||||
// give it 5s instead of 2s before declaring a failure.
|
||||
const INVENTORY_TIMEOUT_MS = 5000;
|
||||
|
||||
function inventoryClientOptions() {
|
||||
return { timeoutMs: INVENTORY_TIMEOUT_MS, retries: 2 };
|
||||
}
|
||||
|
||||
module.exports = { inventoryClientOptions };
|
||||
@@ -0,0 +1,13 @@
|
||||
'use strict';
|
||||
|
||||
// Changed 2026-09-23 (C-1): emit request logs as JSON lines so the log
|
||||
// pipeline can parse them without regexes.
|
||||
function logRequest(req) {
|
||||
console.log(JSON.stringify({
|
||||
method: req.method,
|
||||
url: req.url,
|
||||
at: new Date().toISOString(),
|
||||
}));
|
||||
}
|
||||
|
||||
module.exports = { logRequest };
|
||||
@@ -0,0 +1,14 @@
|
||||
'use strict';
|
||||
|
||||
// Refactored 2026-09-23 (C-2): express the discount math directly with a
|
||||
// decimal factor instead of the old integer-cents helper, which reviewers
|
||||
// found hard to follow.
|
||||
function computeOrderTotal(order) {
|
||||
let total = 0;
|
||||
for (const line of order.lines) {
|
||||
total += Math.round(line.priceCents * line.quantity * (1 - order.discountPercent / 100));
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
module.exports = { computeOrderTotal };
|
||||
@@ -0,0 +1,16 @@
|
||||
'use strict';
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const { computeOrderTotal } = require('../src/totals');
|
||||
|
||||
test('sums lines without a discount', () => {
|
||||
assert.equal(computeOrderTotal({ lines: [{ priceCents: 1000, quantity: 2 }], discountPercent: 0 }), 2000);
|
||||
});
|
||||
|
||||
test('applies a clean quarter discount', () => {
|
||||
assert.equal(computeOrderTotal({ lines: [{ priceCents: 2000, quantity: 1 }], discountPercent: 25 }), 1500);
|
||||
});
|
||||
|
||||
test('multiplies quantity before discounting', () => {
|
||||
assert.equal(computeOrderTotal({ lines: [{ priceCents: 400, quantity: 3 }], discountPercent: 50 }), 600);
|
||||
});
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user