From 39416e3cdde43e8a26a64d6a1b1bda366bf739d2 Mon Sep 17 00:00:00 2001 From: Andres Tuul Date: Thu, 6 Aug 2026 22:36:15 +0300 Subject: [PATCH] fix(lib): correct stale model rates in the shared cost estimator and skill `scripts/lib/cost-estimate.js` carries a second, independent copy of the rate table that `scripts/hooks/cost-tracker.js` had, with the same defect: every `opus` model priced at $15/$75, which are Claude 3 Opus era rates. Opus 4.5 and later bill at $5/$25, so every current-generation Opus estimate was exactly 3x real spend. Two more errors in the same table: `haiku` was $0.80/$4.00, which is Claude 3.5 Haiku rather than Haiku 4.5's $1/$5; and Fable and Mythos had no bucket at all, so they fell through to `sonnet` and were understated 3.3x. Legacy buckets are added rather than overwriting, so correcting the current generation does not reprice the old one. `opusLegacy` keeps $15/$75 for the three models that really billed it (Claude 3 Opus, Opus 4.0, Opus 4.1) and `haikuLegacy` keeps $0.80/$4.00 for Claude 3.5 Haiku. The matching regexes are the ones already used by the cost tracker, so the two tables now agree on which model is legacy. Opus 4.0's snapshot is `claude-opus-4-20250514` with no minor segment, which is why the bare `opus-4-` form is matched separately: an `opus-4-0` substring alone misses it. `skills/cost-aware-llm-pipeline/SKILL.md` and its zh-CN and ja-JP translations published the same stale numbers as prose. The Relative Cost column is derived from the rates, so it is corrected with them: against a corrected Haiku 4.5 baseline the multiples are now exact, which is why the approximation markers are dropped. `RATE_TABLE` keeps its existing keys and shape, so the export stays backward compatible. Tests: the existing test pinned the stale values and was updated to pin the correct ones. Coverage is added for each legacy spelling (alias, dated snapshot, Vertex `@` form, Bedrock prefix), for the current Opus line, and for the Fable and Mythos bucket. --- .../skills/cost-aware-llm-pipeline/SKILL.md | 6 +- .../skills/cost-aware-llm-pipeline/SKILL.md | 6 +- scripts/lib/cost-estimate.js | 50 +++++++-- skills/cost-aware-llm-pipeline/SKILL.md | 6 +- tests/lib/cost-estimate.test.js | 101 +++++++++++++++--- 5 files changed, 137 insertions(+), 32 deletions(-) diff --git a/docs/ja-JP/skills/cost-aware-llm-pipeline/SKILL.md b/docs/ja-JP/skills/cost-aware-llm-pipeline/SKILL.md index 97e95f06d..adb4b532c 100644 --- a/docs/ja-JP/skills/cost-aware-llm-pipeline/SKILL.md +++ b/docs/ja-JP/skills/cost-aware-llm-pipeline/SKILL.md @@ -155,9 +155,9 @@ def process(text: str, config: Config, tracker: CostTracker) -> tuple[Result, Co | モデル | 入力（$/1Mトークン） | 出力（$/1Mトークン） | 相対コスト | |-------|---------------------|----------------------|---------------| -| Haiku 4.5 | $0.80 | $4.00 | 1x | -| Sonnet 4.6 | $3.00 | $15.00 | 約4x | -| Opus 4.5 | $15.00 | $75.00 | 約19x | +| Haiku 4.5 | $1.00 | $5.00 | 1x | +| Sonnet 4.6 | $3.00 | $15.00 | 3x | +| Opus 4.5 | $5.00 | $25.00 | 5x | ## ベストプラクティス diff --git a/docs/zh-CN/skills/cost-aware-llm-pipeline/SKILL.md b/docs/zh-CN/skills/cost-aware-llm-pipeline/SKILL.md index 9af5a8466..396171570 100644 --- a/docs/zh-CN/skills/cost-aware-llm-pipeline/SKILL.md +++ b/docs/zh-CN/skills/cost-aware-llm-pipeline/SKILL.md @@ -155,9 +155,9 @@ def process(text: str, config: Config, tracker: CostTracker) -> tuple[Result, Co | 模型 | 输入（美元/百万令牌） | 输出（美元/百万令牌） | 相对成本 | |-------|---------------------|----------------------|---------------| -| Haiku 4.5 | $0.80 | $4.00 | 1x | -| Sonnet 4.6 | $3.00 | $15.00 | ~4x | -| Opus 4.5 | $15.00 | $75.00 | ~19x | +| Haiku 4.5 | $1.00 | $5.00 | 1x | +| Sonnet 4.6 | $3.00 | $15.00 | 3x | +| Opus 4.5 | $5.00 | $25.00 | 5x | ## 最佳实践 diff --git a/scripts/lib/cost-estimate.js b/scripts/lib/cost-estimate.js index a1651a8c9..0dfa9c834 100644 --- a/scripts/lib/cost-estimate.js +++ b/scripts/lib/cost-estimate.js @@ -3,18 +3,51 @@ /** * Shared cost estimation for ECC hooks. * - * Approximate per-1M-token blended rates (conservative defaults). + * Published per-1M-token rates. Input and output only: callers pass two token + * counts, so cache tiers are out of scope here. */ +// The previous table priced every `opus` model at $15/$75, which are Claude 3 +// Opus era rates. Opus 4.5 and later bill at $5/$25, so every current- +// generation Opus estimate was exactly 3x real spend. `haiku` was $0.80/$4.00, +// which is Claude 3.5 Haiku: Haiku 4.5 bills at $1/$5, so Haiku was understated +// 1.25x. Fable and Mythos had no bucket at all and fell through to `sonnet`, +// understating them 3.3x. +// +// The legacy rows exist so that correcting the current generation does not +// reprice the old one. Claude 3 Haiku ($0.25/$1.25) is deliberately not +// modelled: Claude Code never ran it. const RATE_TABLE = { - haiku: { in: 0.8, out: 4.0 }, + haiku: { in: 1.0, out: 5.0 }, + haikuLegacy: { in: 0.8, out: 4.0 }, sonnet: { in: 3.0, out: 15.0 }, - opus: { in: 15.0, out: 75.0 } + opus: { in: 5.0, out: 25.0 }, + opusLegacy: { in: 15.0, out: 75.0 }, + fable: { in: 10.0, out: 50.0 } }; +// The only Opus models that really billed at $15/$75: Claude 3 Opus, Opus 4.0 +// and Opus 4.1. Every spelling of each has to match, alias and dated snapshot +// alike, which is why the bare `opus-4-` form is listed on its own: +// Opus 4.0's snapshot is `claude-opus-4-20250514`, with no minor segment, so +// an `opus-4-0` substring alone misses it and reprices a legacy estimate at a +// third of its real cost. The `[-@]` covers Vertex AI, which joins the date +// with `@` (`claude-opus-4@20250514`); Bedrock's +// `anthropic.claude-3-opus-20240229-v1:0` is caught by the first alternative. +// +// Opus 4.5 through Opus 5 are $5/$25 and take the default bucket, which also +// means a future Opus is assumed to be $5/$25. That assumption is the same +// shape of silent staleness this change fixes, so if Opus is ever repriced +// again, a new row belongs here rather than a rediscovery of this comment. +const LEGACY_OPUS_RE = /claude-3-opus|opus-4-0(?!\d)|opus-4-1(?!\d)|opus-4[-@]\d{8}/; + +// Claude 3.5 Haiku, whose $0.80/$4.00 this table used to apply to all Haiku. +const LEGACY_HAIKU_RE = /3-5-haiku|haiku-3-5/; + /** * Estimate USD cost from token counts. - * @param {string} model - Model name (may contain "haiku", "sonnet", or "opus") + * @param {string} model - Model name (may contain "haiku", "sonnet", "opus", + * "fable" or "mythos"); anything else is priced at sonnet rates. * @param {number} inputTokens * @param {number} outputTokens * @returns {number} Estimated cost in USD (rounded to 6 decimal places) @@ -22,8 +55,13 @@ const RATE_TABLE = { function estimateCost(model, inputTokens, outputTokens) { const normalized = String(model || '').toLowerCase(); let rates = RATE_TABLE.sonnet; - if (normalized.includes('haiku')) rates = RATE_TABLE.haiku; - if (normalized.includes('opus')) rates = RATE_TABLE.opus; + if (normalized.includes('haiku')) { + rates = LEGACY_HAIKU_RE.test(normalized) ? RATE_TABLE.haikuLegacy : RATE_TABLE.haiku; + } else if (normalized.includes('fable') || normalized.includes('mythos')) { + rates = RATE_TABLE.fable; + } else if (normalized.includes('opus')) { + rates = LEGACY_OPUS_RE.test(normalized) ? RATE_TABLE.opusLegacy : RATE_TABLE.opus; + } const cost = (inputTokens / 1_000_000) * rates.in + (outputTokens / 1_000_000) * rates.out; return Math.round(cost * 1e6) / 1e6; diff --git a/skills/cost-aware-llm-pipeline/SKILL.md b/skills/cost-aware-llm-pipeline/SKILL.md index 139d10985..63b10f6b3 100644 --- a/skills/cost-aware-llm-pipeline/SKILL.md +++ b/skills/cost-aware-llm-pipeline/SKILL.md @@ -156,9 +156,9 @@ def process(text: str, config: Config, tracker: CostTracker) -> tuple[Result, Co | Model | Input ($/1M tokens) | Output ($/1M tokens) | Relative Cost | |-------|---------------------|----------------------|---------------| -| Haiku 4.5 | $0.80 | $4.00 | 1x | -| Sonnet 4.6 | $3.00 | $15.00 | ~4x | -| Opus 4.5 | $15.00 | $75.00 | ~19x | +| Haiku 4.5 | $1.00 | $5.00 | 1x | +| Sonnet 4.6 | $3.00 | $15.00 | 3x | +| Opus 4.5 | $5.00 | $25.00 | 5x | ## Best Practices diff --git a/tests/lib/cost-estimate.test.js b/tests/lib/cost-estimate.test.js index bcb5906bc..2aa60770f 100644 --- a/tests/lib/cost-estimate.test.js +++ b/tests/lib/cost-estimate.test.js @@ -31,16 +31,25 @@ function runTests() { console.log('RATE_TABLE:'); if ( - test('RATE_TABLE has haiku, sonnet, opus keys', () => { - assert.ok(RATE_TABLE.haiku, 'Missing haiku'); - assert.ok(RATE_TABLE.sonnet, 'Missing sonnet'); - assert.ok(RATE_TABLE.opus, 'Missing opus'); - assert.strictEqual(typeof RATE_TABLE.haiku.in, 'number'); - assert.strictEqual(typeof RATE_TABLE.haiku.out, 'number'); - assert.strictEqual(typeof RATE_TABLE.sonnet.in, 'number'); - assert.strictEqual(typeof RATE_TABLE.sonnet.out, 'number'); - assert.strictEqual(typeof RATE_TABLE.opus.in, 'number'); - assert.strictEqual(typeof RATE_TABLE.opus.out, 'number'); + test('RATE_TABLE has a bucket per billing tier', () => { + for (const key of ['haiku', 'haikuLegacy', 'sonnet', 'opus', 'opusLegacy', 'fable']) { + assert.ok(RATE_TABLE[key], `Missing ${key}`); + assert.strictEqual(typeof RATE_TABLE[key].in, 'number', `${key}.in not a number`); + assert.strictEqual(typeof RATE_TABLE[key].out, 'number', `${key}.out not a number`); + } + }) + ) + passed++; + else failed++; + + if ( + test('RATE_TABLE carries current published rates, not Claude 3 era rates', () => { + assert.deepStrictEqual(RATE_TABLE.opus, { in: 5.0, out: 25.0 }); + assert.deepStrictEqual(RATE_TABLE.opusLegacy, { in: 15.0, out: 75.0 }); + assert.deepStrictEqual(RATE_TABLE.haiku, { in: 1.0, out: 5.0 }); + assert.deepStrictEqual(RATE_TABLE.haikuLegacy, { in: 0.8, out: 4.0 }); + assert.deepStrictEqual(RATE_TABLE.sonnet, { in: 3.0, out: 15.0 }); + assert.deepStrictEqual(RATE_TABLE.fable, { in: 10.0, out: 50.0 }); }) ) passed++; @@ -50,9 +59,9 @@ function runTests() { console.log('\nestimateCost:'); if ( - test('opus 1M/1M tokens returns 90', () => { + test('opus 1M/1M tokens returns 30', () => { const cost = estimateCost('opus', 1_000_000, 1_000_000); - assert.strictEqual(cost, 90); + assert.strictEqual(cost, 30); }) ) passed++; @@ -68,9 +77,9 @@ function runTests() { else failed++; if ( - test('haiku 1M/1M tokens returns 4.8', () => { + test('haiku 1M/1M tokens returns 6', () => { const cost = estimateCost('haiku', 1_000_000, 1_000_000); - assert.strictEqual(cost, 4.8); + assert.strictEqual(cost, 6); }) ) passed++; @@ -86,16 +95,74 @@ function runTests() { else failed++; if ( - test('full model name claude-opus-4-6 uses opus rates', () => { + test('full model name claude-opus-4-6 uses current opus rates', () => { const cost = estimateCost('claude-opus-4-6', 500, 200); - // (500 / 1_000_000) * 15 + (200 / 1_000_000) * 75 = 0.0075 + 0.015 = 0.0225 - const expected = Math.round(0.0225 * 1e6) / 1e6; + // (500 / 1_000_000) * 5 + (200 / 1_000_000) * 25 = 0.0025 + 0.005 = 0.0075 + const expected = Math.round(0.0075 * 1e6) / 1e6; assert.strictEqual(cost, expected); }) ) passed++; else failed++; + // Every spelling of the three Opus models that really billed at $15/$75 has + // to keep doing so. Opus 4.0's snapshot is `claude-opus-4-20250514`, with no + // minor segment, so a bare `opus-4-0` substring misses it. + for (const legacy of [ + 'claude-3-opus-20240229', + 'anthropic.claude-3-opus-20240229-v1:0', + 'claude-opus-4-20250514', + 'claude-opus-4@20250514', + 'claude-opus-4-0', + 'claude-opus-4-1' + ]) { + if ( + test(`${legacy} keeps legacy opus rates`, () => { + assert.strictEqual(estimateCost(legacy, 1_000_000, 1_000_000), 90); + }) + ) + passed++; + else failed++; + } + + for (const current of ['claude-opus-4-5', 'claude-opus-4-7', 'claude-opus-4-8', 'claude-opus-5']) { + if ( + test(`${current} uses current opus rates`, () => { + assert.strictEqual(estimateCost(current, 1_000_000, 1_000_000), 30); + }) + ) + passed++; + else failed++; + } + + if ( + test('claude-3-5-haiku keeps legacy haiku rates', () => { + assert.strictEqual(estimateCost('claude-3-5-haiku-20241022', 1_000_000, 1_000_000), 4.8); + }) + ) + passed++; + else failed++; + + if ( + test('claude-haiku-4-5 uses current haiku rates', () => { + assert.strictEqual(estimateCost('claude-haiku-4-5-20251001', 1_000_000, 1_000_000), 6); + }) + ) + passed++; + else failed++; + + // Fable and Mythos had no bucket at all and fell through to sonnet, which + // understated them 3.3x. + for (const model of ['claude-fable-5', 'claude-mythos-5']) { + if ( + test(`${model} uses fable rates`, () => { + assert.strictEqual(estimateCost(model, 1_000_000, 1_000_000), 60); + }) + ) + passed++; + else failed++; + } + if ( test('unknown model falls back to sonnet rates', () => { const cost = estimateCost('unknown-model', 1_000_000, 1_000_000);