From c751b9c4d5d252ca3a9b8fbfd2e01f675af0c627 Mon Sep 17 00:00:00 2001 From: can1357 Date: Tue, 28 Apr 2026 18:11:11 +0200 Subject: [PATCH] fix(stats): resolved stats openai-codex cost fallback to OpenAI rates - Updated Codex model pricing data to include non-zero input/output/cacheRead rates. - Added generate-models fallback to copy billable OpenAI costs into bundled openai-codex models. - Added catalog-cost helpers to fallback openai-codex pricing to openai and compute token totals. - Added regression tests for openai-codex cost parity and db backfill, and documented the fix in changelog. --- packages/ai/CHANGELOG.md | 4 + packages/ai/scripts/generate-models.ts | 32 ++++++ packages/ai/src/models.json | 80 +++++++-------- packages/ai/test/models-cost.test.ts | 19 ++++ packages/stats/CHANGELOG.md | 4 + packages/stats/src/db.ts | 115 ++++++++++++++++++++-- packages/stats/test/db-cost.test.ts | 130 +++++++++++++++++++++++++ 7 files changed, 337 insertions(+), 47 deletions(-) create mode 100644 packages/stats/test/db-cost.test.ts diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index a547ecaa4..cce0e98d2 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Fixed + +- Fixed OpenAI Codex GPT model pricing by inheriting matching OpenAI catalog rates for zero-priced discovered Codex entries. + ## [14.5.3] - 2026-04-27 ### Added diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index 1a6645087..340a3ba1a 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -179,6 +179,37 @@ function applyPremiumMultiplierOverrides(models: readonly Model[]): Model[] { }; }); } +function hasBillableCost(cost: Model["cost"]): boolean { + return cost.input !== 0 || cost.output !== 0 || cost.cacheRead !== 0 || cost.cacheWrite !== 0; +} + +function applyCodexPricingFallback(models: readonly Model[]): Model[] { + const openAIModels = new Map( + models + .filter(model => model.provider === "openai" && hasBillableCost(model.cost)) + .map(model => [model.id, model.cost]), + ); + + return models.map(model => { + if (model.provider !== "openai-codex" || model.api !== "openai-codex-responses") { + return model; + } + if (hasBillableCost(model.cost)) { + return model; + } + + const openAICost = openAIModels.get(model.id); + if (!openAICost) { + return model; + } + + return { + ...model, + cost: { ...openAICost }, + }; + }); +} + const ANTIGRAVITY_ENDPOINT = "https://daily-cloudcode-pa.sandbox.googleapis.com"; async function getOAuthCredentialsFromStorage(provider: OAuthProvider): Promise { @@ -334,6 +365,7 @@ async function generateModels() { allModels = applyGlobalModelsDevFallback(allModels, modelsDevModels); allModels = applyPremiumMultiplierOverrides(allModels); + allModels = applyCodexPricingFallback(allModels); applyGeneratedModelPolicies(allModels); linkOpenAIPromotionTargets(allModels); diff --git a/packages/ai/src/models.json b/packages/ai/src/models.json index 3f5b5ba39..9dd91d2c0 100644 --- a/packages/ai/src/models.json +++ b/packages/ai/src/models.json @@ -33899,9 +33899,9 @@ "image" ], "cost": { - "input": 0, - "output": 0, - "cacheRead": 0, + "input": 1.25, + "output": 10, + "cacheRead": 0.125, "cacheWrite": 0 }, "contextWindow": 400000, @@ -33927,9 +33927,9 @@ "image" ], "cost": { - "input": 0, - "output": 0, - "cacheRead": 0, + "input": 1.25, + "output": 10, + "cacheRead": 0.125, "cacheWrite": 0 }, "contextWindow": 272000, @@ -33983,9 +33983,9 @@ "image" ], "cost": { - "input": 0, - "output": 0, - "cacheRead": 0, + "input": 1.25, + "output": 10, + "cacheRead": 0.13, "cacheWrite": 0 }, "contextWindow": 400000, @@ -34011,9 +34011,9 @@ "image" ], "cost": { - "input": 0, - "output": 0, - "cacheRead": 0, + "input": 1.25, + "output": 10, + "cacheRead": 0.125, "cacheWrite": 0 }, "contextWindow": 272000, @@ -34039,9 +34039,9 @@ "image" ], "cost": { - "input": 0, - "output": 0, - "cacheRead": 0, + "input": 1.25, + "output": 10, + "cacheRead": 0.125, "cacheWrite": 0 }, "contextWindow": 272000, @@ -34067,9 +34067,9 @@ "image" ], "cost": { - "input": 0, - "output": 0, - "cacheRead": 0, + "input": 0.25, + "output": 2, + "cacheRead": 0.025, "cacheWrite": 0 }, "contextWindow": 272000, @@ -34095,9 +34095,9 @@ "image" ], "cost": { - "input": 0, - "output": 0, - "cacheRead": 0, + "input": 1.75, + "output": 14, + "cacheRead": 0.175, "cacheWrite": 0 }, "contextWindow": 272000, @@ -34123,9 +34123,9 @@ "image" ], "cost": { - "input": 0, - "output": 0, - "cacheRead": 0, + "input": 1.75, + "output": 14, + "cacheRead": 0.175, "cacheWrite": 0 }, "contextWindow": 272000, @@ -34151,9 +34151,9 @@ "image" ], "cost": { - "input": 0, - "output": 0, - "cacheRead": 0, + "input": 1.75, + "output": 14, + "cacheRead": 0.175, "cacheWrite": 0 }, "contextWindow": 272000, @@ -34206,9 +34206,9 @@ "image" ], "cost": { - "input": 0, - "output": 0, - "cacheRead": 0, + "input": 2.5, + "output": 15, + "cacheRead": 0.25, "cacheWrite": 0 }, "contextWindow": 272000, @@ -34234,9 +34234,9 @@ "image" ], "cost": { - "input": 0, - "output": 0, - "cacheRead": 0, + "input": 0.75, + "output": 4.5, + "cacheRead": 0.075, "cacheWrite": 0 }, "contextWindow": 272000, @@ -34262,9 +34262,9 @@ "image" ], "cost": { - "input": 0, - "output": 0, - "cacheRead": 0, + "input": 0.2, + "output": 1.25, + "cacheRead": 0.02, "cacheWrite": 0 }, "contextWindow": 272000, @@ -34290,9 +34290,9 @@ "image" ], "cost": { - "input": 0, - "output": 0, - "cacheRead": 0, + "input": 5, + "output": 30, + "cacheRead": 0.5, "cacheWrite": 0 }, "contextWindow": 272000, @@ -52775,4 +52775,4 @@ } } } -} \ No newline at end of file +} diff --git a/packages/ai/test/models-cost.test.ts b/packages/ai/test/models-cost.test.ts index ab8ec29cf..07b80d523 100644 --- a/packages/ai/test/models-cost.test.ts +++ b/packages/ai/test/models-cost.test.ts @@ -70,4 +70,23 @@ describe("calculateCost", () => { expect(usage.cost.cacheWrite).toBeCloseTo(0.08, 8); expect(usage.cost.total).toBeCloseTo(2.18, 8); }); + + it("prices OpenAI Codex GPT models from the matching OpenAI catalog entry", () => { + const openAIModel = getBundledModel("openai", "gpt-5.4"); + const codexModel = getBundledModel("openai-codex", "gpt-5.4"); + const usage: Usage = { + input: 1000, + output: 500, + cacheRead: 200, + cacheWrite: 0, + totalTokens: 1700, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }; + + expect(codexModel.cost).toEqual(openAIModel.cost); + + calculateCost(codexModel, usage); + + expect(usage.cost.total).toBeCloseTo(0.01005, 8); + }); }); diff --git a/packages/stats/CHANGELOG.md b/packages/stats/CHANGELOG.md index 303576da7..cc745ef4b 100644 --- a/packages/stats/CHANGELOG.md +++ b/packages/stats/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Fixed + +- Fixed GPT cost reporting by deriving missing OpenAI Codex costs from the model catalog and backfilling existing zero-cost rows. + ## [13.6.0] - 2026-03-03 ### Fixed diff --git a/packages/stats/src/db.ts b/packages/stats/src/db.ts index 30f2d0fe8..c38dec254 100644 --- a/packages/stats/src/db.ts +++ b/packages/stats/src/db.ts @@ -1,5 +1,6 @@ import { Database } from "bun:sqlite"; import * as fs from "node:fs/promises"; +import { type GeneratedProvider, getBundledModel, type Usage } from "@oh-my-pi/pi-ai"; import { getConfigRootDir, getStatsDbPath } from "@oh-my-pi/pi-utils"; import type { AggregatedStats, @@ -12,7 +13,19 @@ import type { TimeSeriesPoint, } from "./types"; -const DB_PATH = getStatsDbPath(); +type ModelCost = { input: number; output: number; cacheRead: number; cacheWrite: number }; +type UsageCost = Usage["cost"]; +type CostTokens = Pick; + +interface CostBackfillRow { + id: number; + provider: string; + model: string; + input_tokens: number; + output_tokens: number; + cache_read_tokens: number; + cache_write_tokens: number; +} let db: Database | null = null; @@ -25,7 +38,7 @@ export async function initDb(): Promise { // Ensure directory exists await fs.mkdir(getConfigRootDir(), { recursive: true }); - db = new Database(DB_PATH); + db = new Database(getStatsDbPath()); db.exec("PRAGMA journal_mode = WAL"); // Create tables @@ -74,9 +87,96 @@ export async function initDb(): Promise { db.exec("ALTER TABLE messages ADD COLUMN premium_requests REAL NOT NULL DEFAULT 0"); } db.exec("UPDATE messages SET premium_requests = 0 WHERE premium_requests IS NULL"); + backfillMissingCatalogCosts(db); return db; } +function hasBillableCost(cost: ModelCost): boolean { + return cost.input !== 0 || cost.output !== 0 || cost.cacheRead !== 0 || cost.cacheWrite !== 0; +} + +function getBundledModelCost(provider: string, modelId: string): ModelCost | null { + const model = getBundledModel(provider as GeneratedProvider, modelId); + return model?.cost ?? null; +} + +function getCatalogCost(provider: string, modelId: string): ModelCost | null { + const primaryCost = getBundledModelCost(provider, modelId); + if (primaryCost && hasBillableCost(primaryCost)) { + return primaryCost; + } + + if (provider === "openai-codex") { + const openAICost = getBundledModelCost("openai", modelId); + if (openAICost && hasBillableCost(openAICost)) { + return openAICost; + } + } + + return null; +} + +function calculateCatalogCost(provider: string, modelId: string, tokens: CostTokens): UsageCost | null { + const cost = getCatalogCost(provider, modelId); + if (!cost) return null; + + const input = (cost.input / 1_000_000) * tokens.input; + const output = (cost.output / 1_000_000) * tokens.output; + const cacheRead = (cost.cacheRead / 1_000_000) * tokens.cacheRead; + const cacheWrite = (cost.cacheWrite / 1_000_000) * tokens.cacheWrite; + + return { + input, + output, + cacheRead, + cacheWrite, + total: input + output + cacheRead + cacheWrite, + }; +} + +function resolveStoredCost(stats: MessageStats): UsageCost { + if (stats.usage.cost.total !== 0) { + return stats.usage.cost; + } + + return calculateCatalogCost(stats.provider, stats.model, stats.usage) ?? stats.usage.cost; +} + +function backfillMissingCatalogCosts(database: Database): void { + const rows = database + .prepare(` + SELECT id, provider, model, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens + FROM messages + WHERE cost_total = 0 AND total_tokens > 0 + `) + .all() as CostBackfillRow[]; + + if (rows.length === 0) return; + + const update = database.prepare(` + UPDATE messages + SET cost_input = ?, cost_output = ?, cost_cache_read = ?, cost_cache_write = ?, cost_total = ? + WHERE id = ? + `); + + const applyBackfill = database.transaction(() => { + for (const row of rows) { + const cost = calculateCatalogCost(row.provider, row.model, { + input: row.input_tokens, + output: row.output_tokens, + cacheRead: row.cache_read_tokens, + cacheWrite: row.cache_write_tokens, + }); + + if (!cost || cost.total === 0) continue; + + update.run(cost.input, cost.output, cost.cacheRead, cost.cacheWrite, cost.total, row.id); + } + }); + + applyBackfill(); +} + /** * Get the stored offset for a session file. */ @@ -120,6 +220,7 @@ export function insertMessageStats(stats: MessageStats[]): number { let inserted = 0; const insert = db.transaction(() => { for (const s of stats) { + const cost = resolveStoredCost(s); const result = stmt.run( s.sessionFile, s.entryId, @@ -138,11 +239,11 @@ export function insertMessageStats(stats: MessageStats[]): number { s.usage.cacheWrite, s.usage.totalTokens, s.usage.premiumRequests ?? 0, - s.usage.cost.input, - s.usage.cost.output, - s.usage.cost.cacheRead, - s.usage.cost.cacheWrite, - s.usage.cost.total, + cost.input, + cost.output, + cost.cacheRead, + cost.cacheWrite, + cost.total, ); if (result.changes > 0) inserted++; } diff --git a/packages/stats/test/db-cost.test.ts b/packages/stats/test/db-cost.test.ts new file mode 100644 index 000000000..31a1f428f --- /dev/null +++ b/packages/stats/test/db-cost.test.ts @@ -0,0 +1,130 @@ +import { Database } from "bun:sqlite"; +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import * as os from "node:os"; +import * as path from "node:path"; +import { getBundledModel } from "@oh-my-pi/pi-ai"; +import { getAgentDir, getStatsDbPath, setAgentDir, TempDir } from "@oh-my-pi/pi-utils"; +import { closeDb, getRecentRequests, initDb, insertMessageStats } from "../src/db"; +import type { MessageStats } from "../src/types"; + +const originalConfigDir = process.env.PI_CONFIG_DIR; +const originalAgentDir = getAgentDir(); +let tempDir: TempDir | null = null; + +beforeEach(() => { + tempDir = TempDir.createSync("@pi-stats-db-"); + const configDir = path.relative(os.homedir(), tempDir.join("config")); + process.env.PI_CONFIG_DIR = configDir; + setAgentDir(path.join(os.homedir(), configDir, "agent")); +}); + +afterEach(() => { + closeDb(); + if (originalConfigDir === undefined) { + delete process.env.PI_CONFIG_DIR; + } else { + process.env.PI_CONFIG_DIR = originalConfigDir; + } + setAgentDir(originalAgentDir); + tempDir?.removeSync(); + tempDir = null; +}); + +function createCodexGptStats(entryId: string): MessageStats { + return { + sessionFile: "/tmp/session.jsonl", + entryId, + folder: "/tmp/project", + model: "gpt-5.4", + provider: "openai-codex", + api: "openai-codex-responses", + timestamp: Date.now(), + duration: 1000, + ttft: 100, + stopReason: "stop", + errorMessage: null, + usage: { + input: 1000, + output: 500, + cacheRead: 200, + cacheWrite: 0, + totalTokens: 1700, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + }; +} + +function expectedCodexGptCost() { + const cost = getBundledModel("openai-codex", "gpt-5.4").cost; + const input = (cost.input / 1_000_000) * 1000; + const output = (cost.output / 1_000_000) * 500; + const cacheRead = (cost.cacheRead / 1_000_000) * 200; + return { + input, + output, + cacheRead, + total: input + output + cacheRead, + }; +} + +describe("stats GPT cost correction", () => { + it("stores catalog-derived cost when OpenAI Codex session usage has zero cost", async () => { + await initDb(); + + insertMessageStats([createCodexGptStats("inserted")]); + + const expected = expectedCodexGptCost(); + const request = getRecentRequests(1)[0]; + expect(expected.total).toBeGreaterThan(0); + expect(request?.usage.cost.input).toBeCloseTo(expected.input, 8); + expect(request?.usage.cost.output).toBeCloseTo(expected.output, 8); + expect(request?.usage.cost.cacheRead).toBeCloseTo(expected.cacheRead, 8); + expect(request?.usage.cost.total).toBeCloseTo(expected.total, 8); + }); + + it("backfills existing zero-cost OpenAI Codex GPT rows on database init", async () => { + await initDb(); + closeDb(); + + const database = new Database(getStatsDbPath()); + database + .prepare(` + INSERT INTO messages ( + session_file, entry_id, folder, model, provider, api, timestamp, + duration, ttft, stop_reason, error_message, + input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, total_tokens, premium_requests, + cost_input, cost_output, cost_cache_read, cost_cache_write, cost_total + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `) + .run( + "/tmp/session.jsonl", + "backfilled", + "/tmp/project", + "gpt-5.4", + "openai-codex", + "openai-codex-responses", + Date.now(), + 1000, + 100, + "stop", + null, + 1000, + 500, + 200, + 0, + 1700, + 0, + 0, + 0, + 0, + 0, + 0, + ); + database.close(); + + await initDb(); + + const request = getRecentRequests(1)[0]; + expect(request?.usage.cost.total).toBeCloseTo(expectedCodexGptCost().total, 8); + }); +});