fix: correct context window percentage and provider token mapping (#306)
The context fullness gauge was driven by output token count, causing erratic jumps between turns (e.g. 84% -> 64%) with no compaction. Status bar and estimateContextTokens now use calculatePromptTokens() which returns input + cacheRead + cacheWrite — the actual input context size. Previously both used a formula that included the final output token count, which fluctuates with response length and is not part of the context window for the current request. isContextOverflow's usage-based fallback (z.ai silent overflow) was also missing cacheWrite (cache_creation_input_tokens). Per Anthropic docs the threshold is input + cache_read + cache_creation — all three. Ref: https://platform.claude.com/docs/en/about-claude/pricing#long-context-pricing google.ts and google-vertex.ts were double-counting cached tokens. Gemini's promptTokenCount already includes cachedContentTokenCount, so assigning input = promptTokenCount and cacheRead = cachedContentTokenCount overcounted by cachedContentTokenCount on every cached request. Fixed by subtracting first, matching the OpenAI convention: input = promptTokenCount - cachedContentTokenCount cacheRead = cachedContentTokenCount => input + cacheRead = promptTokenCount (total prompt, no double-count) Ref: https://ai.google.dev/api/generate-content#v1beta.GenerateContentResponse.UsageMetadata All other providers validated: amazon-bedrock (inputTokens is uncached by API contract), openai-completions/responses/azure (already subtract cached), kimi/gitlab-duo (delegate to correct implementations), cursor (API exposes output tokens only — input stays 0 by design). Co-authored-by: Miroslav Drbal <miroslav.drbal@gendigital.com>
This commit is contained in:
committed by
GitHub
parent
8320641e24
commit
a2223cef60
@@ -234,11 +234,17 @@ export const streamGoogleVertex: StreamFunction<"google-vertex"> = (
|
||||
}
|
||||
|
||||
if (chunk.usageMetadata) {
|
||||
// promptTokenCount includes cachedContentTokenCount when cached content is used.
|
||||
// Subtract to get non-cached input, matching the OpenAI convention where
|
||||
// input = uncached prompt tokens and cacheRead = cached tokens so that
|
||||
// input + cacheRead = total prompt tokens (no double-counting).
|
||||
// Ref: https://ai.google.dev/api/generate-content#v1beta.GenerateContentResponse.UsageMetadata
|
||||
const cachedTokens = chunk.usageMetadata.cachedContentTokenCount || 0;
|
||||
output.usage = {
|
||||
input: chunk.usageMetadata.promptTokenCount || 0,
|
||||
input: (chunk.usageMetadata.promptTokenCount || 0) - cachedTokens,
|
||||
output:
|
||||
(chunk.usageMetadata.candidatesTokenCount || 0) + (chunk.usageMetadata.thoughtsTokenCount || 0),
|
||||
cacheRead: chunk.usageMetadata.cachedContentTokenCount || 0,
|
||||
cacheRead: cachedTokens,
|
||||
cacheWrite: 0,
|
||||
totalTokens: chunk.usageMetadata.totalTokenCount || 0,
|
||||
cost: {
|
||||
|
||||
@@ -217,11 +217,17 @@ export const streamGoogle: StreamFunction<"google-generative-ai"> = (
|
||||
}
|
||||
|
||||
if (chunk.usageMetadata) {
|
||||
// promptTokenCount includes cachedContentTokenCount when cached content is used.
|
||||
// Subtract to get non-cached input, matching the OpenAI convention where
|
||||
// input = uncached prompt tokens and cacheRead = cached tokens so that
|
||||
// input + cacheRead = total prompt tokens (no double-counting).
|
||||
// Ref: https://ai.google.dev/api/generate-content#v1beta.GenerateContentResponse.UsageMetadata
|
||||
const cachedTokens = chunk.usageMetadata.cachedContentTokenCount || 0;
|
||||
output.usage = {
|
||||
input: chunk.usageMetadata.promptTokenCount || 0,
|
||||
input: (chunk.usageMetadata.promptTokenCount || 0) - cachedTokens,
|
||||
output:
|
||||
(chunk.usageMetadata.candidatesTokenCount || 0) + (chunk.usageMetadata.thoughtsTokenCount || 0),
|
||||
cacheRead: chunk.usageMetadata.cachedContentTokenCount || 0,
|
||||
cacheRead: cachedTokens,
|
||||
cacheWrite: 0,
|
||||
totalTokens: chunk.usageMetadata.totalTokenCount || 0,
|
||||
cost: {
|
||||
|
||||
@@ -115,7 +115,7 @@ export function isContextOverflow(message: AssistantMessage, contextWindow?: num
|
||||
|
||||
// Case 2: Usage-based overflow (silent or provider-specific)
|
||||
if (contextWindow) {
|
||||
const inputTokens = message.usage.input + message.usage.cacheRead;
|
||||
const inputTokens = message.usage.input + message.usage.cacheRead + message.usage.cacheWrite;
|
||||
if (inputTokens > contextWindow) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import { settings } from "../../config/settings";
|
||||
import type { StatusLinePreset, StatusLineSegmentId, StatusLineSeparatorStyle } from "../../config/settings-schema";
|
||||
import { theme } from "../../modes/theme/theme";
|
||||
import type { AgentSession } from "../../session/agent-session";
|
||||
import { calculatePromptTokens } from "../../session/compaction/compaction";
|
||||
import { findGitHeadPathSync, sanitizeStatusText } from "../shared";
|
||||
import {
|
||||
canReuseCachedPr,
|
||||
@@ -365,12 +366,7 @@ export class StatusLineComponent implements Component {
|
||||
.reverse()
|
||||
.find(m => m.role === "assistant" && m.stopReason !== "aborted") as AssistantMessage | undefined;
|
||||
|
||||
const contextTokens = lastAssistantMessage
|
||||
? lastAssistantMessage.usage.input +
|
||||
lastAssistantMessage.usage.output +
|
||||
lastAssistantMessage.usage.cacheRead +
|
||||
lastAssistantMessage.usage.cacheWrite
|
||||
: 0;
|
||||
const contextTokens = lastAssistantMessage ? calculatePromptTokens(lastAssistantMessage.usage) : 0;
|
||||
const contextWindow = state.model?.contextWindow || 0;
|
||||
const contextPercent = contextWindow > 0 ? (contextTokens / contextWindow) * 100 : 0;
|
||||
|
||||
|
||||
@@ -108,6 +108,7 @@ import { extractFileMentions, generateFileMentionMessages } from "../utils/file-
|
||||
import {
|
||||
type CompactionResult,
|
||||
calculateContextTokens,
|
||||
calculatePromptTokens,
|
||||
collectEntriesForBranchSummary,
|
||||
compact,
|
||||
estimateTokens,
|
||||
@@ -5076,7 +5077,7 @@ Be thorough - include exact file paths, function names, error messages, and tech
|
||||
};
|
||||
}
|
||||
|
||||
const usageTokens = calculateContextTokens(lastUsage);
|
||||
const usageTokens = calculatePromptTokens(lastUsage);
|
||||
let trailingTokens = 0;
|
||||
for (let i = lastUsageIndex + 1; i < messages.length; i++) {
|
||||
trailingTokens += estimateTokens(messages[i]);
|
||||
|
||||
Reference in New Issue
Block a user