From fbf3a6eb3f65ed69ed2df3b9ca0af4d260b6fb6e Mon Sep 17 00:00:00 2001 From: can1357 Date: Thu, 2 Jul 2026 08:31:18 +0200 Subject: [PATCH] feat(stats): added tool usage dashboard - Parsed assistant `toolCall` blocks and `toolResult` messages into persisted `tool_calls` rows with one-shot historical backfill. - Added tool aggregate queries, model breakdowns, call time-series data, and the `/api/stats/tools` dashboard endpoint. - Added the `/#/tools` route with summary metrics, stacked calls-over-time chart, per-tool table, and model breakdown panel. - Added end-to-end stats coverage for tool ingestion, result/error linkage, fork deduplication, incremental updates, and dashboard shaping. --- packages/stats/CHANGELOG.md | 4 + packages/stats/src/aggregator.ts | 23 +- packages/stats/src/client/App.tsx | 3 + packages/stats/src/client/api.ts | 8 + packages/stats/src/client/app/routes.ts | 8 +- .../stats/src/client/data/useHashRoute.ts | 1 + packages/stats/src/client/data/view-models.ts | 18 + .../stats/src/client/routes/ToolsRoute.tsx | 463 ++++++++++++++++++ packages/stats/src/client/routes/index.ts | 1 + packages/stats/src/db.ts | 252 ++++++++++ packages/stats/src/index.ts | 5 + packages/stats/src/parser.ts | 86 +++- packages/stats/src/server.ts | 6 + packages/stats/src/shared-types.ts | 52 ++ packages/stats/src/types.ts | 43 ++ packages/stats/test/tool-stats.test.ts | 346 +++++++++++++ 16 files changed, 1315 insertions(+), 4 deletions(-) create mode 100644 packages/stats/src/client/routes/ToolsRoute.tsx create mode 100644 packages/stats/test/tool-stats.test.ts diff --git a/packages/stats/CHANGELOG.md b/packages/stats/CHANGELOG.md index 1cbaed0ca..9170dfad8 100644 --- a/packages/stats/CHANGELOG.md +++ b/packages/stats/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Added + +- Added a Tools tab to the `omp stats` dashboard (`/#/tools`): per-tool call counts, error rates, result/argument payload sizes, per-model breakdown, and a stacked calls-over-time chart. Token and cost columns attribute each invoking turn's real provider usage evenly across that turn's tool calls. Existing databases re-parse sessions once on the next sync to backfill historical tool calls. + ## [16.2.7] - 2026-06-30 ### Fixed diff --git a/packages/stats/src/aggregator.ts b/packages/stats/src/aggregator.ts index 1aec92565..553033314 100644 --- a/packages/stats/src/aggregator.ts +++ b/packages/stats/src/aggregator.ts @@ -17,10 +17,15 @@ import { getStatsByFolder, getStatsByModel, getTimeSeries, + getToolStats, + getToolStatsByModel, + getToolTimeSeries, initDb, insertMessageStats, + insertToolCalls, insertUserMessageStats, setFileOffset, + updateToolResults, updateUserMessageLinks, } from "./db"; import { getSessionEntry, listAllSessionFiles, type ParseSessionResult, parseSessionFile } from "./parser"; @@ -29,7 +34,7 @@ import type { SyncWorkerRequest, SyncWorkerResponse } from "./sync-worker"; // hidden argv mode, so the compiled binary and npm bundle only need one // JavaScript entry. Standalone source `omp-stats` keeps using this package's // own sync-worker source file. -import type { BehaviorDashboardStats, DashboardStats, MessageStats, RequestDetails } from "./types"; +import type { BehaviorDashboardStats, DashboardStats, MessageStats, RequestDetails, ToolDashboardStats } from "./types"; /** * Apply a freshly parsed result to the database. Runs entirely on the @@ -39,6 +44,8 @@ function applyParseResult(sessionFile: string, lastModified: number, result: Par if (result.stats.length > 0) insertMessageStats(result.stats); if (result.userStats.length > 0) insertUserMessageStats(result.userStats); if (result.userLinks.length > 0) updateUserMessageLinks(result.userLinks); + if (result.toolCalls.length > 0) insertToolCalls(result.toolCalls); + if (result.toolResults.length > 0) updateToolResults(result.toolResults); setFileOffset(sessionFile, result.newOffset, lastModified); return result.stats.length + result.userStats.length; } @@ -478,3 +485,17 @@ export async function getBehaviorDashboardStats(range?: string | null): Promise< behaviorSeries: getBehaviorTimeSeries(cutoff), }; } + +/** + * Get the tools dashboard payload: per-tool totals, per-(tool, model) + * breakdown, and the call time series (bucketed like the model series). + */ +export async function getToolDashboardStats(range?: string | null): Promise { + await initDb(); + const { modelSeriesDays, modelSeriesBucketMs, cutoff } = getTimeRangeConfig(range); + return { + byTool: getToolStats(cutoff ?? undefined), + byToolModel: getToolStatsByModel(cutoff ?? undefined), + series: getToolTimeSeries(modelSeriesDays, cutoff, modelSeriesBucketMs), + }; +} diff --git a/packages/stats/src/client/App.tsx b/packages/stats/src/client/App.tsx index 4b4639a26..4a48034be 100644 --- a/packages/stats/src/client/App.tsx +++ b/packages/stats/src/client/App.tsx @@ -11,6 +11,7 @@ import { OverviewRoute, ProjectsRoute, RequestsRoute, + ToolsRoute, } from "./routes"; import { RequestDrawer } from "./ui/RequestDrawer"; @@ -72,6 +73,8 @@ export default function App() { ); case "models": return ; + case "tools": + return ; case "costs": return ; case "behavior": diff --git a/packages/stats/src/client/api.ts b/packages/stats/src/client/api.ts index 07caed2ba..eef020b20 100644 --- a/packages/stats/src/client/api.ts +++ b/packages/stats/src/client/api.ts @@ -8,6 +8,7 @@ import type { OverviewStats, RequestDetails, TimeRange, + ToolDashboardStats, } from "./types"; const API_BASE = "/api"; @@ -92,3 +93,10 @@ export async function getGainDashboardStats( if (project) params.set("project", project); return fetchJson(`${API_BASE}/stats/gain?${params}`, { signal }); } + +export async function getToolDashboardStats( + range: TimeRange = "24h", + signal?: AbortSignal, +): Promise { + return fetchJson(`${API_BASE}/stats/tools?range=${encodeURIComponent(range)}`, { signal }); +} diff --git a/packages/stats/src/client/app/routes.ts b/packages/stats/src/client/app/routes.ts index 909a8d591..ae5ec89da 100644 --- a/packages/stats/src/client/app/routes.ts +++ b/packages/stats/src/client/app/routes.ts @@ -1,4 +1,4 @@ -import { Activity, AlertCircle, Coins, Cpu, Folder, LayoutDashboard, Smile, TrendingUp } from "lucide-react"; +import { Activity, AlertCircle, Coins, Cpu, Folder, LayoutDashboard, Smile, TrendingUp, Wrench } from "lucide-react"; import type React from "react"; export type DashboardSection = @@ -6,6 +6,7 @@ export type DashboardSection = | "requests" | "errors" | "models" + | "tools" | "costs" | "behavior" | "projects" @@ -39,6 +40,11 @@ export const routes: DashboardRoute[] = [ label: "Models", icon: Cpu, }, + { + id: "tools", + label: "Tools", + icon: Wrench, + }, { id: "costs", label: "Costs", diff --git a/packages/stats/src/client/data/useHashRoute.ts b/packages/stats/src/client/data/useHashRoute.ts index bb78f8429..6f0575e18 100644 --- a/packages/stats/src/client/data/useHashRoute.ts +++ b/packages/stats/src/client/data/useHashRoute.ts @@ -7,6 +7,7 @@ const VALID_SECTIONS: DashboardSection[] = [ "requests", "errors", "models", + "tools", "costs", "behavior", "projects", diff --git a/packages/stats/src/client/data/view-models.ts b/packages/stats/src/client/data/view-models.ts index 33340a37b..c4e95b7b7 100644 --- a/packages/stats/src/client/data/view-models.ts +++ b/packages/stats/src/client/data/view-models.ts @@ -8,6 +8,7 @@ import type { FolderStats, ModelPerformancePoint, TimeRange, + ToolUsageStats, } from "../types"; /** Fixed display order for the agent-token-share breakdown. */ @@ -231,3 +232,20 @@ export function buildFolderRows(folders: FolderStats[]): FolderRowView[] { requestsPercentage: maxRequests > 0 ? (f.totalRequests / maxRequests) * 100 : 0, })); } + +/** Table row for the Tools route: usage stats plus derived rates/shares. */ +export interface ToolRowView extends ToolUsageStats { + /** errors / calls (0 for zero calls). */ + errorRate: number; + /** Calls relative to the busiest tool, 0-100, for the share bar. */ + callsPercentage: number; +} + +export function buildToolRows(tools: ToolUsageStats[]): ToolRowView[] { + const maxCalls = tools.reduce((max, t) => Math.max(max, t.calls), 0); + return tools.map(t => ({ + ...t, + errorRate: t.calls > 0 ? t.errors / t.calls : 0, + callsPercentage: maxCalls > 0 ? (t.calls / maxCalls) * 100 : 0, + })); +} diff --git a/packages/stats/src/client/routes/ToolsRoute.tsx b/packages/stats/src/client/routes/ToolsRoute.tsx new file mode 100644 index 000000000..8c47cc689 --- /dev/null +++ b/packages/stats/src/client/routes/ToolsRoute.tsx @@ -0,0 +1,463 @@ +import { useMemo, useState } from "react"; +import { Line } from "react-chartjs-2"; +import { getToolDashboardStats } from "../api"; +import { CHART_THEMES, MODEL_COLORS } from "../components/chart-shared"; +import { formatRangeTick, rangeMeta } from "../components/range-meta"; +import { formatCompact, formatCost, formatInteger, formatPercent, formatRelativeTime } from "../data/formatters"; +import { useResource } from "../data/useResource"; +import { buildToolRows, type ToolRowView } from "../data/view-models"; +import type { TimeRange, ToolModelStats, ToolTimeSeriesPoint, ToolUsageStats } from "../types"; +import { AsyncBoundary, DataTable, Panel, StatusPill } from "../ui"; +import { useSystemTheme } from "../useSystemTheme"; + +export interface ToolsRouteProps { + active: boolean; + range: TimeRange; + refreshTrigger: number; +} + +export function ToolsRoute({ active, range, refreshTrigger }: ToolsRouteProps) { + const { + data: stats, + error, + loading, + } = useResource(["tools", range, refreshTrigger], signal => getToolDashboardStats(range, signal), { + pollMs: 30000, + enabled: active, + }); + + return ( +
+ + {stats && ( + <> + + + + + + )} + +
+ ); +} + +// --------------------------------------------------------------------------- +// Summary metrics +// --------------------------------------------------------------------------- + +function ToolsSummaryPanel({ byTool }: { byTool: ToolUsageStats[] }) { + const totals = useMemo(() => { + let calls = 0; + let errors = 0; + let tokens = 0; + let output = 0; + let cost = 0; + let resultChars = 0; + let argsChars = 0; + for (const t of byTool) { + calls += t.calls; + errors += t.errors; + tokens += t.totalTokensShare; + output += t.outputTokensShare; + cost += t.costShare; + resultChars += t.resultChars; + argsChars += t.argsChars; + } + return { calls, errors, tokens, output, cost, resultChars, argsChars, tools: byTool.length }; + }, [byTool]); + + return ( + +
+
+
+
Tool Calls
+
{formatInteger(totals.calls)}
+
+
+
Tools Used
+
{formatInteger(totals.tools)}
+
+
+
Error Rate
+
+ {formatPercent(totals.calls > 0 ? totals.errors / totals.calls : 0)} +
+
+
+
Attributed Cost
+
{formatCost(totals.cost)}
+
+
+ +
+
+
Attributed Tokens
+
{formatCompact(Math.round(totals.tokens))}
+
+
+
Attributed Output
+
{formatCompact(Math.round(totals.output))}
+
+
+
Result Text
+
{formatCompact(totals.resultChars)} chars
+
+
+
Call Arguments
+
{formatCompact(totals.argsChars)} chars
+
+
+
+ + ); +} + +// --------------------------------------------------------------------------- +// Calls over time (stacked by top tools) +// --------------------------------------------------------------------------- + +const TOP_TOOLS = 6; + +function buildToolCallSeries(points: ToolTimeSeriesPoint[]): { + buckets: number[]; + tools: string[]; + data: Map>; +} { + const totals = new Map(); + for (const p of points) totals.set(p.tool, (totals.get(p.tool) ?? 0) + p.calls); + const ranked = [...totals.entries()].sort((a, b) => b[1] - a[1]); + const top = ranked.slice(0, TOP_TOOLS).map(([tool]) => tool); + const topSet = new Set(top); + const hasOther = ranked.length > top.length; + const tools = hasOther ? [...top, "Other"] : top; + + const buckets = [...new Set(points.map(p => p.timestamp))].sort((a, b) => a - b); + const data = new Map>(); + for (const bucket of buckets) data.set(bucket, {}); + for (const p of points) { + const label = topSet.has(p.tool) ? p.tool : "Other"; + const row = data.get(p.timestamp); + if (row) row[label] = (row[label] ?? 0) + p.calls; + } + return { buckets, tools, data }; +} + +function ToolCallsChart({ series, timeRange }: { series: ToolTimeSeriesPoint[]; timeRange: TimeRange }) { + const theme = useSystemTheme(); + const chartTheme = CHART_THEMES[theme]; + const meta = rangeMeta(timeRange); + + const chartSeries = useMemo(() => buildToolCallSeries(series), [series]); + + const data = useMemo( + () => ({ + labels: chartSeries.buckets.map(ts => formatRangeTick(ts, timeRange)), + datasets: chartSeries.tools.map((tool, index) => ({ + label: tool, + data: chartSeries.buckets.map(bucket => chartSeries.data.get(bucket)?.[tool] ?? 0), + borderColor: MODEL_COLORS[index % MODEL_COLORS.length], + backgroundColor: `${MODEL_COLORS[index % MODEL_COLORS.length]}30`, + fill: true, + tension: 0.4, + pointRadius: 0, + pointHoverRadius: 4, + borderWidth: 2, + })), + }), + [chartSeries, timeRange], + ); + + const options = useMemo( + () => ({ + responsive: true, + maintainAspectRatio: false, + interaction: { mode: "index" as const, intersect: false }, + plugins: { + legend: { + position: "top" as const, + align: "start" as const, + labels: { + color: chartTheme.legendLabel, + usePointStyle: true, + padding: 16, + font: { size: 12 }, + boxWidth: 8, + }, + }, + tooltip: { + backgroundColor: chartTheme.tooltipBackground, + titleColor: chartTheme.tooltipTitle, + bodyColor: chartTheme.tooltipBody, + borderColor: chartTheme.tooltipBorder, + borderWidth: 1, + padding: 12, + cornerRadius: 8, + callbacks: { + label: (context: { dataset: { label?: string }; parsed: { y: number | null } }) => + `${context.dataset.label ?? ""}: ${formatInteger(context.parsed.y ?? 0)} calls`, + }, + }, + }, + scales: { + x: { + stacked: true, + grid: { color: chartTheme.grid, drawBorder: false }, + ticks: { color: chartTheme.tick, font: { size: 11 } }, + }, + y: { + stacked: true, + grid: { color: chartTheme.grid, drawBorder: false }, + ticks: { color: chartTheme.tick, font: { size: 11 }, precision: 0 }, + min: 0, + }, + }, + }), + [chartTheme], + ); + + return ( + +
+ {chartSeries.buckets.length === 0 ? ( +
No data available
+ ) : ( + + )} +
+ + ); +} + +// --------------------------------------------------------------------------- +// Per-tool table +// --------------------------------------------------------------------------- + +function errorPillVariant(errorRate: number): "danger" | "warning" | "success" { + return errorRate > 0.1 ? "danger" : errorRate > 0 ? "warning" : "success"; +} + +function ToolsTable({ byTool }: { byTool: ToolUsageStats[] }) { + const rows = useMemo(() => buildToolRows(byTool), [byTool]); + + const columns = useMemo( + () => [ + { + key: "tool", + header: "Tool", + render: (item: ToolRowView) => ( +
+ {item.tool} +
+ ), + }, + { + key: "calls", + header: "Calls", + numeric: true, + render: (item: ToolRowView) => ( +
+
{formatInteger(item.calls)}
+
+
+
+
+ ), + }, + { + key: "errorRate", + header: "Error Rate", + numeric: true, + render: (item: ToolRowView) => ( + {formatPercent(item.errorRate)} + ), + }, + { + key: "tokens", + header: "Attr. Tokens", + numeric: true, + render: (item: ToolRowView) => ( + + {formatCompact(Math.round(item.totalTokensShare))} + + ), + }, + { + key: "cost", + header: "Attr. Cost", + numeric: true, + render: (item: ToolRowView) => {formatCost(item.costShare)}, + }, + { + key: "resultChars", + header: "Result Text", + numeric: true, + render: (item: ToolRowView) => ( + + {formatCompact(item.resultChars)} + + ), + }, + { + key: "lastUsed", + header: "Last Used", + numeric: true, + render: (item: ToolRowView) => ( + {formatRelativeTime(item.lastUsed)} + ), + }, + ], + [], + ); + + const renderMobileCard = (item: ToolRowView) => ( +
+
+
{item.tool}
+ {formatPercent(item.errorRate)} Err +
+
+
+
Calls
+
{formatInteger(item.calls)}
+
+
+
Attr. Tokens
+
+ {formatCompact(Math.round(item.totalTokensShare))} +
+
+
+
Attr. Cost
+
{formatCost(item.costShare)}
+
+
+
Result Text
+
{formatCompact(item.resultChars)}
+
+
+
+ ); + + return ( + + item.tool} + renderMobileCard={renderMobileCard} + emptyText="No tool calls recorded for this range." + /> + + ); +} + +// --------------------------------------------------------------------------- +// Per-(tool, model) breakdown +// --------------------------------------------------------------------------- + +function ToolModelPanel({ byToolModel }: { byToolModel: ToolModelStats[] }) { + const [tool, setTool] = useState(null); + + const tools = useMemo(() => [...new Set(byToolModel.map(row => row.tool))].sort(), [byToolModel]); + + const rows = useMemo(() => { + const filtered = tool ? byToolModel.filter(row => row.tool === tool) : byToolModel; + return filtered.map(row => ({ + ...row, + errorRate: row.calls > 0 ? row.errors / row.calls : 0, + })); + }, [byToolModel, tool]); + + const columns = useMemo( + () => [ + { + key: "tool", + header: "Tool", + render: (item: ToolModelStats & { errorRate: number }) => ( + {item.tool} + ), + }, + { + key: "model", + header: "Model", + render: (item: ToolModelStats & { errorRate: number }) => ( +
+
{item.model || "(unknown)"}
+
{item.provider}
+
+ ), + }, + { + key: "calls", + header: "Calls", + numeric: true, + render: (item: ToolModelStats & { errorRate: number }) => ( + {formatInteger(item.calls)} + ), + }, + { + key: "errorRate", + header: "Error Rate", + numeric: true, + render: (item: ToolModelStats & { errorRate: number }) => ( + {formatPercent(item.errorRate)} + ), + }, + { + key: "tokens", + header: "Attr. Tokens", + numeric: true, + render: (item: ToolModelStats & { errorRate: number }) => ( + {formatCompact(Math.round(item.totalTokensShare))} + ), + }, + { + key: "cost", + header: "Attr. Cost", + numeric: true, + render: (item: ToolModelStats & { errorRate: number }) => ( + {formatCost(item.costShare)} + ), + }, + ], + [], + ); + + return ( + +
+ + Tool + + +
+ `${item.tool}::${item.model}::${item.provider}`} + emptyText="No tool calls recorded for this range." + /> + + ); +} diff --git a/packages/stats/src/client/routes/index.ts b/packages/stats/src/client/routes/index.ts index 2c97e38a6..9f8236f8d 100644 --- a/packages/stats/src/client/routes/index.ts +++ b/packages/stats/src/client/routes/index.ts @@ -6,3 +6,4 @@ export * from "./ModelsRoute"; export * from "./OverviewRoute"; export * from "./ProjectsRoute"; export * from "./RequestsRoute"; +export * from "./ToolsRoute"; diff --git a/packages/stats/src/db.ts b/packages/stats/src/db.ts index a680436fa..be0fff856 100644 --- a/packages/stats/src/db.ts +++ b/packages/stats/src/db.ts @@ -19,6 +19,11 @@ import type { ModelStats, ModelTimeSeriesPoint, TimeSeriesPoint, + ToolCallStats, + ToolModelStats, + ToolResultLink, + ToolTimeSeriesPoint, + ToolUsageStats, UserMessageLink, UserMessageStats, } from "./types"; @@ -46,6 +51,7 @@ const USER_MESSAGE_LINKS_REPAIR_KEY = "user_message_links_v1"; const PRIORITY_PREMIUM_REQUESTS_BACKFILL_KEY = "premium_requests_priority_v1"; const AGENT_TYPE_BACKFILL_KEY = "agent_type_v1"; const FORK_DEDUPE_KEY = "fork_dedupe_v1"; +const TOOL_CALLS_BACKFILL_KEY = "tool_calls_v1"; function shouldResetBackfill(value: string | undefined): boolean { return value !== BACKFILL_COMPLETE && value !== BACKFILL_PENDING; } @@ -135,6 +141,27 @@ export async function initDb(): Promise { CREATE INDEX IF NOT EXISTS idx_user_messages_timestamp ON user_messages(timestamp); CREATE INDEX IF NOT EXISTS idx_user_messages_timestamp_model ON user_messages(timestamp, model, provider); + CREATE TABLE IF NOT EXISTS tool_calls ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + session_file TEXT NOT NULL, + entry_id TEXT NOT NULL, + tool_call_id TEXT NOT NULL, + folder TEXT NOT NULL, + tool_name TEXT NOT NULL, + model TEXT NOT NULL, + provider TEXT NOT NULL, + timestamp INTEGER NOT NULL, + agent_type TEXT NOT NULL DEFAULT 'main', + calls_in_turn INTEGER NOT NULL DEFAULT 1, + args_chars INTEGER NOT NULL DEFAULT 0, + result_chars INTEGER, + is_error INTEGER, + UNIQUE(session_file, tool_call_id) + ); + + CREATE INDEX IF NOT EXISTS idx_tool_calls_timestamp ON tool_calls(timestamp); + CREATE INDEX IF NOT EXISTS idx_tool_calls_tool_timestamp ON tool_calls(tool_name, timestamp); + CREATE TABLE IF NOT EXISTS meta ( key TEXT PRIMARY KEY, value TEXT NOT NULL @@ -215,6 +242,7 @@ export async function initDb(): Promise { `); } backfillUserMessages(db); + backfillToolCalls(db); repairUserMessageLinks(db); backfillPriorityPremiumRequests(db); backfillAgentType(db); @@ -881,6 +909,27 @@ function backfillUserMessages(database: Database): void { .run(USER_MESSAGES_BACKFILL_KEY, BACKFILL_PENDING); } +/** + * One-shot wipe of `tool_calls` + `file_offsets` when the `tool_calls` table + * is introduced (or its schema version bumps), so the next sync re-parses + * every session and ingests historical tool calls. `messages` and + * `user_messages` re-inserts are idempotent, so the offset reset is safe. + * Same sentinel protocol as {@link backfillUserMessages}: the PENDING value + * written here prevents re-wiping on subsequent inits. + */ +function backfillToolCalls(database: Database): void { + const row = database.prepare("SELECT value FROM meta WHERE key = ?").get(TOOL_CALLS_BACKFILL_KEY) as + | { value: string } + | undefined; + if (!shouldResetBackfill(row?.value)) return; + + database.run("DELETE FROM tool_calls"); + database.run("DELETE FROM file_offsets"); + database + .prepare("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)") + .run(TOOL_CALLS_BACKFILL_KEY, BACKFILL_PENDING); +} + /** * Reclassify pre-existing `messages` rows by agent type once, after the * `agent_type` column is added to an older database (every prior row defaulted @@ -1276,3 +1325,206 @@ export function getBehaviorByModel(cutoff?: number | null): BehaviorModelStats[] lastTimestamp: row.last_timestamp ?? 0, })); } + +/** + * Insert tool-call rows. Idempotent via UNIQUE(session_file, tool_call_id); + * the `WHERE NOT EXISTS` guard mirrors {@link insertMessageStats}: forked + * sessions deep-copy assistant entries (same `entry_id`, `timestamp`, and + * tool-call ids under a new file), so first-write-wins across the lineage + * keeps aggregates from double counting. Keyed on the assistant entry + * identity, not the call id alone — provider call ids are not a global + * namespace across unrelated sessions. + */ +export function insertToolCalls(calls: ToolCallStats[]): number { + if (!db || calls.length === 0) return 0; + + const stmt = db.prepare(` + INSERT OR IGNORE INTO tool_calls ( + session_file, entry_id, tool_call_id, folder, tool_name, + model, provider, timestamp, agent_type, calls_in_turn, args_chars + ) + SELECT ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + WHERE NOT EXISTS ( + SELECT 1 FROM tool_calls + WHERE entry_id = ? AND timestamp = ? AND tool_call_id = ? AND session_file <> ? + ) + `); + + let inserted = 0; + const insert = db.transaction(() => { + for (const c of calls) { + const result = stmt.run( + c.sessionFile, + c.entryId, + c.toolCallId, + c.folder, + c.toolName, + c.model, + c.provider, + c.timestamp, + c.agentType, + c.callsInTurn, + c.argsChars, + // `WHERE NOT EXISTS` binds: skip when a different session_file + // already holds this (entry_id, timestamp, tool_call_id). + c.entryId, + c.timestamp, + c.toolCallId, + c.sessionFile, + ); + if (result.changes > 0) inserted++; + } + }); + insert(); + return inserted; +} + +/** + * Attach result size / error flag to persisted tool-call rows. Results can + * land in a later incremental sync pass than the call that produced them, so + * this is an UPDATE keyed by (session_file, tool_call_id). The `IS NULL` + * guard makes re-syncs idempotent; rows skipped by the fork guard simply + * never match. + */ +export function updateToolResults(links: ToolResultLink[]): number { + if (!db || links.length === 0) return 0; + + const stmt = db.prepare(` + UPDATE tool_calls + SET result_chars = ?, is_error = ? + WHERE session_file = ? AND tool_call_id = ? AND result_chars IS NULL + `); + + let updated = 0; + const apply = db.transaction(() => { + for (const link of links) { + const result = stmt.run(link.resultChars, link.isError ? 1 : 0, link.sessionFile, link.toolCallId); + updated += result.changes; + } + }); + apply(); + return updated; +} + +/** + * Shared SELECT list for tool aggregates. Real provider usage comes from the + * invoking assistant turn (`messages` join) divided by `calls_in_turn`, so + * per-tool token/cost shares stay additive across tools. + */ +const TOOL_AGGREGATE_COLUMNS = ` + COUNT(*) as calls, + SUM(CASE WHEN t.is_error = 1 THEN 1 ELSE 0 END) as errors, + SUM(t.args_chars) as args_chars, + SUM(COALESCE(t.result_chars, 0)) as result_chars, + SUM(COALESCE(m.total_tokens, 0) * 1.0 / t.calls_in_turn) as total_tokens_share, + SUM(COALESCE(m.output_tokens, 0) * 1.0 / t.calls_in_turn) as output_tokens_share, + SUM(COALESCE(m.cost_total, 0) / t.calls_in_turn) as cost_share, + MAX(t.timestamp) as last_used +`; + +interface ToolAggregateRow { + tool_name: string; + model?: string; + provider?: string; + calls: number; + errors: number; + args_chars: number | null; + result_chars: number | null; + total_tokens_share: number | null; + output_tokens_share: number | null; + cost_share: number | null; + last_used: number; +} + +function rowToToolUsage(row: ToolAggregateRow): ToolUsageStats { + return { + tool: row.tool_name, + calls: row.calls, + errors: row.errors, + argsChars: row.args_chars ?? 0, + resultChars: row.result_chars ?? 0, + totalTokensShare: row.total_tokens_share ?? 0, + outputTokensShare: row.output_tokens_share ?? 0, + costShare: row.cost_share ?? 0, + lastUsed: row.last_used, + }; +} + +/** + * Get tool usage aggregated by tool name. + */ +export function getToolStats(cutoff?: number): ToolUsageStats[] { + if (!db) return []; + + const hasCutoff = cutoff !== undefined && cutoff > 0; + const stmt = db.prepare(` + SELECT t.tool_name, ${TOOL_AGGREGATE_COLUMNS} + FROM tool_calls t + LEFT JOIN messages m ON m.session_file = t.session_file AND m.entry_id = t.entry_id + ${hasCutoff ? "WHERE t.timestamp >= ?" : ""} + GROUP BY t.tool_name + ORDER BY calls DESC + `); + + const rows = (hasCutoff ? stmt.all(cutoff) : stmt.all()) as ToolAggregateRow[]; + return rows.map(rowToToolUsage); +} + +/** + * Get tool usage aggregated by (tool, model, provider). + */ +export function getToolStatsByModel(cutoff?: number): ToolModelStats[] { + if (!db) return []; + + const hasCutoff = cutoff !== undefined && cutoff > 0; + const stmt = db.prepare(` + SELECT t.tool_name, t.model, t.provider, ${TOOL_AGGREGATE_COLUMNS} + FROM tool_calls t + LEFT JOIN messages m ON m.session_file = t.session_file AND m.entry_id = t.entry_id + ${hasCutoff ? "WHERE t.timestamp >= ?" : ""} + GROUP BY t.tool_name, t.model, t.provider + ORDER BY calls DESC + `); + + const rows = (hasCutoff ? stmt.all(cutoff) : stmt.all()) as ToolAggregateRow[]; + return rows.map(row => ({ + ...rowToToolUsage(row), + model: row.model ?? "", + provider: row.provider ?? "", + })); +} + +/** + * Get tool-call time series (one point per bucket per tool). + */ +export function getToolTimeSeries( + days = 14, + cutoff?: number | null, + bucketMs = 24 * 60 * 60 * 1000, +): ToolTimeSeriesPoint[] { + if (!db) return []; + + const hasCutoff = cutoff !== null; + const seriesCutoff = hasCutoff ? (cutoff ?? Date.now() - days * 24 * 60 * 60 * 1000) : 0; + + const stmt = db.prepare(` + SELECT + (timestamp / ?) * ? as bucket, + tool_name, + COUNT(*) as calls, + SUM(CASE WHEN is_error = 1 THEN 1 ELSE 0 END) as errors + FROM tool_calls + ${hasCutoff ? "WHERE timestamp >= ?" : ""} + GROUP BY bucket, tool_name + ORDER BY bucket ASC + `); + + const rowsRaw = hasCutoff ? stmt.all(bucketMs, bucketMs, seriesCutoff) : stmt.all(bucketMs, bucketMs); + const rows = rowsRaw as Array<{ bucket: number; tool_name: string; calls: number; errors: number }>; + return rows.map(row => ({ + timestamp: row.bucket, + tool: row.tool_name, + calls: row.calls, + errors: row.errors, + })); +} diff --git a/packages/stats/src/index.ts b/packages/stats/src/index.ts index aa053adf2..d74c4080b 100755 --- a/packages/stats/src/index.ts +++ b/packages/stats/src/index.ts @@ -8,6 +8,7 @@ import { startServer } from "./server"; export { getDashboardStats, + getToolDashboardStats, getTotalMessageCount, type SyncOptions, type SyncProgress, @@ -32,6 +33,10 @@ export type { ModelStats, ModelTimeSeriesPoint, TimeSeriesPoint, + ToolDashboardStats, + ToolModelStats, + ToolTimeSeriesPoint, + ToolUsageStats, } from "./types"; /** diff --git a/packages/stats/src/parser.ts b/packages/stats/src/parser.ts index b77f9e549..ca958f695 100644 --- a/packages/stats/src/parser.ts +++ b/packages/stats/src/parser.ts @@ -6,6 +6,7 @@ import { getPriorityPremiumRequests, resolveModelServiceTier, type ServiceTierByFamily, + type ToolResultMessage, } from "@oh-my-pi/pi-ai"; import { getSessionsDir, isEnoent } from "@oh-my-pi/pi-utils"; import type { @@ -14,6 +15,8 @@ import type { SessionEntry, SessionMessageEntry, SessionServiceTierChangeEntry, + ToolCallStats, + ToolResultLink, UserMessageLink, UserMessageStats, } from "./types"; @@ -85,6 +88,14 @@ function isServiceTierChange(entry: SessionEntry): entry is SessionServiceTierCh return entry.type === "service_tier_change"; } +/** + * Check if an entry is a tool-result message. + */ +function isToolResultMessage(entry: SessionEntry): entry is SessionMessageEntry { + if (entry.type !== "message") return false; + return (entry as SessionMessageEntry).message?.role === "toolResult"; +} + /** * Extract plain text from a user message content payload. */ @@ -171,6 +182,66 @@ function extractStats( }; } +/** + * Extract one {@link ToolCallStats} per `toolCall` content block of an + * assistant message. Returns an empty array for turns without tool calls. + */ +function extractToolCalls( + sessionFile: string, + folder: string, + entry: SessionMessageEntry, + agentType: AgentType, +): ToolCallStats[] { + const msg = entry.message as AssistantMessage; + if (msg?.role !== "assistant" || !Array.isArray(msg.content)) return []; + + const blocks = msg.content.filter(block => block.type === "toolCall"); + if (blocks.length === 0) return []; + + return blocks.map(block => { + let argsChars = 0; + try { + argsChars = JSON.stringify(block.arguments ?? {}).length; + } catch { + // Non-serializable arguments (shouldn't happen in persisted JSONL); size unknown. + } + return { + sessionFile, + entryId: entry.id, + toolCallId: block.id, + folder, + toolName: block.name, + model: msg.model, + provider: msg.provider, + timestamp: msg.timestamp, + agentType, + callsInTurn: blocks.length, + argsChars, + }; + }); +} + +/** + * Build the result linkage for a `toolResult` entry: text characters fed back + * into context plus the error flag, keyed to the originating call. + */ +function extractToolResultLink(sessionFile: string, entry: SessionMessageEntry): ToolResultLink | null { + const msg = entry.message as ToolResultMessage; + if (msg.role !== "toolResult" || typeof msg.toolCallId !== "string" || msg.toolCallId.length === 0) return null; + let resultChars = 0; + if (Array.isArray(msg.content)) { + for (const block of msg.content) { + if (block.type === "text" && typeof block.text === "string") resultChars += block.text.length; + } + } + return { + sessionFile, + toolCallId: msg.toolCallId, + resultChars, + isError: msg.isError === true, + }; +} + const LF = 0x0a; const CR = 0x0d; const jsonLineDecoder = new TextDecoder(); @@ -241,6 +312,8 @@ export interface ParseSessionResult { stats: MessageStats[]; userStats: UserMessageStats[]; userLinks: UserMessageLink[]; + toolCalls: ToolCallStats[]; + toolResults: ToolResultLink[]; newOffset: number; } export async function parseSessionFile(sessionPath: string, fromOffset = 0): Promise { @@ -248,7 +321,8 @@ export async function parseSessionFile(sessionPath: string, fromOffset = 0): Pro try { bytes = await Bun.file(sessionPath).bytes(); } catch (err) { - if (isEnoent(err)) return { stats: [], userStats: [], userLinks: [], newOffset: fromOffset }; + if (isEnoent(err)) + return { stats: [], userStats: [], userLinks: [], toolCalls: [], toolResults: [], newOffset: fromOffset }; throw err; } @@ -257,6 +331,8 @@ export async function parseSessionFile(sessionPath: string, fromOffset = 0): Pro const stats: MessageStats[] = []; const userStats: UserMessageStats[] = []; const userLinks: UserMessageLink[] = []; + const toolCalls: ToolCallStats[] = []; + const toolResults: ToolResultLink[] = []; const userByEntryId = new Map(); const start = Math.max(0, Math.min(fromOffset, bytes.length)); const unprocessed = bytes.subarray(start); @@ -278,9 +354,15 @@ export async function parseSessionFile(sessionPath: string, fromOffset = 0): Pro } continue; } + if (isToolResultMessage(entry)) { + const link = extractToolResultLink(sessionPath, entry); + if (link) toolResults.push(link); + continue; + } if (isAssistantMessage(entry)) { const msgStats = extractStats(sessionPath, folder, entry, currentServiceTier, agentType); if (msgStats) stats.push(msgStats); + toolCalls.push(...extractToolCalls(sessionPath, folder, entry, agentType)); // Link assistant's responding model back to the user message it answered. const parentId = (entry as SessionMessageEntry).parentId; if (parentId) { @@ -303,7 +385,7 @@ export async function parseSessionFile(sessionPath: string, fromOffset = 0): Pro } } - return { stats, userStats, userLinks, newOffset: start + read }; + return { stats, userStats, userLinks, toolCalls, toolResults, newOffset: start + read }; } /** diff --git a/packages/stats/src/server.ts b/packages/stats/src/server.ts index 8af71ffdb..17fc9fc40 100644 --- a/packages/stats/src/server.ts +++ b/packages/stats/src/server.ts @@ -13,6 +13,7 @@ import { getRecentErrors, getRecentRequests, getRequestDetails, + getToolDashboardStats, getTotalMessageCount, syncAllSessions, } from "./aggregator"; @@ -215,6 +216,11 @@ async function handleApi(req: Request): Promise { return Response.json(stats); } + if (path === "/api/stats/tools") { + const stats = await getToolDashboardStats(range); + return Response.json(stats); + } + if (path === "/api/stats/recent") { const limit = url.searchParams.get("limit"); const stats = await getRecentRequests(limit ? parseInt(limit, 10) : undefined); diff --git a/packages/stats/src/shared-types.ts b/packages/stats/src/shared-types.ts index c0defd4f7..db84fd2f7 100644 --- a/packages/stats/src/shared-types.ts +++ b/packages/stats/src/shared-types.ts @@ -269,3 +269,55 @@ export interface GainDashboardStats { /** All distinct projects seen in the data, for the selector. */ projects: string[]; } + +/** + * Aggregated usage for a single tool over the active range. + * + * Token/cost fields are the *real* provider usage of the assistant turns that + * invoked the tool, split evenly across that turn's tool calls so the numbers + * stay additive (a turn with 3 calls contributes a third of its usage to each + * tool). Payload fields (`argsChars`/`resultChars`) are raw character counts + * of the serialized arguments and the text fed back into context — a size + * proxy, not provider-counted tokens. + */ +export interface ToolUsageStats { + /** Tool name as recorded on the tool call. */ + tool: string; + /** Number of tool calls. */ + calls: number; + /** Calls whose result came back with `isError`. */ + errors: number; + /** Serialized tool-call argument characters. */ + argsChars: number; + /** Text characters of tool results fed back into context. */ + resultChars: number; + /** Total provider tokens of invoking turns, attributed per call share. */ + totalTokensShare: number; + /** Output tokens of invoking turns, attributed per call share. */ + outputTokensShare: number; + /** Cost (USD) of invoking turns, attributed per call share. */ + costShare: number; + /** Unix ms of the most recent call in range. */ + lastUsed: number; +} + +/** Per-(tool, model) breakdown with the same attribution as {@link ToolUsageStats}. */ +export interface ToolModelStats extends ToolUsageStats { + model: string; + provider: string; +} + +/** Tool-call time-series point (one bucket per tool). */ +export interface ToolTimeSeriesPoint { + timestamp: number; + tool: string; + calls: number; + errors: number; +} + +/** Complete tools dashboard payload. */ +export interface ToolDashboardStats { + byTool: ToolUsageStats[]; + byToolModel: ToolModelStats[]; + series: ToolTimeSeriesPoint[]; +} diff --git a/packages/stats/src/types.ts b/packages/stats/src/types.ts index 8036e5077..d2d6c0ccb 100644 --- a/packages/stats/src/types.ts +++ b/packages/stats/src/types.ts @@ -126,3 +126,46 @@ export interface UserMessageLink { model: string; provider: string; } + +/** + * One tool call extracted from an assistant message's `toolCall` content + * blocks. `callsInTurn` records how many calls that assistant turn contained + * so aggregation can split the turn's real provider usage evenly per call. + */ +export interface ToolCallStats { + /** Session file path */ + sessionFile: string; + /** Assistant-message entry ID that emitted the call */ + entryId: string; + /** Provider-assigned tool call ID (unique within a session) */ + toolCallId: string; + /** Folder/project path (extracted from session filename) */ + folder: string; + /** Tool name */ + toolName: string; + /** Model that emitted the call */ + model: string; + /** Provider name */ + provider: string; + /** Assistant-message timestamp (Unix ms) */ + timestamp: number; + /** Which agent produced the call */ + agentType: AgentType; + /** Total tool calls in the same assistant turn (>= 1) */ + callsInTurn: number; + /** Serialized argument characters */ + argsChars: number; +} + +/** + * Result linkage emitted when the parser sees a `toolResult` message entry. + * Applied as an UPDATE on the persisted tool-call row — results can land in a + * later incremental sync pass than the call that produced them. + */ +export interface ToolResultLink { + sessionFile: string; + toolCallId: string; + /** Text characters fed back into context */ + resultChars: number; + isError: boolean; +} diff --git a/packages/stats/test/tool-stats.test.ts b/packages/stats/test/tool-stats.test.ts new file mode 100644 index 000000000..6b6be833e --- /dev/null +++ b/packages/stats/test/tool-stats.test.ts @@ -0,0 +1,346 @@ +import { describe, expect, it } from "bun:test"; +import * as fs from "node:fs/promises"; +import * as path from "node:path"; +import { getToolDashboardStats, syncAllSessions } from "@oh-my-pi/omp-stats/aggregator"; +import { getToolStats, getToolStatsByModel } from "@oh-my-pi/omp-stats/db"; +import type { ToolUsageStats } from "@oh-my-pi/omp-stats/types"; +import { getSessionsDir } from "@oh-my-pi/pi-utils"; +import { installStatsTestIsolation } from "./helpers/temp-agent"; + +installStatsTestIsolation("@pi-stats-tool-stats-"); + +const FOLDER_SLUG = "--tmp--tool-stats"; +const MODEL = "gpt-5.4"; +const PROVIDER = "openai"; + +const TS1 = "2026-06-24T10:00:00.000Z"; +const TS2 = "2026-06-24T10:05:00.000Z"; + +// Turn 1: two toolCall blocks (grep + read) sharing one provider request. +const TURN1_TOTAL_TOKENS = 100; +const TURN1_OUTPUT_TOKENS = 20; +const TURN1_COST = 0.01; +// Turn 2: a single grep toolCall owning the whole request. +const TURN2_TOTAL_TOKENS = 40; +const TURN2_OUTPUT_TOKENS = 8; +const TURN2_COST = 0.004; + +const GREP_ARGS_1 = { pattern: "x" }; +const READ_ARGS = { path: "/tmp/f" }; +const GREP_ARGS_2 = { pattern: "yz" }; + +const GREP_RESULT_1 = "grep hit: src/index.ts:42"; +const READ_ERROR_RESULT = "read failed: ENOENT"; +const GREP_RESULT_2 = "ok"; + +interface ToolCallBlock { + id: string; + name: string; + arguments: Record; +} + +interface AssistantTurnOptions { + entryId: string; + parentId?: string | null; + timestamp: string; + toolCalls: ToolCallBlock[]; + totalTokens: number; + outputTokens: number; + costTotal: number; +} + +function buildAssistantEntry(opts: AssistantTurnOptions) { + return { + type: "message", + id: opts.entryId, + parentId: opts.parentId ?? null, + timestamp: opts.timestamp, + message: { + role: "assistant", + content: [ + { type: "text", text: "ok" }, + ...opts.toolCalls.map(call => ({ + type: "toolCall", + id: call.id, + name: call.name, + arguments: call.arguments, + })), + ], + api: "openai-responses", + provider: PROVIDER, + model: MODEL, + usage: { + input: 10, + output: opts.outputTokens, + cacheRead: 0, + cacheWrite: 0, + totalTokens: opts.totalTokens, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: opts.costTotal }, + }, + stopReason: "toolUse", + timestamp: Date.parse(opts.timestamp), + duration: 10, + ttft: 5, + }, + }; +} + +interface ToolResultOptions { + entryId: string; + parentId: string; + timestamp: string; + toolCallId: string; + toolName: string; + text: string; + isError?: boolean; +} + +function buildToolResultEntry(opts: ToolResultOptions) { + return { + type: "message", + id: opts.entryId, + parentId: opts.parentId, + timestamp: opts.timestamp, + message: { + role: "toolResult", + toolCallId: opts.toolCallId, + toolName: opts.toolName, + content: [{ type: "text", text: opts.text }], + isError: opts.isError ?? false, + timestamp: Date.parse(opts.timestamp), + }, + }; +} + +async function writeSessionFile( + fileName: string, + header: { id: string; parentSession?: string }, + entries: unknown[], +): Promise { + const sessionDir = path.join(getSessionsDir(), FOLDER_SLUG); + await fs.mkdir(sessionDir, { recursive: true }); + const sessionFile = path.join(sessionDir, fileName); + const headerEntry = { + type: "session", + version: 3, + id: header.id, + timestamp: new Date().toISOString(), + cwd: "/tmp/project", + ...(header.parentSession ? { parentSession: header.parentSession } : {}), + }; + const lines = [headerEntry, ...entries].map(entry => JSON.stringify(entry)).join("\n"); + await Bun.write(sessionFile, `${lines}\n`); + return sessionFile; +} + +/** + * Standard fixture: turn 1 calls grep+read (grep succeeds, read errors), + * turn 2 calls grep again and succeeds. + */ +function buildStandardEntries(): unknown[] { + return [ + buildAssistantEntry({ + entryId: "asst-1", + timestamp: TS1, + toolCalls: [ + { id: "call-1", name: "grep", arguments: GREP_ARGS_1 }, + { id: "call-2", name: "read", arguments: READ_ARGS }, + ], + totalTokens: TURN1_TOTAL_TOKENS, + outputTokens: TURN1_OUTPUT_TOKENS, + costTotal: TURN1_COST, + }), + buildToolResultEntry({ + entryId: "tr-1", + parentId: "asst-1", + timestamp: TS1, + toolCallId: "call-1", + toolName: "grep", + text: GREP_RESULT_1, + }), + buildToolResultEntry({ + entryId: "tr-2", + parentId: "asst-1", + timestamp: TS1, + toolCallId: "call-2", + toolName: "read", + text: READ_ERROR_RESULT, + isError: true, + }), + buildAssistantEntry({ + entryId: "asst-2", + parentId: "tr-2", + timestamp: TS2, + toolCalls: [{ id: "call-3", name: "grep", arguments: GREP_ARGS_2 }], + totalTokens: TURN2_TOTAL_TOKENS, + outputTokens: TURN2_OUTPUT_TOKENS, + costTotal: TURN2_COST, + }), + buildToolResultEntry({ + entryId: "tr-3", + parentId: "asst-2", + timestamp: TS2, + toolCallId: "call-3", + toolName: "grep", + text: GREP_RESULT_2, + }), + ]; +} + +function toolRow(rows: ToolUsageStats[], tool: string): ToolUsageStats { + const row = rows.find(r => r.tool === tool); + if (!row) throw new Error(`missing aggregate row for tool "${tool}"`); + return row; +} + +describe("tool usage stats pipeline", () => { + it("ingests tool calls and results end-to-end and splits turn usage across calls", async () => { + await writeSessionFile("session.jsonl", { id: "sess0001" }, buildStandardEntries()); + await syncAllSessions({ workers: 1 }); + + const stats = getToolStats(); + expect(stats).toHaveLength(2); + + const grep = toolRow(stats, "grep"); + expect(grep.calls).toBe(2); + expect(grep.errors).toBe(0); + expect(grep.resultChars).toBe(GREP_RESULT_1.length + GREP_RESULT_2.length); + expect(grep.argsChars).toBe(JSON.stringify(GREP_ARGS_1).length + JSON.stringify(GREP_ARGS_2).length); + // Turn 1's request is split across its two toolCall blocks; turn 2 is + // grep's alone: 100/2 + 40 = 90, 20/2 + 8 = 18, 0.01/2 + 0.004 = 0.009. + expect(grep.totalTokensShare).toBeCloseTo(90, 6); + expect(grep.outputTokensShare).toBeCloseTo(18, 6); + expect(grep.costShare).toBeCloseTo(0.009, 8); + expect(grep.lastUsed).toBe(Date.parse(TS2)); + + const read = toolRow(stats, "read"); + expect(read.calls).toBe(1); + expect(read.errors).toBe(1); + expect(read.resultChars).toBe(READ_ERROR_RESULT.length); + expect(read.argsChars).toBe(JSON.stringify(READ_ARGS).length); + expect(read.totalTokensShare).toBeCloseTo(50, 6); + expect(read.outputTokensShare).toBeCloseTo(10, 6); + expect(read.costShare).toBeCloseTo(0.005, 8); + expect(read.lastUsed).toBe(Date.parse(TS1)); + + // Per-model breakdown carries the fixture model/provider with the same split. + const byModel = getToolStatsByModel(); + expect(byModel).toHaveLength(2); + for (const row of byModel) { + expect(row.model).toBe(MODEL); + expect(row.provider).toBe(PROVIDER); + } + expect(toolRow(byModel, "grep").calls).toBe(2); + expect(toolRow(byModel, "grep").totalTokensShare).toBeCloseTo(90, 6); + expect(toolRow(byModel, "read").calls).toBe(1); + expect(toolRow(byModel, "read").totalTokensShare).toBeCloseTo(50, 6); + + // Dashboard payload reuses the same aggregates and buckets the calls. + const dashboard = await getToolDashboardStats("all"); + expect(dashboard.byTool).toEqual(stats); + expect(dashboard.series.length).toBeGreaterThan(0); + const seriesCalls = new Map(); + const seriesErrors = new Map(); + for (const point of dashboard.series) { + seriesCalls.set(point.tool, (seriesCalls.get(point.tool) ?? 0) + point.calls); + seriesErrors.set(point.tool, (seriesErrors.get(point.tool) ?? 0) + point.errors); + } + expect(seriesCalls.get("grep")).toBe(2); + expect(seriesCalls.get("read")).toBe(1); + expect(seriesErrors.get("grep")).toBe(0); + expect(seriesErrors.get("read")).toBe(1); + }); + + it("links a result that lands in a later sync pass without duplicating the call", async () => { + const lateResultText = "late failure output"; + const sessionFile = await writeSessionFile("session.jsonl", { id: "sess0002" }, [ + buildAssistantEntry({ + entryId: "asst-1", + timestamp: TS1, + toolCalls: [{ id: "call-1", name: "grep", arguments: GREP_ARGS_1 }], + totalTokens: TURN2_TOTAL_TOKENS, + outputTokens: TURN2_OUTPUT_TOKENS, + costTotal: TURN2_COST, + }), + ]); + await syncAllSessions({ workers: 1 }); + + // First pass: the call is recorded but no result has arrived yet. + const pending = getToolStats(); + expect(pending).toHaveLength(1); + expect(pending[0].tool).toBe("grep"); + expect(pending[0].calls).toBe(1); + expect(pending[0].resultChars).toBe(0); + expect(pending[0].errors).toBe(0); + + // The toolResult is appended after the first pass consumed the file. + const lateResult = buildToolResultEntry({ + entryId: "tr-1", + parentId: "asst-1", + timestamp: TS2, + toolCallId: "call-1", + toolName: "grep", + text: lateResultText, + isError: true, + }); + await fs.appendFile(sessionFile, `${JSON.stringify(lateResult)}\n`); + // Guarantee the stored mtime is strictly older than the file's, so the + // incremental sync re-reads it regardless of filesystem granularity. + const bumped = new Date(Date.now() + 1_000); + await fs.utimes(sessionFile, bumped, bumped); + + await syncAllSessions({ workers: 1 }); + + const linked = getToolStats(); + expect(linked).toHaveLength(1); + expect(linked[0].tool).toBe("grep"); + expect(linked[0].calls).toBe(1); + expect(linked[0].resultChars).toBe(lateResultText.length); + expect(linked[0].errors).toBe(1); + }); + + it("does not double-count tool calls copied into a forked session file", async () => { + const entries = buildStandardEntries(); + const parentFile = await writeSessionFile("01_parent.jsonl", { id: "parent00" }, entries); + // `createBranchedSession` deep-copies the parent's entries (same entry + // ids, timestamps, and tool call ids) into the child file. + await writeSessionFile("02_fork.jsonl", { id: "fork0000", parentSession: parentFile }, entries); + + await syncAllSessions({ workers: 1 }); + + const stats = getToolStats(); + expect(stats).toHaveLength(2); + + const grep = toolRow(stats, "grep"); + expect(grep.calls).toBe(2); + expect(grep.errors).toBe(0); + expect(grep.resultChars).toBe(GREP_RESULT_1.length + GREP_RESULT_2.length); + expect(grep.totalTokensShare).toBeCloseTo(90, 6); + + const read = toolRow(stats, "read"); + expect(read.calls).toBe(1); + expect(read.errors).toBe(1); + expect(read.resultChars).toBe(READ_ERROR_RESULT.length); + expect(read.totalTokensShare).toBeCloseTo(50, 6); + }); + + it("keeps tool aggregates stable across repeated syncs of unchanged data", async () => { + const sessionFile = await writeSessionFile("session.jsonl", { id: "sess0003" }, buildStandardEntries()); + await syncAllSessions({ workers: 1 }); + + const first = getToolStats(); + expect(toolRow(first, "grep").calls).toBe(2); + expect(toolRow(first, "read").calls).toBe(1); + + // Plain re-sync: the offset table short-circuits the unchanged file. + await syncAllSessions({ workers: 1 }); + expect(getToolStats()).toEqual(first); + + // Bumped mtime with identical content: the file is re-examined from its + // stored offset and must not re-ingest anything. + const bumped = new Date(Date.now() + 1_000); + await fs.utimes(sessionFile, bumped, bumped); + await syncAllSessions({ workers: 1 }); + expect(getToolStats()).toEqual(first); + }); +});