feat(stats): added Providers dashboard with per-provider analytics and utilization insights
- Added `/api/stats/providers` endpoint and database queries returning per-provider request/token/cost totals, hourly burn, and time-series data. - Created ProvidersRoute UI component with totals panel, trend charts, peak-hours histogram, and window-insights visualizations. - Added usage-window analytics computing peak utilization fractions, ideal account counts, and exhaustion events from usage snapshots. - Included comprehensive tests for the aggregation logic, API, and window-stat computations.
This commit is contained in:
can1357
@@ -2,6 +2,10 @@
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
|
||||
- Added a Providers dashboard section: per-provider totals, stacked token/cost burn over time, peak-burn-hours histogram, subscription-window insights (windows burned, estimated tokens per window, peak concurrent utilization, ideal account count, exhaustion events), and latest window utilization per account — window analytics are derived from the auth store's recorded usage-limit snapshots.
|
||||
|
||||
## [17.1.0] - 2026-07-24
|
||||
|
||||
### Fixed
|
||||
|
||||
@@ -13,9 +13,12 @@ import {
|
||||
getModelPerformanceSeries,
|
||||
getModelTimeSeries,
|
||||
getOverallStats,
|
||||
getProviderHourlyBurn,
|
||||
getProviderTimeSeries,
|
||||
getStatsByAgentType,
|
||||
getStatsByFolder,
|
||||
getStatsByModel,
|
||||
getStatsByProvider,
|
||||
getTimeSeries,
|
||||
getToolStats,
|
||||
getToolStatsByModel,
|
||||
@@ -35,7 +38,15 @@ import type { SyncWorkerRequest, SyncWorkerResponse } from "./sync-worker";
|
||||
// hidden argv mode, so the compiled binary and npm bundle only need one
|
||||
// JavaScript entry. Standalone source `omp-stats` keeps using this package's
|
||||
// own sync-worker source file.
|
||||
import type { BehaviorDashboardStats, DashboardStats, MessageStats, RequestDetails, ToolDashboardStats } from "./types";
|
||||
import type {
|
||||
BehaviorDashboardStats,
|
||||
DashboardStats,
|
||||
MessageStats,
|
||||
ProviderDashboardStats,
|
||||
RequestDetails,
|
||||
ToolDashboardStats,
|
||||
} from "./types";
|
||||
import { computeUsageWindowStats, readUsageSnapshots } from "./usage-windows";
|
||||
|
||||
/**
|
||||
* Apply a freshly parsed result to the database. Runs entirely on the
|
||||
@@ -504,3 +515,23 @@ export async function getToolDashboardStats(range?: string | null): Promise<Tool
|
||||
series: getToolTimeSeries(modelSeriesDays, cutoff, modelSeriesBucketMs),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the providers dashboard payload: per-provider totals, peak-burn-hours
|
||||
* histogram, provider token time series, and subscription-window analytics
|
||||
* (utilization series + insights) derived from recorded usage-limit snapshots.
|
||||
*/
|
||||
export async function getProviderDashboardStats(range?: string | null): Promise<ProviderDashboardStats> {
|
||||
await initDb();
|
||||
const { modelSeriesDays, modelSeriesBucketMs, cutoff } = getTimeRangeConfig(range);
|
||||
const providers = getStatsByProvider(cutoff ?? undefined);
|
||||
const tokensByProvider = new Map(providers.map(p => [p.provider, p.totalTokens]));
|
||||
const { usageSeries, windowInsights } = computeUsageWindowStats(readUsageSnapshots(cutoff ?? 0), tokensByProvider);
|
||||
return {
|
||||
providers,
|
||||
hourly: getProviderHourlyBurn(cutoff ?? undefined),
|
||||
series: getProviderTimeSeries(modelSeriesDays, cutoff, modelSeriesBucketMs),
|
||||
usageSeries,
|
||||
windowInsights,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
ModelsRoute,
|
||||
OverviewRoute,
|
||||
ProjectsRoute,
|
||||
ProvidersRoute,
|
||||
RequestsRoute,
|
||||
ToolsRoute,
|
||||
} from "./routes";
|
||||
@@ -73,6 +74,8 @@ export default function App() {
|
||||
);
|
||||
case "models":
|
||||
return <ModelsRoute active={isActive} range={range} refreshTrigger={refreshTrigger} />;
|
||||
case "providers":
|
||||
return <ProvidersRoute active={isActive} range={range} refreshTrigger={refreshTrigger} />;
|
||||
case "tools":
|
||||
return <ToolsRoute active={isActive} range={range} refreshTrigger={refreshTrigger} />;
|
||||
case "costs":
|
||||
|
||||
@@ -6,6 +6,7 @@ import type {
|
||||
MessageStats,
|
||||
ModelDashboardStats,
|
||||
OverviewStats,
|
||||
ProviderDashboardStats,
|
||||
RequestDetails,
|
||||
TimeRange,
|
||||
ToolDashboardStats,
|
||||
@@ -106,3 +107,12 @@ export async function getToolDashboardStats(
|
||||
): Promise<ToolDashboardStats> {
|
||||
return fetchJson<ToolDashboardStats>(`${API_BASE}/stats/tools?range=${encodeURIComponent(range)}`, { signal });
|
||||
}
|
||||
|
||||
export async function getProviderDashboardStats(
|
||||
range: TimeRange = "24h",
|
||||
signal?: AbortSignal,
|
||||
): Promise<ProviderDashboardStats> {
|
||||
return fetchJson<ProviderDashboardStats>(`${API_BASE}/stats/providers?range=${encodeURIComponent(range)}`, {
|
||||
signal,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,4 +1,15 @@
|
||||
import { Activity, AlertCircle, Coins, Cpu, Folder, LayoutDashboard, Smile, TrendingUp, Wrench } from "lucide-react";
|
||||
import {
|
||||
Activity,
|
||||
AlertCircle,
|
||||
Coins,
|
||||
Cpu,
|
||||
Folder,
|
||||
LayoutDashboard,
|
||||
Plug,
|
||||
Smile,
|
||||
TrendingUp,
|
||||
Wrench,
|
||||
} from "lucide-react";
|
||||
import type React from "react";
|
||||
|
||||
export type DashboardSection =
|
||||
@@ -6,6 +17,7 @@ export type DashboardSection =
|
||||
| "requests"
|
||||
| "errors"
|
||||
| "models"
|
||||
| "providers"
|
||||
| "tools"
|
||||
| "costs"
|
||||
| "behavior"
|
||||
@@ -40,6 +52,11 @@ export const routes: DashboardRoute[] = [
|
||||
label: "Models",
|
||||
icon: Cpu,
|
||||
},
|
||||
{
|
||||
id: "providers",
|
||||
label: "Providers",
|
||||
icon: Plug,
|
||||
},
|
||||
{
|
||||
id: "tools",
|
||||
label: "Tools",
|
||||
|
||||
@@ -7,6 +7,7 @@ const VALID_SECTIONS: DashboardSection[] = [
|
||||
"requests",
|
||||
"errors",
|
||||
"models",
|
||||
"providers",
|
||||
"tools",
|
||||
"costs",
|
||||
"behavior",
|
||||
|
||||
@@ -0,0 +1,492 @@
|
||||
import { useMemo, useState } from "react";
|
||||
import { Bar } from "react-chartjs-2";
|
||||
import { getProviderDashboardStats } from "../api";
|
||||
import {
|
||||
barDatasetStyle,
|
||||
buildSharedPlugins,
|
||||
buildSharedScales,
|
||||
buildTopNByModelSeries,
|
||||
CHART_THEMES,
|
||||
MODEL_COLORS,
|
||||
styleDatasets,
|
||||
} from "../components/chart-shared";
|
||||
import {
|
||||
formatCompact,
|
||||
formatCost,
|
||||
formatInteger,
|
||||
formatPercent,
|
||||
formatRelativeTime,
|
||||
formatTokensPerSecond,
|
||||
} from "../data/formatters";
|
||||
import { useResource } from "../data/useResource";
|
||||
import type {
|
||||
ProviderAggregate,
|
||||
ProviderDashboardStats,
|
||||
ProviderHourlyPoint,
|
||||
ProviderWindowInsight,
|
||||
TimeRange,
|
||||
UsageWindowSeries,
|
||||
} from "../types";
|
||||
import { AsyncBoundary, DataTable, type DataTableColumn, EmptyState, Panel, SegmentedControl } from "../ui";
|
||||
import { useSystemTheme } from "../useSystemTheme";
|
||||
|
||||
export interface ProvidersRouteProps {
|
||||
active: boolean;
|
||||
range: TimeRange;
|
||||
refreshTrigger: number;
|
||||
}
|
||||
|
||||
export function ProvidersRoute({ active, range, refreshTrigger }: ProvidersRouteProps) {
|
||||
const {
|
||||
data: stats,
|
||||
error,
|
||||
loading,
|
||||
} = useResource(["providers", range, refreshTrigger], signal => getProviderDashboardStats(range, signal), {
|
||||
pollMs: 30000,
|
||||
enabled: active,
|
||||
});
|
||||
|
||||
return (
|
||||
<div className="stats-route-container space-y-6">
|
||||
<AsyncBoundary loading={loading} error={error} data={stats}>
|
||||
{stats && (
|
||||
<>
|
||||
<ProviderTotalsPanel providers={stats.providers} />
|
||||
<ProviderTrendPanel stats={stats} />
|
||||
<PeakHoursPanel hourly={stats.hourly} providers={stats.providers} />
|
||||
<WindowInsightsPanel insights={stats.windowInsights} />
|
||||
<WindowUtilizationPanel usageSeries={stats.usageSeries} />
|
||||
</>
|
||||
)}
|
||||
</AsyncBoundary>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Provider totals
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function ProviderTotalsPanel({ providers }: { providers: ProviderAggregate[] }) {
|
||||
const grandTotal = useMemo(() => providers.reduce((sum, p) => sum + p.totalTokens, 0), [providers]);
|
||||
|
||||
const columns: DataTableColumn<ProviderAggregate>[] = [
|
||||
{ key: "provider", header: "Provider", render: p => <span className="font-medium">{p.provider}</span> },
|
||||
{ key: "requests", header: "Requests", numeric: true, render: p => formatInteger(p.totalRequests) },
|
||||
{
|
||||
key: "errors",
|
||||
header: "Error Rate",
|
||||
numeric: true,
|
||||
render: p => formatPercent(p.totalRequests > 0 ? p.failedRequests / p.totalRequests : 0),
|
||||
},
|
||||
{ key: "models", header: "Models", numeric: true, render: p => formatInteger(p.models) },
|
||||
{
|
||||
key: "tokens",
|
||||
header: "Tokens",
|
||||
numeric: true,
|
||||
render: p => (
|
||||
<span
|
||||
title={`Input ${formatCompact(p.totalInputTokens)} · Output ${formatCompact(p.totalOutputTokens)} · Cache read ${formatCompact(p.totalCacheReadTokens)} · Cache write ${formatCompact(p.totalCacheWriteTokens)}`}
|
||||
>
|
||||
{formatCompact(p.totalTokens)}
|
||||
</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
key: "share",
|
||||
header: "Share",
|
||||
numeric: true,
|
||||
render: p => formatPercent(grandTotal > 0 ? p.totalTokens / grandTotal : 0),
|
||||
},
|
||||
{ key: "cost", header: "Cost", numeric: true, render: p => formatCost(p.totalCost) },
|
||||
{ key: "tps", header: "Tok/s", numeric: true, render: p => formatTokensPerSecond(p.avgTokensPerSecond) },
|
||||
];
|
||||
|
||||
return (
|
||||
<Panel title="Provider Totals" subtitle="Token, request, and cost totals per provider over the active range">
|
||||
<DataTable
|
||||
columns={columns}
|
||||
data={providers}
|
||||
keyExtractor={p => p.provider}
|
||||
emptyText="No requests recorded in this range"
|
||||
/>
|
||||
</Panel>
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Token / cost trend by provider
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function ProviderTrendPanel({ stats }: { stats: ProviderDashboardStats }) {
|
||||
const [metric, setMetric] = useState<"tokens" | "cost">("tokens");
|
||||
const theme = useSystemTheme();
|
||||
const chartTheme = CHART_THEMES[theme];
|
||||
|
||||
// buildTopNByModelSeries keys on `model`; feed it the provider name so we
|
||||
// get the same top-N + "Other" rollup without a parallel implementation.
|
||||
const chartData = useMemo(() => {
|
||||
const points = stats.series.map(p => ({ ...p, model: p.provider }));
|
||||
return buildTopNByModelSeries<(typeof points)[number], { total: number }>(points, {
|
||||
topN: 6,
|
||||
rankWeight: p => (metric === "tokens" ? p.totalTokens : p.cost),
|
||||
initBucket: () => ({ total: 0 }),
|
||||
accumulate: (bucket, p) => {
|
||||
bucket.total += metric === "tokens" ? p.totalTokens : p.cost;
|
||||
},
|
||||
bucketToValue: bucket => bucket.total,
|
||||
});
|
||||
}, [stats.series, metric]);
|
||||
|
||||
const formatValue = metric === "tokens" ? formatCompact : (v: number) => formatCost(v);
|
||||
const options = useMemo(() => {
|
||||
const { sharedScaleBase, yScale } = buildSharedScales({ chartTheme, formatY: formatValue });
|
||||
return {
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
interaction: { mode: "index" as const, intersect: false },
|
||||
plugins: buildSharedPlugins({
|
||||
chartTheme,
|
||||
showLegend: true,
|
||||
defaultLabel: metric === "tokens" ? "Tokens" : "Cost",
|
||||
formatValue,
|
||||
footer: items => {
|
||||
if (items.length < 2) return undefined;
|
||||
const total = items.reduce((sum, item) => sum + (item.parsed.y ?? 0), 0);
|
||||
return `Total: ${formatValue(total)}`;
|
||||
},
|
||||
}),
|
||||
scales: {
|
||||
x: { ...sharedScaleBase, stacked: true },
|
||||
y: { ...yScale, stacked: true },
|
||||
},
|
||||
};
|
||||
}, [chartTheme, metric, formatValue]);
|
||||
|
||||
const data = useMemo(
|
||||
() => ({
|
||||
labels: chartData.labels,
|
||||
datasets: styleDatasets(chartData, i => barDatasetStyle(MODEL_COLORS[i % MODEL_COLORS.length])),
|
||||
}),
|
||||
[chartData],
|
||||
);
|
||||
|
||||
return (
|
||||
<Panel
|
||||
title="Burn by Provider"
|
||||
subtitle="Stacked token/cost burn per provider over time"
|
||||
actions={
|
||||
<SegmentedControl
|
||||
options={[
|
||||
{ value: "tokens" as const, label: "Tokens" },
|
||||
{ value: "cost" as const, label: "Cost" },
|
||||
]}
|
||||
value={metric}
|
||||
onChange={setMetric}
|
||||
/>
|
||||
}
|
||||
>
|
||||
<div className="h-[300px]">
|
||||
{chartData.labels.length === 0 ? (
|
||||
<EmptyState message="No provider activity in this range" />
|
||||
) : (
|
||||
<Bar data={data} options={options} />
|
||||
)}
|
||||
</div>
|
||||
</Panel>
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Peak burn hours
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const ALL_PROVIDERS = "__all__";
|
||||
|
||||
function PeakHoursPanel({ hourly, providers }: { hourly: ProviderHourlyPoint[]; providers: ProviderAggregate[] }) {
|
||||
const [provider, setProvider] = useState(ALL_PROVIDERS);
|
||||
const theme = useSystemTheme();
|
||||
const chartTheme = CHART_THEMES[theme];
|
||||
|
||||
const { tokensByHour, peakHour } = useMemo(() => {
|
||||
const tokens = new Array<number>(24).fill(0);
|
||||
for (const point of hourly) {
|
||||
if (provider !== ALL_PROVIDERS && point.provider !== provider) continue;
|
||||
tokens[point.hour] += point.totalTokens;
|
||||
}
|
||||
let peak = 0;
|
||||
for (let hour = 1; hour < 24; hour++) {
|
||||
if (tokens[hour] > tokens[peak]) peak = hour;
|
||||
}
|
||||
return { tokensByHour: tokens, peakHour: peak };
|
||||
}, [hourly, provider]);
|
||||
|
||||
const hasData = tokensByHour.some(v => v > 0);
|
||||
|
||||
const data = useMemo(
|
||||
() => ({
|
||||
labels: Array.from({ length: 24 }, (_, hour) => `${String(hour).padStart(2, "0")}:00`),
|
||||
datasets: [
|
||||
{
|
||||
label: "Tokens",
|
||||
data: tokensByHour,
|
||||
...barDatasetStyle(MODEL_COLORS[2]),
|
||||
// Highlight the peak hour in the brand accent color.
|
||||
backgroundColor: tokensByHour.map((_, hour) => (hour === peakHour ? MODEL_COLORS[0] : MODEL_COLORS[2])),
|
||||
},
|
||||
],
|
||||
}),
|
||||
[tokensByHour, peakHour],
|
||||
);
|
||||
|
||||
const options = useMemo(() => {
|
||||
const { sharedScaleBase, yScale } = buildSharedScales({ chartTheme, formatY: formatCompact });
|
||||
return {
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
plugins: buildSharedPlugins({
|
||||
chartTheme,
|
||||
showLegend: false,
|
||||
defaultLabel: "Tokens",
|
||||
formatValue: formatCompact,
|
||||
}),
|
||||
scales: { x: sharedScaleBase, y: yScale },
|
||||
};
|
||||
}, [chartTheme]);
|
||||
|
||||
return (
|
||||
<Panel
|
||||
title="Peak Burn Hours"
|
||||
subtitle={
|
||||
hasData
|
||||
? `Token burn by local hour of day — peak at ${String(peakHour).padStart(2, "0")}:00`
|
||||
: "Token burn by local hour of day"
|
||||
}
|
||||
actions={
|
||||
<select
|
||||
className="stats-select"
|
||||
value={provider}
|
||||
onChange={e => setProvider(e.target.value)}
|
||||
aria-label="Provider"
|
||||
>
|
||||
<option value={ALL_PROVIDERS}>All providers</option>
|
||||
{providers.map(p => (
|
||||
<option key={p.provider} value={p.provider}>
|
||||
{p.provider}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
}
|
||||
>
|
||||
<div className="h-[260px]">
|
||||
{hasData ? <Bar data={data} options={options} /> : <EmptyState message="No activity in this range" />}
|
||||
</div>
|
||||
</Panel>
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Subscription window insights
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function WindowInsightsPanel({ insights }: { insights: ProviderWindowInsight[] }) {
|
||||
const columns: DataTableColumn<ProviderWindowInsight>[] = [
|
||||
{ key: "provider", header: "Provider", render: i => <span className="font-medium">{i.provider}</span> },
|
||||
{ key: "window", header: "Window", render: i => i.windowLabel },
|
||||
{ key: "accounts", header: "Accounts", numeric: true, render: i => formatInteger(i.accounts) },
|
||||
{
|
||||
key: "consumed",
|
||||
header: "Windows Burned",
|
||||
numeric: true,
|
||||
render: i => (
|
||||
<span title="Subscription-window equivalents consumed in range (sum of used-fraction increases across accounts)">
|
||||
{i.fractionConsumed.toFixed(2)}
|
||||
</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
key: "capacity",
|
||||
header: "Est. Tokens / Window",
|
||||
numeric: true,
|
||||
render: i => (
|
||||
<span title="Provider tokens burned in range ÷ windows burned — what one full window is worth">
|
||||
{i.estTokensPerWindow !== null ? formatCompact(i.estTokensPerWindow) : "—"}
|
||||
</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
key: "peak",
|
||||
header: "Peak Utilization",
|
||||
numeric: true,
|
||||
render: i => (
|
||||
<span title="Peak of summed used fraction across accounts at any sampled instant">
|
||||
{formatPercent(i.peakConcurrentFraction)}
|
||||
</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
key: "ideal",
|
||||
header: "Ideal Accounts",
|
||||
numeric: true,
|
||||
render: i => (
|
||||
<span
|
||||
title="Accounts needed to keep peak demand under 90% of fleet capacity"
|
||||
className={i.idealAccounts > i.accounts ? "stats-text-warning font-semibold" : undefined}
|
||||
>
|
||||
{formatInteger(i.idealAccounts)}
|
||||
{i.idealAccounts > i.accounts ? ` (have ${i.accounts})` : ""}
|
||||
</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
key: "exhausted",
|
||||
header: "Exhaustions",
|
||||
numeric: true,
|
||||
render: i => (
|
||||
<span className={i.exhaustedEvents > 0 ? "stats-text-warning" : undefined}>
|
||||
{formatInteger(i.exhaustedEvents)}
|
||||
</span>
|
||||
),
|
||||
},
|
||||
];
|
||||
|
||||
return (
|
||||
<Panel
|
||||
title="Subscription Windows"
|
||||
subtitle="What each usage window buys you, and how many accounts peak demand needs"
|
||||
>
|
||||
<DataTable
|
||||
columns={columns}
|
||||
data={insights}
|
||||
keyExtractor={i => `${i.provider}::${i.windowKey}`}
|
||||
emptyText="No usage snapshots recorded yet — they accumulate whenever usage is fetched (TUI footer, /usage, omp usage)"
|
||||
/>
|
||||
</Panel>
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Window utilization
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const UTILIZATION_COLORS = {
|
||||
ok: "#62d394",
|
||||
warning: "#f5c14b",
|
||||
exhausted: "#ff6b7d",
|
||||
} as const;
|
||||
|
||||
function WindowUtilizationPanel({ usageSeries }: { usageSeries: UsageWindowSeries[] }) {
|
||||
const providers = useMemo(() => [...new Set(usageSeries.map(s => s.provider))], [usageSeries]);
|
||||
const [selected, setSelected] = useState<string | null>(null);
|
||||
const provider = selected !== null && providers.includes(selected) ? selected : (providers[0] ?? null);
|
||||
const theme = useSystemTheme();
|
||||
const chartTheme = CHART_THEMES[theme];
|
||||
|
||||
// One row per (window, account): the latest recorded fraction. Snapshot
|
||||
// history is bursty (rows appear whenever usage is fetched), so a "how full
|
||||
// is each window right now" bar reads far better than a time axis.
|
||||
const rows = useMemo(() => {
|
||||
return usageSeries
|
||||
.filter(s => s.provider === provider)
|
||||
.map(s => {
|
||||
const latest = [...s.points].reverse().find(p => p.usedFraction !== null);
|
||||
return latest
|
||||
? {
|
||||
label: `${s.windowLabel} · ${s.accountLabel}`,
|
||||
fraction: latest.usedFraction ?? 0,
|
||||
exhausted: latest.exhausted,
|
||||
recordedAt: latest.timestamp,
|
||||
}
|
||||
: null;
|
||||
})
|
||||
.filter(row => row !== null)
|
||||
.sort((a, b) => b.fraction - a.fraction);
|
||||
}, [usageSeries, provider]);
|
||||
|
||||
const data = useMemo(
|
||||
() => ({
|
||||
labels: rows.map(r => r.label),
|
||||
datasets: [
|
||||
{
|
||||
label: "Used",
|
||||
data: rows.map(r => r.fraction * 100),
|
||||
backgroundColor: rows.map(r =>
|
||||
r.exhausted
|
||||
? UTILIZATION_COLORS.exhausted
|
||||
: r.fraction >= 0.8
|
||||
? UTILIZATION_COLORS.warning
|
||||
: UTILIZATION_COLORS.ok,
|
||||
),
|
||||
borderWidth: 0,
|
||||
borderRadius: 4,
|
||||
barThickness: 18,
|
||||
},
|
||||
],
|
||||
}),
|
||||
[rows],
|
||||
);
|
||||
|
||||
const options = useMemo(() => {
|
||||
const { sharedScaleBase, yScale } = buildSharedScales({ chartTheme, formatY: v => `${Math.round(v)}%` });
|
||||
const xMax = Math.max(100, ...rows.map(r => r.fraction * 100));
|
||||
const shared = buildSharedPlugins({
|
||||
chartTheme,
|
||||
showLegend: false,
|
||||
defaultLabel: "Used",
|
||||
formatValue: v => `${v.toFixed(1)}%`,
|
||||
});
|
||||
return {
|
||||
indexAxis: "y" as const,
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
plugins: {
|
||||
...shared,
|
||||
tooltip: {
|
||||
...shared.tooltip,
|
||||
callbacks: {
|
||||
label: (ctx: { dataIndex: number; parsed: { x: number | null } }) => {
|
||||
const row = rows[ctx.dataIndex];
|
||||
const used = `${(ctx.parsed.x ?? 0).toFixed(1)}% used`;
|
||||
return row ? `${used} · recorded ${formatRelativeTime(row.recordedAt)}` : used;
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
scales: {
|
||||
x: { ...yScale, max: xMax },
|
||||
y: { ...sharedScaleBase, grid: { display: false } },
|
||||
},
|
||||
};
|
||||
}, [chartTheme, rows]);
|
||||
|
||||
return (
|
||||
<Panel
|
||||
title="Window Utilization"
|
||||
subtitle="Latest recorded limit utilization per account and window — red bars are exhausted, amber above 80%"
|
||||
actions={
|
||||
providers.length > 1 ? (
|
||||
<select
|
||||
className="stats-select"
|
||||
value={provider ?? ""}
|
||||
onChange={e => setSelected(e.target.value)}
|
||||
aria-label="Provider"
|
||||
>
|
||||
{providers.map(p => (
|
||||
<option key={p} value={p}>
|
||||
{p}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
) : undefined
|
||||
}
|
||||
>
|
||||
<div style={{ height: Math.max(160, rows.length * 34 + 60) }}>
|
||||
{rows.length === 0 ? (
|
||||
<EmptyState message="No usage snapshots recorded yet — they accumulate whenever usage is fetched" />
|
||||
) : (
|
||||
<Bar data={data} options={options} />
|
||||
)}
|
||||
</div>
|
||||
</Panel>
|
||||
);
|
||||
}
|
||||
@@ -5,5 +5,6 @@ export * from "./GainRoute";
|
||||
export * from "./ModelsRoute";
|
||||
export * from "./OverviewRoute";
|
||||
export * from "./ProjectsRoute";
|
||||
export * from "./ProvidersRoute";
|
||||
export * from "./RequestsRoute";
|
||||
export * from "./ToolsRoute";
|
||||
|
||||
@@ -1172,6 +1172,7 @@
|
||||
.stats-text-primary { color: var(--text); }
|
||||
.stats-text-secondary { color: var(--muted); }
|
||||
.stats-text-muted { color: var(--dim); }
|
||||
.stats-text-warning { color: var(--warning); }
|
||||
.stats-text-xs { font-size: 12px; }
|
||||
.stats-font-medium { font-weight: 500; }
|
||||
.stats-font-semibold { font-weight: 600; }
|
||||
|
||||
@@ -18,6 +18,9 @@ import type {
|
||||
ModelPerformancePoint,
|
||||
ModelStats,
|
||||
ModelTimeSeriesPoint,
|
||||
ProviderAggregate,
|
||||
ProviderHourlyPoint,
|
||||
ProviderTimeSeriesPoint,
|
||||
TimeSeriesPoint,
|
||||
ToolCallStats,
|
||||
ToolModelStats,
|
||||
@@ -723,6 +726,144 @@ export function getModelTimeSeries(
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get request/token/cost totals grouped by provider.
|
||||
*/
|
||||
export function getStatsByProvider(cutoff?: number | null): ProviderAggregate[] {
|
||||
if (!db) return [];
|
||||
|
||||
const hasCutoff = cutoff !== undefined && cutoff !== null && cutoff > 0;
|
||||
const stmt = db.prepare(`
|
||||
SELECT
|
||||
provider,
|
||||
COUNT(*) as total_requests,
|
||||
SUM(CASE WHEN stop_reason = 'error' THEN 1 ELSE 0 END) as failed_requests,
|
||||
COUNT(DISTINCT model) as models,
|
||||
SUM(input_tokens) as total_input_tokens,
|
||||
SUM(output_tokens) as total_output_tokens,
|
||||
SUM(cache_read_tokens) as total_cache_read_tokens,
|
||||
SUM(cache_write_tokens) as total_cache_write_tokens,
|
||||
SUM(input_tokens + output_tokens + cache_read_tokens + cache_write_tokens) as total_tokens,
|
||||
SUM(cost_total) as total_cost,
|
||||
SUM(premium_requests) as total_premium_requests,
|
||||
AVG(CASE WHEN duration > 0 THEN output_tokens * 1000.0 / duration ELSE NULL END) as avg_tokens_per_second
|
||||
FROM messages
|
||||
${hasCutoff ? "WHERE timestamp >= ?" : ""}
|
||||
GROUP BY provider
|
||||
ORDER BY total_tokens DESC
|
||||
`);
|
||||
|
||||
const rows = (hasCutoff ? stmt.all(cutoff) : stmt.all()) as Array<{
|
||||
provider: string;
|
||||
total_requests: number;
|
||||
failed_requests: number;
|
||||
models: number;
|
||||
total_input_tokens: number | null;
|
||||
total_output_tokens: number | null;
|
||||
total_cache_read_tokens: number | null;
|
||||
total_cache_write_tokens: number | null;
|
||||
total_tokens: number | null;
|
||||
total_cost: number | null;
|
||||
total_premium_requests: number | null;
|
||||
avg_tokens_per_second: number | null;
|
||||
}>;
|
||||
return rows.map(row => ({
|
||||
provider: row.provider,
|
||||
totalRequests: row.total_requests,
|
||||
failedRequests: row.failed_requests,
|
||||
models: row.models,
|
||||
totalInputTokens: row.total_input_tokens ?? 0,
|
||||
totalOutputTokens: row.total_output_tokens ?? 0,
|
||||
totalCacheReadTokens: row.total_cache_read_tokens ?? 0,
|
||||
totalCacheWriteTokens: row.total_cache_write_tokens ?? 0,
|
||||
totalTokens: row.total_tokens ?? 0,
|
||||
totalCost: row.total_cost ?? 0,
|
||||
totalPremiumRequests: row.total_premium_requests ?? 0,
|
||||
avgTokensPerSecond: row.avg_tokens_per_second,
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get token burn grouped by provider and local hour of day (0-23).
|
||||
* Hours use the server's timezone — the dashboard is a localhost tool, so
|
||||
* server-local and viewer-local time coincide.
|
||||
*/
|
||||
export function getProviderHourlyBurn(cutoff?: number | null): ProviderHourlyPoint[] {
|
||||
if (!db) return [];
|
||||
|
||||
const hasCutoff = cutoff !== undefined && cutoff !== null && cutoff > 0;
|
||||
const stmt = db.prepare(`
|
||||
SELECT
|
||||
provider,
|
||||
CAST(strftime('%H', timestamp / 1000, 'unixepoch', 'localtime') AS INTEGER) as hour,
|
||||
SUM(input_tokens + output_tokens + cache_read_tokens + cache_write_tokens) as total_tokens,
|
||||
SUM(output_tokens) as output_tokens,
|
||||
COUNT(*) as requests
|
||||
FROM messages
|
||||
${hasCutoff ? "WHERE timestamp >= ?" : ""}
|
||||
GROUP BY provider, hour
|
||||
ORDER BY provider, hour
|
||||
`);
|
||||
|
||||
const rows = (hasCutoff ? stmt.all(cutoff) : stmt.all()) as Array<{
|
||||
provider: string;
|
||||
hour: number;
|
||||
total_tokens: number | null;
|
||||
output_tokens: number | null;
|
||||
requests: number;
|
||||
}>;
|
||||
return rows.map(row => ({
|
||||
provider: row.provider,
|
||||
hour: row.hour,
|
||||
totalTokens: row.total_tokens ?? 0,
|
||||
outputTokens: row.output_tokens ?? 0,
|
||||
requests: row.requests,
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get token/cost time series grouped by provider (bucketed like the model series).
|
||||
*/
|
||||
export function getProviderTimeSeries(
|
||||
days = 14,
|
||||
cutoff?: number | null,
|
||||
bucketMs = 24 * 60 * 60 * 1000,
|
||||
): ProviderTimeSeriesPoint[] {
|
||||
if (!db) return [];
|
||||
|
||||
const hasCutoff = cutoff !== null;
|
||||
const seriesCutoff = hasCutoff ? (cutoff ?? Date.now() - days * 24 * 60 * 60 * 1000) : 0;
|
||||
|
||||
const stmt = db.prepare(`
|
||||
SELECT
|
||||
(timestamp / ?) * ? as bucket,
|
||||
provider,
|
||||
SUM(input_tokens + output_tokens + cache_read_tokens + cache_write_tokens) as total_tokens,
|
||||
SUM(cost_total) as cost,
|
||||
COUNT(*) as requests
|
||||
FROM messages
|
||||
${hasCutoff ? "WHERE timestamp >= ?" : ""}
|
||||
GROUP BY bucket, provider
|
||||
ORDER BY bucket ASC
|
||||
`);
|
||||
|
||||
const rowsRaw = hasCutoff ? stmt.all(bucketMs, bucketMs, seriesCutoff) : stmt.all(bucketMs, bucketMs);
|
||||
const rows = rowsRaw as Array<{
|
||||
bucket: number;
|
||||
provider: string;
|
||||
total_tokens: number | null;
|
||||
cost: number | null;
|
||||
requests: number;
|
||||
}>;
|
||||
return rows.map(row => ({
|
||||
timestamp: row.bucket,
|
||||
provider: row.provider,
|
||||
totalTokens: row.total_tokens ?? 0,
|
||||
cost: row.cost ?? 0,
|
||||
requests: row.requests,
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get daily model performance time series data for the last N days.
|
||||
*/
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
getDashboardStats,
|
||||
getModelDashboardStats,
|
||||
getOverviewStats,
|
||||
getProviderDashboardStats,
|
||||
getRecentErrors,
|
||||
getRecentRequests,
|
||||
getRequestDetails,
|
||||
@@ -222,6 +223,11 @@ export async function handleApi(req: Request): Promise<Response> {
|
||||
return Response.json(stats);
|
||||
}
|
||||
|
||||
if (path === "/api/stats/providers") {
|
||||
const stats = await getProviderDashboardStats(range);
|
||||
return Response.json(stats);
|
||||
}
|
||||
|
||||
if (path === "/api/stats/recent") {
|
||||
const limit = url.searchParams.get("limit");
|
||||
const stats = await getRecentRequests(limit ? parseInt(limit, 10) : undefined);
|
||||
|
||||
@@ -321,3 +321,112 @@ export interface ToolDashboardStats {
|
||||
byToolModel: ToolModelStats[];
|
||||
series: ToolTimeSeriesPoint[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Aggregated request/token/cost totals for one provider over the active range.
|
||||
*/
|
||||
export interface ProviderAggregate {
|
||||
provider: string;
|
||||
totalRequests: number;
|
||||
failedRequests: number;
|
||||
/** Distinct models used through this provider in the range. */
|
||||
models: number;
|
||||
totalInputTokens: number;
|
||||
totalOutputTokens: number;
|
||||
totalCacheReadTokens: number;
|
||||
totalCacheWriteTokens: number;
|
||||
/** Uncached input + cache reads + cache writes + output. */
|
||||
totalTokens: number;
|
||||
totalCost: number;
|
||||
totalPremiumRequests: number;
|
||||
avgTokensPerSecond: number | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Token burn attributed to one local hour-of-day (0-23) for one provider.
|
||||
* Powers the "peak burn hours" histogram.
|
||||
*/
|
||||
export interface ProviderHourlyPoint {
|
||||
provider: string;
|
||||
/** Local hour of day, 0-23. */
|
||||
hour: number;
|
||||
totalTokens: number;
|
||||
outputTokens: number;
|
||||
requests: number;
|
||||
}
|
||||
|
||||
/** Provider token/cost time-series point (bucketed like the model series). */
|
||||
export interface ProviderTimeSeriesPoint {
|
||||
timestamp: number;
|
||||
provider: string;
|
||||
totalTokens: number;
|
||||
cost: number;
|
||||
requests: number;
|
||||
}
|
||||
|
||||
/** One recorded usage-limit snapshot for an (account, window) series. */
|
||||
export interface UsageWindowPoint {
|
||||
timestamp: number;
|
||||
/** Used fraction 0..1 (>1 = overage) when the provider reported one. */
|
||||
usedFraction: number | null;
|
||||
exhausted: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Utilization history for one (account, limit window) pair of a provider,
|
||||
* sourced from the auth store's recorded usage-limit snapshots.
|
||||
*/
|
||||
export interface UsageWindowSeries {
|
||||
provider: string;
|
||||
accountKey: string;
|
||||
/** Email/account id when known, else the stable account key. */
|
||||
accountLabel: string;
|
||||
/** Groups the same limit window across accounts (window label or limit id). */
|
||||
windowKey: string;
|
||||
windowLabel: string;
|
||||
points: UsageWindowPoint[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Derived subscription insight for one provider limit window across all
|
||||
* accounts: how much of the window was consumed, what one window is worth in
|
||||
* tokens, and how many accounts peak demand would have needed.
|
||||
*/
|
||||
export interface ProviderWindowInsight {
|
||||
provider: string;
|
||||
windowKey: string;
|
||||
windowLabel: string;
|
||||
/** Accounts with at least one snapshot for this window in range. */
|
||||
accounts: number;
|
||||
/** Window resets observed (drops in used fraction). */
|
||||
cycles: number;
|
||||
/**
|
||||
* Subscription-window equivalents consumed in range: sum of positive
|
||||
* used-fraction deltas across accounts (1.0 = one full window burned).
|
||||
*/
|
||||
fractionConsumed: number;
|
||||
/**
|
||||
* Estimated tokens one full window buys: provider tokens burned in range
|
||||
* divided by {@link fractionConsumed}. Null when too little of the window
|
||||
* was consumed to extrapolate.
|
||||
*/
|
||||
estTokensPerWindow: number | null;
|
||||
/** Peak of sum-across-accounts used fraction at any sampled instant. */
|
||||
peakConcurrentFraction: number;
|
||||
/**
|
||||
* Accounts needed to keep peak demand under 90% of fleet capacity:
|
||||
* max(1, ceil(peakConcurrentFraction / 0.9)).
|
||||
*/
|
||||
idealAccounts: number;
|
||||
/** Transitions into an exhausted state observed in range. */
|
||||
exhaustedEvents: number;
|
||||
}
|
||||
|
||||
/** Complete providers dashboard payload. */
|
||||
export interface ProviderDashboardStats {
|
||||
providers: ProviderAggregate[];
|
||||
hourly: ProviderHourlyPoint[];
|
||||
series: ProviderTimeSeriesPoint[];
|
||||
usageSeries: UsageWindowSeries[];
|
||||
windowInsights: ProviderWindowInsight[];
|
||||
}
|
||||
|
||||
@@ -0,0 +1,270 @@
|
||||
/**
|
||||
* Provider subscription-window analytics for the stats dashboard.
|
||||
*
|
||||
* The auth layer appends one row to `usage_history` in agent.db every time a
|
||||
* provider usage report is fetched (see AuthStorage's usage recording). This
|
||||
* module reads those snapshots read-only and derives:
|
||||
* - utilization series (used fraction over time per account and limit window),
|
||||
* - per-window subscription insights: window-equivalents consumed, an
|
||||
* estimate of how many tokens one full window buys, peak concurrent
|
||||
* utilization across accounts, and the account count that peak implies.
|
||||
*
|
||||
* A missing agent DB or `usage_history` table yields empty results — the
|
||||
* dashboard must keep working for API-key-only setups that never record usage.
|
||||
*/
|
||||
import { Database } from "bun:sqlite";
|
||||
import { getAgentDbPath, logger } from "@oh-my-pi/pi-utils";
|
||||
import type { ProviderWindowInsight, UsageWindowPoint, UsageWindowSeries } from "./shared-types";
|
||||
|
||||
/** Subset of a `usage_history` row consumed by the window analytics. */
|
||||
export interface UsageSnapshotRow {
|
||||
/** Epoch ms the report was fetched. */
|
||||
recordedAt: number;
|
||||
provider: string;
|
||||
/** Stable credential identity key. */
|
||||
accountKey: string;
|
||||
email: string | null;
|
||||
accountId: string | null;
|
||||
limitId: string;
|
||||
label: string;
|
||||
windowLabel: string | null;
|
||||
/** Used fraction (0..1, >1 = overage) when the provider reported one. */
|
||||
usedFraction: number | null;
|
||||
status: string | null;
|
||||
}
|
||||
|
||||
/** Utilization series + derived insights for every provider window in range. */
|
||||
export interface UsageWindowStats {
|
||||
usageSeries: UsageWindowSeries[];
|
||||
windowInsights: ProviderWindowInsight[];
|
||||
}
|
||||
|
||||
/** A used-fraction drop smaller than this is jitter, not a window reset. */
|
||||
const RESET_DROP_THRESHOLD = 0.05;
|
||||
/** Minimum window-equivalents consumed before extrapolating tokens/window. */
|
||||
const MIN_EXTRAPOLATION_FRACTION = 0.1;
|
||||
/** Fleet-capacity headroom target: peak demand should stay under 90%. */
|
||||
const TARGET_PEAK_UTILIZATION = 0.9;
|
||||
/** Used fraction at or above this counts as exhausted even without a status. */
|
||||
const EXHAUSTED_FRACTION = 0.999;
|
||||
/** Utilization series are downsampled (peak per bucket) to at most this many points. */
|
||||
const MAX_SERIES_POINTS = 400;
|
||||
|
||||
/**
|
||||
* Read usage-limit snapshots recorded at or after `sinceMs`, oldest first.
|
||||
* Opens the agent DB read-only; returns `[]` when the DB or table is absent.
|
||||
*/
|
||||
export function readUsageSnapshots(sinceMs: number, dbPath = getAgentDbPath()): UsageSnapshotRow[] {
|
||||
let db: Database | null = null;
|
||||
try {
|
||||
db = new Database(dbPath, { readonly: true });
|
||||
const rows = db
|
||||
.prepare(
|
||||
`SELECT recorded_at, provider, account_key, email, account_id, limit_id, label, window_label, used_fraction, status
|
||||
FROM usage_history
|
||||
WHERE recorded_at >= ?
|
||||
ORDER BY recorded_at ASC`,
|
||||
)
|
||||
.all(sinceMs) as Array<{
|
||||
recorded_at: number;
|
||||
provider: string;
|
||||
account_key: string;
|
||||
email: string | null;
|
||||
account_id: string | null;
|
||||
limit_id: string;
|
||||
label: string;
|
||||
window_label: string | null;
|
||||
used_fraction: number | null;
|
||||
status: string | null;
|
||||
}>;
|
||||
return rows.map(row => ({
|
||||
recordedAt: row.recorded_at,
|
||||
provider: row.provider,
|
||||
accountKey: row.account_key,
|
||||
email: row.email,
|
||||
accountId: row.account_id,
|
||||
limitId: row.limit_id,
|
||||
label: row.label,
|
||||
windowLabel: row.window_label,
|
||||
usedFraction: row.used_fraction,
|
||||
status: row.status,
|
||||
}));
|
||||
} catch (err) {
|
||||
// Expected for fresh installs (no agent.db) or pre-usage-history schemas.
|
||||
logger.debug("usage_history unavailable for provider stats", { dbPath, error: String(err) });
|
||||
return [];
|
||||
} finally {
|
||||
db?.close();
|
||||
}
|
||||
}
|
||||
|
||||
/** True when a snapshot reports an exhausted window, by status or by fraction. */
|
||||
function isExhausted(fraction: number | null, status: string | null): boolean {
|
||||
if (status === "exhausted") return true;
|
||||
return fraction !== null && fraction >= EXHAUSTED_FRACTION;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reduce a point list to at most {@link MAX_SERIES_POINTS} by keeping the
|
||||
* peak-fraction point per time bucket, so utilization peaks survive downsampling.
|
||||
*/
|
||||
function downsamplePoints(points: UsageWindowPoint[]): UsageWindowPoint[] {
|
||||
if (points.length <= MAX_SERIES_POINTS) return points;
|
||||
const first = points[0].timestamp;
|
||||
const span = points[points.length - 1].timestamp - first;
|
||||
const bucketMs = Math.max(1, Math.ceil(span / MAX_SERIES_POINTS));
|
||||
const out: UsageWindowPoint[] = [];
|
||||
let bucket = -1;
|
||||
for (const point of points) {
|
||||
const b = Math.floor((point.timestamp - first) / bucketMs);
|
||||
if (b !== bucket) {
|
||||
out.push(point);
|
||||
bucket = b;
|
||||
continue;
|
||||
}
|
||||
const last = out[out.length - 1];
|
||||
if ((point.usedFraction ?? -1) >= (last.usedFraction ?? -1)) out[out.length - 1] = point;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
interface AccountSeries {
|
||||
accountKey: string;
|
||||
accountLabel: string;
|
||||
rows: UsageSnapshotRow[];
|
||||
}
|
||||
|
||||
interface WindowGroup {
|
||||
provider: string;
|
||||
windowKey: string;
|
||||
windowLabel: string;
|
||||
accounts: Map<string, AccountSeries>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive utilization series and per-window insights from raw snapshots.
|
||||
*
|
||||
* `tokensByProvider` supplies each provider's token burn over the same time
|
||||
* range (from the local message stats); it converts consumed window fraction
|
||||
* into an estimated token capacity per window. Attribution note: tokens are
|
||||
* per provider, not per account, so the estimate treats the account fleet as
|
||||
* one pooled subscription — which is exactly how round-robin auth uses it.
|
||||
*/
|
||||
export function computeUsageWindowStats(
|
||||
rows: UsageSnapshotRow[],
|
||||
tokensByProvider: ReadonlyMap<string, number>,
|
||||
): UsageWindowStats {
|
||||
const groups = new Map<string, WindowGroup>();
|
||||
for (const row of rows) {
|
||||
const windowKey = row.windowLabel ?? row.limitId;
|
||||
const groupKey = `${row.provider}\u0000${windowKey}`;
|
||||
let group = groups.get(groupKey);
|
||||
if (!group) {
|
||||
group = { provider: row.provider, windowKey, windowLabel: row.windowLabel ?? row.label, accounts: new Map() };
|
||||
groups.set(groupKey, group);
|
||||
}
|
||||
// Labels can change across snapshots (provider renames); latest wins.
|
||||
group.windowLabel = row.windowLabel ?? row.label;
|
||||
let account = group.accounts.get(row.accountKey);
|
||||
if (!account) {
|
||||
account = { accountKey: row.accountKey, accountLabel: row.email ?? row.accountId ?? row.accountKey, rows: [] };
|
||||
group.accounts.set(row.accountKey, account);
|
||||
}
|
||||
if (row.email || row.accountId) account.accountLabel = row.email ?? row.accountId ?? row.accountKey;
|
||||
account.rows.push(row);
|
||||
}
|
||||
|
||||
const usageSeries: UsageWindowSeries[] = [];
|
||||
const windowInsights: ProviderWindowInsight[] = [];
|
||||
|
||||
for (const group of groups.values()) {
|
||||
let fractionConsumed = 0;
|
||||
let cycles = 0;
|
||||
let exhaustedEvents = 0;
|
||||
|
||||
for (const account of group.accounts.values()) {
|
||||
const points: UsageWindowPoint[] = account.rows.map(row => ({
|
||||
timestamp: row.recordedAt,
|
||||
usedFraction: row.usedFraction,
|
||||
exhausted: isExhausted(row.usedFraction, row.status),
|
||||
}));
|
||||
usageSeries.push({
|
||||
provider: group.provider,
|
||||
accountKey: account.accountKey,
|
||||
accountLabel: account.accountLabel,
|
||||
windowKey: group.windowKey,
|
||||
windowLabel: group.windowLabel,
|
||||
points: downsamplePoints(points),
|
||||
});
|
||||
|
||||
let prevFraction: number | null = null;
|
||||
let prevExhausted = false;
|
||||
for (const row of account.rows) {
|
||||
const exhausted = isExhausted(row.usedFraction, row.status);
|
||||
if (exhausted && !prevExhausted) exhaustedEvents++;
|
||||
prevExhausted = exhausted;
|
||||
if (row.usedFraction === null) continue;
|
||||
if (prevFraction !== null) {
|
||||
const delta = row.usedFraction - prevFraction;
|
||||
if (delta > 0) fractionConsumed += delta;
|
||||
else if (delta < -RESET_DROP_THRESHOLD) cycles++;
|
||||
}
|
||||
prevFraction = row.usedFraction;
|
||||
}
|
||||
}
|
||||
|
||||
const providerTokens = tokensByProvider.get(group.provider) ?? 0;
|
||||
const peak = peakConcurrentFraction(group);
|
||||
windowInsights.push({
|
||||
provider: group.provider,
|
||||
windowKey: group.windowKey,
|
||||
windowLabel: group.windowLabel,
|
||||
accounts: group.accounts.size,
|
||||
cycles,
|
||||
fractionConsumed,
|
||||
estTokensPerWindow:
|
||||
providerTokens > 0 && fractionConsumed >= MIN_EXTRAPOLATION_FRACTION
|
||||
? Math.round(providerTokens / fractionConsumed)
|
||||
: null,
|
||||
peakConcurrentFraction: peak,
|
||||
idealAccounts: Math.max(1, Math.ceil(peak / TARGET_PEAK_UTILIZATION)),
|
||||
exhaustedEvents,
|
||||
});
|
||||
}
|
||||
|
||||
usageSeries.sort(
|
||||
(a, b) =>
|
||||
a.provider.localeCompare(b.provider) ||
|
||||
a.windowKey.localeCompare(b.windowKey) ||
|
||||
a.accountLabel.localeCompare(b.accountLabel),
|
||||
);
|
||||
windowInsights.sort((a, b) => a.provider.localeCompare(b.provider) || b.fractionConsumed - a.fractionConsumed);
|
||||
return { usageSeries, windowInsights };
|
||||
}
|
||||
|
||||
/**
|
||||
* Peak of sum-across-accounts used fraction at any sampled instant: sweep all
|
||||
* snapshot times, forward-filling each account's last known fraction. A peak
|
||||
* of 1.7 means demand simultaneously held 1.7 windows' worth of quota.
|
||||
*/
|
||||
function peakConcurrentFraction(group: WindowGroup): number {
|
||||
type Event = { timestamp: number; account: string; fraction: number };
|
||||
const events: Event[] = [];
|
||||
for (const account of group.accounts.values()) {
|
||||
for (const row of account.rows) {
|
||||
if (row.usedFraction === null) continue;
|
||||
events.push({ timestamp: row.recordedAt, account: account.accountKey, fraction: row.usedFraction });
|
||||
}
|
||||
}
|
||||
events.sort((a, b) => a.timestamp - b.timestamp);
|
||||
|
||||
const current = new Map<string, number>();
|
||||
let sum = 0;
|
||||
let peak = 0;
|
||||
for (const event of events) {
|
||||
sum += event.fraction - (current.get(event.account) ?? 0);
|
||||
current.set(event.account, event.fraction);
|
||||
if (sum > peak) peak = sum;
|
||||
}
|
||||
return peak;
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
import { Database } from "bun:sqlite";
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { getProviderDashboardStats } from "@oh-my-pi/omp-stats/aggregator";
|
||||
import { initDb, insertMessageStats } from "@oh-my-pi/omp-stats/db";
|
||||
import type { MessageStats } from "@oh-my-pi/omp-stats/types";
|
||||
import { computeUsageWindowStats, readUsageSnapshots, type UsageSnapshotRow } from "@oh-my-pi/omp-stats/usage-windows";
|
||||
import { getAgentDbPath } from "@oh-my-pi/pi-utils";
|
||||
import { installStatsTestIsolation } from "./helpers/temp-agent";
|
||||
|
||||
installStatsTestIsolation("@pi-stats-providers-");
|
||||
|
||||
const T0 = Date.UTC(2026, 6, 20, 10, 0, 0);
|
||||
const MINUTE = 60_000;
|
||||
|
||||
function snapshot(overrides: Partial<UsageSnapshotRow> & { recordedAt: number }): UsageSnapshotRow {
|
||||
return {
|
||||
provider: "prov-a",
|
||||
accountKey: "acct-1",
|
||||
email: null,
|
||||
accountId: null,
|
||||
limitId: "5h",
|
||||
label: "5h limit",
|
||||
windowLabel: "5h",
|
||||
usedFraction: null,
|
||||
status: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function message(overrides: Partial<MessageStats> & { entryId: string }): MessageStats {
|
||||
return {
|
||||
sessionFile: "/tmp/session.jsonl",
|
||||
folder: "/tmp/project",
|
||||
model: "model-x",
|
||||
provider: "prov-a",
|
||||
api: "openai-completions",
|
||||
timestamp: T0,
|
||||
duration: 1000,
|
||||
ttft: 100,
|
||||
stopReason: "stop",
|
||||
errorMessage: null,
|
||||
usage: {
|
||||
input: 600,
|
||||
output: 300,
|
||||
cacheRead: 100,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 1000,
|
||||
cost: { input: 0.01, output: 0.02, cacheRead: 0.001, cacheWrite: 0, total: 0.031 },
|
||||
},
|
||||
agentType: "main",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function createAgentDb(rows: UsageSnapshotRow[]): void {
|
||||
const dbPath = getAgentDbPath();
|
||||
fs.mkdirSync(path.dirname(dbPath), { recursive: true });
|
||||
const db = new Database(dbPath);
|
||||
try {
|
||||
db.run(`
|
||||
CREATE TABLE usage_history (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
recorded_at INTEGER NOT NULL,
|
||||
provider TEXT NOT NULL,
|
||||
account_key TEXT NOT NULL,
|
||||
email TEXT,
|
||||
account_id TEXT,
|
||||
limit_id TEXT NOT NULL,
|
||||
label TEXT NOT NULL,
|
||||
window_label TEXT,
|
||||
used_fraction REAL,
|
||||
status TEXT,
|
||||
resets_at INTEGER
|
||||
)
|
||||
`);
|
||||
const insert = db.prepare(
|
||||
`INSERT INTO usage_history (recorded_at, provider, account_key, email, account_id, limit_id, label, window_label, used_fraction, status, resets_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL)`,
|
||||
);
|
||||
for (const row of rows) {
|
||||
insert.run(
|
||||
row.recordedAt,
|
||||
row.provider,
|
||||
row.accountKey,
|
||||
row.email,
|
||||
row.accountId,
|
||||
row.limitId,
|
||||
row.label,
|
||||
row.windowLabel,
|
||||
row.usedFraction,
|
||||
row.status,
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
describe("computeUsageWindowStats", () => {
|
||||
it("derives consumption, cycles, peak utilization, and capacity from snapshot deltas", () => {
|
||||
// Account 1 burns 0.1→0.5→0.9, resets to 0.2, climbs to 0.4: 1.0 windows.
|
||||
// Account 2 burns 0.3→0.8→1.0 (exhausted), resets to 0.1: 0.7 windows.
|
||||
const rows: UsageSnapshotRow[] = [
|
||||
snapshot({ recordedAt: T0 + 0 * MINUTE, accountKey: "acct-1", usedFraction: 0.1 }),
|
||||
snapshot({ recordedAt: T0 + 1 * MINUTE, accountKey: "acct-2", usedFraction: 0.3 }),
|
||||
snapshot({ recordedAt: T0 + 2 * MINUTE, accountKey: "acct-1", usedFraction: 0.5 }),
|
||||
snapshot({ recordedAt: T0 + 3 * MINUTE, accountKey: "acct-2", usedFraction: 0.8 }),
|
||||
snapshot({ recordedAt: T0 + 4 * MINUTE, accountKey: "acct-1", usedFraction: 0.9 }),
|
||||
snapshot({ recordedAt: T0 + 5 * MINUTE, accountKey: "acct-2", usedFraction: 1.0, status: "exhausted" }),
|
||||
snapshot({ recordedAt: T0 + 6 * MINUTE, accountKey: "acct-1", usedFraction: 0.2 }),
|
||||
snapshot({ recordedAt: T0 + 7 * MINUTE, accountKey: "acct-2", usedFraction: 0.1 }),
|
||||
snapshot({ recordedAt: T0 + 8 * MINUTE, accountKey: "acct-1", usedFraction: 0.4 }),
|
||||
];
|
||||
const { usageSeries, windowInsights } = computeUsageWindowStats(rows, new Map([["prov-a", 1_700_000]]));
|
||||
|
||||
expect(windowInsights).toHaveLength(1);
|
||||
const insight = windowInsights[0];
|
||||
expect(insight.provider).toBe("prov-a");
|
||||
expect(insight.windowKey).toBe("5h");
|
||||
expect(insight.accounts).toBe(2);
|
||||
expect(insight.cycles).toBe(2);
|
||||
expect(insight.fractionConsumed).toBeCloseTo(1.7, 10);
|
||||
// 1.7M provider tokens over 1.7 windows → one window ≈ 1M tokens.
|
||||
expect(insight.estTokensPerWindow).toBe(1_000_000);
|
||||
// Peak: acct-1 at 0.9 while acct-2 hits 1.0 → 1.9 combined.
|
||||
expect(insight.peakConcurrentFraction).toBeCloseTo(1.9, 10);
|
||||
// ceil(1.9 / 0.9) = 3 accounts to keep peak under 90% of fleet capacity.
|
||||
expect(insight.idealAccounts).toBe(3);
|
||||
expect(insight.exhaustedEvents).toBe(1);
|
||||
|
||||
expect(usageSeries).toHaveLength(2);
|
||||
const acct1 = usageSeries.find(s => s.accountKey === "acct-1");
|
||||
expect(acct1?.points.map(p => p.usedFraction)).toEqual([0.1, 0.5, 0.9, 0.2, 0.4]);
|
||||
expect(usageSeries.find(s => s.accountKey === "acct-2")?.points.some(p => p.exhausted)).toBe(true);
|
||||
});
|
||||
|
||||
it("withholds capacity extrapolation when too little of the window was consumed", () => {
|
||||
const rows: UsageSnapshotRow[] = [
|
||||
snapshot({ recordedAt: T0, usedFraction: 0.5 }),
|
||||
snapshot({ recordedAt: T0 + MINUTE, usedFraction: 0.52 }),
|
||||
];
|
||||
const { windowInsights } = computeUsageWindowStats(rows, new Map([["prov-a", 1_000_000]]));
|
||||
expect(windowInsights[0].fractionConsumed).toBeCloseTo(0.02, 10);
|
||||
expect(windowInsights[0].estTokensPerWindow).toBeNull();
|
||||
expect(windowInsights[0].idealAccounts).toBe(1);
|
||||
});
|
||||
|
||||
it("keeps windows with distinct labels separate", () => {
|
||||
const rows: UsageSnapshotRow[] = [
|
||||
snapshot({ recordedAt: T0, usedFraction: 0.2, limitId: "5h", windowLabel: "5h" }),
|
||||
snapshot({ recordedAt: T0, usedFraction: 0.1, limitId: "weekly", windowLabel: "Weekly", label: "Weekly" }),
|
||||
snapshot({ recordedAt: T0 + MINUTE, usedFraction: 0.6, limitId: "5h", windowLabel: "5h" }),
|
||||
snapshot({
|
||||
recordedAt: T0 + MINUTE,
|
||||
usedFraction: 0.15,
|
||||
limitId: "weekly",
|
||||
windowLabel: "Weekly",
|
||||
label: "Weekly",
|
||||
}),
|
||||
];
|
||||
const { windowInsights } = computeUsageWindowStats(rows, new Map());
|
||||
expect(windowInsights.map(i => i.windowKey).sort()).toEqual(["5h", "Weekly"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("readUsageSnapshots", () => {
|
||||
it("returns rows at or after sinceMs and empty results without an agent db", () => {
|
||||
// No agent.db yet — must not throw.
|
||||
expect(readUsageSnapshots(0)).toEqual([]);
|
||||
|
||||
createAgentDb([
|
||||
snapshot({ recordedAt: T0 - MINUTE, usedFraction: 0.1 }),
|
||||
snapshot({ recordedAt: T0 + MINUTE, usedFraction: 0.3, email: "a@example.com" }),
|
||||
]);
|
||||
const rows = readUsageSnapshots(T0);
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0]).toMatchObject({
|
||||
provider: "prov-a",
|
||||
accountKey: "acct-1",
|
||||
email: "a@example.com",
|
||||
usedFraction: 0.3,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("getProviderDashboardStats", () => {
|
||||
it("aggregates per-provider totals, hourly burn, and window insights end to end", async () => {
|
||||
await initDb();
|
||||
insertMessageStats([
|
||||
message({ entryId: "a1", provider: "prov-a", timestamp: T0 }),
|
||||
message({ entryId: "a2", provider: "prov-a", timestamp: T0 + MINUTE, stopReason: "error" }),
|
||||
message({
|
||||
entryId: "b1",
|
||||
provider: "prov-b",
|
||||
model: "model-y",
|
||||
timestamp: T0 + 2 * MINUTE,
|
||||
usage: {
|
||||
input: 100,
|
||||
output: 50,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 150,
|
||||
cost: { input: 0.001, output: 0.002, cacheRead: 0, cacheWrite: 0, total: 0.003 },
|
||||
},
|
||||
}),
|
||||
]);
|
||||
createAgentDb([
|
||||
snapshot({ recordedAt: T0, usedFraction: 0.1 }),
|
||||
snapshot({ recordedAt: T0 + MINUTE, usedFraction: 0.6 }),
|
||||
]);
|
||||
|
||||
const stats = await getProviderDashboardStats("all");
|
||||
|
||||
expect(stats.providers.map(p => p.provider)).toEqual(["prov-a", "prov-b"]);
|
||||
const provA = stats.providers[0];
|
||||
expect(provA.totalRequests).toBe(2);
|
||||
expect(provA.failedRequests).toBe(1);
|
||||
expect(provA.totalTokens).toBe(2000);
|
||||
expect(provA.models).toBe(1);
|
||||
|
||||
// All prov-a messages land in one hour bucket. Bun test pins JS `Date`
|
||||
// to UTC while SQLite 'localtime' uses the OS timezone, so assert the
|
||||
// grouping/summing contract rather than a specific hour value.
|
||||
const provAHours = stats.hourly.filter(p => p.provider === "prov-a");
|
||||
expect(provAHours).toHaveLength(1);
|
||||
expect(provAHours[0].hour).toBeGreaterThanOrEqual(0);
|
||||
expect(provAHours[0].hour).toBeLessThan(24);
|
||||
expect(provAHours[0].totalTokens).toBe(2000);
|
||||
expect(provAHours[0].outputTokens).toBe(600);
|
||||
expect(provAHours[0].requests).toBe(2);
|
||||
expect(stats.series.some(p => p.provider === "prov-b" && p.totalTokens === 150)).toBe(true);
|
||||
|
||||
expect(stats.windowInsights).toHaveLength(1);
|
||||
const insight = stats.windowInsights[0];
|
||||
expect(insight.fractionConsumed).toBeCloseTo(0.5, 10);
|
||||
// prov-a burned 2000 tokens over 0.5 windows → 4000 tokens per window.
|
||||
expect(insight.estTokensPerWindow).toBe(4000);
|
||||
expect(stats.usageSeries).toHaveLength(1);
|
||||
expect(stats.usageSeries[0].accountLabel).toBe("acct-1");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user