can1357
d20e6c0829
- Migrated global service tier settings to a per-model-family architecture (OpenAI, Anthropic, Google). - Implemented `ServiceTierByFamily` mapping to allow independent configuration and resolution per provider. - Added automatic migration logic for legacy service tier and fast-mode application settings. - Updated telemetry, session management, and task execution to support provider-specific tier resolution.
52 lines
1.9 KiB
TypeScript
52 lines
1.9 KiB
TypeScript
import { Args, Command, Flags } from "@oh-my-pi/pi-utils/cli";
|
|
import { runBenchCommand } from "../cli/bench-cli";
|
|
import { SERVICE_TIER_OPENAI_VALUES } from "../config/service-tier";
|
|
|
|
export default class Bench extends Command {
|
|
static description =
|
|
"Benchmark models with the same prompt: time-to-first-token and generation throughput (tokens/s)";
|
|
|
|
static args = {
|
|
models: Args.string({
|
|
description: "Model selectors (provider/model or fuzzy id, e.g. opus)",
|
|
required: true,
|
|
multiple: true,
|
|
}),
|
|
};
|
|
|
|
static flags = {
|
|
runs: Flags.integer({ description: "Requests per model (results are averaged)", default: 10 }),
|
|
"max-tokens": Flags.integer({ description: "Max output tokens per request", default: 512 }),
|
|
prompt: Flags.string({ description: "Custom prompt text (default: bundled bench prompt)" }),
|
|
"service-tier": Flags.string({
|
|
description: "Service tier applied per model family (default: configured `tier.*` settings; `none` omits it)",
|
|
options: SERVICE_TIER_OPENAI_VALUES,
|
|
}),
|
|
json: Flags.boolean({ description: "Output JSON" }),
|
|
par: Flags.integer({ description: "Execute runs with N parallel queries/requests", default: 4 }),
|
|
};
|
|
|
|
static examples = [
|
|
"# Compare two models\n omp bench anthropic/claude-opus-4-5 openai/gpt-5.2",
|
|
"# Fuzzy selectors work\n omp bench opus sonnet",
|
|
"# Average over 3 runs each\n omp bench opus gpt-5.2 --runs 3",
|
|
"# Force priority serving tier\n omp bench openai-codex/gpt-5.5:low --runs 10 --service-tier priority",
|
|
"# Machine-readable output\n omp bench opus --json",
|
|
];
|
|
|
|
async run(): Promise<void> {
|
|
const { args, flags } = await this.parse(Bench);
|
|
await runBenchCommand({
|
|
models: args.models ?? [],
|
|
flags: {
|
|
runs: flags.runs,
|
|
maxTokens: flags["max-tokens"],
|
|
prompt: flags.prompt,
|
|
serviceTier: flags["service-tier"],
|
|
json: flags.json,
|
|
par: flags.par,
|
|
},
|
|
});
|
|
}
|
|
}
|