diff --git a/packages/coding-agent/src/config/settings-schema.ts b/packages/coding-agent/src/config/settings-schema.ts index 79d1fd8ec..40de23cd6 100644 --- a/packages/coding-agent/src/config/settings-schema.ts +++ b/packages/coding-agent/src/config/settings-schema.ts @@ -3,6 +3,7 @@ import { DEFAULT_SHARE_URL } from "@oh-my-pi/pi-wire"; import { SHAPE_VARIANT_NAMES } from "@oh-my-pi/snapcompact"; import { DEFAULT_RELAY_URL } from "../collab/protocol"; import { DEFAULT_STT_MODEL_KEY, STT_MODEL_OPTIONS, STT_MODEL_VALUES } from "../stt/models"; +import { STT_SUBMIT_TRIGGER_OPTIONS, STT_SUBMIT_TRIGGER_VALUES } from "../stt/submit-trigger"; import { AUTO_THINKING, getConfiguredThinkingLevelMetadata, getThinkingLevelMetadata } from "../thinking"; import { TINY_MODEL_DEVICE_DEFAULT, @@ -1794,6 +1795,19 @@ export const SETTINGS_SCHEMA = { options: STT_MODEL_OPTIONS, }, }, + "stt.submitTrigger": { + type: "enum", + values: STT_SUBMIT_TRIGGER_VALUES, + default: "never", + ui: { + tab: "interaction", + group: "Speech", + label: "TTS Submit Trigger", + description: + "Choose when speech dictation automatically submits: Never, Release (2+ words), Release with complete sentence, or When I Say Submit.", + options: STT_SUBMIT_TRIGGER_OPTIONS, + }, + }, // ──────────────────────────────────────────────────────────────────────── // Context @@ -4081,7 +4095,7 @@ export const SETTINGS_SCHEMA = { group: "Subagents", label: "Soft Subagent Request Budget", description: - "Soft per-subagent request budget (assistant requests per run). Crossing it injects one steering notice asking the subagent to wrap up; at 1.5x the budget the run is aborted gracefully, salvaging partial output. 0 disables the guard. Bundled explore/quick_task agents use a lower built-in budget.", + "Soft per-subagent request budget (assistant requests per run). Crossing it injects one steering notice asking the subagent to wrap up; at 1.5x the budget the run is aborted gracefully, salvaging partial output. 0 disables the guard. Bundled explore/sonic agents use a lower built-in budget.", options: [ { value: "0", label: "Disabled" }, { value: "40", label: "40 requests" }, diff --git a/packages/coding-agent/src/stt/index.ts b/packages/coding-agent/src/stt/index.ts index b8da2129a..27fcfe67a 100644 --- a/packages/coding-agent/src/stt/index.ts +++ b/packages/coding-agent/src/stt/index.ts @@ -3,5 +3,6 @@ export * from "./asr-protocol"; export * from "./downloader"; export * from "./models"; export * from "./stt-controller"; +export * from "./submit-trigger"; export * from "./transcriber"; export * from "./wav"; diff --git a/packages/coding-agent/src/stt/stt-controller.ts b/packages/coding-agent/src/stt/stt-controller.ts index 5b4dad504..5fa5b8e9a 100644 --- a/packages/coding-agent/src/stt/stt-controller.ts +++ b/packages/coding-agent/src/stt/stt-controller.ts @@ -15,6 +15,7 @@ import { startStreamingRecording, verifyRecordingFile, } from "./recorder"; +import { evaluateSubmitTrigger, type SttSubmitTrigger } from "./submit-trigger"; import { transcribe } from "./transcriber"; export type SttState = "idle" | "recording" | "transcribing"; @@ -33,6 +34,10 @@ interface Editor { setVolatileText(text: string): void; clearVolatileText(): void; commitVolatileText(text: string): void; + getText(): string; + setText(text: string): void; + submit(): void; + deleteBeforeCursor(count: number): void; } export class STTController { @@ -53,6 +58,7 @@ export class STTController { #streamEditor: Editor | null = null; #streamCommitted = false; #streamAbort: AbortController | null = null; + #streamUtterance = ""; get state(): SttState { return this.#state; @@ -190,6 +196,7 @@ export class STTController { const language = settings.get("stt.language") as string | undefined; this.#streamEditor = editor; this.#streamCommitted = false; + this.#streamUtterance = ""; this.#streamAbort = new AbortController(); const stream = sttClient.startStream(modelKey, { language: language || undefined, @@ -205,6 +212,7 @@ export class STTController { if (prefixed) { this.#streamEditor?.commitVolatileText(prefixed); this.#streamCommitted = true; + this.#streamUtterance += prefixed; } else { this.#streamEditor?.clearVolatileText(); } @@ -266,13 +274,27 @@ export class STTController { return; } if (!this.#streamCommitted && finalText) { - this.#streamEditor?.commitVolatileText(this.#prefixed(finalText)); + const prefixed = this.#prefixed(finalText); + this.#streamEditor?.commitVolatileText(prefixed); this.#streamCommitted = true; + this.#streamUtterance = prefixed; } else { this.#streamEditor?.clearVolatileText(); } options.requestRender?.(); if (!failed) options.showStatus(this.#streamCommitted ? "" : "No speech detected."); + + if (this.#streamCommitted && !failed && this.#streamEditor) { + const trigger = settings.get("stt.submitTrigger") as SttSubmitTrigger; + const { submit, trimTrailing } = evaluateSubmitTrigger(this.#streamUtterance, trigger); + if (trimTrailing > 0) { + this.#streamEditor.deleteBeforeCursor(trimTrailing); + } + if (submit) { + this.#streamEditor.submit(); + } + } + this.#cleanupStream(); this.#setState("idle", options); } @@ -283,6 +305,7 @@ export class STTController { this.#streamEditor = null; this.#streamCommitted = false; this.#streamAbort = null; + this.#streamUtterance = ""; } // ── Batch (single-shot) ───────────────────────────────────────── @@ -327,8 +350,16 @@ export class STTController { this.#transcriptionAbort = null; if (this.#disposed) return; if (text.length > 0) { - editor.insertText(text); + const trigger = settings.get("stt.submitTrigger") as SttSubmitTrigger; + const { submit, trimTrailing } = evaluateSubmitTrigger(text, trigger); + const textToInsert = trimTrailing > 0 ? text.slice(0, -trimTrailing) : text; + if (textToInsert.length > 0) { + editor.insertText(textToInsert); + } options.showStatus(""); + if (submit) { + editor.submit(); + } } else { options.showStatus("No speech detected."); } diff --git a/packages/coding-agent/src/stt/submit-trigger.ts b/packages/coding-agent/src/stt/submit-trigger.ts new file mode 100644 index 000000000..dc2f2331f --- /dev/null +++ b/packages/coding-agent/src/stt/submit-trigger.ts @@ -0,0 +1,74 @@ +/** + * TTS/STT Submit Trigger options and evaluation logic. + */ + +export const STT_SUBMIT_TRIGGER_VALUES = ["never", "release", "release-complete", "say-submit"] as const; + +export type SttSubmitTrigger = (typeof STT_SUBMIT_TRIGGER_VALUES)[number]; + +export const STT_SUBMIT_TRIGGER_OPTIONS = [ + { + value: "never", + label: "Never", + description: "Never automatically submit; insert dictation and remain in editor.", + }, + { + value: "release", + label: "Release", + description: "Submit on release if the utterance has 2+ words to avoid accidental sends.", + }, + { + value: "release-complete", + label: "Release with complete sentence", + description: "Submit on release if the utterance ends with sentence-terminal punctuation (. ? ! etc.).", + }, + { + value: "say-submit", + label: "When I Say Submit", + description: "Submit if the utterance ends with a word containing 'submit' (strips that word before submitting).", + }, +] satisfies ReadonlyArray<{ value: SttSubmitTrigger; label: string; description: string }>; + +/** + * Evaluate the submit trigger against a transcribed utterance. + * Returns whether to submit, and the number of characters to trim from the end of the utterance. + */ +export function evaluateSubmitTrigger( + utterance: string, + trigger: SttSubmitTrigger, +): { submit: boolean; trimTrailing: number } { + const trimmed = utterance.trim(); + if (!trimmed) { + return { submit: false, trimTrailing: 0 }; + } + + if (trigger === "never") { + return { submit: false, trimTrailing: 0 }; + } + + if (trigger === "release") { + // Split by whitespace and count words + const words = trimmed.split(/\s+/).filter(Boolean); + const submit = words.length >= 2; + return { submit, trimTrailing: 0 }; + } + + if (trigger === "release-complete") { + // Matches typical sentence terminators: . ? ! ... or full-width equivalents, optionally followed by space + const hasTerminalPunctuation = /[.?!…。？！]\s*$/.test(trimmed); + return { submit: hasTerminalPunctuation, trimTrailing: 0 }; + } + + if (trigger === "say-submit") { + // Matches space followed by any word containing "submit" (case-insensitive), optionally followed by punctuation/spaces + // Also handles the case where "submit" is the only word in the utterance (no leading space) + const match = utterance.match(/(?:^|\s+)(\S*submit\S*)[.?!…。？！]*\s*$/i); + if (match && match.index !== undefined) { + const trimTrailing = utterance.length - match.index; + return { submit: true, trimTrailing }; + } + return { submit: false, trimTrailing: 0 }; + } + + return { submit: false, trimTrailing: 0 }; +} diff --git a/packages/coding-agent/test/stt-preflight.test.ts b/packages/coding-agent/test/stt-preflight.test.ts index a13c00375..33c1c3ec6 100644 --- a/packages/coding-agent/test/stt-preflight.test.ts +++ b/packages/coding-agent/test/stt-preflight.test.ts @@ -73,6 +73,10 @@ describe("STTController preflight", () => { setVolatileText: vi.fn(), clearVolatileText: vi.fn(), commitVolatileText: vi.fn(), + getText: vi.fn().mockReturnValue(""), + setText: vi.fn(), + submit: vi.fn(), + deleteBeforeCursor: vi.fn(), }; } diff --git a/packages/coding-agent/test/stt-submit-trigger.test.ts b/packages/coding-agent/test/stt-submit-trigger.test.ts new file mode 100644 index 000000000..ab8b3b638 --- /dev/null +++ b/packages/coding-agent/test/stt-submit-trigger.test.ts @@ -0,0 +1,160 @@ +import { describe, expect, it } from "bun:test"; +import { evaluateSubmitTrigger } from "../src/stt/submit-trigger"; + +describe("STT Submit Trigger Evaluation", () => { + describe("never trigger", () => { + it("should never submit", () => { + expect(evaluateSubmitTrigger("hello world", "never")).toEqual({ + submit: false, + trimTrailing: 0, + }); + expect(evaluateSubmitTrigger("submit", "never")).toEqual({ + submit: false, + trimTrailing: 0, + }); + expect(evaluateSubmitTrigger("", "never")).toEqual({ + submit: false, + trimTrailing: 0, + }); + }); + }); + + describe("release trigger", () => { + it("should only submit if utterance has 2+ words", () => { + expect(evaluateSubmitTrigger("hello", "release")).toEqual({ + submit: false, + trimTrailing: 0, + }); + expect(evaluateSubmitTrigger(" hello ", "release")).toEqual({ + submit: false, + trimTrailing: 0, + }); + expect(evaluateSubmitTrigger("hello world", "release")).toEqual({ + submit: true, + trimTrailing: 0, + }); + expect(evaluateSubmitTrigger("hello world!", "release")).toEqual({ + submit: true, + trimTrailing: 0, + }); + expect(evaluateSubmitTrigger("one two three", "release")).toEqual({ + submit: true, + trimTrailing: 0, + }); + expect(evaluateSubmitTrigger("", "release")).toEqual({ + submit: false, + trimTrailing: 0, + }); + }); + }); + + describe("release-complete trigger", () => { + it("should submit only if utterance ends with terminal punctuation", () => { + expect(evaluateSubmitTrigger("hello", "release-complete")).toEqual({ + submit: false, + trimTrailing: 0, + }); + expect(evaluateSubmitTrigger("hello world", "release-complete")).toEqual({ + submit: false, + trimTrailing: 0, + }); + expect(evaluateSubmitTrigger("hello.", "release-complete")).toEqual({ + submit: true, + trimTrailing: 0, + }); + expect(evaluateSubmitTrigger("hello?", "release-complete")).toEqual({ + submit: true, + trimTrailing: 0, + }); + expect(evaluateSubmitTrigger("hello!", "release-complete")).toEqual({ + submit: true, + trimTrailing: 0, + }); + expect(evaluateSubmitTrigger("hello...", "release-complete")).toEqual({ + submit: true, + trimTrailing: 0, + }); + // Full-width punctuation + expect(evaluateSubmitTrigger("hello。", "release-complete")).toEqual({ + submit: true, + trimTrailing: 0, + }); + expect(evaluateSubmitTrigger("hello？", "release-complete")).toEqual({ + submit: true, + trimTrailing: 0, + }); + expect(evaluateSubmitTrigger("hello！", "release-complete")).toEqual({ + submit: true, + trimTrailing: 0, + }); + expect(evaluateSubmitTrigger("hello…", "release-complete")).toEqual({ + submit: true, + trimTrailing: 0, + }); + expect(evaluateSubmitTrigger("", "release-complete")).toEqual({ + submit: false, + trimTrailing: 0, + }); + }); + }); + + describe("say-submit trigger", () => { + it("should submit and trim trailing word when last word contains submit", () => { + // Single word + expect(evaluateSubmitTrigger("submit", "say-submit")).toEqual({ + submit: true, + trimTrailing: 6, + }); + expect(evaluateSubmitTrigger("SUBMIT", "say-submit")).toEqual({ + submit: true, + trimTrailing: 6, + }); + expect(evaluateSubmitTrigger("submit!", "say-submit")).toEqual({ + submit: true, + trimTrailing: 7, + }); + + // Multi word + expect(evaluateSubmitTrigger("please submit", "say-submit")).toEqual({ + submit: true, + trimTrailing: 7, // " submit" has length 7 + }); + expect(evaluateSubmitTrigger("please submit.", "say-submit")).toEqual({ + submit: true, + trimTrailing: 8, // " submit." has length 8 + }); + expect(evaluateSubmitTrigger("please submit?", "say-submit")).toEqual({ + submit: true, + trimTrailing: 8, + }); + expect(evaluateSubmitTrigger("please submit ", "say-submit")).toEqual({ + submit: true, + trimTrailing: 9, // " submit " has length 9 + }); + + // Word containing submit + expect(evaluateSubmitTrigger("please autosubmit", "say-submit")).toEqual({ + submit: true, + trimTrailing: 11, // " autosubmit" has length 11 + }); + expect(evaluateSubmitTrigger("please submitting", "say-submit")).toEqual({ + submit: true, + trimTrailing: 11, + }); + + // Negative cases + expect(evaluateSubmitTrigger("submit please", "say-submit")).toEqual({ + submit: false, + trimTrailing: 0, + }); + expect(evaluateSubmitTrigger("hello", "say-submit")).toEqual({ + submit: false, + trimTrailing: 0, + }); + expect(evaluateSubmitTrigger("", "say-submit")).toEqual({ + submit: false, + trimTrailing: 0, + }); + }); + }); +});