From 8b3870514648fa2fbb302a4e285adb81c69ec318 Mon Sep 17 00:00:00 2001 From: can1357 Date: Mon, 5 Jan 2026 15:11:28 +0100 Subject: [PATCH] feat(coding-agent): implemented OpenAI Realtime API voice mode - Added VoiceSupervisor and VoiceController classes for realtime voice mode using OpenAI Realtime API. - Added echo suppression, noise floor filtering, and fallback transcript handling for voice input. - Changed voice mode toggle from Caps Lock to Ctrl+Y with auto-send on silence behavior. - Changed default TTS model from gpt-4o-mini-tts to tts-1. - Added kwaipilot/kat-coder-pro model and removed deprecated OpenRouter models. --- bun.lock | 130 ++- packages/ai/CHANGELOG.md | 13 + packages/ai/src/models.generated.ts | 142 +-- packages/coding-agent/CHANGELOG.md | 16 +- packages/coding-agent/package.json | 6 +- .../coding-agent/src/core/settings-manager.ts | 2 +- .../coding-agent/src/core/voice-controller.ts | 135 +++ .../coding-agent/src/core/voice-supervisor.ts | 1003 +++++++++++++++++ packages/coding-agent/src/core/voice.ts | 4 +- .../interactive/components/custom-editor.ts | 8 + .../interactive/components/settings-defs.ts | 2 +- .../src/modes/interactive/interactive-mode.ts | 231 ++-- .../coding-agent/src/prompts/voice-summary.md | 1 + 13 files changed, 1487 insertions(+), 206 deletions(-) create mode 100644 packages/coding-agent/src/core/voice-controller.ts create mode 100644 packages/coding-agent/src/core/voice-supervisor.ts diff --git a/bun.lock b/bun.lock index 56d77d837..6308dd1f2 100644 --- a/bun.lock +++ b/bun.lock @@ -64,6 +64,7 @@ "@oh-my-pi/pi-agent-core": "workspace:*", "@oh-my-pi/pi-ai": "workspace:*", "@oh-my-pi/pi-tui": "workspace:*", + "@openai/agents": "^0.3.7", "@sinclair/typebox": "^0.34.46", "ajv": "^8.17.1", "chalk": "^5.5.0", @@ -83,6 +84,7 @@ "winston": "^3.17.0", "winston-daily-rotate-file": "^5.0.0", "yaml": "^2.8.2", + "zod": "^4.3.5", }, "devDependencies": { "@types/diff": "^7.0.2", @@ -259,6 +261,8 @@ "@google/genai": ["@google/genai@1.34.0", "", { "dependencies": { "google-auth-library": "^10.3.0", "ws": "^8.18.0" }, "peerDependencies": { "@modelcontextprotocol/sdk": "^1.24.0" }, "optionalPeers": ["@modelcontextprotocol/sdk"] }, "sha512-vu53UMPvjmb7PGzlYu6Tzxso8Dfhn+a7eQFaS2uNemVtDZKwzSpJ5+ikqBbXplF7RGB1STcVDqCkPvquiwb2sw=="], + "@hono/node-server": ["@hono/node-server@1.19.7", "", { "peerDependencies": { "hono": "^4" } }, "sha512-vUcD0uauS7EU2caukW8z5lJKtoGMokxNbJtBiwHgpqxEXokaHCBkQUmCHhjFB1VUTWdqj25QoMkMKzgjq+uhrw=="], + "@img/colour": ["@img/colour@1.0.0", "", {}, "sha512-A5P/LfWGFSl6nsckYtjw9da+19jB8hkJ6ACTGcDfEJ0aE+l2n2El7dsVM7UVHZQ9s2lmYMWlrS21YLy2IR1LUw=="], "@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.2.4" }, "os": "darwin", "cpu": "arm64" }, "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w=="], @@ -337,6 +341,8 @@ "@mistralai/mistralai": ["@mistralai/mistralai@1.10.0", "", { "dependencies": { "zod": "^3.20.0", "zod-to-json-schema": "^3.24.1" } }, "sha512-tdIgWs4Le8vpvPiUEWne6tK0qbVc+jMenujnvTqOjogrJUsCSQhus0tHTU1avDDh5//Rq2dFgP9mWRAdIEoBqg=="], + "@modelcontextprotocol/sdk": ["@modelcontextprotocol/sdk@1.25.1", "", { "dependencies": { "@hono/node-server": "^1.19.7", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "content-type": "^1.0.5", "cors": "^2.8.5", "cross-spawn": "^7.0.5", "eventsource": "^3.0.2", "eventsource-parser": "^3.0.0", "express": "^5.0.1", "express-rate-limit": "^7.5.0", "jose": "^6.1.1", "json-schema-typed": "^8.0.2", "pkce-challenge": "^5.0.0", "raw-body": "^3.0.0", "zod": "^3.25 || ^4.0", "zod-to-json-schema": "^3.25.0" }, "peerDependencies": { "@cfworker/json-schema": "^4.1.1" }, "optionalPeers": ["@cfworker/json-schema"] }, "sha512-yO28oVFFC7EBoiKdAn+VqRm+plcfv4v0xp6osG/VsCB0NlPZWi87ajbCZZ8f/RvOFLEu7//rSRmuZZ7lMoe3gQ=="], + "@napi-rs/canvas": ["@napi-rs/canvas@0.1.88", "", { "optionalDependencies": { "@napi-rs/canvas-android-arm64": "0.1.88", "@napi-rs/canvas-darwin-arm64": "0.1.88", "@napi-rs/canvas-darwin-x64": "0.1.88", "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.88", "@napi-rs/canvas-linux-arm64-gnu": "0.1.88", "@napi-rs/canvas-linux-arm64-musl": "0.1.88", "@napi-rs/canvas-linux-riscv64-gnu": "0.1.88", "@napi-rs/canvas-linux-x64-gnu": "0.1.88", "@napi-rs/canvas-linux-x64-musl": "0.1.88", "@napi-rs/canvas-win32-arm64-msvc": "0.1.88", "@napi-rs/canvas-win32-x64-msvc": "0.1.88" } }, "sha512-/p08f93LEbsL5mDZFQ3DBxcPv/I4QG9EDYRRq1WNlCOXVfAHBTHMSVMwxlqG/AtnSfUr9+vgfN7MKiyDo0+Weg=="], "@napi-rs/canvas-android-arm64": ["@napi-rs/canvas-android-arm64@0.1.88", "", { "os": "android", "cpu": "arm64" }, "sha512-KEaClPnZuVxJ8smUWjV1wWFkByBO/D+vy4lN+Dm5DFH514oqwukxKGeck9xcKJhaWJGjfruGmYGiwRe//+/zQQ=="], @@ -373,6 +379,14 @@ "@oh-my-pi/pi-web-ui": ["@oh-my-pi/pi-web-ui@workspace:packages/web-ui"], + "@openai/agents": ["@openai/agents@0.3.7", "", { "dependencies": { "@openai/agents-core": "0.3.7", "@openai/agents-openai": "0.3.7", "@openai/agents-realtime": "0.3.7", "debug": "^4.4.0", "openai": "^6" }, "peerDependencies": { "zod": "^3.25.40 || ^4.0" } }, "sha512-9QKr6Gqb0yy0VLsyKSjxUoHHxN8j0fqtlZ1YTNmo2j5SqRcFG5LdSVyaQbkeJ3sWsCUugCXvEBEir7yAbxjSRw=="], + + "@openai/agents-core": ["@openai/agents-core@0.3.7", "", { "dependencies": { "debug": "^4.4.0", "openai": "^6" }, "optionalDependencies": { "@modelcontextprotocol/sdk": "^1.24.0" }, "peerDependencies": { "zod": "^3.25.40 || ^4.0" }, "optionalPeers": ["zod"] }, "sha512-Ya1sPc8UyHXxgQjNWN1sn/vFNP4CRdT/0SWd0tIc+kHgYuxrvRjwSfWNBqXc70XOZGS5bWWaRTqJBMwQt6AlYg=="], + + "@openai/agents-openai": ["@openai/agents-openai@0.3.7", "", { "dependencies": { "@openai/agents-core": "0.3.7", "debug": "^4.4.0", "openai": "^6" }, "peerDependencies": { "zod": "^3.25.40 || ^4.0" } }, "sha512-xjFNFeKDt3TkAWfoXhvUG3sdBQJ/R3FWd/s3Dv1rt1fdcFCaS1BUKkXX1fu79oo24H5DV8zZJfuUu6Bgi0ZAHw=="], + + "@openai/agents-realtime": ["@openai/agents-realtime@0.3.7", "", { "dependencies": { "@openai/agents-core": "0.3.7", "@types/ws": "^8.18.1", "debug": "^4.4.0", "ws": "^8.18.1" }, "peerDependencies": { "zod": "^3.25.40 || ^4.0" } }, "sha512-8zKu33Q8bmXCkReR7UOWQa1rNbIaLwqI4QgjMD20ALaEkuRglRZeaJyfVTIhTcw9COIF35uyiq+GugxFUvPE5A=="], + "@parcel/watcher": ["@parcel/watcher@2.5.1", "", { "dependencies": { "detect-libc": "^1.0.3", "is-glob": "^4.0.3", "micromatch": "^4.0.5", "node-addon-api": "^7.0.0" }, "optionalDependencies": { "@parcel/watcher-android-arm64": "2.5.1", "@parcel/watcher-darwin-arm64": "2.5.1", "@parcel/watcher-darwin-x64": "2.5.1", "@parcel/watcher-freebsd-x64": "2.5.1", "@parcel/watcher-linux-arm-glibc": "2.5.1", "@parcel/watcher-linux-arm-musl": "2.5.1", "@parcel/watcher-linux-arm64-glibc": "2.5.1", "@parcel/watcher-linux-arm64-musl": "2.5.1", "@parcel/watcher-linux-x64-glibc": "2.5.1", "@parcel/watcher-linux-x64-musl": "2.5.1", "@parcel/watcher-win32-arm64": "2.5.1", "@parcel/watcher-win32-ia32": "2.5.1", "@parcel/watcher-win32-x64": "2.5.1" } }, "sha512-dfUnCxiN9H4ap84DvD2ubjw+3vUNpstxa0TneY/Paat8a3R4uQZDLSvWjmznAY/DoahqTHl9V46HF/Zs3F29pg=="], "@parcel/watcher-android-arm64": ["@parcel/watcher-android-arm64@2.5.1", "", { "os": "android", "cpu": "arm64" }, "sha512-KF8+j9nNbUN8vzOFDpRMsaKBHZ/mcjEjMToVMJOhTozkDonQFFrRcfdLWn6yWKCmJKmdVxSgHiYvTCef4/qcBA=="], @@ -565,6 +579,8 @@ "@xterm/xterm": ["@xterm/xterm@5.5.0", "", {}, "sha512-hqJHYaQb5OptNunnyAnkHyM8aCjZ1MEIDTQu1iIbbTD/xops91NB5yq1ZK/dC2JDbVWtF23zUtl9JE2NqwT87A=="], + "accepts": ["accepts@2.0.0", "", { "dependencies": { "mime-types": "^3.0.0", "negotiator": "^1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="], + "agent-base": ["agent-base@7.1.4", "", {}, "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ=="], "ajv": ["ajv@8.17.1", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g=="], @@ -599,6 +615,8 @@ "bl": ["bl@4.1.0", "", { "dependencies": { "buffer": "^5.5.0", "inherits": "^2.0.4", "readable-stream": "^3.4.0" } }, "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w=="], + "body-parser": ["body-parser@2.2.1", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.3", "http-errors": "^2.0.0", "iconv-lite": "^0.7.0", "on-finished": "^2.4.1", "qs": "^6.14.0", "raw-body": "^3.0.1", "type-is": "^2.0.1" } }, "sha512-nfDwkulwiZYQIGwxdy0RUmowMhKcFVcYXUU7m4QlKYim1rUtg83xm2yjZ40QjDuc291AJjjeSc9b++AWHSgSHw=="], + "boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="], "brace-expansion": ["brace-expansion@2.0.2", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ=="], @@ -611,10 +629,14 @@ "bun-types": ["bun-types@1.3.5", "", { "dependencies": { "@types/node": "*" } }, "sha512-inmAYe2PFLs0SUbFOWSVD24sg1jFlMPxOjOSSCYqUgn4Hsc3rDc7dFvfVYjFPNHtov6kgUeulV4SxbuIV/stPw=="], + "bytes": ["bytes@3.1.2", "", {}, "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg=="], + "cac": ["cac@6.7.14", "", {}, "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ=="], "call-bind-apply-helpers": ["call-bind-apply-helpers@1.0.2", "", { "dependencies": { "es-errors": "^1.3.0", "function-bind": "^1.1.2" } }, "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ=="], + "call-bound": ["call-bound@1.0.4", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "get-intrinsic": "^1.3.0" } }, "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg=="], + "canvas": ["canvas@3.2.0", "", { "dependencies": { "node-addon-api": "^7.0.0", "prebuild-install": "^7.1.3" } }, "sha512-jk0GxrLtUEmW/TmFsk2WghvgHe8B0pxGilqCL21y8lHkPUGa6FTsnCNtHPOzT8O3y+N+m3espawV80bbBlgfTA=="], "chai": ["chai@5.3.3", "", { "dependencies": { "assertion-error": "^2.0.1", "check-error": "^2.1.1", "deep-eql": "^5.0.1", "loupe": "^3.1.0", "pathval": "^2.0.0" } }, "sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw=="], @@ -653,8 +675,18 @@ "concurrently": ["concurrently@9.2.1", "", { "dependencies": { "chalk": "4.1.2", "rxjs": "7.8.2", "shell-quote": "1.8.3", "supports-color": "8.1.1", "tree-kill": "1.2.2", "yargs": "17.7.2" }, "bin": { "conc": "dist/bin/concurrently.js", "concurrently": "dist/bin/concurrently.js" } }, "sha512-fsfrO0MxV64Znoy8/l1vVIjjHa29SZyyqPgQBwhiDcaW8wJc2W3XWVOGx4M3oJBnv/zdUZIIp1gDeS98GzP8Ng=="], + "content-disposition": ["content-disposition@1.0.1", "", {}, "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q=="], + + "content-type": ["content-type@1.0.5", "", {}, "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA=="], + + "cookie": ["cookie@0.7.2", "", {}, "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w=="], + + "cookie-signature": ["cookie-signature@1.2.2", "", {}, "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg=="], + "core-util-is": ["core-util-is@1.0.3", "", {}, "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ=="], + "cors": ["cors@2.8.5", "", { "dependencies": { "object-assign": "^4", "vary": "^1" } }, "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g=="], + "croner": ["croner@9.1.0", "", {}, "sha512-p9nwwR4qyT5W996vBZhdvBCnMhicY5ytZkR4D1Xj0wuTDEiMnjwR57Q3RXYY/s0EpX6Ay3vgIcfaR+ewGHsi+g=="], "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], @@ -675,6 +707,8 @@ "delayed-stream": ["delayed-stream@1.0.0", "", {}, "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="], + "depd": ["depd@2.0.0", "", {}, "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="], + "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], "diff": ["diff@8.0.2", "", {}, "sha512-sSuxWU5j5SR9QQji/o2qMvqRNYRDOcBTgsJ/DeCf4iSN4gW+gNMXM7wFIP+fdXZxoNiAnHUTGjCr+TSWXdRDKg=="], @@ -695,10 +729,14 @@ "ecdsa-sig-formatter": ["ecdsa-sig-formatter@1.0.11", "", { "dependencies": { "safe-buffer": "^5.0.1" } }, "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ=="], + "ee-first": ["ee-first@1.1.1", "", {}, "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow=="], + "emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="], "enabled": ["enabled@2.0.0", "", {}, "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ=="], + "encodeurl": ["encodeurl@2.0.0", "", {}, "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg=="], + "end-of-stream": ["end-of-stream@1.4.5", "", { "dependencies": { "once": "^1.4.0" } }, "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg=="], "enhanced-resolve": ["enhanced-resolve@5.18.4", "", { "dependencies": { "graceful-fs": "^4.2.4", "tapable": "^2.2.0" } }, "sha512-LgQMM4WXU3QI+SYgEc2liRgznaD5ojbmY3sb8LxyguVkIg5FxdpTkvk72te2R38/TGKxH634oLxXRGY6d7AP+Q=="], @@ -721,14 +759,26 @@ "escalade": ["escalade@3.2.0", "", {}, "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA=="], + "escape-html": ["escape-html@1.0.3", "", {}, "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="], + "estree-walker": ["estree-walker@3.0.3", "", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g=="], + "etag": ["etag@1.8.1", "", {}, "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg=="], + "eventemitter3": ["eventemitter3@5.0.1", "", {}, "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA=="], + "eventsource": ["eventsource@3.0.7", "", { "dependencies": { "eventsource-parser": "^3.0.1" } }, "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA=="], + + "eventsource-parser": ["eventsource-parser@3.0.6", "", {}, "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg=="], + "expand-template": ["expand-template@2.0.3", "", {}, "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg=="], "expect-type": ["expect-type@1.3.0", "", {}, "sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA=="], + "express": ["express@5.2.1", "", { "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", "content-disposition": "^1.0.0", "content-type": "^1.0.5", "cookie": "^0.7.1", "cookie-signature": "^1.2.1", "debug": "^4.4.0", "depd": "^2.0.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "finalhandler": "^2.1.0", "fresh": "^2.0.0", "http-errors": "^2.0.0", "merge-descriptors": "^2.0.0", "mime-types": "^3.0.0", "on-finished": "^2.4.1", "once": "^1.4.0", "parseurl": "^1.3.3", "proxy-addr": "^2.0.7", "qs": "^6.14.0", "range-parser": "^1.2.1", "router": "^2.2.0", "send": "^1.1.0", "serve-static": "^2.2.0", "statuses": "^2.0.1", "type-is": "^2.0.1", "vary": "^1.1.2" } }, "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw=="], + + "express-rate-limit": ["express-rate-limit@7.5.1", "", { "peerDependencies": { "express": ">= 4.11" } }, "sha512-7iN8iPMDzOMHPUYllBEsQdWVB6fPDMPqwjBaFrgr4Jgr/+okjvzAy+UHlYYL/Vs0OsOrMkwS6PJDkFlJwoxUnw=="], + "extend": ["extend@3.0.2", "", {}, "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="], "fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="], @@ -747,6 +797,8 @@ "fill-range": ["fill-range@7.1.1", "", { "dependencies": { "to-regex-range": "^5.0.1" } }, "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg=="], + "finalhandler": ["finalhandler@2.1.1", "", { "dependencies": { "debug": "^4.4.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "on-finished": "^2.4.1", "parseurl": "^1.3.3", "statuses": "^2.0.1" } }, "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA=="], + "fn.name": ["fn.name@1.1.0", "", {}, "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw=="], "follow-redirects": ["follow-redirects@1.15.11", "", {}, "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ=="], @@ -757,6 +809,10 @@ "formdata-polyfill": ["formdata-polyfill@4.0.10", "", { "dependencies": { "fetch-blob": "^3.1.2" } }, "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g=="], + "forwarded": ["forwarded@0.2.0", "", {}, "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow=="], + + "fresh": ["fresh@2.0.0", "", {}, "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A=="], + "fs-constants": ["fs-constants@1.0.0", "", {}, "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="], "fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], @@ -801,10 +857,16 @@ "highlight.js": ["highlight.js@11.11.1", "", {}, "sha512-Xwwo44whKBVCYoliBQwaPvtd/2tYFkRQtXDWj1nackaV2JPXx3L0+Jvd8/qCJ2p+ML0/XVkJ2q+Mr+UVdpJK5w=="], + "hono": ["hono@4.11.3", "", {}, "sha512-PmQi306+M/ct/m5s66Hrg+adPnkD5jiO6IjA7WhWw0gSBSo1EcRegwuI1deZ+wd5pzCGynCcn2DprnE4/yEV4w=="], + "html-parse-string": ["html-parse-string@0.0.9", "", {}, "sha512-wyGnsOolHbNrcb8N6bdJF4EHyzd3zVGCb9/mBxeNjAYBDOZqD7YkqLBz7kXtdgHwNnV8lN/BpSDpsI1zm8Sd8g=="], + "http-errors": ["http-errors@2.0.1", "", { "dependencies": { "depd": "~2.0.0", "inherits": "~2.0.4", "setprototypeof": "~1.2.0", "statuses": "~2.0.2", "toidentifier": "~1.0.1" } }, "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ=="], + "https-proxy-agent": ["https-proxy-agent@7.0.6", "", { "dependencies": { "agent-base": "^7.1.2", "debug": "4" } }, "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw=="], + "iconv-lite": ["iconv-lite@0.7.1", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-2Tth85cXwGFHfvRgZWszZSvdo+0Xsqmw8k8ZwxScfcBneNUraK+dxRxRm24nszx80Y0TVio8kKLt5sLE7ZCLlw=="], + "ieee754": ["ieee754@1.2.1", "", {}, "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="], "immediate": ["immediate@3.0.6", "", {}, "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ=="], @@ -813,6 +875,8 @@ "ini": ["ini@1.3.8", "", {}, "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew=="], + "ipaddr.js": ["ipaddr.js@1.9.1", "", {}, "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="], + "is-electron": ["is-electron@2.2.2", "", {}, "sha512-FO/Rhvz5tuw4MCWkpMzHFKWD2LsfHzIb7i6MdPYZ/KW7AlxawyLkqdy+jPZP1WubqEADE3O4FUENlJHDfQASRg=="], "is-extglob": ["is-extglob@2.1.1", "", {}, "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ=="], @@ -823,6 +887,8 @@ "is-number": ["is-number@7.0.0", "", {}, "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng=="], + "is-promise": ["is-promise@4.0.0", "", {}, "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ=="], + "is-stream": ["is-stream@2.0.1", "", {}, "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg=="], "isarray": ["isarray@1.0.0", "", {}, "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="], @@ -833,6 +899,8 @@ "jiti": ["jiti@2.6.1", "", { "bin": { "jiti": "lib/jiti-cli.mjs" } }, "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ=="], + "jose": ["jose@6.1.3", "", {}, "sha512-0TpaTfihd4QMNwrz/ob2Bp7X04yuxJkjRGi4aKmOqwhov54i6u79oCv7T+C7lo70MKH6BesI3vscD1yb/yzKXQ=="], + "js-tokens": ["js-tokens@9.0.1", "", {}, "sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ=="], "json-bigint": ["json-bigint@1.0.0", "", { "dependencies": { "bignumber.js": "^9.0.0" } }, "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ=="], @@ -841,6 +909,8 @@ "json-schema-traverse": ["json-schema-traverse@1.0.0", "", {}, "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="], + "json-schema-typed": ["json-schema-typed@8.0.2", "", {}, "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA=="], + "json-stringify-safe": ["json-stringify-safe@5.0.1", "", {}, "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA=="], "jsonschema": ["jsonschema@1.5.0", "", {}, "sha512-K+A9hhqbn0f3pJX17Q/7H6yQfD/5OXgdrR5UE12gMXCiN9D5Xq2o5mddV2QEcX/bjla99ASsAAQUyMCCRWAEhw=="], @@ -909,6 +979,10 @@ "math-intrinsics": ["math-intrinsics@1.1.0", "", {}, "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="], + "media-typer": ["media-typer@1.1.0", "", {}, "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw=="], + + "merge-descriptors": ["merge-descriptors@2.0.0", "", {}, "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g=="], + "micromatch": ["micromatch@4.0.8", "", { "dependencies": { "braces": "^3.0.3", "picomatch": "^2.3.1" } }, "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA=="], "mime-db": ["mime-db@1.54.0", "", {}, "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ=="], @@ -943,6 +1017,8 @@ "ndjson": ["ndjson@2.0.0", "", { "dependencies": { "json-stringify-safe": "^5.0.1", "minimist": "^1.2.5", "readable-stream": "^3.6.0", "split2": "^3.0.0", "through2": "^4.0.0" }, "bin": { "ndjson": "cli.js" } }, "sha512-nGl7LRGrzugTtaFcJMhLbpzJM6XdivmbkdlaGcrk/LXg2KL/YBC6z1g70xh0/al+oFuVFP8N8kiWRucmeEH/qQ=="], + "negotiator": ["negotiator@1.0.0", "", {}, "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg=="], + "node-abi": ["node-abi@3.85.0", "", { "dependencies": { "semver": "^7.3.5" } }, "sha512-zsFhmbkAzwhTft6nd3VxcG0cvJsT70rL+BIGHWVq5fi6MwGrHwzqKaxXE+Hl2GmnGItnDKPPkO5/LQqjVkIdFg=="], "node-addon-api": ["node-addon-api@7.1.1", "", {}, "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ=="], @@ -959,8 +1035,12 @@ "object-hash": ["object-hash@3.0.0", "", {}, "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw=="], + "object-inspect": ["object-inspect@1.13.4", "", {}, "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew=="], + "ollama": ["ollama@0.6.3", "", { "dependencies": { "whatwg-fetch": "^3.6.20" } }, "sha512-KEWEhIqE5wtfzEIZbDCLH51VFZ6Z3ZSa6sIOg/E/tBV8S51flyqBOXi+bRxlOYKDf8i327zG9eSTb8IJxvm3Zg=="], + "on-finished": ["on-finished@2.4.1", "", { "dependencies": { "ee-first": "1.1.1" } }, "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg=="], + "once": ["once@1.4.0", "", { "dependencies": { "wrappy": "1" } }, "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w=="], "one-time": ["one-time@1.0.0", "", { "dependencies": { "fn.name": "1.x.x" } }, "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g=="], @@ -985,12 +1065,16 @@ "parse5-htmlparser2-tree-adapter": ["parse5-htmlparser2-tree-adapter@6.0.1", "", { "dependencies": { "parse5": "^6.0.1" } }, "sha512-qPuWvbLgvDGilKc5BoicRovlT4MtYT6JfJyBOMDsKoiT+GiuP5qyrPCnR9HcPECIJJmZh5jRndyNThnhhb/vlA=="], + "parseurl": ["parseurl@1.3.3", "", {}, "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ=="], + "partial-json": ["partial-json@0.1.7", "", {}, "sha512-Njv/59hHaokb/hRUjce3Hdv12wd60MtM9Z5Olmn+nehe0QDAsRtRbJPvJ0Z91TusF0SuZRIvnM+S4l6EIP8leA=="], "path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="], "path-scurry": ["path-scurry@2.0.1", "", { "dependencies": { "lru-cache": "^11.0.0", "minipass": "^7.1.2" } }, "sha512-oWyT4gICAu+kaA7QWk/jvCHWarMKNs6pXOGWKDTr7cw4IGcUbW+PeTfbaQiLGheFRpjo6O9J0PmyMfQPjH71oA=="], + "path-to-regexp": ["path-to-regexp@8.3.0", "", {}, "sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA=="], + "pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], "pathval": ["pathval@2.0.1", "", {}, "sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ=="], @@ -1005,16 +1089,26 @@ "pidtree": ["pidtree@0.6.0", "", { "bin": { "pidtree": "bin/pidtree.js" } }, "sha512-eG2dWTVw5bzqGRztnHExczNxt5VGsE6OwTeCG3fdUf9KBsZzO3R5OIIIzWR+iZA0NtZ+RDVdaoE2dK1cn6jH4g=="], + "pkce-challenge": ["pkce-challenge@5.0.1", "", {}, "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ=="], + "postcss": ["postcss@8.5.6", "", { "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg=="], "prebuild-install": ["prebuild-install@7.1.3", "", { "dependencies": { "detect-libc": "^2.0.0", "expand-template": "^2.0.3", "github-from-package": "0.0.0", "minimist": "^1.2.3", "mkdirp-classic": "^0.5.3", "napi-build-utils": "^2.0.0", "node-abi": "^3.3.0", "pump": "^3.0.0", "rc": "^1.2.7", "simple-get": "^4.0.0", "tar-fs": "^2.0.0", "tunnel-agent": "^0.6.0" }, "bin": { "prebuild-install": "bin.js" } }, "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug=="], "process-nextick-args": ["process-nextick-args@2.0.1", "", {}, "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="], + "proxy-addr": ["proxy-addr@2.0.7", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg=="], + "proxy-from-env": ["proxy-from-env@1.1.0", "", {}, "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="], "pump": ["pump@3.0.3", "", { "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" } }, "sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA=="], + "qs": ["qs@6.14.1", "", { "dependencies": { "side-channel": "^1.1.0" } }, "sha512-4EK3+xJl8Ts67nLYNwqw/dsFVnCf+qR7RgXSK9jEEm9unao3njwMDdmsdvoKBKHzxd7tCYz5e5M+SnMjdtXGQQ=="], + + "range-parser": ["range-parser@1.2.1", "", {}, "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="], + + "raw-body": ["raw-body@3.0.2", "", { "dependencies": { "bytes": "~3.1.2", "http-errors": "~2.0.1", "iconv-lite": "~0.7.0", "unpipe": "~1.0.0" } }, "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA=="], + "rc": ["rc@1.2.8", "", { "dependencies": { "deep-extend": "^0.6.0", "ini": "~1.3.0", "minimist": "^1.2.0", "strip-json-comments": "~2.0.1" }, "bin": { "rc": "./cli.js" } }, "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw=="], "readable-stream": ["readable-stream@3.6.2", "", { "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" } }, "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA=="], @@ -1033,16 +1127,26 @@ "rollup": ["rollup@4.54.0", "", { "dependencies": { "@types/estree": "1.0.8" }, "optionalDependencies": { "@rollup/rollup-android-arm-eabi": "4.54.0", "@rollup/rollup-android-arm64": "4.54.0", "@rollup/rollup-darwin-arm64": "4.54.0", "@rollup/rollup-darwin-x64": "4.54.0", "@rollup/rollup-freebsd-arm64": "4.54.0", "@rollup/rollup-freebsd-x64": "4.54.0", "@rollup/rollup-linux-arm-gnueabihf": "4.54.0", "@rollup/rollup-linux-arm-musleabihf": "4.54.0", "@rollup/rollup-linux-arm64-gnu": "4.54.0", "@rollup/rollup-linux-arm64-musl": "4.54.0", "@rollup/rollup-linux-loong64-gnu": "4.54.0", "@rollup/rollup-linux-ppc64-gnu": "4.54.0", "@rollup/rollup-linux-riscv64-gnu": "4.54.0", "@rollup/rollup-linux-riscv64-musl": "4.54.0", "@rollup/rollup-linux-s390x-gnu": "4.54.0", "@rollup/rollup-linux-x64-gnu": "4.54.0", "@rollup/rollup-linux-x64-musl": "4.54.0", "@rollup/rollup-openharmony-arm64": "4.54.0", "@rollup/rollup-win32-arm64-msvc": "4.54.0", "@rollup/rollup-win32-ia32-msvc": "4.54.0", "@rollup/rollup-win32-x64-gnu": "4.54.0", "@rollup/rollup-win32-x64-msvc": "4.54.0", "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-3nk8Y3a9Ea8szgKhinMlGMhGMw89mqule3KWczxhIzqudyHdCIOHw8WJlj/r329fACjKLEh13ZSk7oE22kyeIw=="], + "router": ["router@2.2.0", "", { "dependencies": { "debug": "^4.4.0", "depd": "^2.0.0", "is-promise": "^4.0.0", "parseurl": "^1.3.3", "path-to-regexp": "^8.0.0" } }, "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ=="], + "rxjs": ["rxjs@7.8.2", "", { "dependencies": { "tslib": "^2.1.0" } }, "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA=="], "safe-buffer": ["safe-buffer@5.1.2", "", {}, "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="], "safe-stable-stringify": ["safe-stable-stringify@2.5.0", "", {}, "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA=="], + "safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="], + "semver": ["semver@7.7.3", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q=="], + "send": ["send@1.2.1", "", { "dependencies": { "debug": "^4.4.3", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "fresh": "^2.0.0", "http-errors": "^2.0.1", "mime-types": "^3.0.2", "ms": "^2.1.3", "on-finished": "^2.4.1", "range-parser": "^1.2.1", "statuses": "^2.0.2" } }, "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ=="], + + "serve-static": ["serve-static@2.2.1", "", { "dependencies": { "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "parseurl": "^1.3.3", "send": "^1.2.0" } }, "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw=="], + "setimmediate": ["setimmediate@1.0.5", "", {}, "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA=="], + "setprototypeof": ["setprototypeof@1.2.0", "", {}, "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="], + "sharp": ["sharp@0.34.5", "", { "dependencies": { "@img/colour": "^1.0.0", "detect-libc": "^2.1.2", "semver": "^7.7.3" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.34.5", "@img/sharp-darwin-x64": "0.34.5", "@img/sharp-libvips-darwin-arm64": "1.2.4", "@img/sharp-libvips-darwin-x64": "1.2.4", "@img/sharp-libvips-linux-arm": "1.2.4", "@img/sharp-libvips-linux-arm64": "1.2.4", "@img/sharp-libvips-linux-ppc64": "1.2.4", "@img/sharp-libvips-linux-riscv64": "1.2.4", "@img/sharp-libvips-linux-s390x": "1.2.4", "@img/sharp-libvips-linux-x64": "1.2.4", "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", "@img/sharp-libvips-linuxmusl-x64": "1.2.4", "@img/sharp-linux-arm": "0.34.5", "@img/sharp-linux-arm64": "0.34.5", "@img/sharp-linux-ppc64": "0.34.5", "@img/sharp-linux-riscv64": "0.34.5", "@img/sharp-linux-s390x": "0.34.5", "@img/sharp-linux-x64": "0.34.5", "@img/sharp-linuxmusl-arm64": "0.34.5", "@img/sharp-linuxmusl-x64": "0.34.5", "@img/sharp-wasm32": "0.34.5", "@img/sharp-win32-arm64": "0.34.5", "@img/sharp-win32-ia32": "0.34.5", "@img/sharp-win32-x64": "0.34.5" } }, "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg=="], "shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="], @@ -1051,6 +1155,14 @@ "shell-quote": ["shell-quote@1.8.3", "", {}, "sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw=="], + "side-channel": ["side-channel@1.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3", "side-channel-list": "^1.0.0", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" } }, "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw=="], + + "side-channel-list": ["side-channel-list@1.0.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3" } }, "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA=="], + + "side-channel-map": ["side-channel-map@1.0.1", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3" } }, "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA=="], + + "side-channel-weakmap": ["side-channel-weakmap@1.0.2", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3", "side-channel-map": "^1.0.1" } }, "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A=="], + "siginfo": ["siginfo@2.0.0", "", {}, "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g=="], "signal-exit": ["signal-exit@4.1.0", "", {}, "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw=="], @@ -1071,6 +1183,8 @@ "stackback": ["stackback@0.0.2", "", {}, "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw=="], + "statuses": ["statuses@2.0.2", "", {}, "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw=="], + "std-env": ["std-env@3.10.0", "", {}, "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg=="], "string-argv": ["string-argv@0.3.2", "", {}, "sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q=="], @@ -1127,6 +1241,8 @@ "to-regex-range": ["to-regex-range@5.0.1", "", { "dependencies": { "is-number": "^7.0.0" } }, "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ=="], + "toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="], + "token-types": ["token-types@6.1.2", "", { "dependencies": { "@borewit/text-codec": "^0.2.1", "@tokenizer/token": "^0.3.0", "ieee754": "^1.2.1" } }, "sha512-dRXchy+C0IgK8WPC6xvCHFRIWYUbqqdEIKPaKo/AcTUNzwLTK6AH7RjdLWsEZcAN/TBdtfUw3PYEgPr5VPr6ww=="], "tree-kill": ["tree-kill@1.2.2", "", { "bin": { "tree-kill": "cli.js" } }, "sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A=="], @@ -1139,14 +1255,20 @@ "tunnel-agent": ["tunnel-agent@0.6.0", "", { "dependencies": { "safe-buffer": "^5.0.1" } }, "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w=="], + "type-is": ["type-is@2.0.1", "", { "dependencies": { "content-type": "^1.0.5", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw=="], + "uhtml": ["uhtml@5.0.9", "", { "dependencies": { "@webreflection/alien-signals": "^0.3.2" } }, "sha512-qPyu3vGilaLe6zrjOCD/xezWEHLwdevxmbY3hzyhT25KBDF4F7YYW3YZcL3kylD/6dMoVISHjn8ggV3+9FY+5g=="], "uint8array-extras": ["uint8array-extras@1.5.0", "", {}, "sha512-rvKSBiC5zqCCiDZ9kAOszZcDvdAHwwIKJG33Ykj43OKcWsnmcBRL09YTU4nOeHZ8Y2a7l1MgTd08SBe9A8Qj6A=="], "undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="], + "unpipe": ["unpipe@1.0.0", "", {}, "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ=="], + "util-deprecate": ["util-deprecate@1.0.2", "", {}, "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="], + "vary": ["vary@1.1.2", "", {}, "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg=="], + "vite": ["vite@7.3.0", "", { "dependencies": { "esbuild": "^0.27.0", "fdir": "^6.5.0", "picomatch": "^4.0.3", "postcss": "^8.5.6", "rollup": "^4.43.0", "tinyglobby": "^0.2.15" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "jiti": ">=1.21.0", "less": "^4.0.0", "lightningcss": "^1.21.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "jiti", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-dZwN5L1VlUBewiP6H9s2+B3e3Jg96D0vzN+Ry73sOefebhYr9f94wwkMNN/9ouoU8pV1BqA1d1zGk8928cx0rg=="], "vite-node": ["vite-node@3.2.4", "", { "dependencies": { "cac": "^6.7.14", "debug": "^4.4.1", "es-module-lexer": "^1.7.0", "pathe": "^2.0.3", "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" }, "bin": { "vite-node": "vite-node.mjs" } }, "sha512-EbKSKh+bh1E1IFxeO0pg1n4dvoOTt0UDiXMd/qn++r98+jPO1xtJilvXldeuQ8giIB5IkpjCgMleHMNEsGH6pg=="], @@ -1185,20 +1307,26 @@ "yargs-parser": ["yargs-parser@21.1.1", "", {}, "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw=="], - "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], + "zod": ["zod@4.3.5", "", {}, "sha512-k7Nwx6vuWx1IJ9Bjuf4Zt1PEllcwe7cls3VNzm4CQ1/hgtFUK2bRNG3rvnpPUhFjmqJKAKtjV576KnUkHocg/g=="], "zod-to-json-schema": ["zod-to-json-schema@3.25.1", "", { "peerDependencies": { "zod": "^3.25 || ^4" } }, "sha512-pM/SU9d3YAggzi6MtR4h7ruuQlqKtad8e9S0fmxcMi+ueAK5Korys/aWcV9LIIHTVbj01NdzxcnXSN+O74ZIVA=="], "@anthropic-ai/sandbox-runtime/commander": ["commander@12.1.0", "", {}, "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA=="], + "@anthropic-ai/sandbox-runtime/zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], + "@isaacs/cliui/string-width": ["string-width@5.1.2", "", { "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", "strip-ansi": "^7.0.1" } }, "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA=="], "@isaacs/cliui/wrap-ansi": ["wrap-ansi@8.1.0", "", { "dependencies": { "ansi-styles": "^6.1.0", "string-width": "^5.0.1", "strip-ansi": "^7.0.1" } }, "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ=="], "@lmstudio/sdk/chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="], + "@lmstudio/sdk/zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], + "@mariozechner/mini-lit/marked": ["marked@16.4.2", "", { "bin": { "marked": "bin/marked.js" } }, "sha512-TI3V8YYWvkVf3KJe1dRkpnjs68JUPyEa5vjKrp1XEEJUAOaQc+Qj+L1qWbPd0SJuAdQkFU0h73sXXqwDYxsiDA=="], + "@mistralai/mistralai/zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], + "@oh-my-pi/pi-agent-core/@types/node": ["@types/node@24.10.4", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-vnDVpYPMzs4wunl27jHrfmwojOGKya0xyM3sH+UE5iv5uPS6vX7UIoh6m+vQc5LGBq52HBKPIn/zcSZVzeDEZg=="], "@oh-my-pi/pi-ai/@types/node": ["@types/node@24.10.4", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-vnDVpYPMzs4wunl27jHrfmwojOGKya0xyM3sH+UE5iv5uPS6vX7UIoh6m+vQc5LGBq52HBKPIn/zcSZVzeDEZg=="], diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index fd2153d61..43bcda6ae 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -1,6 +1,19 @@ # Changelog ## [Unreleased] +### Added + +- Added support for kwaipilot/kat-coder-pro model via OpenRouter + +### Changed + +- Updated model specifications including context windows, max tokens, and pricing for multiple OpenRouter models + +### Removed + +- Removed alibaba/tongyi-deepresearch-30b-a3b:free model from OpenRouter +- Removed nousresearch/hermes-4-405b model from OpenRouter +- Removed tngtech/tng-r1t-chimera:free model from OpenRouter ## [3.15.1] - 2026-01-05 diff --git a/packages/ai/src/models.generated.ts b/packages/ai/src/models.generated.ts index dd9525798..9c03c7b4c 100644 --- a/packages/ai/src/models.generated.ts +++ b/packages/ai/src/models.generated.ts @@ -1387,7 +1387,7 @@ export const MODELS = { cacheWrite: 0, }, contextWindow: 131072, - maxTokens: 8192, + maxTokens: 131072, } satisfies Model<"openai-completions">, "llama-3.3-70b-versatile": { id: "llama-3.3-70b-versatile", @@ -1535,12 +1535,12 @@ export const MODELS = { input: ["text"], cost: { input: 0.15, - output: 0.75, + output: 0.6, cacheRead: 0, cacheWrite: 0, }, contextWindow: 131072, - maxTokens: 32768, + maxTokens: 65536, } satisfies Model<"openai-completions">, "openai/gpt-oss-20b": { id: "openai/gpt-oss-20b", @@ -1551,13 +1551,13 @@ export const MODELS = { reasoning: true, input: ["text"], cost: { - input: 0.1, - output: 0.5, + input: 0.075, + output: 0.3, cacheRead: 0, cacheWrite: 0, }, contextWindow: 131072, - maxTokens: 32768, + maxTokens: 65536, } satisfies Model<"openai-completions">, "qwen-qwq-32b": { id: "qwen-qwq-32b", @@ -2636,23 +2636,6 @@ export const MODELS = { contextWindow: 131072, maxTokens: 131072, } satisfies Model<"openai-completions">, - "alibaba/tongyi-deepresearch-30b-a3b:free": { - id: "alibaba/tongyi-deepresearch-30b-a3b:free", - name: "Tongyi DeepResearch 30B A3B (free)", - api: "openai-completions", - provider: "openrouter", - baseUrl: "https://openrouter.ai/api/v1", - reasoning: true, - input: ["text"], - cost: { - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - }, - contextWindow: 131072, - maxTokens: 131072, - } satisfies Model<"openai-completions">, "allenai/olmo-3-7b-instruct": { id: "allenai/olmo-3-7b-instruct", name: "AllenAI: Olmo 3 7B Instruct", @@ -3206,13 +3189,13 @@ export const MODELS = { reasoning: true, input: ["text"], cost: { - input: 0.19999999999999998, - output: 0.88, - cacheRead: 0.106, + input: 0.19, + output: 0.87, + cacheRead: 0, cacheWrite: 0, }, contextWindow: 163840, - maxTokens: 4096, + maxTokens: 65536, } satisfies Model<"openai-completions">, "deepseek/deepseek-chat-v3.1": { id: "deepseek/deepseek-chat-v3.1", @@ -3240,13 +3223,13 @@ export const MODELS = { reasoning: true, input: ["text"], cost: { - input: 0.3, - output: 1.2, + input: 0.7, + output: 2.4, cacheRead: 0, cacheWrite: 0, }, contextWindow: 163840, - maxTokens: 4096, + maxTokens: 163840, } satisfies Model<"openai-completions">, "deepseek/deepseek-r1-0528": { id: "deepseek/deepseek-r1-0528", @@ -3622,6 +3605,23 @@ export const MODELS = { contextWindow: 128000, maxTokens: 16384, } satisfies Model<"openai-completions">, + "kwaipilot/kat-coder-pro": { + id: "kwaipilot/kat-coder-pro", + name: "Kwaipilot: KAT-Coder-Pro V1", + api: "openai-completions", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + reasoning: false, + input: ["text"], + cost: { + input: 0.207, + output: 0.828, + cacheRead: 0.0414, + cacheWrite: 0, + }, + contextWindow: 256000, + maxTokens: 128000, + } satisfies Model<"openai-completions">, "kwaipilot/kat-coder-pro:free": { id: "kwaipilot/kat-coder-pro:free", name: "Kwaipilot: KAT-Coder-Pro V1 (free)", @@ -3903,13 +3903,13 @@ export const MODELS = { reasoning: true, input: ["text"], cost: { - input: 0.3, - output: 1.2, - cacheRead: 0.03, - cacheWrite: 0.375, + input: 0.12, + output: 0.48, + cacheRead: 0, + cacheWrite: 0, }, - contextWindow: 204800, - maxTokens: 131072, + contextWindow: 196608, + maxTokens: 4096, } satisfies Model<"openai-completions">, "mistralai/codestral-2508": { id: "mistralai/codestral-2508", @@ -4447,13 +4447,13 @@ export const MODELS = { reasoning: false, input: ["text"], cost: { - input: 0.456, - output: 1.8399999999999999, + input: 0.5, + output: 2.4, cacheRead: 0, cacheWrite: 0, }, contextWindow: 131072, - maxTokens: 131072, + maxTokens: 4096, } satisfies Model<"openai-completions">, "moonshotai/kimi-k2-0905": { id: "moonshotai/kimi-k2-0905", @@ -4498,13 +4498,13 @@ export const MODELS = { reasoning: true, input: ["text"], cost: { - input: 0.39999999999999997, - output: 1.75, + input: 0.32, + output: 0.48, cacheRead: 0, cacheWrite: 0, }, contextWindow: 262144, - maxTokens: 65535, + maxTokens: 4096, } satisfies Model<"openai-completions">, "nex-agi/deepseek-v3.1-nex-n1:free": { id: "nex-agi/deepseek-v3.1-nex-n1:free", @@ -4540,23 +4540,6 @@ export const MODELS = { contextWindow: 32768, maxTokens: 32768, } satisfies Model<"openai-completions">, - "nousresearch/hermes-4-405b": { - id: "nousresearch/hermes-4-405b", - name: "Nous: Hermes 4 405B", - api: "openai-completions", - provider: "openrouter", - baseUrl: "https://openrouter.ai/api/v1", - reasoning: true, - input: ["text"], - cost: { - input: 0.3, - output: 1.2, - cacheRead: 0, - cacheWrite: 0, - }, - contextWindow: 131072, - maxTokens: 131072, - } satisfies Model<"openai-completions">, "nousresearch/hermes-4-70b": { id: "nousresearch/hermes-4-70b", name: "Nous: Hermes 4 70B", @@ -5371,7 +5354,7 @@ export const MODELS = { cacheWrite: 0, }, contextWindow: 131072, - maxTokens: 131072, + maxTokens: 4096, } satisfies Model<"openai-completions">, "openai/gpt-oss-safeguard-20b": { id: "openai/gpt-oss-safeguard-20b", @@ -6028,8 +6011,8 @@ export const MODELS = { reasoning: false, input: ["text", "image"], cost: { - input: 0.19999999999999998, - output: 1.2, + input: 0.12, + output: 0.56, cacheRead: 0, cacheWrite: 0, }, @@ -6045,8 +6028,8 @@ export const MODELS = { reasoning: true, input: ["text", "image"], cost: { - input: 0.3, - output: 1.2, + input: 0.44999999999999996, + output: 3.5, cacheRead: 0, cacheWrite: 0, }, @@ -6274,23 +6257,6 @@ export const MODELS = { contextWindow: 163840, maxTokens: 65536, } satisfies Model<"openai-completions">, - "tngtech/tng-r1t-chimera:free": { - id: "tngtech/tng-r1t-chimera:free", - name: "TNG: R1T Chimera (free)", - api: "openai-completions", - provider: "openrouter", - baseUrl: "https://openrouter.ai/api/v1", - reasoning: true, - input: ["text"], - cost: { - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - }, - contextWindow: 163840, - maxTokens: 163840, - } satisfies Model<"openai-completions">, "x-ai/grok-3": { id: "x-ai/grok-3", name: "xAI: Grok 3", @@ -6487,13 +6453,13 @@ export const MODELS = { reasoning: true, input: ["text"], cost: { - input: 0.13, - output: 0.85, + input: 0.049999999999999996, + output: 0.22, cacheRead: 0, cacheWrite: 0, }, contextWindow: 131072, - maxTokens: 98304, + maxTokens: 131072, } satisfies Model<"openai-completions">, "z-ai/glm-4.5-air:free": { id: "z-ai/glm-4.5-air:free", @@ -6510,7 +6476,7 @@ export const MODELS = { cacheWrite: 0, }, contextWindow: 131072, - maxTokens: 131072, + maxTokens: 96000, } satisfies Model<"openai-completions">, "z-ai/glm-4.5v": { id: "z-ai/glm-4.5v", @@ -6589,13 +6555,13 @@ export const MODELS = { reasoning: true, input: ["text"], cost: { - input: 0.39999999999999997, - output: 1.5, + input: 0.16, + output: 0.7999999999999999, cacheRead: 0, cacheWrite: 0, }, contextWindow: 202752, - maxTokens: 65535, + maxTokens: 4096, } satisfies Model<"openai-completions">, }, "xai": { diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 0c8358178..f793d01e4 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -1,10 +1,15 @@ # Changelog ## [Unreleased] - ### Added -- Added voice mode with Caps Lock toggle for speech-to-text input and text-to-speech response summaries using OpenAI Whisper and TTS APIs +- Added VoiceSupervisor class for realtime voice mode using OpenAI Realtime API with continuous mic streaming and semantic VAD turn detection +- Added VoiceController class for steering user input and deciding presentation of assistant responses +- Added echo suppression and noise floor filtering for microphone input during voice playback +- Added fallback transcript handling when realtime assistant produces no tool call or audio output +- Added voice progress notifications that speak partial results after 15 seconds of streaming +- Added platform-specific audio tool detection with helpful installation instructions for missing tools +- Added realtime voice mode using OpenAI gpt-5-realtime with continuous mic streaming, interruptible input, and supervisor-controlled spoken updates - Added `gemini_image` tool for Gemini Nano Banana image generation when `GEMINI_API_KEY` (or `GOOGLE_API_KEY`) is set - Added `description` field to task tool for displaying short user-facing summaries in progress output - Added `getApiKeyForProvider()` method to ModelRegistry for retrieving API keys by provider name @@ -17,6 +22,11 @@ ### Changed +- Changed voice mode toggle from Caps Lock to Ctrl+Y with auto-send on silence behavior +- Changed default TTS model from gpt-4o-mini-tts to tts-1 +- Changed voice mode description to reflect realtime input/output with auto-send on silence +- Updated hotkeys help to show Ctrl+Y for voice mode toggle instead of Caps Lock +- Voice mode now uses OpenAI Realtime (gpt-5-realtime) with Ctrl+Y toggle and auto-send on silence - Updated web search tool to support `auto` as explicit provider option for auto-detection - Standardized tool result rendering across grep, find, ls, notebook, ask, output, and web search tools with consistent tree formatting and expand hints - Updated grep and find tool output to display language-specific icons for files and folder icons for directories @@ -1656,4 +1666,4 @@ Initial public release. - Git branch display in footer - Message queueing during streaming responses - OAuth integration for Gmail and Google Calendar access -- HTML export with syntax highlighting and collapsible sections +- HTML export with syntax highlighting and collapsible sections \ No newline at end of file diff --git a/packages/coding-agent/package.json b/packages/coding-agent/package.json index b12201a07..fa621691b 100644 --- a/packages/coding-agent/package.json +++ b/packages/coding-agent/package.json @@ -42,6 +42,7 @@ "@oh-my-pi/pi-agent-core": "workspace:*", "@oh-my-pi/pi-ai": "workspace:*", "@oh-my-pi/pi-tui": "workspace:*", + "@openai/agents": "^0.3.7", "@sinclair/typebox": "^0.34.46", "ajv": "^8.17.1", "chalk": "^5.5.0", @@ -55,12 +56,13 @@ "nanoid": "^5.1.6", "ndjson": "^2.0.0", "node-html-parser": "^6.1.13", + "sharp": "^0.34.2", "smol-toml": "^1.6.0", "strip-ansi": "^7.1.2", - "sharp": "^0.34.2", "winston": "^3.17.0", "winston-daily-rotate-file": "^5.0.0", - "yaml": "^2.8.2" + "yaml": "^2.8.2", + "zod": "^4.3.5" }, "devDependencies": { "@types/diff": "^7.0.2", diff --git a/packages/coding-agent/src/core/settings-manager.ts b/packages/coding-agent/src/core/settings-manager.ts index ca8418454..f397403bd 100644 --- a/packages/coding-agent/src/core/settings-manager.ts +++ b/packages/coding-agent/src/core/settings-manager.ts @@ -752,7 +752,7 @@ export class SettingsManager { enabled: this.settings.voice?.enabled ?? false, transcriptionModel: this.settings.voice?.transcriptionModel ?? "whisper-1", transcriptionLanguage: this.settings.voice?.transcriptionLanguage ?? "", - ttsModel: this.settings.voice?.ttsModel ?? "gpt-4o-mini-tts", + ttsModel: this.settings.voice?.ttsModel ?? "tts-1", ttsVoice: this.settings.voice?.ttsVoice ?? "alloy", ttsFormat: this.settings.voice?.ttsFormat ?? "wav", }; diff --git a/packages/coding-agent/src/core/voice-controller.ts b/packages/coding-agent/src/core/voice-controller.ts new file mode 100644 index 000000000..1873b4ec1 --- /dev/null +++ b/packages/coding-agent/src/core/voice-controller.ts @@ -0,0 +1,135 @@ +import { Agent, run, setDefaultOpenAIKey } from "@openai/agents"; +import { z } from "zod"; +import { logger } from "./logger"; +import type { ModelRegistry } from "./model-registry"; + +const DEFAULT_CONTROLLER_MODEL = process.env.OMP_VOICE_CONTROLLER_MODEL ?? "gpt-4o-mini"; +const DEFAULT_SUMMARY_MODEL = process.env.OMP_VOICE_SUMMARY_MODEL ?? DEFAULT_CONTROLLER_MODEL; +const MAX_INPUT_CHARS = 8000; + +export type VoiceSteeringDecision = { action: "pass" | "ask"; text: string }; +export type VoicePresentationDecision = { action: "skip" | "speak"; text?: string }; +type VoiceSummaryOutput = { text: string }; + +const steeringSchema: z.ZodType = z.object({ + action: z.enum(["pass", "ask"]), + text: z.string().min(1), +}); + +const presentationSchema: z.ZodType = z.object({ + action: z.enum(["skip", "speak"]), + text: z.string().min(1).optional(), +}); + +const summarySchema: z.ZodType = z.object({ + text: z.string().min(1), +}); + +function normalizeText(text: string): string { + return text.replace(/\s+/g, " ").trim(); +} + +function truncateText(text: string, maxChars: number): string { + if (text.length <= maxChars) return text; + return `${text.slice(0, maxChars)}...`; +} + +export class VoiceController { + private lastApiKey: string | undefined; + + constructor(private registry: ModelRegistry) {} + + private async ensureApiKey(): Promise { + const apiKey = await this.registry.getApiKeyForProvider("openai"); + if (!apiKey) { + logger.debug("voice-controller: no OpenAI API key available"); + return null; + } + if (apiKey !== this.lastApiKey) { + setDefaultOpenAIKey(apiKey); + this.lastApiKey = apiKey; + } + return apiKey; + } + + async steerUserInput(text: string): Promise { + if (!(await this.ensureApiKey())) return null; + + const normalized = truncateText(normalizeText(text), MAX_INPUT_CHARS); + const agent = new Agent({ + name: "Voice Input Steering", + instructions: + "You are a voice-input controller for a coding agent. " + + "Given a user's speech transcript, decide if it is clear enough to send to the agent. " + + "If unclear or missing key details, ask exactly one short question. " + + "If clear, rewrite it as a concise instruction for the agent. " + + "Keep it short and preserve intent.", + model: DEFAULT_CONTROLLER_MODEL, + outputType: steeringSchema, + }); + + try { + const result = await run(agent, normalized); + return result.finalOutput ?? null; + } catch (error) { + logger.debug("voice-controller: steering error", { + error: error instanceof Error ? error.message : String(error), + }); + return null; + } + } + + async decidePresentation(text: string): Promise { + if (!(await this.ensureApiKey())) return null; + + const normalized = truncateText(normalizeText(text), MAX_INPUT_CHARS); + const agent = new Agent({ + name: "Voice Presentation Gate", + instructions: + "You are a voice presentation gate for a coding agent. " + + "Decide whether to speak the assistant response to the user. " + + "Speak when there is a decision, summary, or a question for the user. " + + "Skip if it is mostly tool output, verbose logs, or not useful to speak. " + + "When speaking, respond in 1-3 short sentences (<=45 words) in a casual, concise tone. " + + "If user input is needed, ask exactly one short question.", + model: DEFAULT_CONTROLLER_MODEL, + outputType: presentationSchema, + }); + + try { + const result = await run(agent, normalized); + return result.finalOutput ?? null; + } catch (error) { + logger.debug("voice-controller: presentation error", { + error: error instanceof Error ? error.message : String(error), + }); + return null; + } + } + + async summarizeForVoice(text: string): Promise { + if (!(await this.ensureApiKey())) return null; + + const normalized = truncateText(normalizeText(text), MAX_INPUT_CHARS); + const agent = new Agent({ + name: "Voice Summary", + instructions: + "Summarize the assistant response for voice playback. " + + "Use 1-2 short sentences. " + + "If a question is required from the user, ask one short question.", + model: DEFAULT_SUMMARY_MODEL, + outputType: summarySchema, + }); + + try { + const result = await run(agent, normalized); + const output = result.finalOutput?.text ?? ""; + return output.trim() || null; + } catch (error) { + logger.debug("voice-controller: summary error", { + error: error instanceof Error ? error.message : String(error), + }); + return null; + } + } +} diff --git a/packages/coding-agent/src/core/voice-supervisor.ts b/packages/coding-agent/src/core/voice-supervisor.ts new file mode 100644 index 000000000..ad02b9198 --- /dev/null +++ b/packages/coding-agent/src/core/voice-supervisor.ts @@ -0,0 +1,1003 @@ +import { + RealtimeAgent, + RealtimeSession, + type RealtimeSessionConfig, + type TransportEvent, + type TransportLayerAudio, + tool, +} from "@openai/agents/realtime"; +import type { Subprocess } from "bun"; +import type { ReadableStreamDefaultReader as WebReadableStreamDefaultReader } from "stream/web"; +import { z } from "zod"; +import { logger } from "./logger"; +import type { ModelRegistry } from "./model-registry"; + +const DEFAULT_REALTIME_MODEL = process.env.OMP_VOICE_REALTIME_MODEL ?? "gpt-realtime"; +const DEFAULT_REALTIME_VOICE = process.env.OMP_VOICE_REALTIME_VOICE ?? "marin"; +const DEFAULT_SAMPLE_RATE = 24000; +const DEFAULT_CHANNELS = 1; +const DEFAULT_BITS = 16; +const INTERRUPT_DEBOUNCE_MS = 200; +const MAX_RESULT_CHARS = 6000; +const MAX_PROGRESS_CHARS = 1400; +const PLAYBACK_ACTIVE_WINDOW_MS = 350; +// Echo cancellation: only suppress mic when playback is active and mic is much quieter +const ECHO_SUPPRESSION_RATIO = 2.5; +// Minimum RMS to ever send (absolute noise floor) +const MIC_NOISE_FLOOR = 0.005; +const PLAYBACK_ERROR_COOLDOWN_MS = 2000; + +const SUPERVISOR_INSTRUCTIONS = [ + "You are the realtime voice supervisor for a terminal coding agent.", + "Manage conversation flow, turn-taking, and what gets spoken aloud.", + "For user speech: if unclear, ask exactly one short question.", + "If clear, call send_to_agent with a concise instruction for the coding agent.", + "If the user is greeting/smalltalk or gives no actionable request, respond briefly and do not call send_to_agent.", + "Keep spoken responses to 1-2 short sentences (<=40 words).", + "You will receive system updates prefixed with SYSTEM_EVENT, PROGRESS_UPDATE, or AGENT_OUTPUT.", + "For AGENT_OUTPUT, always respond with a brief spoken summary and any single question needed.", + "For PROGRESS_UPDATE, speak a short update only if it helps the user stay oriented.", + "Do not call send_to_agent for system updates.", + "If the user asks to stop or cancel work, call interrupt_agent.", +].join(" "); + +type VoiceSupervisorCallbacks = { + onSendToAgent: (text: string) => Promise | void; + onInterruptAgent: (reason?: string) => Promise | void; + onStatus: (status?: string) => void; + onError: (error: Error) => void; + onWarning?: (message: string) => void; +}; + +function normalizeText(text: string): string { + return text.replace(/\s+/g, " ").trim(); +} + +function truncateText(text: string, maxChars: number): string { + if (text.length <= maxChars) return text; + return `${text.slice(0, maxChars)}...`; +} + +function toArrayBuffer(chunk: Uint8Array): ArrayBuffer { + const buffer = chunk.buffer; + if (buffer instanceof ArrayBuffer) { + if (chunk.byteOffset === 0 && chunk.byteLength === buffer.byteLength) { + return buffer; + } + return buffer.slice(chunk.byteOffset, chunk.byteOffset + chunk.byteLength); + } + const copy = new Uint8Array(chunk.byteLength); + copy.set(chunk); + return copy.buffer; +} + +function describeError(error: unknown): string { + if (error instanceof Error) return error.message; + if (typeof error === "string") return error; + if (error && typeof error === "object") { + const maybeMessage = (error as { message?: unknown }).message; + if (typeof maybeMessage === "string") return maybeMessage; + const nested = (error as { error?: unknown }).error; + if (nested) return describeError(nested); + try { + return JSON.stringify(error); + } catch { + return String(error); + } + } + return String(error); +} + +type AudioToolStatus = { + capture: { available: boolean; tool?: string; command?: string[] }; + playback: { available: boolean; tool?: string; command?: string[] }; +}; + +function checkAudioTools(sampleRate: number, channels: number): AudioToolStatus { + const captureResult = buildCaptureCommand(sampleRate, channels); + const playbackCmd = buildPlaybackCommand(sampleRate, channels); + + return { + capture: { + available: captureResult !== null, + tool: captureResult?.command[0], + command: captureResult?.command, + }, + playback: { + available: playbackCmd !== null, + tool: playbackCmd?.[0], + command: playbackCmd ?? undefined, + }, + }; +} + +function getMissingToolsMessage(): string { + const platform = process.platform; + const lines: string[] = ["Voice mode requires audio tools. Install one of the following:"]; + + if (platform === "linux") { + lines.push(""); + lines.push(" For capture (microphone):"); + lines.push(" • sox (recommended): sudo dnf install sox"); + lines.push(" • pulseaudio-utils: sudo dnf install pulseaudio-utils"); + lines.push(" • alsa-utils: sudo dnf install alsa-utils"); + lines.push(" • ffmpeg: sudo dnf install ffmpeg"); + lines.push(""); + lines.push(" For playback (speaker):"); + lines.push(" • sox (recommended): sudo dnf install sox"); + lines.push(" • ffmpeg: sudo dnf install ffmpeg"); + lines.push(""); + lines.push(" Set OMP_VOICE_CAPTURE_DEVICE to override the default capture device."); + lines.push(" (Applies to all tools; for sox, this sets AUDIODEV internally.)"); + } else if (platform === "darwin") { + lines.push(""); + lines.push(" • sox (recommended): brew install sox"); + lines.push(" • ffmpeg: brew install ffmpeg"); + } else if (platform === "win32") { + lines.push(""); + lines.push(" • sox: choco install sox"); + lines.push(" • ffmpeg: choco install ffmpeg"); + } + + return lines.join("\n"); +} + +type CaptureCommand = { command: string[]; env?: Record }; + +function buildCaptureCommand(sampleRate: number, channels: number): CaptureCommand | null { + const platform = process.platform; + // Allow user to override capture device via environment + const captureDevice = process.env.OMP_VOICE_CAPTURE_DEVICE; + + // Prefer sox/rec as they work well across platforms + const soxPath = Bun.which("sox") ?? Bun.which("rec"); + if (soxPath) { + const command = [ + soxPath, + "-q", + "-d", + "-t", + "raw", + "-r", + String(sampleRate), + "-e", + "signed-integer", + "-b", + String(DEFAULT_BITS), + "-c", + String(channels), + "-", + ]; + // sox uses AUDIODEV env var to override the default device + const env = captureDevice ? { AUDIODEV: captureDevice } : undefined; + return { command, env }; + } + + // On Linux, try PulseAudio first (parecord) + if (platform === "linux") { + const parecordPath = Bun.which("parecord"); + if (parecordPath) { + const command = [parecordPath, "--raw", "--format=s16le", `--rate=${sampleRate}`, `--channels=${channels}`]; + if (captureDevice) { + command.push(`--device=${captureDevice}`); + } + return { command }; + } + } + + // ALSA arecord as fallback on Linux + const arecordPath = Bun.which("arecord"); + if (arecordPath) { + const device = captureDevice ?? "default"; + return { + command: [ + arecordPath, + "-q", + "-D", + device, + "-f", + "S16_LE", + "-r", + String(sampleRate), + "-c", + String(channels), + "-t", + "raw", + ], + }; + } + + // ffmpeg fallback with platform-specific input + const ffmpegPath = Bun.which("ffmpeg"); + if (ffmpegPath) { + if (platform === "darwin") { + const device = captureDevice ?? ":0"; + return { + command: [ + ffmpegPath, + "-hide_banner", + "-loglevel", + "error", + "-f", + "avfoundation", + "-i", + device, + "-ac", + String(channels), + "-ar", + String(sampleRate), + "-f", + "s16le", + "-", + ], + }; + } + if (platform === "linux") { + // Try PulseAudio format first, fall back to ALSA + const hasPulse = Bun.which("pulseaudio") || Bun.which("pipewire-pulse") || process.env.PULSE_SERVER; + const format = hasPulse ? "pulse" : "alsa"; + const device = captureDevice ?? "default"; + return { + command: [ + ffmpegPath, + "-hide_banner", + "-loglevel", + "error", + "-f", + format, + "-i", + device, + "-ac", + String(channels), + "-ar", + String(sampleRate), + "-f", + "s16le", + "-", + ], + }; + } + if (platform === "win32") { + const device = captureDevice ?? "audio=default"; + return { + command: [ + ffmpegPath, + "-hide_banner", + "-loglevel", + "error", + "-f", + "dshow", + "-i", + device, + "-ac", + String(channels), + "-ar", + String(sampleRate), + "-f", + "s16le", + "-", + ], + }; + } + } + + return null; +} + +function buildPlaybackCommand(sampleRate: number, channels: number): string[] | null { + const preferred = process.env.OMP_VOICE_PLAYBACK?.toLowerCase(); + const ffplayPath = Bun.which("ffplay"); + const playPath = Bun.which("play"); + const soxPath = Bun.which("sox"); + + const playCommand = playPath + ? [ + playPath, + "-q", + "-t", + "raw", + "-r", + String(sampleRate), + "-e", + "signed-integer", + "-b", + String(DEFAULT_BITS), + "-c", + String(channels), + "-", + ] + : null; + + const soxCommand = soxPath + ? [ + soxPath, + "-q", + "-t", + "raw", + "-r", + String(sampleRate), + "-e", + "signed-integer", + "-b", + String(DEFAULT_BITS), + "-c", + String(channels), + "-", + "-d", + ] + : null; + + const ffplayCommand = ffplayPath + ? [ + ffplayPath, + "-nodisp", + "-autoexit", + "-hide_banner", + "-loglevel", + "error", + "-fflags", + "nobuffer", + "-flags", + "low_delay", + "-f", + "s16le", + "-ar", + String(sampleRate), + "-ac", + String(channels), + "-", + ] + : null; + + if (preferred === "ffplay") return ffplayCommand; + if (preferred === "play") return playCommand ?? soxCommand; + if (preferred === "sox") return soxCommand ?? playCommand; + + return playCommand ?? soxCommand ?? ffplayCommand; +} + +function rms16le(buffer: Uint8Array): number { + if (buffer.byteLength < 2) return 0; + const view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength); + let sum = 0; + let count = 0; + for (let i = 0; i + 1 < buffer.byteLength; i += 2) { + const sample = view.getInt16(i, true) / 32768; + sum += sample * sample; + count += 1; + } + if (count === 0) return 0; + return Math.sqrt(sum / count); +} + +export class VoiceSupervisor { + private session: RealtimeSession | undefined = undefined; + private captureProcess: Subprocess | undefined = undefined; + private captureReader: WebReadableStreamDefaultReader | undefined = undefined; + private playbackProcess: Subprocess | undefined = undefined; + private playbackWriter: + | { + write: (chunk: Uint8Array) => Promise; + close: () => Promise; + } + | undefined = undefined; + private active = false; + private connected = false; + private sessionReady = false; + private lastInterruptAt = 0; + private lastPlaybackAt = 0; + private lastPlaybackRms = 0; + private lastPlaybackErrorAt = 0; + // Fallback transcript handling: track user speech when no tool call is made + private pendingTranscript = ""; + private pendingResponseHasToolCall = false; + private pendingResponseHasAudioOutput = false; + + constructor( + private registry: ModelRegistry, + private callbacks: VoiceSupervisorCallbacks, + ) {} + + /** + * Check if audio tools are available for voice mode. + * Returns null if all tools are available, or an error message if not. + */ + static checkAvailability(): { available: boolean; error?: string; tools?: AudioToolStatus } { + const status = checkAudioTools(DEFAULT_SAMPLE_RATE, DEFAULT_CHANNELS); + if (status.capture.available && status.playback.available) { + return { available: true, tools: status }; + } + + const missing: string[] = []; + if (!status.capture.available) missing.push("capture"); + if (!status.playback.available) missing.push("playback"); + + return { + available: false, + error: `Missing audio ${missing.join(" and ")} tools.\n\n${getMissingToolsMessage()}`, + tools: status, + }; + } + + get isActive(): boolean { + return this.active; + } + + async start(): Promise { + if (this.active) return; + + const apiKey = await this.registry.getApiKeyForProvider("openai"); + if (!apiKey) { + throw new Error("OpenAI API key not found (set OPENAI_API_KEY or login)."); + } + + this.active = true; + this.lastInterruptAt = 0; + this.sessionReady = false; + this.lastPlaybackErrorAt = 0; + this.pendingTranscript = ""; + this.pendingResponseHasToolCall = false; + this.pendingResponseHasAudioOutput = false; + this.callbacks.onStatus("Connecting realtime voice..."); + + try { + const agent = this.createSupervisorAgent(); + const session = new RealtimeSession(agent, { + transport: "websocket", + model: DEFAULT_REALTIME_MODEL, + config: this.buildSessionConfig(), + }); + + this.session = session; + this.bindSessionEvents(session); + await session.connect({ apiKey }); + this.connected = session.transport.status === "connected"; + this.sessionReady = this.connected; + if (!this.connected) { + await this.waitForConnection(session, 5000); + } + await this.waitForSessionReady(session, 5000); + await this.startCapture(); + await this.ensurePlayback(); + this.callbacks.onStatus("Listening... (auto-send on silence, Ctrl+Y to stop)"); + } catch (error) { + await this.stop(); + throw new Error(describeError(error)); + } + } + + async stop(): Promise { + if (!this.active) return; + this.active = false; + this.connected = false; + this.sessionReady = false; + await this.stopCapture(); + await this.resetPlayback(); + if (this.session) { + this.session.close(); + this.session = undefined; + } + this.callbacks.onStatus(undefined); + } + + notifyProgress(text: string): void { + this.sendSystemMessage("PROGRESS_UPDATE", text, MAX_PROGRESS_CHARS); + } + + notifyResult(text: string): void { + this.sendSystemMessage("AGENT_OUTPUT", text, MAX_RESULT_CHARS); + } + + private sendSystemMessage(prefix: string, text: string, maxChars: number): void { + if (!this.session || !this.active) return; + if (!this.connected || !this.sessionReady || this.session.transport.status !== "connected") return; + const trimmed = normalizeText(text); + if (!trimmed) return; + const payload = `${prefix}: ${truncateText(trimmed, maxChars)}`; + try { + this.session.transport.sendEvent({ + type: "conversation.item.create", + item: { + type: "message", + role: "system", + content: [{ type: "input_text", text: payload }], + }, + }); + this.session.transport.sendEvent({ type: "response.create" }); + } catch (error) { + const message = describeError(error); + if (message.includes("WebSocket is not connected")) return; + this.callbacks.onError(error instanceof Error ? error : new Error(message)); + } + } + + private createSupervisorAgent(): RealtimeAgent { + const sendToAgentTool = tool({ + name: "send_to_agent", + description: "Send a concise instruction to the coding agent.", + parameters: z.object({ + text: z.string().min(1), + }), + execute: async ({ text }) => { + const cleaned = normalizeText(text); + if (cleaned) { + await this.callbacks.onSendToAgent(cleaned); + } + return "sent"; + }, + }); + + const interruptAgentTool = tool({ + name: "interrupt_agent", + description: "Interrupt the coding agent immediately.", + parameters: z.object({ + reason: z.string().optional(), + }), + execute: async ({ reason }) => { + await this.callbacks.onInterruptAgent(reason); + return "interrupted"; + }, + }); + + return new RealtimeAgent({ + name: "Voice Supervisor", + instructions: SUPERVISOR_INSTRUCTIONS, + tools: [sendToAgentTool, interruptAgentTool], + voice: DEFAULT_REALTIME_VOICE, + }); + } + + private buildSessionConfig(): Partial { + return { + outputModalities: ["audio"], + audio: { + input: { + format: { type: "audio/pcm", rate: DEFAULT_SAMPLE_RATE }, + noiseReduction: { type: "near_field" }, + turnDetection: { + type: "semantic_vad", + createResponse: true, + interruptResponse: true, + }, + }, + output: { + format: { type: "audio/pcm", rate: DEFAULT_SAMPLE_RATE }, + ...(DEFAULT_REALTIME_VOICE ? { voice: DEFAULT_REALTIME_VOICE } : {}), + }, + }, + }; + } + + private bindSessionEvents(session: RealtimeSession): void { + session.transport.on("connection_change", (status) => { + this.connected = status === "connected"; + if (this.connected) { + this.sessionReady = true; + } else { + this.sessionReady = false; + } + if (!this.active) return; + if (this.connected) { + this.callbacks.onStatus("Listening... (auto-send on silence, Ctrl+Y to stop)"); + } else { + this.callbacks.onStatus("Reconnecting realtime voice..."); + } + }); + + session.on("audio", (event: TransportLayerAudio) => { + void this.handleAudio(event); + }); + + session.on("audio_start", () => { + if (!this.active) return; + this.pendingResponseHasAudioOutput = true; + this.callbacks.onStatus("Speaking..."); + }); + + session.on("audio_stopped", () => { + if (!this.active) return; + this.callbacks.onStatus("Listening... (auto-send on silence, Ctrl+Y to stop)"); + }); + + session.on("audio_interrupted", () => { + void this.resetPlayback(); + if (!this.active) return; + this.callbacks.onStatus("Listening... (auto-send on silence, Ctrl+Y to stop)"); + }); + + session.on("transport_event", (event: TransportEvent) => { + this.handleTransportEvent(event); + }); + + session.on("error", (error) => { + const message = describeError(error); + logger.debug("voice-supervisor: realtime error", { error: message }); + if (message.includes("WebSocket is not connected")) { + if (this.active) { + this.callbacks.onStatus("Reconnecting realtime voice..."); + } + return; + } + this.callbacks.onError(new Error(message)); + }); + } + + private handleTransportEvent(event: TransportEvent): void { + if (!this.active) return; + + // Session ready + if (event.type === "session.created") { + this.sessionReady = true; + return; + } + + // User speech started - interrupt agent and reset tracking + if (event.type === "input_audio_buffer.speech_started") { + const now = Date.now(); + if (now - this.lastInterruptAt < INTERRUPT_DEBOUNCE_MS) return; + this.lastInterruptAt = now; + this.pendingTranscript = ""; + this.pendingResponseHasToolCall = false; + this.pendingResponseHasAudioOutput = false; + void this.callbacks.onInterruptAgent(); + return; + } + + // User speech transcript completed - store for fallback + if (event.type === "conversation.item.input_audio_transcription.completed") { + const transcript = (event as { transcript?: string }).transcript; + if (transcript && typeof transcript === "string") { + this.pendingTranscript = normalizeText(transcript); + logger.debug("voice-supervisor: transcript captured", { transcript: this.pendingTranscript }); + } + return; + } + + // Response started - begin tracking + if (event.type === "response.created") { + this.pendingResponseHasToolCall = false; + this.pendingResponseHasAudioOutput = false; + return; + } + + // Tool call detected - mark so we know not to use fallback + // Check multiple event types for robustness against API changes + if ( + event.type === "function_call" || + event.type === "response.function_call_arguments.done" || + event.type === "response.function_call_arguments.delta" || + event.type === "response.output_item.added" + ) { + // For output_item.added, only mark if it's a function_call type + if (event.type === "response.output_item.added") { + const item = (event as { item?: { type?: string } }).item; + if (item?.type === "function_call") { + this.pendingResponseHasToolCall = true; + } + } else { + this.pendingResponseHasToolCall = true; + } + return; + } + + // Audio output detected - mark so we don't fallback + if ( + event.type === "response.output_audio.delta" || + event.type === "response.output_audio.done" || + event.type === "response.output_audio_transcript.delta" || + event.type === "response.output_audio_transcript.done" || + event.type === "response.content_part.added" || + event.type === "response.content_part.done" + ) { + this.pendingResponseHasAudioOutput = true; + return; + } + + // Response completed - check if we need fallback + if (event.type === "response.done") { + // Only use fallback if we have a transcript AND there was no tool call AND no audio output + // This prevents duplicate responses when the realtime assistant already spoke + if (this.pendingTranscript && !this.pendingResponseHasToolCall && !this.pendingResponseHasAudioOutput) { + logger.debug("voice-supervisor: using fallback transcript path", { + transcript: this.pendingTranscript, + }); + const transcript = this.pendingTranscript; + this.pendingTranscript = ""; + // Queue the fallback asynchronously to avoid blocking + setImmediate(() => { + if (this.active) { + void this.callbacks.onSendToAgent(transcript); + } + }); + } + return; + } + } + + private async handleAudio(event: TransportLayerAudio): Promise { + if (!this.active) return; + const now = Date.now(); + try { + await this.ensurePlayback(); + } catch (error) { + this.callbacks.onError(new Error(describeError(error))); + return; + } + if (!this.playbackWriter) return; + try { + await this.playbackWriter.write(new Uint8Array(event.data)); + this.lastPlaybackAt = now; + this.lastPlaybackRms = rms16le(new Uint8Array(event.data)); + } catch (error) { + logger.debug("voice-supervisor: playback write failed", { + error: describeError(error), + }); + void this.resetPlayback(); + } + } + + private async startCapture(): Promise { + const captureResult = buildCaptureCommand(DEFAULT_SAMPLE_RATE, DEFAULT_CHANNELS); + if (!captureResult) { + throw new Error(`No audio capture tool found.\n\n${getMissingToolsMessage()}`); + } + + const { command, env: captureEnv } = captureResult; + logger.debug("voice-supervisor: starting mic capture", { command, env: captureEnv }); + const proc = Bun.spawn(command, { + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + env: captureEnv ? { ...process.env, ...captureEnv } : undefined, + }); + this.captureProcess = proc; + + const reader = (proc.stdout as ReadableStream).getReader(); + this.captureReader = reader; + + (async () => { + while (this.active) { + const { value, done } = await reader.read(); + if (done || !this.active) break; + if (!value || !this.session) continue; + if (!this.connected || !this.sessionReady || this.session.transport.status !== "connected") { + continue; + } + + const micRms = rms16le(value); + const now = Date.now(); + const playbackActive = now - this.lastPlaybackAt < PLAYBACK_ACTIVE_WINDOW_MS; + + // Echo suppression: only skip if playback is active AND mic is very quiet relative to playback + // This prevents feedback loops while allowing user to speak over the assistant + if (playbackActive && micRms < MIC_NOISE_FLOOR && micRms < this.lastPlaybackRms / ECHO_SUPPRESSION_RATIO) { + continue; + } + + // Send all audio to realtime API - let semantic_vad handle turn detection + const buffer = toArrayBuffer(value); + if (buffer.byteLength === 0) continue; + try { + this.session.sendAudio(buffer); + } catch (error) { + const message = describeError(error); + logger.debug("voice-supervisor: sendAudio failed", { error: message }); + if (message.includes("WebSocket is not connected")) { + continue; + } + this.callbacks.onError(error instanceof Error ? error : new Error(message)); + return; + } + } + if (this.active) { + this.callbacks.onError(new Error("Voice capture stopped unexpectedly.")); + } + })().catch((error) => { + if (!this.active) return; + logger.debug("voice-supervisor: capture loop error", { + error: describeError(error), + }); + this.callbacks.onError(new Error(describeError(error))); + }); + } + + private async stopCapture(): Promise { + if (this.captureReader) { + try { + await this.captureReader.cancel(); + } catch { + // ignore + } + this.captureReader = undefined; + } + if (this.captureProcess) { + try { + this.captureProcess.kill(); + } catch { + // ignore + } + await this.captureProcess.exited; + this.captureProcess = undefined; + } + } + + private async ensurePlayback(): Promise { + if (this.playbackProcess && this.playbackWriter) return; + const command = buildPlaybackCommand(DEFAULT_SAMPLE_RATE, DEFAULT_CHANNELS); + if (!command) { + throw new Error(`No audio playback tool found.\n\n${getMissingToolsMessage()}`); + } + + logger.debug("voice-supervisor: starting audio playback", { command }); + const proc = Bun.spawn(command, { + stdin: "pipe", + stdout: "ignore", + stderr: "pipe", + }); + const startedAt = Date.now(); + const stderrBuffer = { text: "" }; + this.readStderr(proc.stderr, stderrBuffer); + + this.playbackProcess = proc; + const stdin = proc.stdin; + if (!stdin) { + throw new Error("Audio playback stdin unavailable."); + } + if ("getWriter" in stdin && typeof stdin.getWriter === "function") { + const writer = (stdin as unknown as WritableStream).getWriter(); + this.playbackWriter = { + write: async (chunk) => { + await writer.write(chunk); + }, + close: async () => { + await writer.close(); + }, + }; + } else if ("write" in stdin && typeof (stdin as { write?: unknown }).write === "function") { + const sink = stdin as unknown as { + write: (chunk: Uint8Array) => void | number | Promise; + end?: () => void | number | Promise; + close?: () => void | number | Promise; + }; + this.playbackWriter = { + write: async (chunk) => { + await sink.write(chunk); + }, + close: async () => { + if (sink.end) { + await sink.end(); + } else if (sink.close) { + await sink.close(); + } + }, + }; + } else { + throw new Error("Audio playback stdin is not writable."); + } + + proc.exited + .then((code) => { + if (this.playbackProcess === proc) { + this.playbackProcess = undefined; + this.playbackWriter = undefined; + } + const trimmed = stderrBuffer.text.trim(); + if (trimmed) { + logger.debug("voice-supervisor: playback stderr", { stderr: trimmed }); + } + const elapsed = Date.now() - startedAt; + if (code !== 0 && elapsed < 2000 && this.active) { + this.maybeWarnPlaybackFailure(trimmed || `exit code ${code}`); + } + }) + .catch(() => { + // ignore + }); + } + + private async resetPlayback(): Promise { + if (this.playbackWriter) { + try { + await this.playbackWriter.close(); + } catch { + // ignore + } + } + if (this.playbackProcess) { + try { + this.playbackProcess.kill(); + } catch { + // ignore + } + await this.playbackProcess.exited; + } + this.playbackProcess = undefined; + this.playbackWriter = undefined; + } + + private readStderr(stderr: Subprocess["stderr"], buffer: { text: string }): void { + if (!stderr || typeof stderr === "number") return; + const reader = (stderr as ReadableStream).getReader(); + const decoder = new TextDecoder(); + (async () => { + while (true) { + const { value, done } = await reader.read(); + if (done || !value) break; + buffer.text += decoder.decode(value, { stream: true }); + if (buffer.text.length > 4000) { + buffer.text = buffer.text.slice(0, 4000); + break; + } + } + })().catch(() => { + // ignore + }); + } + + private maybeWarnPlaybackFailure(message: string): void { + if (!this.callbacks.onWarning) return; + const now = Date.now(); + if (now - this.lastPlaybackErrorAt < PLAYBACK_ERROR_COOLDOWN_MS) return; + this.lastPlaybackErrorAt = now; + this.callbacks.onWarning(`Audio playback failed: ${message}`); + } + + private async waitForConnection(session: RealtimeSession, timeoutMs: number): Promise { + if (session.transport.status === "connected") { + this.connected = true; + return; + } + await new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + cleanup(); + reject(new Error("Realtime voice connection timed out.")); + }, timeoutMs); + + const cleanup = () => { + clearTimeout(timeout); + session.transport.off("connection_change", onChange); + }; + + const onChange = (status: string) => { + if (status === "connected") { + this.connected = true; + cleanup(); + resolve(); + } + }; + + session.transport.on("connection_change", onChange); + }); + } + + private async waitForSessionReady(session: RealtimeSession, timeoutMs: number): Promise { + if (this.sessionReady) return; + await new Promise((resolve, reject) => { + let resolved = false; + + const cleanup = () => { + clearTimeout(timeout); + session.off("transport_event", onEvent); + }; + + const timeout = setTimeout(() => { + if (resolved) return; + resolved = true; + cleanup(); + reject(new Error("Realtime voice session not ready.")); + }, timeoutMs); + + const onEvent = (event: TransportEvent) => { + if (resolved) return; + if (event.type === "session.created") { + this.sessionReady = true; + resolved = true; + cleanup(); + resolve(); + } + }; + + session.on("transport_event", onEvent); + }); + } +} diff --git a/packages/coding-agent/src/core/voice.ts b/packages/coding-agent/src/core/voice.ts index 4b6c303c8..c0428c0cf 100644 --- a/packages/coding-agent/src/core/voice.ts +++ b/packages/coding-agent/src/core/voice.ts @@ -96,7 +96,7 @@ function buildRecordingCommand(filePath: string, sampleRate: number, channels: n return null; } -export async function startVoiceRecording(settings: VoiceSettings): Promise { +export async function startVoiceRecording(_settings: VoiceSettings): Promise { const sampleRate = DEFAULT_SAMPLE_RATE; const channels = DEFAULT_CHANNELS; const filePath = join(tmpdir(), `omp-voice-${Date.now()}.wav`); @@ -180,7 +180,7 @@ export async function synthesizeSpeech( "Content-Type": "application/json", }, body: JSON.stringify({ - model: settings.ttsModel ?? "gpt-4o-mini-tts", + model: settings.ttsModel ?? "tts-1", voice: settings.ttsVoice ?? "alloy", format, input: text, diff --git a/packages/coding-agent/src/modes/interactive/components/custom-editor.ts b/packages/coding-agent/src/modes/interactive/components/custom-editor.ts index e4404b7a9..39f6eb3b5 100644 --- a/packages/coding-agent/src/modes/interactive/components/custom-editor.ts +++ b/packages/coding-agent/src/modes/interactive/components/custom-editor.ts @@ -9,6 +9,7 @@ import { isCtrlP, isCtrlT, isCtrlV, + isCtrlY, isCtrlZ, isEscape, isShiftCtrlP, @@ -32,6 +33,7 @@ export class CustomEditor extends Editor { public onCtrlZ?: () => void; public onQuestionMark?: () => void; public onCapsLock?: () => void; + public onCtrlY?: () => void; /** Called when Ctrl+V is pressed. Returns true if handled (image found), false to fall through to text paste. */ public onCtrlV?: () => Promise; @@ -53,6 +55,12 @@ export class CustomEditor extends Editor { return; } + // Intercept Ctrl+Y for voice input + if (isCtrlY(data) && this.onCtrlY) { + this.onCtrlY(); + return; + } + // Intercept Ctrl+Z for suspend if (isCtrlZ(data) && this.onCtrlZ) { this.onCtrlZ(); diff --git a/packages/coding-agent/src/modes/interactive/components/settings-defs.ts b/packages/coding-agent/src/modes/interactive/components/settings-defs.ts index 631971db1..ef0caf28c 100644 --- a/packages/coding-agent/src/modes/interactive/components/settings-defs.ts +++ b/packages/coding-agent/src/modes/interactive/components/settings-defs.ts @@ -100,7 +100,7 @@ export const SETTINGS_DEFS: SettingDef[] = [ tab: "config", type: "boolean", label: "Voice mode", - description: "Enable Caps Lock voice input and spoken summaries", + description: "Enable realtime voice input/output (Ctrl+Y toggle, auto-send on silence)", get: (sm) => sm.getVoiceEnabled(), set: (sm, v) => sm.setVoiceEnabled(v), }, diff --git a/packages/coding-agent/src/modes/interactive/interactive-mode.ts b/packages/coding-agent/src/modes/interactive/interactive-mode.ts index c0e27592c..480eab971 100644 --- a/packages/coding-agent/src/modes/interactive/interactive-mode.ts +++ b/packages/coding-agent/src/modes/interactive/interactive-mode.ts @@ -31,14 +31,7 @@ import { createCompactionSummaryMessage } from "../../core/messages"; import { getRecentSessions, type SessionContext, SessionManager } from "../../core/session-manager"; import { generateSessionTitle, setTerminalTitle } from "../../core/title-generator"; import type { TruncationResult } from "../../core/tools/truncate"; -import { - playAudio, - startVoiceRecording, - summarizeForVoice, - synthesizeSpeech, - transcribeAudio, - type VoiceRecordingHandle, -} from "../../core/voice"; +import { VoiceSupervisor } from "../../core/voice-supervisor"; import { disableProvider, enableProvider } from "../../discovery"; import { getChangelogPath, parseChangelog } from "../../utils/changelog"; import { copyToClipboard, readImageFromClipboard } from "../../utils/clipboard"; @@ -88,6 +81,10 @@ function isExpandable(obj: unknown): obj is Expandable { return typeof obj === "object" && obj !== null && "setExpanded" in obj && typeof obj.setExpanded === "function"; } +const VOICE_PROGRESS_DELAY_MS = 15000; +const VOICE_PROGRESS_MIN_CHARS = 160; +const VOICE_PROGRESS_DELTA_CHARS = 120; + export class InteractiveMode { private session: AgentSession; private ui: TUI; @@ -142,8 +139,12 @@ export class InteractiveMode { private pendingImages: ImageContent[] = []; // Voice mode state - private voiceRecording: VoiceRecordingHandle | undefined = undefined; - private voiceOutputQueue: Promise = Promise.resolve(); + private voiceSupervisor: VoiceSupervisor; + private voiceAutoModeEnabled = false; + private voiceProgressTimer: ReturnType | undefined = undefined; + private voiceProgressSpoken = false; + private voiceProgressLastLength = 0; + private lastVoiceInterruptAt = 0; // Auto-compaction state private autoCompactionLoader: Loader | undefined = undefined; @@ -197,6 +198,26 @@ export class InteractiveMode { this.editorContainer.addChild(this.editor); this.statusLine = new StatusLineComponent(session); this.statusLine.setAutoCompactEnabled(session.autoCompactionEnabled); + this.voiceSupervisor = new VoiceSupervisor(this.session.modelRegistry, { + onSendToAgent: async (text) => { + await this.submitVoiceText(text); + }, + onInterruptAgent: async (reason) => { + await this.handleVoiceInterrupt(reason); + }, + onStatus: (status) => { + this.setVoiceStatus(status); + }, + onError: (error) => { + this.showError(error.message); + this.voiceAutoModeEnabled = false; + void this.voiceSupervisor.stop(); + this.setVoiceStatus(undefined); + }, + onWarning: (message) => { + this.showWarning(message); + }, + }); // Define slash commands for autocomplete const slashCommands: SlashCommand[] = [ @@ -749,11 +770,11 @@ export class InteractiveMode { this.editor.onCtrlO = () => this.toggleToolOutputExpansion(); this.editor.onCtrlT = () => this.toggleThinkingBlockVisibility(); this.editor.onCtrlG = () => this.openExternalEditor(); - this.editor.onQuestionMark = () => this.handleHotkeysCommand(); - this.editor.onCtrlV = () => this.handleImagePaste(); - this.editor.onCapsLock = () => { + this.editor.onCtrlY = () => { void this.toggleVoiceListening(); }; + this.editor.onQuestionMark = () => this.handleHotkeysCommand(); + this.editor.onCtrlV = () => this.handleImagePaste(); this.editor.onChange = (text: string) => { const wasBashMode = this.isBashMode; @@ -988,6 +1009,7 @@ export class InteractiveMode { getSymbolTheme().spinnerFrames, ); this.statusContainer.addChild(this.loadingAnimation); + this.startVoiceProgressTimer(); this.ui.requestRender(); break; @@ -1123,6 +1145,7 @@ export class InteractiveMode { } case "agent_end": + this.stopVoiceProgressTimer(); if (this.loadingAnimation) { this.loadingAnimation.stop(); this.loadingAnimation = undefined; @@ -1134,12 +1157,12 @@ export class InteractiveMode { this.streamingMessage = undefined; } this.pendingTools.clear(); - if (this.settingsManager.getVoiceEnabled()) { + if (this.settingsManager.getVoiceEnabled() && this.voiceAutoModeEnabled) { const lastAssistant = this.findLastAssistantMessage(); if (lastAssistant && lastAssistant.stopReason !== "aborted" && lastAssistant.stopReason !== "error") { const text = this.extractAssistantText(lastAssistant); if (text) { - this.enqueueVoiceOutput(text); + this.voiceSupervisor.notifyResult(text); } } } @@ -1479,10 +1502,8 @@ export class InteractiveMode { * Emits shutdown event to hooks and tools, then exits. */ private async shutdown(): Promise { - if (this.voiceRecording) { - await this.voiceRecording.cancel(); - this.voiceRecording = undefined; - } + this.voiceAutoModeEnabled = false; + await this.voiceSupervisor.stop(); // Flush pending session writes before shutdown await this.sessionManager.flush(); @@ -1550,84 +1571,110 @@ export class InteractiveMode { this.ui.requestRender(); } + private async handleVoiceInterrupt(reason?: string): Promise { + const now = Date.now(); + if (now - this.lastVoiceInterruptAt < 200) return; + this.lastVoiceInterruptAt = now; + if (this.session.isBashRunning) { + this.session.abortBash(); + } + if (this.session.isStreaming) { + await this.session.abort(); + } + if (reason) { + this.showStatus(reason); + } + } + + private stopVoiceProgressTimer(): void { + if (this.voiceProgressTimer) { + clearTimeout(this.voiceProgressTimer); + this.voiceProgressTimer = undefined; + } + } + + private startVoiceProgressTimer(): void { + this.stopVoiceProgressTimer(); + if (!this.settingsManager.getVoiceEnabled() || !this.voiceAutoModeEnabled) return; + this.voiceProgressSpoken = false; + this.voiceProgressLastLength = 0; + this.voiceProgressTimer = setTimeout(() => { + void this.maybeSpeakProgress(); + }, VOICE_PROGRESS_DELAY_MS); + } + + private async maybeSpeakProgress(): Promise { + if (!this.session.isStreaming || this.voiceProgressSpoken || !this.voiceAutoModeEnabled) return; + const streaming = this.streamingMessage; + if (!streaming) return; + const text = this.extractAssistantText(streaming); + if (!text || text.length < VOICE_PROGRESS_MIN_CHARS) { + if (this.session.isStreaming) { + this.voiceProgressTimer = setTimeout(() => { + void this.maybeSpeakProgress(); + }, VOICE_PROGRESS_DELAY_MS); + } + return; + } + + const delta = text.length - this.voiceProgressLastLength; + if (delta < VOICE_PROGRESS_DELTA_CHARS) { + if (this.session.isStreaming) { + this.voiceProgressTimer = setTimeout(() => { + void this.maybeSpeakProgress(); + }, VOICE_PROGRESS_DELAY_MS); + } + return; + } + + this.voiceProgressLastLength = text.length; + this.voiceProgressSpoken = true; + this.voiceSupervisor.notifyProgress(text); + } + private async toggleVoiceListening(): Promise { if (!this.settingsManager.getVoiceEnabled()) { - this.showStatus("Voice mode is disabled. Enable it in /settings."); + this.settingsManager.setVoiceEnabled(true); + this.showStatus("Voice mode enabled."); + } + + if (this.voiceAutoModeEnabled) { + this.voiceAutoModeEnabled = false; + this.stopVoiceProgressTimer(); + await this.voiceSupervisor.stop(); + this.setVoiceStatus(undefined); + this.showStatus("Voice mode disabled."); return; } - if (this.voiceRecording) { - await this.stopVoiceListening(); - return; - } - - await this.startVoiceListening(); - } - - private async startVoiceListening(): Promise { - if (this.voiceRecording) return; + this.voiceAutoModeEnabled = true; try { - this.voiceRecording = await startVoiceRecording(this.settingsManager.getVoiceSettings()); - this.setVoiceStatus("Listening... (Caps Lock to stop)"); + await this.voiceSupervisor.start(); } catch (error) { - this.voiceRecording = undefined; + this.voiceAutoModeEnabled = false; this.setVoiceStatus(undefined); this.showError(error instanceof Error ? error.message : String(error)); } } - private async stopVoiceListening(): Promise { - const recording = this.voiceRecording; - if (!recording) return; - this.voiceRecording = undefined; - this.setVoiceStatus("Transcribing..."); - - try { - await recording.stop(); - const apiKey = await this.session.modelRegistry.getApiKeyForProvider("openai"); - if (!apiKey) { - throw new Error("OpenAI API key not found (set OPENAI_API_KEY or login)."); - } - - const { text } = await transcribeAudio(recording.filePath, apiKey, this.settingsManager.getVoiceSettings()); - if (!text) { - this.showWarning("No speech detected. Try again."); - return; - } - await this.submitVoiceText(text); - } catch (error) { - this.showError(error instanceof Error ? error.message : String(error)); - } finally { - recording.cleanup(); - this.setVoiceStatus(undefined); - } - } - - private async cancelVoiceRecording(): Promise { - const recording = this.voiceRecording; - if (!recording) return; - this.voiceRecording = undefined; - await recording.cancel(); - this.setVoiceStatus(undefined); - } - private async submitVoiceText(text: string): Promise { const cleaned = text.trim(); if (!cleaned) { this.showWarning("No speech detected. Try again."); return; } - - this.editor.addToHistory(cleaned); + const toSend = cleaned; + this.editor.addToHistory(toSend); if (this.session.isStreaming) { - await this.session.queueMessage(cleaned); + await this.session.abort(); + await this.session.queueMessage(toSend); this.updatePendingMessagesDisplay(); return; } if (this.onInputCallback) { - this.onInputCallback({ text: cleaned }); + this.onInputCallback({ text: toSend }); } } @@ -1651,38 +1698,6 @@ export class InteractiveMode { return text.trim(); } - private enqueueVoiceOutput(text: string): void { - this.voiceOutputQueue = this.voiceOutputQueue - .then(() => this.speakVoiceSummary(text)) - .catch((error) => { - this.showError(error instanceof Error ? error.message : String(error)); - }); - } - - private async speakVoiceSummary(text: string): Promise { - if (!this.settingsManager.getVoiceEnabled()) { - return; - } - const apiKey = await this.session.modelRegistry.getApiKeyForProvider("openai"); - if (!apiKey) { - this.showWarning("OpenAI API key not found for voice playback."); - return; - } - - const smolModel = this.settingsManager.getModelRole("smol"); - const summary = (await summarizeForVoice(text, this.session.modelRegistry, smolModel)) ?? text; - const cleaned = summary.trim(); - if (!cleaned) return; - - this.setVoiceStatus("Speaking..."); - try { - const synthesis = await synthesizeSpeech(cleaned, apiKey, this.settingsManager.getVoiceSettings()); - await playAudio(synthesis.audio, synthesis.format); - } finally { - this.setVoiceStatus(undefined); - } - } - private updateEditorBorderColor(): void { if (this.isBashMode) { this.editor.borderColor = theme.getBashModeBorderColor(); @@ -2020,9 +2035,9 @@ export class InteractiveMode { } case "voiceEnabled": { if (!value) { - if (this.voiceRecording) { - void this.cancelVoiceRecording(); - } + this.voiceAutoModeEnabled = false; + this.stopVoiceProgressTimer(); + void this.voiceSupervisor.stop(); this.setVoiceStatus(undefined); } break; @@ -2636,7 +2651,7 @@ export class InteractiveMode { | \`Ctrl+P\` | Cycle models | | \`Ctrl+O\` | Toggle tool output expansion | | \`Ctrl+T\` | Toggle thinking block visibility | -| \`Caps Lock\` | Voice input (start/stop) | +| \`Ctrl+Y\` | Voice mode toggle (auto-send on silence) | | \`Ctrl+G\` | Edit message in external editor | | \`/\` | Slash commands | | \`!\` | Run bash command | diff --git a/packages/coding-agent/src/prompts/voice-summary.md b/packages/coding-agent/src/prompts/voice-summary.md index b997cfc2e..b3cf298d2 100644 --- a/packages/coding-agent/src/prompts/voice-summary.md +++ b/packages/coding-agent/src/prompts/voice-summary.md @@ -7,5 +7,6 @@ Rules: - No markdown, no code blocks, no inline code, no URLs. - If the response already fits, return it unchanged. - Preserve the most important question if one is asked. +- If a decision or missing info is required, ask one short question. Return only the spoken text.