fix(coding-agent): keep smoke-test fallback for uncovered UI surfaces

When a session has browser but not computer (or vice versa), the
behavioral/smoke-test fallback for UI changes was suppressed entirely,
leaving native-desktop (or web) verification guidance undefined. Render
the fallback whenever any visual runtime tool is missing.

Also strengthen the plan-mode tests per review: exercise the iterative
branch gating with iterative:true (previously passed trivially) and
restore a typed Overrides record for the render helper.
This commit is contained in:
Slava Zavadsky
2026-08-10 20:37:28 -04:00
parent b6a3862ebc
commit 0bbdfe2eaa
3 changed files with 21 additions and 11 deletions
@@ -214,9 +214,8 @@ EXECUTION WORKFLOW
- **Native desktop UI** → drive it with `{{toolRefs.computer}}`; ground every claim in fresh screenshot or accessibility evidence.
{{/has}}
- **TUI/CLI** → launch the actual program and verify terminal interaction, output, or state.
{{#ifAny (includes tools "browser") (includes tools "computer")}}
{{else}}
- If no suitable runtime tool is available, verify with a behavioral test or smoke test and explicitly report when visual verification cannot be performed.
{{#ifAny (not (includes tools "browser")) (not (includes tools "computer"))}}
- If no suitable runtime tool is available for the surface being changed, verify with a behavioral test or smoke test and explicitly report when visual verification cannot be performed.
{{/ifAny}}
- **Bug fix** → reproduce the bug, apply the fix, confirm the reproduction no longer triggers.
- **Permanent feature / API change** → existing tests that cover the changed contract. Add a test only when the change introduces a new observable contract not already covered, or the user asked for one.
@@ -12,34 +12,42 @@ const BASE = {
askAvailable: true,
taskAvailable: true,
scoutAvailable: true,
reentry: false,
planExists: true,
} as const;
function render(overrides: Partial<Record<string, unknown>>): string {
type Overrides = Partial<Record<keyof typeof BASE, boolean | string>>;
function render(overrides: Overrides = {}): string {
return prompt.render(planModeActivePrompt, { ...BASE, ...overrides });
}
describe("plan-mode re-entry prompt", () => {
it("only emits the Re-entry section when re-entering", () => {
expect(render({ reentry: false, planExists: true })).not.toContain("## Re-entry");
expect(render({ reentry: true, planExists: true })).toContain("## Re-entry");
expect(render({ reentry: false })).not.toContain("## Re-entry");
expect(render({ reentry: true })).toContain("## Re-entry");
});
});
describe("plan-mode-active tool availability", () => {
it("omits ask-tool directives when ask is unavailable", () => {
const withoutAsk = render({ askAvailable: false });
const withoutAsk = render({ askAvailable: false, iterative: true });
expect(withoutAsk).not.toContain("`ask` with 2–4 mutually exclusive options");
expect(withoutAsk).not.toContain("use `ask` for preferences and tradeoffs");
expect(withoutAsk).not.toContain("Using `ask` to gather requirements");
const withAsk = render({ askAvailable: true });
const withAsk = render({ askAvailable: true, iterative: true });
expect(withAsk).toContain("`ask` with 2–4 mutually exclusive options");
expect(withAsk).toContain("use `ask` for preferences and tradeoffs");
});
it("provides a prose fallback for preference collection when ask is unavailable", () => {
const withoutAsk = render({ askAvailable: false });
expect(withoutAsk).toContain("present the candidates with a recommendation in prose");
expect(withoutAsk).toContain("surface any remaining preference questions with a recommendation in prose");
const iterativeWithoutAsk = render({ askAvailable: false, iterative: true });
expect(iterativeWithoutAsk).toContain("present the candidates with a recommendation in prose");
expect(iterativeWithoutAsk).not.toContain("`ask` for preferences and tradeoffs only");
const parallelWithoutAsk = render({ askAvailable: false, iterative: false });
expect(parallelWithoutAsk).toContain("surface any remaining preference questions with a recommendation in prose");
});
it("omits scout-via-task dispatch when the task tool is unavailable", () => {
@@ -724,6 +724,9 @@ describe("system prompt tool inventory", () => {
).systemPrompt.join("\n\n");
expect(withBrowser).toContain("drive it in `browser`");
// A browser-only session still needs the smoke-test fallback for
// native-desktop surfaces (no computer tool).
expect(withBrowser).toContain("behavioral test or smoke test");
});
it("omits todo workflow guidance when the todo tool is absent", async () => {