From 3d5572a0fc1fa9dbb1625e1d53695db076aba571 Mon Sep 17 00:00:00 2001 From: can1357 Date: Mon, 8 Jun 2026 14:59:20 +0200 Subject: [PATCH] test(natives): documented AST conditions and expanded astMatch tests - Updated `docs/ttsr-injection-lifecycle.md` to document `astCondition` behavior, including registration rules and tool-stream matching. - Extended `packages/natives/test/native.test.ts` with new `astMatch` coverage for Smart matching, metavariable consistency, parse errors, and empty-language rejection. --- docs/rulebook-matching-pipeline.md | 14 ++++--- docs/ttsr-injection-lifecycle.md | 13 ++++-- packages/agent/src/agent-loop.ts | 7 +++- packages/ai/src/providers/anthropic.ts | 38 ++++++++++++----- .../coding-agent/src/utils/image-loading.ts | 2 +- packages/natives/test/native.test.ts | 42 +++++++++++++++++++ 6 files changed, 93 insertions(+), 23 deletions(-) diff --git a/docs/rulebook-matching-pipeline.md b/docs/rulebook-matching-pipeline.md index edf7889dd..089652fe7 100644 --- a/docs/rulebook-matching-pipeline.md +++ b/docs/rulebook-matching-pipeline.md @@ -38,6 +38,7 @@ interface Rule { alwaysApply?: boolean; description?: string; condition?: string[]; + astCondition?: string[]; scope?: string[]; interruptMode?: "never" | "prose-only" | "tool-only" | "always"; _source: SourceMeta; @@ -183,16 +184,16 @@ After rule discovery in `createAgentSession` (`sdk.ts`), `bucketRules(...)` appl 1. Drop rules listed in `ttsr.disabledRules`. 2. Drop rules from the `builtin-defaults` provider when `ttsr.builtinRules === false`. -3. Register rules with non-empty `condition` into `TtsrManager`; if registration succeeds, the rule is TTSR-only. +3. Register rules with a non-empty `condition` or `astCondition` into `TtsrManager`; if registration succeeds, the rule is TTSR-only. 4. Put remaining `alwaysApply === true` rules into `alwaysApplyRules`. 5. Put remaining rules with `description` into `rulebookRules`. ### Bucket behavior -- **TTSR bucket**: any enabled rule with a non-empty parsed `condition` that `TtsrManager.addRule(...)` accepts. Takes priority over other buckets. +- **TTSR bucket**: any enabled rule with a non-empty parsed `condition` (regex) or `astCondition` (ast-grep patterns) that `TtsrManager.addRule(...)` accepts. Takes priority over other buckets. - **Always-apply bucket**: `alwaysApply === true`, not TTSR. Full content injected into system prompt. Resolvable via `rule://`. - **Rulebook bucket**: must have description, must not be TTSR, must not be `alwaysApply`. Listed in system prompt by name+description; content read on demand via `rule://`. -- A rule with both `condition` and `alwaysApply` goes to TTSR only if TTSR registration accepts it; otherwise it can fall through to always-apply. +- A rule with both a trigger condition and `alwaysApply` goes to TTSR only if TTSR registration accepts it; otherwise it can fall through to always-apply. - A rule with both `alwaysApply` and `description` goes to always-apply only (not rulebook). ## 6. How metadata affects runtime surfaces @@ -219,10 +220,11 @@ After rule discovery in `createAgentSession` (`sdk.ts`), `bucketRules(...)` appl - **Full rule content is auto-injected into the system prompt** (before the rulebook rules section). - Rule is also addressable via `rule://` for re-reading. -### `condition`, `scope`, and `interruptMode` +### `condition`, `astCondition`, `scope`, and `interruptMode` -- `condition` is the current TTSR trigger field; legacy `ttsr_trigger` / `ttsrTrigger` are accepted as fallback inputs during parsing. -- `scope` narrows TTSR matching scope. A condition token that looks like a file glob becomes `tool:edit()` and `tool:write()` scope entries plus catch-all condition `.*`. +- `condition` is the regex TTSR trigger field; legacy `ttsr_trigger` / `ttsrTrigger` are accepted as fallback inputs during parsing. +- `astCondition` is the ast-grep trigger field: a string or list of structural patterns, kept verbatim (no glob inference). It only matches on edit/write tool streams, where the language is inferred from the file path. A rule may set `condition`, `astCondition`, or both. +- `scope` narrows TTSR matching scope. A `condition` token that looks like a file glob becomes `tool:edit()` and `tool:write()` scope entries plus catch-all condition `.*`; `astCondition` tokens never trigger this shorthand. - `interruptMode` can override the global TTSR interrupt mode for the rule. ## 7. System prompt inclusion path diff --git a/docs/ttsr-injection-lifecycle.md b/docs/ttsr-injection-lifecycle.md index 375c2c374..6f6a63682 100644 --- a/docs/ttsr-injection-lifecycle.md +++ b/docs/ttsr-injection-lifecycle.md @@ -43,12 +43,18 @@ const { rulebookRules, alwaysApplyRules } = bucketRules( Registration is skipped when: -- `rule.condition` is absent or all condition regexes fail to compile +- both `rule.condition` (regex) and `rule.astCondition` (ast-grep patterns) are absent, or every regex condition fails to compile and there are no AST conditions - a rule with the same `rule.name` was already registered in this manager - the rule scope excludes all monitored streams Invalid regex conditions and unreachable scopes are logged as warnings and ignored; session startup continues. If a TTSR rule defines `globs`, those globs are compiled as a global file-path gate for matching. +### AST conditions (`astCondition`) + +A rule may carry `astCondition`: a list of [ast-grep](https://ast-grep.github.io/) patterns (OR'd, same as regex `condition`), matched structurally instead of textually. A repeated metavariable inside one pattern requires both occurrences to be equal (`if ($X) clearTimeout($X)` matches but `if ($X) clearTimeout($Y)` does not). + +AST conditions only evaluate on **edit/write tool-argument streams** — they need a language, which is inferred from the file extension on the tool's path argument, and they match against the tool's reconstructed source snapshot (`matcherDigest`), not the raw wire delta. Matching is performed in memory by the native `astMatch` engine (no temp files) with Smart strictness. Streams without a usable file path (prose, thinking, path-less tool calls) skip AST conditions entirely. A rule may mix `condition` and `astCondition`; the regex paths keep working on every scope while AST paths apply only to those tool streams. + ### Setting caveat `TtsrSettings.enabled` is loaded into the manager but is not currently checked in runtime gating. If TTSR rules exist, matching still runs. @@ -69,9 +75,10 @@ When assistant updates arrive and rules exist: - monitor `text_delta`, `thinking_delta`, and `toolcall_delta` - append delta into a source/tool scoped manager buffer -- call `checkDelta(delta, matchContext)` +- call `checkDelta(delta, matchContext)` (synchronous regex matching) +- for edit/write tool streams, when `hasAstRules()` is true, `await checkAstSnapshot(snapshot, matchContext)` (asynchronous AST matching) -`checkDelta()` iterates registered rules and returns all matching rules that pass scope, global path-glob, condition, and repeat policy checks. +`checkDelta()` iterates registered rules and returns all matching rules that pass scope, global path-glob, regex condition, and repeat policy checks. `checkAstSnapshot()` applies the same scope/path/repeat gates, then runs each candidate rule's `astCondition` patterns against the snapshot via the native `astMatch` engine. It is throttled per stream key: an identical consecutive snapshot (common when only non-source arguments change between deltas) is skipped without re-running the matcher. Both paths feed their matches through the same trigger-decision handler. ## 3. Trigger decision and immediate abort path diff --git a/packages/agent/src/agent-loop.ts b/packages/agent/src/agent-loop.ts index c463525a8..921b2a982 100644 --- a/packages/agent/src/agent-loop.ts +++ b/packages/agent/src/agent-loop.ts @@ -20,10 +20,10 @@ import { createHarmonyAuditEvent, detectHarmonyLeakInAssistantMessage, extractHarmonyRemoved, - recoverHarmonyToolCall, type HarmonyDetection, type HarmonyRecoveredToolCall, isHarmonyLeakMitigationTarget, + recoverHarmonyToolCall, signalListLabel, } from "./harmony-leak"; import { type AgentRunCoverage, type AgentRunSummary, ToolCallBlockedError } from "./run-collector"; @@ -1085,7 +1085,10 @@ async function streamAssistantResponse( } } -function retainCompletedToolCalls(message: AssistantMessage, completedToolCallIds: ReadonlySet): AssistantMessage { +function retainCompletedToolCalls( + message: AssistantMessage, + completedToolCallIds: ReadonlySet, +): AssistantMessage { if (message.stopReason !== "error" && message.stopReason !== "aborted") return message; let changed = false; const content = message.content.filter(block => { diff --git a/packages/ai/src/providers/anthropic.ts b/packages/ai/src/providers/anthropic.ts index ec7c35c99..758e0c289 100644 --- a/packages/ai/src/providers/anthropic.ts +++ b/packages/ai/src/providers/anthropic.ts @@ -277,7 +277,6 @@ function cloneAnthropicCacheControl(cacheControl: AnthropicCacheControl): Anthro return { ...cacheControl }; } - type AnthropicOutputConfig = NonNullable; const ANTHROPIC_STOP_SEQUENCES_MAX = 4; @@ -1453,7 +1452,11 @@ export const streamAnthropic: StreamFunction<"anthropic-messages"> = ( const baseUrl = resolveAnthropicBaseUrl(model, options?.apiKey ?? getEnvApiKey(model.provider) ?? "") ?? "https://api.anthropic.com"; - const providerSessionState = getAnthropicProviderSessionState(options?.providerSessionState, baseUrl, model.id); + const providerSessionState = getAnthropicProviderSessionState( + options?.providerSessionState, + baseUrl, + model.id, + ); let disableStrictTools = (providerSessionState?.strictToolsDisabled ?? false) || (model.compat?.disableStrictTools ?? false); let strictFallbackErrorMessage: string | undefined; @@ -1666,7 +1669,9 @@ export const streamAnthropic: StreamFunction<"anthropic-messages"> = ( } const openBlock = openBlocks.get(event.index); if (!openBlock) { - throw createAnthropicStreamEnvelopeError(`received content_block_delta for unopened index ${event.index}`); + throw createAnthropicStreamEnvelopeError( + `received content_block_delta for unopened index ${event.index}`, + ); } const block = blocks[openBlock.contentIndex]; if (event.delta.type === "text_delta") { @@ -1683,7 +1688,9 @@ export const streamAnthropic: StreamFunction<"anthropic-messages"> = ( }); } else if (event.delta.type === "thinking_delta") { if (openBlock.kind !== "thinking" || block?.type !== "thinking") { - throw createAnthropicStreamEnvelopeError(`received thinking_delta for ${openBlock.kind} block`); + throw createAnthropicStreamEnvelopeError( + `received thinking_delta for ${openBlock.kind} block`, + ); } streamedReplayUnsafeContent = true; block.thinking += event.delta.thinking; @@ -1695,7 +1702,9 @@ export const streamAnthropic: StreamFunction<"anthropic-messages"> = ( }); } else if (event.delta.type === "input_json_delta") { if (openBlock.kind !== "toolCall" || block?.type !== "toolCall") { - throw createAnthropicStreamEnvelopeError(`received input_json_delta for ${openBlock.kind} block`); + throw createAnthropicStreamEnvelopeError( + `received input_json_delta for ${openBlock.kind} block`, + ); } streamedReplayUnsafeContent = true; block.partialJson += event.delta.partial_json; @@ -1712,7 +1721,9 @@ export const streamAnthropic: StreamFunction<"anthropic-messages"> = ( }); } else if (event.delta.type === "signature_delta") { if (openBlock.kind !== "thinking" || block?.type !== "thinking") { - throw createAnthropicStreamEnvelopeError(`received signature_delta for ${openBlock.kind} block`); + throw createAnthropicStreamEnvelopeError( + `received signature_delta for ${openBlock.kind} block`, + ); } streamedReplayUnsafeContent = true; block.thinkingSignature = block.thinkingSignature || ""; @@ -1724,11 +1735,15 @@ export const streamAnthropic: StreamFunction<"anthropic-messages"> = ( } const openBlock = openBlocks.get(event.index); if (!openBlock) { - throw createAnthropicStreamEnvelopeError(`received content_block_stop for unopened index ${event.index}`); + throw createAnthropicStreamEnvelopeError( + `received content_block_stop for unopened index ${event.index}`, + ); } const block = blocks[openBlock.contentIndex]; if (!block || block.type !== openBlock.kind) { - throw createAnthropicStreamEnvelopeError(`content_block_stop kind mismatch for index ${event.index}`); + throw createAnthropicStreamEnvelopeError( + `content_block_stop kind mismatch for index ${event.index}`, + ); } openBlocks.delete(event.index); delete (block as { index?: number }).index; @@ -2189,7 +2204,10 @@ function ensureMaxTokensForThinking(params: MessageCreateParamsStreaming, model: const maxAllowedTokens = Math.min(CLAUDE_CODE_MAX_OUTPUT_TOKENS, model.maxTokens); const currentMaxTokens = Math.min(params.max_tokens ?? maxAllowedTokens, maxAllowedTokens); - const raisedMaxTokens = Math.min(Math.max(currentMaxTokens, budgetTokens + OUTPUT_FALLBACK_BUFFER), maxAllowedTokens); + const raisedMaxTokens = Math.min( + Math.max(currentMaxTokens, budgetTokens + OUTPUT_FALLBACK_BUFFER), + maxAllowedTokens, + ); params.max_tokens = raisedMaxTokens; if (budgetTokens + OUTPUT_FALLBACK_BUFFER <= raisedMaxTokens) return; @@ -2900,7 +2918,6 @@ function pickAnthropicEffectiveScalarType(schema: Record): stri return undefined; } - function anthropicPerTypeKeep(scalarType: string | undefined): Set | undefined { switch (scalarType) { case "object": @@ -3071,7 +3088,6 @@ function hasAnthropicSchemaDefiningKeyword(schema: Record): boo return schema.$defs !== undefined || schema.definitions !== undefined; } - function makeAnthropicNullableSchema(schema: unknown, budget: AnthropicStrictBudget): unknown | undefined { if (isRecord(schema)) { if (hasNullVariant(schema)) return schema; diff --git a/packages/coding-agent/src/utils/image-loading.ts b/packages/coding-agent/src/utils/image-loading.ts index 4be0a534e..c643b626f 100644 --- a/packages/coding-agent/src/utils/image-loading.ts +++ b/packages/coding-agent/src/utils/image-loading.ts @@ -2,7 +2,7 @@ import * as fs from "node:fs/promises"; import type { ImageContent } from "@oh-my-pi/pi-ai"; import { formatBytes, readImageMetadata, SUPPORTED_IMAGE_MIME_TYPES } from "@oh-my-pi/pi-utils"; import { resolveReadPath } from "../tools/path-utils"; -import { formatDimensionNote, resizeImage, type ImageResizeOptions } from "./image-resize"; +import { formatDimensionNote, type ImageResizeOptions, resizeImage } from "./image-resize"; export const MAX_IMAGE_INPUT_BYTES = 20 * 1024 * 1024; export const SUPPORTED_INPUT_IMAGE_MIME_TYPES = SUPPORTED_IMAGE_MIME_TYPES; diff --git a/packages/natives/test/native.test.ts b/packages/natives/test/native.test.ts index f659d765a..832997e22 100644 --- a/packages/natives/test/native.test.ts +++ b/packages/natives/test/native.test.ts @@ -3,6 +3,8 @@ import * as fs from "node:fs/promises"; import * as os from "node:os"; import * as path from "node:path"; import { + AstMatchStrictness, + astMatch, executeShell, FileType, fuzzyFind, @@ -674,4 +676,44 @@ describe("pi-natives", () => { assertion.stop(); }); }); + + describe("astMatch", () => { + it("matches a pattern against an in-memory source string", async () => { + const result = await astMatch({ + source: 'function greet() {\n\tconsole.log("hi");\n}', + lang: "ts", + patterns: ["console.log($MSG)"], + strictness: AstMatchStrictness.Smart, + includeMeta: true, + }); + expect(result.totalMatches).toBe(1); + expect(result.matches[0]?.text).toBe('console.log("hi")'); + expect(result.matches[0]?.metaVariables?.MSG).toBe('"hi"'); + }); + + it("enforces metavariable equality within a pattern", async () => { + const same = await astMatch({ + source: "if (x) clearTimeout(x);", + lang: "ts", + patterns: ["if ($X) clearTimeout($X)"], + }); + const diff = await astMatch({ + source: "if (x) clearTimeout(y);", + lang: "ts", + patterns: ["if ($X) clearTimeout($X)"], + }); + expect(same.totalMatches).toBe(1); + expect(diff.totalMatches).toBe(0); + }); + + it("reports parse errors for incomplete source without throwing", async () => { + const result = await astMatch({ source: "console.log(", lang: "ts", patterns: ["console.log($A)"] }); + expect(result.totalMatches).toBe(0); + expect(result.parseErrors?.length).toBeGreaterThan(0); + }); + + it("rejects an empty language", async () => { + await expect(astMatch({ source: "const a = 1;", lang: " ", patterns: ["const $A = $B"] })).rejects.toThrow(); + }); + }); });