test(natives): documented AST conditions and expanded astMatch tests
- Updated `docs/ttsr-injection-lifecycle.md` to document `astCondition` behavior, including registration rules and tool-stream matching. - Extended `packages/natives/test/native.test.ts` with new `astMatch` coverage for Smart matching, metavariable consistency, parse errors, and empty-language rejection.
This commit is contained in:
@@ -38,6 +38,7 @@ interface Rule {
|
||||
alwaysApply?: boolean;
|
||||
description?: string;
|
||||
condition?: string[];
|
||||
astCondition?: string[];
|
||||
scope?: string[];
|
||||
interruptMode?: "never" | "prose-only" | "tool-only" | "always";
|
||||
_source: SourceMeta;
|
||||
@@ -183,16 +184,16 @@ After rule discovery in `createAgentSession` (`sdk.ts`), `bucketRules(...)` appl
|
||||
|
||||
1. Drop rules listed in `ttsr.disabledRules`.
|
||||
2. Drop rules from the `builtin-defaults` provider when `ttsr.builtinRules === false`.
|
||||
3. Register rules with non-empty `condition` into `TtsrManager`; if registration succeeds, the rule is TTSR-only.
|
||||
3. Register rules with a non-empty `condition` or `astCondition` into `TtsrManager`; if registration succeeds, the rule is TTSR-only.
|
||||
4. Put remaining `alwaysApply === true` rules into `alwaysApplyRules`.
|
||||
5. Put remaining rules with `description` into `rulebookRules`.
|
||||
|
||||
### Bucket behavior
|
||||
|
||||
- **TTSR bucket**: any enabled rule with a non-empty parsed `condition` that `TtsrManager.addRule(...)` accepts. Takes priority over other buckets.
|
||||
- **TTSR bucket**: any enabled rule with a non-empty parsed `condition` (regex) or `astCondition` (ast-grep patterns) that `TtsrManager.addRule(...)` accepts. Takes priority over other buckets.
|
||||
- **Always-apply bucket**: `alwaysApply === true`, not TTSR. Full content injected into system prompt. Resolvable via `rule://`.
|
||||
- **Rulebook bucket**: must have description, must not be TTSR, must not be `alwaysApply`. Listed in system prompt by name+description; content read on demand via `rule://`.
|
||||
- A rule with both `condition` and `alwaysApply` goes to TTSR only if TTSR registration accepts it; otherwise it can fall through to always-apply.
|
||||
- A rule with both a trigger condition and `alwaysApply` goes to TTSR only if TTSR registration accepts it; otherwise it can fall through to always-apply.
|
||||
- A rule with both `alwaysApply` and `description` goes to always-apply only (not rulebook).
|
||||
|
||||
## 6. How metadata affects runtime surfaces
|
||||
@@ -219,10 +220,11 @@ After rule discovery in `createAgentSession` (`sdk.ts`), `bucketRules(...)` appl
|
||||
- **Full rule content is auto-injected into the system prompt** (before the rulebook rules section).
|
||||
- Rule is also addressable via `rule://<name>` for re-reading.
|
||||
|
||||
### `condition`, `scope`, and `interruptMode`
|
||||
### `condition`, `astCondition`, `scope`, and `interruptMode`
|
||||
|
||||
- `condition` is the current TTSR trigger field; legacy `ttsr_trigger` / `ttsrTrigger` are accepted as fallback inputs during parsing.
|
||||
- `scope` narrows TTSR matching scope. A condition token that looks like a file glob becomes `tool:edit(<glob>)` and `tool:write(<glob>)` scope entries plus catch-all condition `.*`.
|
||||
- `condition` is the regex TTSR trigger field; legacy `ttsr_trigger` / `ttsrTrigger` are accepted as fallback inputs during parsing.
|
||||
- `astCondition` is the ast-grep trigger field: a string or list of structural patterns, kept verbatim (no glob inference). It only matches on edit/write tool streams, where the language is inferred from the file path. A rule may set `condition`, `astCondition`, or both.
|
||||
- `scope` narrows TTSR matching scope. A `condition` token that looks like a file glob becomes `tool:edit(<glob>)` and `tool:write(<glob>)` scope entries plus catch-all condition `.*`; `astCondition` tokens never trigger this shorthand.
|
||||
- `interruptMode` can override the global TTSR interrupt mode for the rule.
|
||||
|
||||
## 7. System prompt inclusion path
|
||||
|
||||
@@ -43,12 +43,18 @@ const { rulebookRules, alwaysApplyRules } = bucketRules(
|
||||
|
||||
Registration is skipped when:
|
||||
|
||||
- `rule.condition` is absent or all condition regexes fail to compile
|
||||
- both `rule.condition` (regex) and `rule.astCondition` (ast-grep patterns) are absent, or every regex condition fails to compile and there are no AST conditions
|
||||
- a rule with the same `rule.name` was already registered in this manager
|
||||
- the rule scope excludes all monitored streams
|
||||
|
||||
Invalid regex conditions and unreachable scopes are logged as warnings and ignored; session startup continues. If a TTSR rule defines `globs`, those globs are compiled as a global file-path gate for matching.
|
||||
|
||||
### AST conditions (`astCondition`)
|
||||
|
||||
A rule may carry `astCondition`: a list of [ast-grep](https://ast-grep.github.io/) patterns (OR'd, same as regex `condition`), matched structurally instead of textually. A repeated metavariable inside one pattern requires both occurrences to be equal (`if ($X) clearTimeout($X)` matches but `if ($X) clearTimeout($Y)` does not).
|
||||
|
||||
AST conditions only evaluate on **edit/write tool-argument streams** — they need a language, which is inferred from the file extension on the tool's path argument, and they match against the tool's reconstructed source snapshot (`matcherDigest`), not the raw wire delta. Matching is performed in memory by the native `astMatch` engine (no temp files) with Smart strictness. Streams without a usable file path (prose, thinking, path-less tool calls) skip AST conditions entirely. A rule may mix `condition` and `astCondition`; the regex paths keep working on every scope while AST paths apply only to those tool streams.
|
||||
|
||||
### Setting caveat
|
||||
|
||||
`TtsrSettings.enabled` is loaded into the manager but is not currently checked in runtime gating. If TTSR rules exist, matching still runs.
|
||||
@@ -69,9 +75,10 @@ When assistant updates arrive and rules exist:
|
||||
|
||||
- monitor `text_delta`, `thinking_delta`, and `toolcall_delta`
|
||||
- append delta into a source/tool scoped manager buffer
|
||||
- call `checkDelta(delta, matchContext)`
|
||||
- call `checkDelta(delta, matchContext)` (synchronous regex matching)
|
||||
- for edit/write tool streams, when `hasAstRules()` is true, `await checkAstSnapshot(snapshot, matchContext)` (asynchronous AST matching)
|
||||
|
||||
`checkDelta()` iterates registered rules and returns all matching rules that pass scope, global path-glob, condition, and repeat policy checks.
|
||||
`checkDelta()` iterates registered rules and returns all matching rules that pass scope, global path-glob, regex condition, and repeat policy checks. `checkAstSnapshot()` applies the same scope/path/repeat gates, then runs each candidate rule's `astCondition` patterns against the snapshot via the native `astMatch` engine. It is throttled per stream key: an identical consecutive snapshot (common when only non-source arguments change between deltas) is skipped without re-running the matcher. Both paths feed their matches through the same trigger-decision handler.
|
||||
|
||||
## 3. Trigger decision and immediate abort path
|
||||
|
||||
|
||||
@@ -20,10 +20,10 @@ import {
|
||||
createHarmonyAuditEvent,
|
||||
detectHarmonyLeakInAssistantMessage,
|
||||
extractHarmonyRemoved,
|
||||
recoverHarmonyToolCall,
|
||||
type HarmonyDetection,
|
||||
type HarmonyRecoveredToolCall,
|
||||
isHarmonyLeakMitigationTarget,
|
||||
recoverHarmonyToolCall,
|
||||
signalListLabel,
|
||||
} from "./harmony-leak";
|
||||
import { type AgentRunCoverage, type AgentRunSummary, ToolCallBlockedError } from "./run-collector";
|
||||
@@ -1085,7 +1085,10 @@ async function streamAssistantResponse(
|
||||
}
|
||||
}
|
||||
|
||||
function retainCompletedToolCalls(message: AssistantMessage, completedToolCallIds: ReadonlySet<string>): AssistantMessage {
|
||||
function retainCompletedToolCalls(
|
||||
message: AssistantMessage,
|
||||
completedToolCallIds: ReadonlySet<string>,
|
||||
): AssistantMessage {
|
||||
if (message.stopReason !== "error" && message.stopReason !== "aborted") return message;
|
||||
let changed = false;
|
||||
const content = message.content.filter(block => {
|
||||
|
||||
@@ -277,7 +277,6 @@ function cloneAnthropicCacheControl(cacheControl: AnthropicCacheControl): Anthro
|
||||
return { ...cacheControl };
|
||||
}
|
||||
|
||||
|
||||
type AnthropicOutputConfig = NonNullable<MessageCreateParamsStreaming["output_config"]>;
|
||||
|
||||
const ANTHROPIC_STOP_SEQUENCES_MAX = 4;
|
||||
@@ -1453,7 +1452,11 @@ export const streamAnthropic: StreamFunction<"anthropic-messages"> = (
|
||||
const baseUrl =
|
||||
resolveAnthropicBaseUrl(model, options?.apiKey ?? getEnvApiKey(model.provider) ?? "") ??
|
||||
"https://api.anthropic.com";
|
||||
const providerSessionState = getAnthropicProviderSessionState(options?.providerSessionState, baseUrl, model.id);
|
||||
const providerSessionState = getAnthropicProviderSessionState(
|
||||
options?.providerSessionState,
|
||||
baseUrl,
|
||||
model.id,
|
||||
);
|
||||
let disableStrictTools =
|
||||
(providerSessionState?.strictToolsDisabled ?? false) || (model.compat?.disableStrictTools ?? false);
|
||||
let strictFallbackErrorMessage: string | undefined;
|
||||
@@ -1666,7 +1669,9 @@ export const streamAnthropic: StreamFunction<"anthropic-messages"> = (
|
||||
}
|
||||
const openBlock = openBlocks.get(event.index);
|
||||
if (!openBlock) {
|
||||
throw createAnthropicStreamEnvelopeError(`received content_block_delta for unopened index ${event.index}`);
|
||||
throw createAnthropicStreamEnvelopeError(
|
||||
`received content_block_delta for unopened index ${event.index}`,
|
||||
);
|
||||
}
|
||||
const block = blocks[openBlock.contentIndex];
|
||||
if (event.delta.type === "text_delta") {
|
||||
@@ -1683,7 +1688,9 @@ export const streamAnthropic: StreamFunction<"anthropic-messages"> = (
|
||||
});
|
||||
} else if (event.delta.type === "thinking_delta") {
|
||||
if (openBlock.kind !== "thinking" || block?.type !== "thinking") {
|
||||
throw createAnthropicStreamEnvelopeError(`received thinking_delta for ${openBlock.kind} block`);
|
||||
throw createAnthropicStreamEnvelopeError(
|
||||
`received thinking_delta for ${openBlock.kind} block`,
|
||||
);
|
||||
}
|
||||
streamedReplayUnsafeContent = true;
|
||||
block.thinking += event.delta.thinking;
|
||||
@@ -1695,7 +1702,9 @@ export const streamAnthropic: StreamFunction<"anthropic-messages"> = (
|
||||
});
|
||||
} else if (event.delta.type === "input_json_delta") {
|
||||
if (openBlock.kind !== "toolCall" || block?.type !== "toolCall") {
|
||||
throw createAnthropicStreamEnvelopeError(`received input_json_delta for ${openBlock.kind} block`);
|
||||
throw createAnthropicStreamEnvelopeError(
|
||||
`received input_json_delta for ${openBlock.kind} block`,
|
||||
);
|
||||
}
|
||||
streamedReplayUnsafeContent = true;
|
||||
block.partialJson += event.delta.partial_json;
|
||||
@@ -1712,7 +1721,9 @@ export const streamAnthropic: StreamFunction<"anthropic-messages"> = (
|
||||
});
|
||||
} else if (event.delta.type === "signature_delta") {
|
||||
if (openBlock.kind !== "thinking" || block?.type !== "thinking") {
|
||||
throw createAnthropicStreamEnvelopeError(`received signature_delta for ${openBlock.kind} block`);
|
||||
throw createAnthropicStreamEnvelopeError(
|
||||
`received signature_delta for ${openBlock.kind} block`,
|
||||
);
|
||||
}
|
||||
streamedReplayUnsafeContent = true;
|
||||
block.thinkingSignature = block.thinkingSignature || "";
|
||||
@@ -1724,11 +1735,15 @@ export const streamAnthropic: StreamFunction<"anthropic-messages"> = (
|
||||
}
|
||||
const openBlock = openBlocks.get(event.index);
|
||||
if (!openBlock) {
|
||||
throw createAnthropicStreamEnvelopeError(`received content_block_stop for unopened index ${event.index}`);
|
||||
throw createAnthropicStreamEnvelopeError(
|
||||
`received content_block_stop for unopened index ${event.index}`,
|
||||
);
|
||||
}
|
||||
const block = blocks[openBlock.contentIndex];
|
||||
if (!block || block.type !== openBlock.kind) {
|
||||
throw createAnthropicStreamEnvelopeError(`content_block_stop kind mismatch for index ${event.index}`);
|
||||
throw createAnthropicStreamEnvelopeError(
|
||||
`content_block_stop kind mismatch for index ${event.index}`,
|
||||
);
|
||||
}
|
||||
openBlocks.delete(event.index);
|
||||
delete (block as { index?: number }).index;
|
||||
@@ -2189,7 +2204,10 @@ function ensureMaxTokensForThinking(params: MessageCreateParamsStreaming, model:
|
||||
|
||||
const maxAllowedTokens = Math.min(CLAUDE_CODE_MAX_OUTPUT_TOKENS, model.maxTokens);
|
||||
const currentMaxTokens = Math.min(params.max_tokens ?? maxAllowedTokens, maxAllowedTokens);
|
||||
const raisedMaxTokens = Math.min(Math.max(currentMaxTokens, budgetTokens + OUTPUT_FALLBACK_BUFFER), maxAllowedTokens);
|
||||
const raisedMaxTokens = Math.min(
|
||||
Math.max(currentMaxTokens, budgetTokens + OUTPUT_FALLBACK_BUFFER),
|
||||
maxAllowedTokens,
|
||||
);
|
||||
params.max_tokens = raisedMaxTokens;
|
||||
|
||||
if (budgetTokens + OUTPUT_FALLBACK_BUFFER <= raisedMaxTokens) return;
|
||||
@@ -2900,7 +2918,6 @@ function pickAnthropicEffectiveScalarType(schema: Record<string, unknown>): stri
|
||||
return undefined;
|
||||
}
|
||||
|
||||
|
||||
function anthropicPerTypeKeep(scalarType: string | undefined): Set<string> | undefined {
|
||||
switch (scalarType) {
|
||||
case "object":
|
||||
@@ -3071,7 +3088,6 @@ function hasAnthropicSchemaDefiningKeyword(schema: Record<string, unknown>): boo
|
||||
return schema.$defs !== undefined || schema.definitions !== undefined;
|
||||
}
|
||||
|
||||
|
||||
function makeAnthropicNullableSchema(schema: unknown, budget: AnthropicStrictBudget): unknown | undefined {
|
||||
if (isRecord(schema)) {
|
||||
if (hasNullVariant(schema)) return schema;
|
||||
|
||||
@@ -2,7 +2,7 @@ import * as fs from "node:fs/promises";
|
||||
import type { ImageContent } from "@oh-my-pi/pi-ai";
|
||||
import { formatBytes, readImageMetadata, SUPPORTED_IMAGE_MIME_TYPES } from "@oh-my-pi/pi-utils";
|
||||
import { resolveReadPath } from "../tools/path-utils";
|
||||
import { formatDimensionNote, resizeImage, type ImageResizeOptions } from "./image-resize";
|
||||
import { formatDimensionNote, type ImageResizeOptions, resizeImage } from "./image-resize";
|
||||
|
||||
export const MAX_IMAGE_INPUT_BYTES = 20 * 1024 * 1024;
|
||||
export const SUPPORTED_INPUT_IMAGE_MIME_TYPES = SUPPORTED_IMAGE_MIME_TYPES;
|
||||
|
||||
@@ -3,6 +3,8 @@ import * as fs from "node:fs/promises";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import {
|
||||
AstMatchStrictness,
|
||||
astMatch,
|
||||
executeShell,
|
||||
FileType,
|
||||
fuzzyFind,
|
||||
@@ -674,4 +676,44 @@ describe("pi-natives", () => {
|
||||
assertion.stop();
|
||||
});
|
||||
});
|
||||
|
||||
describe("astMatch", () => {
|
||||
it("matches a pattern against an in-memory source string", async () => {
|
||||
const result = await astMatch({
|
||||
source: 'function greet() {\n\tconsole.log("hi");\n}',
|
||||
lang: "ts",
|
||||
patterns: ["console.log($MSG)"],
|
||||
strictness: AstMatchStrictness.Smart,
|
||||
includeMeta: true,
|
||||
});
|
||||
expect(result.totalMatches).toBe(1);
|
||||
expect(result.matches[0]?.text).toBe('console.log("hi")');
|
||||
expect(result.matches[0]?.metaVariables?.MSG).toBe('"hi"');
|
||||
});
|
||||
|
||||
it("enforces metavariable equality within a pattern", async () => {
|
||||
const same = await astMatch({
|
||||
source: "if (x) clearTimeout(x);",
|
||||
lang: "ts",
|
||||
patterns: ["if ($X) clearTimeout($X)"],
|
||||
});
|
||||
const diff = await astMatch({
|
||||
source: "if (x) clearTimeout(y);",
|
||||
lang: "ts",
|
||||
patterns: ["if ($X) clearTimeout($X)"],
|
||||
});
|
||||
expect(same.totalMatches).toBe(1);
|
||||
expect(diff.totalMatches).toBe(0);
|
||||
});
|
||||
|
||||
it("reports parse errors for incomplete source without throwing", async () => {
|
||||
const result = await astMatch({ source: "console.log(", lang: "ts", patterns: ["console.log($A)"] });
|
||||
expect(result.totalMatches).toBe(0);
|
||||
expect(result.parseErrors?.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("rejects an empty language", async () => {
|
||||
await expect(astMatch({ source: "const a = 1;", lang: " ", patterns: ["const $A = $B"] })).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user