smashimo/oh-my-pi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: reformat

Browse Source
This commit is contained in:
can1357
2026-07-15 00:08:50 +02:00
parent 7d02778c60
commit 55f5ebec49
97 changed files with 12237 additions and 3179 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/analyze_small_edits.py
+84 -39
View File
@@ -12,9 +12,21 @@ import sys
if __package__ in (None, ""):
sys.path.insert(0, str(Path(__file__).resolve().parent))
from tool_io import ReservoirSample, ToolIOConfig, ToolInvocation, iter_tool_invocations, list_recent_session_files
from tool_io import (
ReservoirSample,
ToolIOConfig,
ToolInvocation,
iter_tool_invocations,
list_recent_session_files,
)
else:
from scripts.tool_io import ReservoirSample, ToolIOConfig, ToolInvocation, iter_tool_invocations, list_recent_session_files
from scripts.tool_io import (
ReservoirSample,
ToolIOConfig,
ToolInvocation,
iter_tool_invocations,
list_recent_session_files,
)
TOOL_NAMES = ("edit", "ast_edit")
@@ -81,10 +93,13 @@ class RunStats:
small_edits_after_same_path_failed_edit: int = 0
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Analyze small edit/ast_edit tool usage in session logs.")
parser.add_argument("--sessions-dir", type=Path, default=Path.home() / ".omp" / "agent" / "sessions")
parser = argparse.ArgumentParser(
description="Analyze small edit/ast_edit tool usage in session logs."
)
parser.add_argument(
"--sessions-dir", type=Path, default=Path.home() / ".omp" / "agent" / "sessions"
)
parser.add_argument("--sample-size", type=positive_int, default=30)
parser.add_argument("--max-files", type=positive_int, default=500)
parser.add_argument("--since-days", type=positive_int, default=30)
@@ -94,7 +109,6 @@ def parse_args() -> argparse.Namespace:
return parser.parse_args()
def positive_int(value: str) -> int:
parsed = int(value)
if parsed <= 0:
@@ -102,7 +116,6 @@ def positive_int(value: str) -> int:
return parsed
def strip_decorations(line: str) -> str:
return re.sub(r"^\s*\d+\s+", "", line).strip()
@@ -111,7 +124,6 @@ def is_delimiter_line(line: str) -> bool:
return bool(re.match(r"^[\]}),;]+$", line))
def is_tiny_structural_line(line: str) -> bool:
if len(line) == 0:
return True
@@ -126,14 +138,16 @@ def is_tiny_structural_line(line: str) -> bool:
return False
def classify_success_issue(summary: DiffSummary) -> str:
previews = summary.changed_preview
if previews and all(len(line) == 0 for line in previews):
return "blank-line-adjustment"
if previews and all(is_delimiter_line(line) for line in previews):
return "delimiter-adjustment"
if previews and all(re.match(r"^(pub\s+mod|pub\s+use|mod|use|import|export)\b", line) for line in previews):
if previews and all(
re.match(r"^(pub\s+mod|pub\s+use|mod|use|import|export)\b", line)
for line in previews
):
return "import-or-module-tweak"
if summary.removed_lines == 1 and summary.added_lines == 0:
return "single-line-delete"
@@ -144,24 +158,32 @@ def classify_success_issue(summary: DiffSummary) -> str:
return "small-structural-fix"
def classify_failure_issue(result_text: str) -> str:
if re.search(r"identical content|No changes made", result_text, re.IGNORECASE):
return "no-op-identical"
if re.search(r"Failed to find context|matches for context|expected lines|tag mismatch|>>>", result_text, re.IGNORECASE):
if re.search(
r"Failed to find context|matches for context|expected lines|tag mismatch|>>>",
result_text,
re.IGNORECASE,
):
return "context-mismatch"
if re.search(r"Unexpected line in hunk|parse error|SyntaxError", result_text, re.IGNORECASE):
if re.search(
r"Unexpected line in hunk|parse error|SyntaxError", result_text, re.IGNORECASE
):
return "invalid-patch-shape"
if re.search(r"File not found", result_text, re.IGNORECASE):
return "missing-file"
if re.search(r"occurrence|ambiguous", result_text, re.IGNORECASE):
return "ambiguous-target"
if re.search(r"Validation failed|required property|must have required property", result_text, re.IGNORECASE):
if re.search(
r"Validation failed|required property|must have required property",
result_text,
re.IGNORECASE,
):
return "invalid-arguments"
return "other-failure"
def summarize_diff(diff: str | None) -> DiffSummary:
if not diff:
return DiffSummary(
@@ -188,7 +210,9 @@ def summarize_diff(diff: str | None) -> DiffSummary:
previews = [line for line in all_changes if line or include_blank]
changed_lines = len(added) + len(removed)
tiny_only = all(is_tiny_structural_line(line) for line in all_changes)
small = changed_lines > 0 and (changed_lines <= 2 or (changed_lines <= 4 and tiny_only))
small = changed_lines > 0 and (
changed_lines <= 2 or (changed_lines <= 4 and tiny_only)
)
preview_slice = previews[:4]
category = None
if small:
@@ -211,18 +235,20 @@ def summarize_diff(diff: str | None) -> DiffSummary:
)
def previews_need_blank_marker(lines: list[str]) -> bool:
return any(len(line) == 0 for line in lines)
def build_completed_edit(invocation: ToolInvocation) -> CompletedEdit | None:
if not invocation.has_result:
return None
diff_summary = summarize_diff(invocation.diff)
is_error = invocation.is_error
issue = classify_failure_issue(invocation.result_text) if is_error else (diff_summary.category or "other-success")
issue = (
classify_failure_issue(invocation.result_text)
if is_error
else (diff_summary.category or "other-success")
)
return CompletedEdit(
session_file=str(invocation.session_file),
tool_call_id=invocation.tool_call_id,
@@ -245,8 +271,9 @@ def build_completed_edit(invocation: ToolInvocation) -> CompletedEdit | None:
)
def analyze_small_edits(stream: Iterable[ToolInvocation], *, sample_size: int, files_scanned: int) -> dict[str, object]:
def analyze_small_edits(
stream: Iterable[ToolInvocation], *, sample_size: int, files_scanned: int
) -> dict[str, object]:
sample: ReservoirSample[Candidate] = ReservoirSample(size=sample_size)
issue_counts: dict[str, int] = {}
stats = RunStats(files_scanned=files_scanned)
@@ -295,7 +322,6 @@ def analyze_small_edits(stream: Iterable[ToolInvocation], *, sample_size: int, f
}
def candidate_to_dict(candidate: Candidate) -> dict[str, object]:
payload = {"kind": candidate.kind, "edit": asdict(candidate.edit)}
if candidate.previous_edit is not None:
@@ -303,19 +329,20 @@ def candidate_to_dict(candidate: Candidate) -> dict[str, object]:
return payload
def top_entries(counts: dict[str, int], limit: int) -> list[dict[str, object]]:
return [
{"name": name, "count": count}
for name, count in sorted(counts.items(), key=lambda entry: (-entry[1], entry[0]))[:limit]
for name, count in sorted(
counts.items(), key=lambda entry: (-entry[1], entry[0])
)[:limit]
]
def short_path(target_path: str) -> str:
home = str(Path.home())
return f"~{target_path[len(home):]}" if target_path.startswith(home) else target_path
return (
f"~{target_path[len(home) :]}" if target_path.startswith(home) else target_path
)
def truncate(text: str, limit: int) -> str:
@@ -324,7 +351,6 @@ def truncate(text: str, limit: int) -> str:
return f"{text[: limit - 1]}…"
def format_sample_entry(candidate: dict[str, object], index: int) -> str:
edit = candidate["edit"]
assert isinstance(edit, dict)
@@ -340,10 +366,16 @@ def format_sample_entry(candidate: dict[str, object], index: int) -> str:
)
changed_preview = edit.get("changed_preview")
if isinstance(changed_preview, list) and changed_preview:
lines.append(f" preview: {' | '.join(str(item) for item in changed_preview)}")
lines.append(
f" preview: {' | '.join(str(item) for item in changed_preview)}"
)
previous = candidate.get("previous_edit")
if isinstance(previous, dict):
path_part = f" ({short_path(str(previous['path']))})" if previous.get("path") else ""
path_part = (
f" ({short_path(str(previous['path']))})"
if previous.get("path")
else ""
)
lines.append(
" previous edit: "
f"{'same-path' if previous.get('same_path') else 'other-path'} "
@@ -351,16 +383,21 @@ def format_sample_entry(candidate: dict[str, object], index: int) -> str:
)
previous_preview = previous.get("changed_preview")
if isinstance(previous_preview, list) and previous_preview:
lines.append(f" previous preview: {' | '.join(str(item) for item in previous_preview)}")
lines.append(
f" previous preview: {' | '.join(str(item) for item in previous_preview)}"
)
else:
lines.append(" previous edit: none")
else:
lines.append(f" result: {truncate(' '.join(str(edit['result_text']).split()), 220)}")
lines.append(
f" result: {truncate(' '.join(str(edit['result_text']).split()), 220)}"
)
changed_preview = edit.get("changed_preview")
if isinstance(changed_preview, list) and changed_preview:
lines.append(f" diff preview: {' | '.join(str(item) for item in changed_preview)}")
return '\n'.join(lines)
lines.append(
f" diff preview: {' | '.join(str(item) for item in changed_preview)}"
)
return "\n".join(lines)
def main() -> None:
@@ -375,7 +412,9 @@ def main() -> None:
)
files = list_recent_session_files(config)
stream = iter_tool_invocations(TOOL_NAMES, config)
analysis = analyze_small_edits(stream, sample_size=options.sample_size, files_scanned=len(files))
analysis = analyze_small_edits(
stream, sample_size=options.sample_size, files_scanned=len(files)
)
if options.json:
print(
@@ -403,14 +442,20 @@ def main() -> None:
assert isinstance(stats, dict)
assert isinstance(top_issues, list)
assert isinstance(sample, list)
print(f"Scanned {stats['files_scanned']} session file(s) from {short_path(str(options.sessions_dir))}")
print(
f"Scanned {stats['files_scanned']} session file(s) from {short_path(str(options.sessions_dir))}"
)
print(f"Edit attempts: {stats['total_edit_attempts']}")
print(f"Failed edits: {stats['failed_edits']}")
print(f"Small edits: {stats['small_edits']}")
print(f"Small edits with previous edit: {stats['small_edits_with_previous_edit']}")
print(f"Small edits with previous same-path edit: {stats['small_edits_with_previous_same_path']}")
print(
f"Small edits with previous same-path edit: {stats['small_edits_with_previous_same_path']}"
)
print(f"Small edits after failed edit: {stats['small_edits_after_failed_edit']}")
print(f"Small edits after same-path failed edit: {stats['small_edits_after_same_path_failed_edit']}")
print(
f"Small edits after same-path failed edit: {stats['small_edits_after_same_path_failed_edit']}"
)
print()
print("Top issues:")
for entry in top_issues[:12]:
scripts/edit-benchmark.py
+54 -45
View File
@@ -6,68 +6,77 @@ Select the edit variant via the PI_EDIT_VARIANT env var (e.g. `vim`,
`hashline`, `replace`, `patch`, `apply_patch`) or `--variant`.
Examples:
PI_EDIT_VARIANT=vim scripts/edit-benchmark.py
scripts/edit-benchmark.py --variant hashline
PI_EDIT_VARIANT=vim scripts/edit-benchmark.py
scripts/edit-benchmark.py --variant hashline
"""
from __future__ import annotations
import os
import sys
from edit_benchmark_common import BenchmarkSpec, EDIT_DIFF, EXPECTED_CONTENT, run_benchmark_main
from edit_benchmark_common import (
BenchmarkSpec,
EDIT_DIFF,
EXPECTED_CONTENT,
run_benchmark_main,
)
def _extract_variant_arg() -> str | None:
"""Pop `--variant <value>` (or `--variant=<value>`) from sys.argv before argparse in common runs."""
argv = sys.argv
for i, arg in enumerate(argv[1:], start=1):
if arg == "--variant" and i + 1 < len(argv):
value = argv[i + 1]
del argv[i : i + 2]
return value
if arg.startswith("--variant="):
value = arg.split("=", 1)[1]
del argv[i]
return value
return None
"""Pop `--variant <value>` (or `--variant=<value>`) from sys.argv before argparse in common runs."""
argv = sys.argv
for i, arg in enumerate(argv[1:], start=1):
if arg == "--variant" and i + 1 < len(argv):
value = argv[i + 1]
del argv[i : i + 2]
return value
if arg.startswith("--variant="):
value = arg.split("=", 1)[1]
del argv[i]
return value
return None
def _resolve_variant() -> str:
cli_variant = _extract_variant_arg()
variant = cli_variant or os.environ.get("PI_EDIT_VARIANT")
if not variant:
raise SystemExit("edit-benchmark: set PI_EDIT_VARIANT=<variant> or pass --variant <variant>.")
return variant
cli_variant = _extract_variant_arg()
variant = cli_variant or os.environ.get("PI_EDIT_VARIANT")
if not variant:
raise SystemExit(
"edit-benchmark: set PI_EDIT_VARIANT=<variant> or pass --variant <variant>."
)
return variant
def build_spec(variant: str) -> BenchmarkSpec:
mode_phrase = f"in {variant} mode"
prompt = (
f"Use the `read` tool to inspect `test.rs`, then use the `edit` tool {mode_phrase} "
f"to make `test.rs` exactly match the requested change.\n"
f"\n"
f"Apply this diff:\n"
f"```diff\n"
f"{EDIT_DIFF}```\n"
f"\n"
f"Final expected file content:\n"
f"```rust\n"
f"{EXPECTED_CONTENT}```\n"
)
retry = f"Please try again using the edit tool {mode_phrase}."
return BenchmarkSpec(
description=f"Benchmark edit tool in {variant} mode across models with simple edit tasks.",
workspace_prefix=f"{variant}-benchmark",
tools=("edit", "read"),
env={"PI_EDIT_VARIANT": variant, "PI_STRICT_EDIT_MODE": "1"},
initial_prompt=prompt,
retry_instruction=retry,
)
mode_phrase = f"in {variant} mode"
prompt = (
f"Use the `read` tool to inspect `test.rs`, then use the `edit` tool {mode_phrase} "
f"to make `test.rs` exactly match the requested change.\n"
f"\n"
f"Apply this diff:\n"
f"```diff\n"
f"{EDIT_DIFF}```\n"
f"\n"
f"Final expected file content:\n"
f"```rust\n"
f"{EXPECTED_CONTENT}```\n"
)
retry = f"Please try again using the edit tool {mode_phrase}."
return BenchmarkSpec(
description=f"Benchmark edit tool in {variant} mode across models with simple edit tasks.",
workspace_prefix=f"{variant}-benchmark",
tools=("edit", "read"),
env={"PI_EDIT_VARIANT": variant, "PI_STRICT_EDIT_MODE": "1"},
initial_prompt=prompt,
retry_instruction=retry,
)
def main() -> int:
variant = _resolve_variant()
return run_benchmark_main(build_spec(variant))
variant = _resolve_variant()
return run_benchmark_main(build_spec(variant))
if __name__ == "__main__":
raise SystemExit(main())
raise SystemExit(main())
scripts/edit_benchmark_common.py
+41 -13
View File
@@ -2,6 +2,7 @@
"""
Shared helpers for edit benchmark scripts.
"""
from __future__ import annotations
import argparse
@@ -21,13 +22,19 @@ from typing import Any, Callable
REPO_ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(REPO_ROOT / "python/omp-rpc/src"))
from omp_rpc import MessageEndEvent, MessageStartEvent, MessageUpdateEvent, RpcClient, ToolExecutionStartEvent # noqa: E402
from omp_rpc import (
MessageEndEvent,
MessageStartEvent,
MessageUpdateEvent,
RpcClient,
ToolExecutionStartEvent,
) # noqa: E402
MODELS = [
"openrouter/moonshotai/kimi-k2.5",
"openrouter/anthropic/claude-haiku-4.5",
"openrouter/google/gemini-3.1-flash-lite-preview",
"openrouter/z-ai/glm-4.7-20251222:nitro"
"openrouter/z-ai/glm-4.7-20251222:nitro",
# "openrouter/anthropic/claude-sonnet-4.6",
# "openrouter/google/gemini-3-flash-preview",
# "openrouter/z-ai/glm-5-turbo",
@@ -457,6 +464,7 @@ mod tests {
}
"""
def _compute_edit_diff() -> str:
initial_lines = INITIAL_CONTENT.splitlines(keepends=True)
expected_lines = EXPECTED_CONTENT.splitlines(keepends=True)
@@ -526,13 +534,17 @@ class VerbosePrinter:
sys.stderr.flush()
self._open_kind = None
def emit_delta(self, kind: str, delta: str, content_index: int | None = None) -> None:
def emit_delta(
self, kind: str, delta: str, content_index: int | None = None
) -> None:
if not delta:
return
if content_index is not None:
key = (kind, content_index)
self._seen_block_lengths[key] = self._seen_block_lengths.get(key, 0) + len(delta)
self._seen_block_lengths[key] = self._seen_block_lengths.get(key, 0) + len(
delta
)
with _PRINT_LOCK:
if self._open_kind != kind:
@@ -553,7 +565,9 @@ class VerbosePrinter:
sys.stderr.flush()
def emit_tool_call(self, tool_name: str, args: Any) -> None:
rendered_args = json.dumps(args, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
rendered_args = json.dumps(
args, ensure_ascii=False, sort_keys=True, separators=(",", ":")
)
with _PRINT_LOCK:
if self._open_kind is not None:
sys.stderr.write("\n")
@@ -597,7 +611,10 @@ class VerbosePrinter:
if not isinstance(content, list):
return
has_redacted = any(isinstance(block, dict) and block.get("type") == "redactedThinking" for block in content)
has_redacted = any(
isinstance(block, dict) and block.get("type") == "redactedThinking"
for block in content
)
if not has_redacted:
return
@@ -624,7 +641,9 @@ def resolve_omp_bin(raw: str | None) -> str:
return repo_bin
found = shutil.which("omp")
if not found:
raise SystemExit("Could not find `omp` on PATH and could not resolve the repo CLI. Set --omp-bin or OMP_BIN.")
raise SystemExit(
"Could not find `omp` on PATH and could not resolve the repo CLI. Set --omp-bin or OMP_BIN."
)
return found
@@ -672,9 +691,13 @@ def install_verbose_logging(
message_event = event.assistant_message_event
event_type = message_event["type"]
if event_type == "text_delta":
printer.emit_delta("text", message_event["delta"], message_event["contentIndex"])
printer.emit_delta(
"text", message_event["delta"], message_event["contentIndex"]
)
elif event_type == "thinking_delta":
printer.emit_delta("thinking", message_event["delta"], message_event["contentIndex"])
printer.emit_delta(
"thinking", message_event["delta"], message_event["contentIndex"]
)
def handle_message_end(event: MessageEndEvent) -> None:
if not include_messages:
@@ -745,7 +768,6 @@ def run_benchmark_for_model(
client.install_headless_ui()
verbose_cleanup = install_verbose_logging(client, model, log_mode, thinking)
def handle_tool_count(event: ToolExecutionStartEvent) -> None:
nonlocal edit_tool_calls, turns_used
if counting_edit_turns:
@@ -816,11 +838,15 @@ def run_benchmark_for_model(
)
async def run_all(spec: BenchmarkSpec, args: argparse.Namespace) -> dict[str, dict[str, Any]]:
async def run_all(
spec: BenchmarkSpec, args: argparse.Namespace
) -> dict[str, dict[str, Any]]:
omp_bin = resolve_omp_bin(args.omp_bin)
timestamp = time.strftime("%Y%m%d-%H%M%S")
workspace_root = Path(tempfile.gettempdir()) / f"{spec.workspace_prefix}-{timestamp}"
workspace_root = (
Path(tempfile.gettempdir()) / f"{spec.workspace_prefix}-{timestamp}"
)
workspace_root.mkdir(parents=True, exist_ok=True)
selected_models = args.models or MODELS
@@ -839,7 +865,9 @@ async def run_all(spec: BenchmarkSpec, args: argparse.Namespace) -> dict[str, di
omp_bin=omp_bin,
workspace=workspace,
timeout=args.timeout,
log_mode="verbose" if args.verbose else ("print" if args.print else None),
log_mode="verbose"
if args.verbose
else ("print" if args.print else None),
thinking=args.thinking,
max_turns=args.max_turns,
)
scripts/rate-edit-tool.py
+7 -3
View File
@@ -48,7 +48,7 @@ MODELS = [
"openrouter/moonshotai/kimi-k2.5",
"openrouter/anthropic/claude-haiku-4.5",
"openrouter/z-ai/glm-4.7",
"openai-codex/gpt-5.4"
"openai-codex/gpt-5.4",
]
ORACLE_MODEL = "openrouter/anthropic/claude-opus-4.6"
@@ -1472,14 +1472,18 @@ async def run_all(args: argparse.Namespace) -> int:
)
except (RpcError, RpcProcessExitError) as exc:
err = f"{type(exc).__name__}: {exc}"
(results_dir / "oracle_error.txt").write_text(err + "\n", encoding="utf-8")
(results_dir / "oracle_error.txt").write_text(
err + "\n", encoding="utf-8"
)
print(f"Oracle synthesis FAILED: {err}", file=sys.stderr)
print(f"Saved error to {results_dir}/oracle_error.txt", file=sys.stderr)
return 2
print(synthesis)
return 0
combined = format_combined_reviews(sources)
(results_dir / "combined_reviews.md").write_text(combined + "\n", encoding="utf-8")
(results_dir / "combined_reviews.md").write_text(
combined + "\n", encoding="utf-8"
)
print(combined)
return 0
scripts/session-stats/analyze_search_relevance.py
+186 -70
View File
@@ -19,6 +19,7 @@ whichever comes first.
Outputs scripts/session-stats/out/search-relevance.png.
"""
from __future__ import annotations
import argparse
@@ -36,8 +37,8 @@ import numpy as np
DB_PATH = Path.home() / ".omp" / "stats.db"
OUT_DIR = Path(__file__).resolve().parent / "out"
DEFAULT_SINCE = "2026-04-01" # search/grep traffic before this is sparse
LOOKAHEAD = 30 # max tool calls to scan after a search
DEFAULT_SINCE = "2026-04-01" # search/grep traffic before this is sparse
LOOKAHEAD = 30 # max tool calls to scan after a search
# --------------------------------------------------------------------------- #
@@ -107,6 +108,7 @@ def extract_paths(result_text: str | None) -> tuple[list[str], dict[str, int]]:
# --------------------------------------------------------------------------- #
# Tool-call helpers
def search_signature(arg_obj: dict) -> tuple:
"""Stable identity key for a search/grep call: (pattern, path-scope).
@@ -154,6 +156,7 @@ def read_path(arg_obj: dict) -> str | None:
# --------------------------------------------------------------------------- #
# Per-session walk
def classify_sessions(conn: sqlite3.Connection, since_ms: int) -> list[dict]:
"""Walks each session in seq order, classifying every search/grep call."""
# Pull calls + paired results in one ordered stream per session.
@@ -301,23 +304,33 @@ def report(records: list[dict]) -> None:
coverage = deepest_1b / result_counts
engaged_n = np.array([r["engaged_count"] for r in engaged], dtype=np.int64)
print(f"\nfor engaged-read calls (n={len(engaged):,}):")
print(f" deepest index reached p50={int(np.median(deepest_1b))} "
f"p75={int(np.percentile(deepest_1b,75))} "
f"p90={int(np.percentile(deepest_1b,90))} "
f"max={int(deepest_1b.max())}")
print(f" result list length p50={int(np.median(result_counts))} "
f"p90={int(np.percentile(result_counts,90))} "
f"max={int(result_counts.max())}")
print(f" deepest / list size p50={np.median(coverage)*100:.0f}% "
f"p25={np.percentile(coverage,25)*100:.0f}%")
print(f" reads per result list p50={int(np.median(engaged_n))} "
f"p90={int(np.percentile(engaged_n,90))}")
print(
f" deepest index reached p50={int(np.median(deepest_1b))} "
f"p75={int(np.percentile(deepest_1b, 75))} "
f"p90={int(np.percentile(deepest_1b, 90))} "
f"max={int(deepest_1b.max())}"
)
print(
f" result list length p50={int(np.median(result_counts))} "
f"p90={int(np.percentile(result_counts, 90))} "
f"max={int(result_counts.max())}"
)
print(
f" deepest / list size p50={np.median(coverage) * 100:.0f}% "
f"p25={np.percentile(coverage, 25) * 100:.0f}%"
)
print(
f" reads per result list p50={int(np.median(engaged_n))} "
f"p90={int(np.percentile(engaged_n, 90))}"
)
next_page = sum(1 for r in records if r["next_page"])
refined = sum(1 for r in records if r["refined"])
print(f"\nbehaviours (not exclusive):")
print(f" any next-page request : {next_page:,} ({100*next_page/total:.1f}%)")
print(f" any refined-query : {refined:,} ({100*refined/total:.1f}%)")
print(
f" any next-page request : {next_page:,} ({100 * next_page / total:.1f}%)"
)
print(f" any refined-query : {refined:,} ({100 * refined / total:.1f}%)")
# Shape of result lists — files per result, matches per file, and whether
# diversity (files-per-result / matches-per-file) correlates with engagement.
@@ -327,26 +340,39 @@ def report(records: list[dict]) -> None:
dtype=np.int64,
)
print(f"\nresult shape across all {total:,} calls:")
print(f" files per result "
f"p50={int(np.median(files_per_result))} "
f"p75={int(np.percentile(files_per_result,75))} "
f"p90={int(np.percentile(files_per_result,90))} "
f"p99={int(np.percentile(files_per_result,99))} "
f"max={int(files_per_result.max())}")
print(
f" files per result "
f"p50={int(np.median(files_per_result))} "
f"p75={int(np.percentile(files_per_result, 75))} "
f"p90={int(np.percentile(files_per_result, 90))} "
f"p99={int(np.percentile(files_per_result, 99))} "
f"max={int(files_per_result.max())}"
)
if matches_per_file_flat.size:
print(f" matches per file (flat) "
f"p50={int(np.median(matches_per_file_flat))} "
f"p75={int(np.percentile(matches_per_file_flat,75))} "
f"p90={int(np.percentile(matches_per_file_flat,90))} "
f"p99={int(np.percentile(matches_per_file_flat,99))} "
f"max={int(matches_per_file_flat.max())}")
print(
f" matches per file (flat) "
f"p50={int(np.median(matches_per_file_flat))} "
f"p75={int(np.percentile(matches_per_file_flat, 75))} "
f"p90={int(np.percentile(matches_per_file_flat, 90))} "
f"p99={int(np.percentile(matches_per_file_flat, 99))} "
f"max={int(matches_per_file_flat.max())}"
)
# Engagement vs shape: is the model more likely to read at all when there
# are more distinct files? When matches are more concentrated per file?
print(f"\nengagement vs result shape:")
print(f" {'files-per-result':<22} {'n calls':>9} {'engaged %':>10} {'p50 deepest':>12}")
bins = [(1, 1, "1"), (2, 2, "2"), (3, 5, "3-5"), (6, 10, "6-10"),
(11, 20, "11-20"), (21, 50, "21-50"), (51, 10**9, "51+")]
print(
f" {'files-per-result':<22} {'n calls':>9} {'engaged %':>10} {'p50 deepest':>12}"
)
bins = [
(1, 1, "1"),
(2, 2, "2"),
(3, 5, "3-5"),
(6, 10, "6-10"),
(11, 20, "11-20"),
(21, 50, "21-50"),
(51, 10**9, "51+"),
]
for lo, hi, label in bins:
bucket = [r for r in records if lo <= r["n_results"] <= hi]
if not bucket:
@@ -359,13 +385,22 @@ def report(records: list[dict]) -> None:
p50_deep = 0
print(f" {label:<22} {len(bucket):>9,} {eng_share:>9.1f}% {p50_deep:>12}")
print(f"\n {'max matches/file':<22} {'n calls':>9} {'engaged %':>10} {'p50 deepest':>12}")
bins = [(1, 1, "1"), (2, 5, "2-5"), (6, 20, "6-20"),
(21, 100, "21-100"), (101, 10**9, "100+")]
print(
f"\n {'max matches/file':<22} {'n calls':>9} {'engaged %':>10} {'p50 deepest':>12}"
)
bins = [
(1, 1, "1"),
(2, 5, "2-5"),
(6, 20, "6-20"),
(21, 100, "21-100"),
(101, 10**9, "100+"),
]
for lo, hi, label in bins:
bucket = [r for r in records
if r["matches_per_file"]
and lo <= max(r["matches_per_file"]) <= hi]
bucket = [
r
for r in records
if r["matches_per_file"] and lo <= max(r["matches_per_file"]) <= hi
]
if not bucket:
continue
eng = [r for r in bucket if r["outcome"] == "engaged-read"]
@@ -380,6 +415,7 @@ def report(records: list[dict]) -> None:
# --------------------------------------------------------------------------- #
# Plot
def plot(records: list[dict], since: str) -> Path | None:
if not records:
return None
@@ -400,8 +436,14 @@ def plot(records: list[dict], since: str) -> Path | None:
colors = [OUTCOME_COLORS[o] for o in ordered]
bars = ax.bar(ordered, pct, color=colors, edgecolor="#1f2937", linewidth=0.5)
for b, p, n in zip(bars, pct, nvals):
ax.text(b.get_x() + b.get_width() / 2, p + 1.5,
f"{p:.1f}%\nn={n:,}", ha="center", va="bottom", fontsize=9)
ax.text(
b.get_x() + b.get_width() / 2,
p + 1.5,
f"{p:.1f}%\nn={n:,}",
ha="center",
va="bottom",
fontsize=9,
)
ax.set_title(f"search outcome (n={total:,})")
ax.set_ylabel("share of calls")
ax.set_ylim(0, max(pct) + 12)
@@ -419,8 +461,14 @@ def plot(records: list[dict], since: str) -> Path | None:
pct = 100 * hist / deepest.size
bars = ax.bar(labels, pct, color="#0f766e", edgecolor="#134e4a", linewidth=0.5)
for b, p, n in zip(bars, pct, hist):
ax.text(b.get_x() + b.get_width() / 2, p + 1.2,
f"{p:.1f}%\nn={n:,}", ha="center", va="bottom", fontsize=8)
ax.text(
b.get_x() + b.get_width() / 2,
p + 1.2,
f"{p:.1f}%\nn={n:,}",
ha="center",
va="bottom",
fontsize=8,
)
ax.set_title("deepest result index the model read")
ax.set_ylabel("share of engaged-read calls")
ax.set_ylim(0, max(pct) + 12)
@@ -437,8 +485,12 @@ def plot(records: list[dict], since: str) -> Path | None:
coverage.sort()
cdf = np.arange(1, coverage.size + 1) / coverage.size
ax.plot(coverage, cdf, color="#7c3aed", linewidth=2.0)
ax.axvline(0.1, color="#9ca3af", linestyle="--", linewidth=1, label="10% of list")
ax.axvline(0.5, color="#9ca3af", linestyle=":", linewidth=1, label="50% of list")
ax.axvline(
0.1, color="#9ca3af", linestyle="--", linewidth=1, label="10% of list"
)
ax.axvline(
0.5, color="#9ca3af", linestyle=":", linewidth=1, label="50% of list"
)
ax.set_title("coverage CDF — deepest read / list size")
ax.set_xlabel("fraction of list reached")
ax.set_ylabel("CDF of engaged-read calls")
@@ -454,9 +506,14 @@ def plot(records: list[dict], since: str) -> Path | None:
sizes = [r["n_results"] for r in records if r["outcome"] == outcome]
if not sizes:
continue
ax.hist(sizes, bins=bins, histtype="step", linewidth=1.8,
color=OUTCOME_COLORS[outcome],
label=f"{outcome} (p50={int(np.median(sizes))})")
ax.hist(
sizes,
bins=bins,
histtype="step",
linewidth=1.8,
color=OUTCOME_COLORS[outcome],
label=f"{outcome} (p50={int(np.median(sizes))})",
)
ax.set_xscale("log")
ax.set_yscale("log")
ax.set_xlabel("result list size")
@@ -467,8 +524,15 @@ def plot(records: list[dict], since: str) -> Path | None:
# Panel E — engagement rate vs files-per-result, with p50 deepest overlay.
ax = axes[2, 0]
bins = [(1, 1, "1"), (2, 2, "2"), (3, 5, "3-5"), (6, 10, "6-10"),
(11, 20, "11-20"), (21, 50, "21-50"), (51, 10**9, "51+")]
bins = [
(1, 1, "1"),
(2, 2, "2"),
(3, 5, "3-5"),
(6, 10, "6-10"),
(11, 20, "11-20"),
(21, 50, "21-50"),
(51, 10**9, "51+"),
]
labels = []
eng_share = []
deep_p50 = []
@@ -481,13 +545,27 @@ def plot(records: list[dict], since: str) -> Path | None:
n_calls.append(len(bucket))
engs = [r for r in bucket if r["outcome"] == "engaged-read"]
eng_share.append(100 * len(engs) / len(bucket))
deep_p50.append(int(np.median([r["deepest_index"] + 1 for r in engs])) if engs else 0)
deep_p50.append(
int(np.median([r["deepest_index"] + 1 for r in engs])) if engs else 0
)
x = np.arange(len(labels))
bars = ax.bar(x, eng_share, color="#16a34a", edgecolor="#14532d",
linewidth=0.5, label="engaged %")
bars = ax.bar(
x,
eng_share,
color="#16a34a",
edgecolor="#14532d",
linewidth=0.5,
label="engaged %",
)
for b, p, n in zip(bars, eng_share, n_calls):
ax.text(b.get_x() + b.get_width() / 2, p + 0.8,
f"{p:.0f}%\nn={n:,}", ha="center", va="bottom", fontsize=8)
ax.text(
b.get_x() + b.get_width() / 2,
p + 0.8,
f"{p:.0f}%\nn={n:,}",
ha="center",
va="bottom",
fontsize=8,
)
ax.set_xticks(x)
ax.set_xticklabels(labels)
ax.set_ylabel("engaged %", color="#15803d")
@@ -496,37 +574,64 @@ def plot(records: list[dict], since: str) -> Path | None:
ax.set_title("engagement vs files-per-result")
ax.set_xlabel("files in result")
ax2 = ax.twinx()
ax2.plot(x, deep_p50, color="#7c3aed", marker="o", linewidth=1.8,
label="p50 deepest index")
ax2.plot(
x,
deep_p50,
color="#7c3aed",
marker="o",
linewidth=1.8,
label="p50 deepest index",
)
ax2.set_ylabel("p50 deepest index", color="#5b21b6")
ax2.tick_params(axis="y", labelcolor="#5b21b6")
ax.grid(True, axis="y", alpha=0.25, linestyle="--")
# Panel F — engagement rate vs max matches-per-file.
ax = axes[2, 1]
bins = [(1, 1, "1"), (2, 5, "2-5"), (6, 20, "6-20"),
(21, 100, "21-100"), (101, 10**9, "100+")]
bins = [
(1, 1, "1"),
(2, 5, "2-5"),
(6, 20, "6-20"),
(21, 100, "21-100"),
(101, 10**9, "100+"),
]
labels = []
eng_share = []
deep_p50 = []
n_calls = []
for lo, hi, label in bins:
bucket = [r for r in records
if r["matches_per_file"]
and lo <= max(r["matches_per_file"]) <= hi]
bucket = [
r
for r in records
if r["matches_per_file"] and lo <= max(r["matches_per_file"]) <= hi
]
if not bucket:
continue
labels.append(label)
n_calls.append(len(bucket))
engs = [r for r in bucket if r["outcome"] == "engaged-read"]
eng_share.append(100 * len(engs) / len(bucket))
deep_p50.append(int(np.median([r["deepest_index"] + 1 for r in engs])) if engs else 0)
deep_p50.append(
int(np.median([r["deepest_index"] + 1 for r in engs])) if engs else 0
)
x = np.arange(len(labels))
bars = ax.bar(x, eng_share, color="#dc2626", edgecolor="#7f1d1d",
linewidth=0.5, label="engaged %")
bars = ax.bar(
x,
eng_share,
color="#dc2626",
edgecolor="#7f1d1d",
linewidth=0.5,
label="engaged %",
)
for b, p, n in zip(bars, eng_share, n_calls):
ax.text(b.get_x() + b.get_width() / 2, p + 0.8,
f"{p:.0f}%\nn={n:,}", ha="center", va="bottom", fontsize=8)
ax.text(
b.get_x() + b.get_width() / 2,
p + 0.8,
f"{p:.0f}%\nn={n:,}",
ha="center",
va="bottom",
fontsize=8,
)
ax.set_xticks(x)
ax.set_xticklabels(labels)
ax.set_ylabel("engaged %", color="#991b1b")
@@ -535,13 +640,20 @@ def plot(records: list[dict], since: str) -> Path | None:
ax.set_title("engagement vs concentration (max matches in one file)")
ax.set_xlabel("max matches in single file")
ax2 = ax.twinx()
ax2.plot(x, deep_p50, color="#7c3aed", marker="o", linewidth=1.8,
label="p50 deepest index")
ax2.plot(
x,
deep_p50,
color="#7c3aed",
marker="o",
linewidth=1.8,
label="p50 deepest index",
)
ax2.set_ylabel("p50 deepest index", color="#5b21b6")
ax2.tick_params(axis="y", labelcolor="#5b21b6")
ax.grid(True, axis="y", alpha=0.25, linestyle="--")
fig.suptitle(f"search/grep result relevance — calls since {since}",
fontsize=13, y=1.0)
fig.suptitle(
f"search/grep result relevance — calls since {since}", fontsize=13, y=1.0
)
fig.tight_layout()
p = OUT_DIR / "search-relevance.png"
fig.savefig(p, bbox_inches="tight")
@@ -552,10 +664,14 @@ def plot(records: list[dict], since: str) -> Path | None:
# --------------------------------------------------------------------------- #
# Entry
def main() -> int:
ap = argparse.ArgumentParser(description="search/grep result relevance analysis")
ap.add_argument("--since", default=DEFAULT_SINCE,
help=f"only calls after this date (default {DEFAULT_SINCE})")
ap.add_argument(
"--since",
default=DEFAULT_SINCE,
help=f"only calls after this date (default {DEFAULT_SINCE})",
)
args = ap.parse_args()
since = datetime.strptime(args.since, "%Y-%m-%d").replace(tzinfo=timezone.utc)
scripts/session-stats/analyze_selector_reads.py
+143 -69
View File
@@ -21,6 +21,7 @@ separate cohort and excluded from interval math.
Outputs:
scripts/session-stats/out/selector-coverage.png
"""
from __future__ import annotations
import argparse
@@ -50,6 +51,7 @@ _RANGE_RE = re.compile(r"^(\d+)(?:([-+])(\d+))?$")
# --------------------------------------------------------------------------- #
# Selector parsing
def parse_selector(path: str) -> tuple[str, int | None, int | None, str]:
"""Returns (base_path, start, end, kind)."""
if not path:
@@ -79,7 +81,9 @@ def parse_selector(path: str) -> tuple[str, int | None, int | None, str]:
return base, start, start + DEFAULT_PAGE - 1, "range"
def args_to_interval(arg_json: str | None) -> tuple[str, int | None, int | None, str] | None:
def args_to_interval(
arg_json: str | None,
) -> tuple[str, int | None, int | None, str] | None:
"""Decode arg_json into (base, start, end, kind)."""
if not arg_json:
return None
@@ -96,7 +100,12 @@ def args_to_interval(arg_json: str | None) -> tuple[str, int | None, int | None,
# Legacy offset/limit.
offset = obj.get("offset")
limit = obj.get("limit")
if isinstance(offset, int) and isinstance(limit, int) and offset >= 1 and limit >= 1:
if (
isinstance(offset, int)
and isinstance(limit, int)
and offset >= 1
and limit >= 1
):
return path, offset, offset + limit - 1, "range"
if isinstance(offset, int) and offset >= 1:
return path, offset, offset + DEFAULT_PAGE - 1, "range"
@@ -108,6 +117,7 @@ def args_to_interval(arg_json: str | None) -> tuple[str, int | None, int | None,
# --------------------------------------------------------------------------- #
# Coverage math
def merge_intervals(ivs: list[tuple[int, int]]) -> list[tuple[int, int]]:
"""Merge overlapping / adjacent intervals. Inclusive bounds."""
if not ivs:
@@ -123,9 +133,7 @@ def merge_intervals(ivs: list[tuple[int, int]]) -> list[tuple[int, int]]:
return out
def classify_followup(
s: int, e: int, init_s: int, init_e: int
) -> str:
def classify_followup(s: int, e: int, init_s: int, init_e: int) -> str:
"""Where does follow-up [s,e] land relative to initial [init_s, init_e]?"""
if s >= init_s and e <= init_e:
return "inside"
@@ -144,6 +152,7 @@ def classify_followup(
# --------------------------------------------------------------------------- #
# Pull
def iter_reads(conn: sqlite3.Connection, since_ms: int):
sql = """
SELECT session_file, seq, timestamp, arg_json
@@ -156,7 +165,9 @@ def iter_reads(conn: sqlite3.Connection, since_ms: int):
def collect(conn, since_ms) -> dict[tuple[str, str], list[tuple[int, int, int, str]]]:
"""key (session, file) -> ordered list of (seq, start, end, kind)."""
by_key: dict[tuple[str, str], list[tuple[int, int | None, int | None, str]]] = defaultdict(list)
by_key: dict[tuple[str, str], list[tuple[int, int | None, int | None, str]]] = (
defaultdict(list)
)
for session, seq, _ts, arg_json in iter_reads(conn, since_ms):
parsed = args_to_interval(arg_json)
if parsed is None:
@@ -171,6 +182,7 @@ def collect(conn, since_ms) -> dict[tuple[str, str], list[tuple[int, int, int, s
# --------------------------------------------------------------------------- #
# Analyze
def analyze(by_key: dict) -> dict:
"""Compute coverage statistics over (session, file) groups whose FIRST
read is a numeric range."""
@@ -213,23 +225,27 @@ def analyze(by_key: dict) -> dict:
gap_lines = span - covered_lines
extra_lines = max(0, covered_lines - init_size) # new lines past initial
eligible.append({
"session": session,
"file": base,
"init_start": s0,
"init_end": e0,
"init_size": init_size,
"n_followups": len(followups),
"n_range_followups": sum(1 for k in followup_kinds if k in ("range", "default")),
"n_raw_followups": sum(1 for k in followup_kinds if k == "raw"),
"first_followup_pos": first_followup_pos,
"intervals": merged,
"regions": regions,
"covered": covered_lines,
"extra_lines": extra_lines,
"span": span,
"gap_lines": gap_lines,
})
eligible.append(
{
"session": session,
"file": base,
"init_start": s0,
"init_end": e0,
"init_size": init_size,
"n_followups": len(followups),
"n_range_followups": sum(
1 for k in followup_kinds if k in ("range", "default")
),
"n_raw_followups": sum(1 for k in followup_kinds if k == "raw"),
"first_followup_pos": first_followup_pos,
"intervals": merged,
"regions": regions,
"covered": covered_lines,
"extra_lines": extra_lines,
"span": span,
"gap_lines": gap_lines,
}
)
return {
"eligible": eligible,
"first_kind": dict(first_kind_counts),
@@ -242,18 +258,18 @@ def analyze(by_key: dict) -> dict:
POS_ORDER = ["forward", "backward", "inside", "both", "gap-above", "gap-below"]
POS_COLORS = {
"forward": "#2563eb",
"backward": "#0f766e",
"inside": "#9ca3af",
"both": "#7c3aed",
"forward": "#2563eb",
"backward": "#0f766e",
"inside": "#9ca3af",
"both": "#7c3aed",
"gap-above": "#dc2626",
"gap-below": "#d97706",
}
POS_HELP = {
"forward": "extended past initial end",
"backward": "extended before initial start",
"inside": "re-read inside the initial range",
"both": "extended on both sides",
"forward": "extended past initial end",
"backward": "extended before initial start",
"inside": "re-read inside the initial range",
"both": "extended on both sides",
"gap-above": "disjoint hop above initial",
"gap-below": "disjoint hop below initial",
}
@@ -269,7 +285,7 @@ def report(stats: dict) -> None:
n = first_kind.get(k, 0)
if n == 0:
continue
print(f" {k:<10} {n:>8,} {100*n/total_pairs:>5.1f}%")
print(f" {k:<10} {n:>8,} {100 * n / total_pairs:>5.1f}%")
print(f" total {total_pairs:>8,}")
if not eligible:
@@ -280,8 +296,12 @@ def report(stats: dict) -> None:
with_followup = sum(1 for e in eligible if e["n_followups"] > 0)
with_range_followup = sum(1 for e in eligible if e["n_range_followups"] > 0)
print(f"\nfor {n:,} (session, file) pairs whose first read was a range:")
print(f" any follow-up read : {with_followup:>8,} ({100*with_followup/n:.1f}%)")
print(f" follow-up with a range : {with_range_followup:>8,} ({100*with_range_followup/n:.1f}%)")
print(
f" any follow-up read : {with_followup:>8,} ({100 * with_followup / n:.1f}%)"
)
print(
f" follow-up with a range : {with_range_followup:>8,} ({100 * with_range_followup / n:.1f}%)"
)
print()
print(" ----- follow-up position breakdown (all follow-up reads) -----")
positions = stats["followup_pos"]
@@ -290,32 +310,36 @@ def report(stats: dict) -> None:
v = positions.get(k, 0)
if v == 0:
continue
print(f" {k:<10} {v:>8,} ({100*v/total_pos:>5.1f}%) -- {POS_HELP[k]}")
print(f" {k:<10} {v:>8,} ({100 * v / total_pos:>5.1f}%) -- {POS_HELP[k]}")
# Region count distribution.
regions = np.array([e["regions"] for e in eligible], dtype=np.int64)
print(f"\ndisjoint regions in final coverage (per session/file):")
print(f" mean={regions.mean():.2f} median={int(np.median(regions))} "
f"p90={int(np.percentile(regions,90))} max={int(regions.max())}")
print(
f" mean={regions.mean():.2f} median={int(np.median(regions))} "
f"p90={int(np.percentile(regions, 90))} max={int(regions.max())}"
)
edges = [1, 2, 3, 4, 6, 11, 10**6]
labels = ["1 (contig)", "2", "3", "4-5", "6-10", "11+"]
hist, _ = np.histogram(regions, bins=edges)
for label, nb in zip(labels, hist):
print(f" {label:<10} {nb:>8,} ({100*nb/regions.size:>5.1f}%)")
print(f" {label:<10} {nb:>8,} ({100 * nb / regions.size:>5.1f}%)")
# Extra lines vs initial (only when follow-ups exist).
fu = [e for e in eligible if e["n_range_followups"] > 0]
extra = np.array([e["extra_lines"] for e in fu], dtype=np.int64)
if extra.size:
print(f"\nextra lines covered beyond initial range (n={extra.size:,}):")
print(f" mean={extra.mean():.0f} median={int(np.median(extra))} "
f"p75={int(np.percentile(extra,75))} p90={int(np.percentile(extra,90))} "
f"max={int(extra.max())}")
print(
f" mean={extra.mean():.0f} median={int(np.median(extra))} "
f"p75={int(np.percentile(extra, 75))} p90={int(np.percentile(extra, 90))} "
f"max={int(extra.max())}"
)
edges = [0, 1, 51, 201, 501, 2001, 10**9]
labels = ["0 (no new)", "1-50", "51-200", "201-500", "501-2000", "2000+"]
hist, _ = np.histogram(extra, bins=edges)
for label, nb in zip(labels, hist):
print(f" {label:<12} {nb:>8,} ({100*nb/extra.size:>5.1f}%)")
print(f" {label:<12} {nb:>8,} ({100 * nb / extra.size:>5.1f}%)")
# Coverage ratio.
init_sizes = np.array([e["init_size"] for e in fu], dtype=np.int64)
@@ -325,13 +349,16 @@ def report(stats: dict) -> None:
ratio = np.where(init_sizes > 0, covered / init_sizes, np.nan)
ratio = ratio[np.isfinite(ratio)]
print(f"\ntotal covered / initial size:")
print(f" mean={ratio.mean():.2f}x median={np.median(ratio):.2f}x "
f"p90={np.percentile(ratio,90):.2f}x")
print(
f" mean={ratio.mean():.2f}x median={np.median(ratio):.2f}x "
f"p90={np.percentile(ratio, 90):.2f}x"
)
# --------------------------------------------------------------------------- #
# Plot
def plot(stats: dict, since: str) -> Path | None:
eligible = stats["eligible"]
if not eligible:
@@ -352,11 +379,15 @@ def plot(stats: dict, since: str) -> Path | None:
colors = [POS_COLORS[k] for k in keys]
bars = ax.bar(keys, vals, color=colors, edgecolor="#111", linewidth=0.5)
for bar, v, k in zip(bars, vals, keys):
ax.text(bar.get_x() + bar.get_width() / 2, v + 1.0,
f"{v:.1f}%\nn={pos[k]:,}", ha="center", va="bottom", fontsize=8)
ax.set_title(
f"where do follow-up reads land vs initial range (n={total:,})"
)
ax.text(
bar.get_x() + bar.get_width() / 2,
v + 1.0,
f"{v:.1f}%\nn={pos[k]:,}",
ha="center",
va="bottom",
fontsize=8,
)
ax.set_title(f"where do follow-up reads land vs initial range (n={total:,})")
ax.set_ylabel("share of follow-up reads")
ax.set_ylim(0, max(vals) + 12)
ax.yaxis.set_major_formatter(plt.FuncFormatter(lambda v, _: f"{v:.0f}%"))
@@ -372,8 +403,14 @@ def plot(stats: dict, since: str) -> Path | None:
colors = ["#16a34a"] + ["#2563eb"] * 5
bars = ax.bar(labels, pct, color=colors, edgecolor="#111", linewidth=0.5)
for bar, p, h in zip(bars, pct, hist):
ax.text(bar.get_x() + bar.get_width() / 2, p + 1.5,
f"{p:.1f}%\nn={h:,}", ha="center", va="bottom", fontsize=8)
ax.text(
bar.get_x() + bar.get_width() / 2,
p + 1.5,
f"{p:.1f}%\nn={h:,}",
ha="center",
va="bottom",
fontsize=8,
)
ax.set_title(f"disjoint regions in final coverage (n={regions.size:,})")
ax.set_ylabel("share of (session, file) pairs")
ax.set_ylim(0, max(pct.max() + 12, 20))
@@ -391,8 +428,14 @@ def plot(stats: dict, since: str) -> Path | None:
pct = 100 * hist / extra.size
bars = ax.bar(labels, pct, color="#d97706", edgecolor="#7c2d12", linewidth=0.5)
for bar, p, h in zip(bars, pct, hist):
ax.text(bar.get_x() + bar.get_width() / 2, p + 1.5,
f"{p:.1f}%\nn={h:,}", ha="center", va="bottom", fontsize=8)
ax.text(
bar.get_x() + bar.get_width() / 2,
p + 1.5,
f"{p:.1f}%\nn={h:,}",
ha="center",
va="bottom",
fontsize=8,
)
ax.set_title(
f"extra lines covered beyond initial range\n"
f"(only pairs that follow up, n={extra.size:,})"
@@ -413,23 +456,37 @@ def plot(stats: dict, since: str) -> Path | None:
ratio.sort()
cdf = np.arange(1, ratio.size + 1) / ratio.size
ax.plot(ratio, cdf, color="#0f766e", linewidth=1.9)
ax.axvline(1.0, color="#9ca3af", linestyle="--", linewidth=1.0,
label="covered = initial")
ax.axvline(
1.0,
color="#9ca3af",
linestyle="--",
linewidth=1.0,
label="covered = initial",
)
for q in (0.5, 0.9):
x = np.interp(q, cdf, ratio)
ax.scatter([x], [q], color="#dc2626", s=22, zorder=3)
ax.annotate(f"p{int(q*100)}={x:.2f}x", (x, q),
textcoords="offset points", xytext=(6, -8), fontsize=9)
ax.annotate(
f"p{int(q * 100)}={x:.2f}x",
(x, q),
textcoords="offset points",
xytext=(6, -8),
fontsize=9,
)
ax.set_xscale("log")
ax.set_xlim(0.8, max(ratio.max(), 10))
ax.set_xlabel("total covered lines / initial range (×, log)")
ax.set_ylabel("CDF of pairs with follow-up")
ax.set_title(f"how much of the file does the session end up reading? (n={ratio.size:,})")
ax.set_title(
f"how much of the file does the session end up reading? (n={ratio.size:,})"
)
ax.set_ylim(0, 1.01)
ax.legend(loc="lower right", frameon=False)
ax.grid(True, which="both", alpha=0.25, linestyle="--")
fig.suptitle(f"selector reads — coverage map analysis (since {since})", fontsize=13, y=0.995)
fig.suptitle(
f"selector reads — coverage map analysis (since {since})", fontsize=13, y=0.995
)
fig.tight_layout()
p = OUT_DIR / "selector-coverage.png"
fig.savefig(p, bbox_inches="tight")
@@ -440,6 +497,7 @@ def plot(stats: dict, since: str) -> Path | None:
# --------------------------------------------------------------------------- #
# Examples
def dump_examples(stats: dict, k: int = 8) -> None:
"""Print a few coverage-map examples for sanity / intuition."""
fu = [e for e in stats["eligible"] if e["n_range_followups"] > 0]
@@ -456,7 +514,11 @@ def dump_examples(stats: dict, k: int = 8) -> None:
for r in sorted(buckets.keys()):
candidates = buckets[r]
# Prefer ones with non-default initial windows.
non_default = [c for c in candidates if (c["init_start"], c["init_end"]) != (1, DEFAULT_PAGE)]
non_default = [
c
for c in candidates
if (c["init_start"], c["init_end"]) != (1, DEFAULT_PAGE)
]
chosen = non_default[0] if non_default else candidates[0]
picks.append(chosen)
if len(picks) >= k:
@@ -483,20 +545,30 @@ def dump_examples(stats: dict, k: int = 8) -> None:
bar[i] = "▓"
bar_str = "".join(bar)
file_short = e["file"][-50:]
print(f" [{bar_str}] regions={e['regions']:>2} "
f"init=[{e['init_start']},{e['init_end']}] "
f"covered={e['covered']:>4} span={e['span']:>4} {file_short}")
print(
f" [{bar_str}] regions={e['regions']:>2} "
f"init=[{e['init_start']},{e['init_end']}] "
f"covered={e['covered']:>4} span={e['span']:>4} {file_short}"
)
# --------------------------------------------------------------------------- #
# Entry
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__.splitlines()[1])
ap.add_argument("--since", default=DEFAULT_SINCE,
help=f"only reads at or after this date (default {DEFAULT_SINCE})")
ap.add_argument("--examples", type=int, default=8,
help="how many coverage-map examples to print (default 8)")
ap.add_argument(
"--since",
default=DEFAULT_SINCE,
help=f"only reads at or after this date (default {DEFAULT_SINCE})",
)
ap.add_argument(
"--examples",
type=int,
default=8,
help="how many coverage-map examples to print (default 8)",
)
args = ap.parse_args()
since = datetime.strptime(args.since, "%Y-%m-%d").replace(tzinfo=timezone.utc)
@@ -508,8 +580,10 @@ def main() -> int:
by_key = collect(conn, since_ms)
conn.close()
print(f"loaded {sum(len(v) for v in by_key.values()):,} read calls across "
f"{len(by_key):,} (session, file) pairs since {args.since}")
print(
f"loaded {sum(len(v) for v in by_key.values()):,} read calls across "
f"{len(by_key):,} (session, file) pairs since {args.since}"
)
stats = analyze(by_key)
report(stats)
scripts/session-stats/harmony_backtest.py
+164 -56
View File
@@ -25,8 +25,12 @@ DB_PATH = Path.home() / ".omp" / "stats.db"
MARKER_RE = re.compile(r"\bto=functions\.[A-Za-z_][A-Za-z0-9_]*")
HARMONY_RE = re.compile(r"<\|(start|end|channel|message|call|return)\|>")
CHANNEL_WORD_RE = re.compile(r"\b(analysis|commentary|assistant|user|system|developer|tool)\s+to=functions\.")
GLITCH_RE = re.compile(r"\b(changedFiles|RTLU|Jsii(?:_commentary)?|Japgolly|tRTLUfunctions|Joshi_commentary|Japgolly_commentary|jsii_commentary|Jsii_commentary|Jsii)\b")
CHANNEL_WORD_RE = re.compile(
r"\b(analysis|commentary|assistant|user|system|developer|tool)\s+to=functions\."
)
GLITCH_RE = re.compile(
r"\b(changedFiles|RTLU|Jsii(?:_commentary)?|Japgolly|tRTLUfunctions|Joshi_commentary|Japgolly_commentary|jsii_commentary|Jsii_commentary|Jsii)\b"
)
NULLISH_RE = re.compile(r"\b(undefined|null)\b")
BODY_CASCADE_RE = re.compile(r"\bto=functions\.[A-Za-z_][A-Za-z0-9_]*\s+code(?:\s|$)")
FAKE_RESULT_RE = re.compile(
@@ -38,18 +42,18 @@ FENCE_RE = re.compile(r"^\s*(```+|~~~+)")
# ranges local and explicit.
SCRIPT_RUN_RE = re.compile(
"["
"\u3400-\u4DBF" # CJK Extension A
"\u4E00-\u9FFF" # CJK Unified Ideographs
"\uF900-\uFAFF" # CJK Compatibility Ideographs
"\u0400-\u04FF" # Cyrillic
"\u0E00-\u0E7F" # Thai
"\u10A0-\u10FF" # Georgian
"\u0530-\u058F" # Armenian
"\u0C80-\u0CFF" # Kannada
"\u0C00-\u0C7F" # Telugu
"\u0900-\u097F" # Devanagari
"\u0600-\u06FF" # Arabic
"\u0D00-\u0D7F" # Malayalam
"\u3400-\u4dbf" # CJK Extension A
"\u4e00-\u9fff" # CJK Unified Ideographs
"\uf900-\ufaff" # CJK Compatibility Ideographs
"\u0400-\u04ff" # Cyrillic
"\u0e00-\u0e7f" # Thai
"\u10a0-\u10ff" # Georgian
"\u0530-\u058f" # Armenian
"\u0c80-\u0cff" # Kannada
"\u0c00-\u0c7f" # Telugu
"\u0900-\u097f" # Devanagari
"\u0600-\u06ff" # Arabic
"\u0d00-\u0d7f" # Malayalam
"]{2,}"
)
@@ -64,9 +68,15 @@ BEGIN_PATCH_RE = re.compile(r"^\*\*\* Begin Patch\s*$")
END_PATCH_RE = re.compile(r"^\*\*\* End Patch\s*$")
# Legacy hashline ops (kept for historical session corpus).
LEGACY_INSERT_RE = re.compile(r"^(?P<op>[«»])\s*(?P<anchor>BOF|EOF|[1-9][0-9]*[A-Za-z]{2})\s*$")
LEGACY_RANGE_RE = re.compile(r"(?P<a>[1-9][0-9]*[A-Za-z]{2})(?:\.\.(?P<b>[1-9][0-9]*[A-Za-z]{2}))?")
LEGACY_REPLACE_RE = re.compile(r"^≔\s*(?P<range>[1-9][0-9]*[A-Za-z]{2}(?:\.\.[1-9][0-9]*[A-Za-z]{2})?)\s*$")
LEGACY_INSERT_RE = re.compile(
r"^(?P<op>[«»])\s*(?P<anchor>BOF|EOF|[1-9][0-9]*[A-Za-z]{2})\s*$"
)
LEGACY_RANGE_RE = re.compile(
r"(?P<a>[1-9][0-9]*[A-Za-z]{2})(?:\.\.(?P<b>[1-9][0-9]*[A-Za-z]{2}))?"
)
LEGACY_REPLACE_RE = re.compile(
r"^≔\s*(?P<range>[1-9][0-9]*[A-Za-z]{2}(?:\.\.[1-9][0-9]*[A-Za-z]{2})?)\s*$"
)
# Current hashline ops.
NEW_INSERT_RE = re.compile(
@@ -183,7 +193,6 @@ def commas(n: int) -> str:
return f"{n:,}"
def one_line(text: str, limit: int = 180) -> str:
text = text.replace("\r", "\\r").replace("\n", " | ").replace("\t", "\\t")
if len(text) <= limit:
@@ -242,7 +251,9 @@ def marker_evidence_for(
window16 = text[max(0, start - 16) : min(len(text), end + 16)]
window200 = text[start : min(len(text), start + 200)]
for c in CHANNEL_WORD_RE.finditer(text[max(0, start - 64) : min(len(text), end + 16)]):
for c in CHANNEL_WORD_RE.finditer(
text[max(0, start - 64) : min(len(text), end + 16)]
):
absolute_start = max(0, start - 64) + c.start()
absolute_end = max(0, start - 64) + c.end()
if absolute_start <= start < absolute_end:
@@ -255,7 +266,9 @@ def marker_evidence_for(
ev.classes.add("N")
if script_mismatch_near(text, start, end):
ev.classes.add("S")
if BODY_CASCADE_RE.match(window200) and MARKER_RE.search(window200[marker.end() - start :]):
if BODY_CASCADE_RE.match(window200) and MARKER_RE.search(
window200[marker.end() - start :]
):
ev.classes.add("B")
if FAKE_RESULT_RE.match(text, start):
ev.classes.add("R")
@@ -280,7 +293,9 @@ def detect_signals(
signals.append(Signal("H", h.start(), h.end(), h.group(0)))
for marker in MARKER_RE.finditer(text):
ev = marker_evidence_for(text, marker, parsed_end, respect_fences, include_nullish)
ev = marker_evidence_for(
text, marker, parsed_end, respect_fences, include_nullish
)
if ev is None:
continue
marker_evidence.append(ev)
@@ -319,6 +334,7 @@ def line_spans(text: str) -> list[tuple[str, int, int]]:
# splitlines(keepends=True) already includes the final unterminated line.
return out
def parse_legacy_diff_boundary(text: str, *, loose_tail: bool = False) -> EditBoundary:
"""Best-effort parser for pre-hashline edit inputs.
@@ -342,7 +358,12 @@ def parse_legacy_diff_boundary(text: str, *, loose_tail: bool = False) -> EditBo
for line, _start, end in line_spans(text):
line_no += 1
if loose_tail and (MARKER_RE.search(line) or HARMONY_RE.search(line)) and cur is not None and cur.op_count > 0:
if (
loose_tail
and (MARKER_RE.search(line) or HARMONY_RE.search(line))
and cur is not None
and cur.op_count > 0
):
break
if line.startswith("---"):
@@ -395,7 +416,9 @@ def parse_legacy_diff_boundary(text: str, *, loose_tail: bool = False) -> EditBo
in_payload = True
continue
if in_payload and (line.startswith("-") or line.startswith(" ") or line.startswith("\\")):
if in_payload and (
line.startswith("-") or line.startswith(" ") or line.startswith("\\")
):
if line.startswith("-") and not line.startswith("---"):
cur.deleted_lines += 1
parsed_end = end
@@ -415,7 +438,9 @@ def parse_legacy_diff_boundary(text: str, *, loose_tail: bool = False) -> EditBo
return EditBoundary(
ok=parsed_end > 0 and bool(sections),
parsed_end=parsed_end,
reason="legacy-edit-ok" if parsed_end > 0 and sections else "no-complete-edit-prefix",
reason="legacy-edit-ok"
if parsed_end > 0 and sections
else "no-complete-edit-prefix",
sections=sections,
line_no=line_no,
)
@@ -547,7 +572,9 @@ def parse_edit_boundary(text: str, *, legacy_loose_tail: bool = False) -> EditBo
if rng:
sigil = rng.group("sigil")
cur.op_count += 1
cur.deleted_lines += new_range_deleted_lines(rng.group("a"), rng.group("b"))
cur.deleted_lines += new_range_deleted_lines(
rng.group("a"), rng.group("b")
)
if sigil == ":" and rng.group("inline"):
cur.payload_lines += 1
parsed_end = end
@@ -620,19 +647,36 @@ def parse_arg_json(raw: str) -> tuple[Any | None, bool, str]:
return None, False, f"json-error:{exc.pos}:{exc.msg}"
def extract_primary_text(tool_name: str, arg_json: str, parsed: Any | None) -> tuple[str, str]:
if tool_name == "edit" and isinstance(parsed, dict) and isinstance(parsed.get("input"), str):
def extract_primary_text(
tool_name: str, arg_json: str, parsed: Any | None
) -> tuple[str, str]:
if (
tool_name == "edit"
and isinstance(parsed, dict)
and isinstance(parsed.get("input"), str)
):
return "edit.input", parsed["input"]
if tool_name == "eval" and isinstance(parsed, dict) and isinstance(parsed.get("input"), str):
if (
tool_name == "eval"
and isinstance(parsed, dict)
and isinstance(parsed.get("input"), str)
):
return "eval.input", parsed["input"]
if tool_name == "write" and isinstance(parsed, dict) and isinstance(parsed.get("content"), str):
if (
tool_name == "write"
and isinstance(parsed, dict)
and isinstance(parsed.get("content"), str)
):
return "write.content", parsed["content"]
if tool_name == "bash" and isinstance(parsed, dict) and isinstance(parsed.get("command"), str):
if (
tool_name == "bash"
and isinstance(parsed, dict)
and isinstance(parsed.get("command"), str)
):
return "bash.command", parsed["command"]
return "arg_json", arg_json
def action_for_tool(
tool_name: str,
surface: str,
@@ -642,7 +686,12 @@ def action_for_tool(
) -> str:
if not signals:
return "allow"
if tool_name == "edit" and surface == "edit.input" and boundary is not None and boundary.ok:
if (
tool_name == "edit"
and surface == "edit.input"
and boundary is not None
and boundary.ok
):
if all(sig.start >= boundary.parsed_end for sig in signals):
return "sanitize_tail"
return "abort_replay"
@@ -782,7 +831,9 @@ def candidate_where(column: str) -> str:
)
def scan_tools(conn: sqlite3.Connection, args: argparse.Namespace) -> list[ToolBacktest]:
def scan_tools(
conn: sqlite3.Connection, args: argparse.Namespace
) -> list[ToolBacktest]:
where = candidate_where("arg_json")
params: list[Any] = []
if args.provider:
@@ -814,7 +865,9 @@ def scan_tools(conn: sqlite3.Connection, args: argparse.Namespace) -> list[ToolB
]
def scan_assistant(conn: sqlite3.Connection, args: argparse.Namespace) -> list[TextBacktest]:
def scan_assistant(
conn: sqlite3.Connection, args: argparse.Namespace
) -> list[TextBacktest]:
if not args.include_assistant:
return []
text_where = candidate_where("text_blob")
@@ -871,8 +924,13 @@ def print_tool_summary(results: list[ToolBacktest]) -> None:
print("=== tool-call scan ===")
print(f"candidate rows: {commas(len(results))}")
print_counter("\nby action:", Counter(r.action for r in results))
print_counter("\nby tool/action:", Counter(f"{r.tool_name}:{r.action}" for r in results))
print_counter("\nby model/action:", Counter(f"{r.model or '<unknown>'}:{r.action}" for r in results))
print_counter(
"\nby tool/action:", Counter(f"{r.tool_name}:{r.action}" for r in results)
)
print_counter(
"\nby model/action:",
Counter(f"{r.model or '<unknown>'}:{r.action}" for r in results),
)
signal_counter: Counter[str] = Counter()
for r in results:
if r.signals:
@@ -890,9 +948,17 @@ def print_tool_summary(results: list[ToolBacktest]) -> None:
print(f" sanitize_tail: {commas(sanitized)}")
print(f" abort_replay: {commas(aborted)}")
if sanitized:
preserved_ops = sum(r.edit_ops for r in edit_results if r.action == "sanitize_tail")
preserved_payload = sum(r.edit_payload_lines for r in edit_results if r.action == "sanitize_tail")
removed = sum(r.removed_len for r in edit_results if r.action == "sanitize_tail")
preserved_ops = sum(
r.edit_ops for r in edit_results if r.action == "sanitize_tail"
)
preserved_payload = sum(
r.edit_payload_lines
for r in edit_results
if r.action == "sanitize_tail"
)
removed = sum(
r.removed_len for r in edit_results if r.action == "sanitize_tail"
)
print(f" ops preserved by sanitize: {commas(preserved_ops)}")
print(f" payload lines preserved: {commas(preserved_payload)}")
print(f" tail bytes removed: {commas(removed)}")
@@ -904,28 +970,40 @@ def print_text_summary(results: list[TextBacktest]) -> None:
print("\n=== assistant message scan ===")
print(f"candidate surfaces: {commas(len(results))}")
print_counter("\nby action:", Counter(r.action for r in results))
print_counter("\nby surface/action:", Counter(f"{r.surface}:{r.action}" for r in results))
print_counter("\nby model/action:", Counter(f"{r.model or '<unknown>'}:{r.action}" for r in results))
print_counter(
"\nby surface/action:", Counter(f"{r.surface}:{r.action}" for r in results)
)
print_counter(
"\nby model/action:",
Counter(f"{r.model or '<unknown>'}:{r.action}" for r in results),
)
def signal_summary(labels: list[str], limit: int = 6) -> str:
if not labels:
return "none"
counts = Counter(labels)
parts = [f"{label}x{count}" if count > 1 else label for label, count in counts.most_common(limit)]
parts = [
f"{label}x{count}" if count > 1 else label
for label, count in counts.most_common(limit)
]
rest = sum(counts.values()) - sum(count for _, count in counts.most_common(limit))
if rest:
parts.append(f"+{rest} more")
return ",".join(parts)
def print_examples(results: list[ToolBacktest], show: int) -> None:
if show <= 0:
return
print(f"\n=== sanitize_tail edit examples (up to {show}) ===")
sanitize_examples = [r for r in results if r.tool_name == "edit" and r.action == "sanitize_tail"]
sanitize_examples = [
r for r in results if r.tool_name == "edit" and r.action == "sanitize_tail"
]
for r in sanitize_examples[:show]:
print(f"\n[id={r.row_id} seq={r.seq} model={r.model} signals={signal_summary(r.signals)}]")
print(
f"\n[id={r.row_id} seq={r.seq} model={r.model} signals={signal_summary(r.signals)}]"
)
print(f"session: {r.session_file}")
print(f"file(s): {', '.join(r.edit_files) if r.edit_files else '<none>'}")
print(
@@ -938,7 +1016,9 @@ def print_examples(results: list[ToolBacktest], show: int) -> None:
print(f"\n=== abort_replay examples (up to {show}) ===")
abort_examples = [r for r in results if r.action == "abort_replay"]
for r in abort_examples[:show]:
print(f"\n[id={r.row_id} tool={r.tool_name} surface={r.surface} seq={r.seq} model={r.model} signals={signal_summary(r.signals)}]")
print(
f"\n[id={r.row_id} tool={r.tool_name} surface={r.surface} seq={r.seq} model={r.model} signals={signal_summary(r.signals)}]"
)
print(f"session: {r.session_file}")
if r.tool_name == "edit":
print(
@@ -948,7 +1028,9 @@ def print_examples(results: list[ToolBacktest], show: int) -> None:
print(f"context: {r.context_preview}")
def write_json_report(path: Path, tools: list[ToolBacktest], texts: list[TextBacktest]) -> None:
def write_json_report(
path: Path, tools: list[ToolBacktest], texts: list[TextBacktest]
) -> None:
def tool_dict(r: ToolBacktest) -> dict[str, Any]:
return {
"surface": r.surface,
@@ -998,7 +1080,9 @@ def write_json_report(path: Path, tools: list[ToolBacktest], texts: list[TextBac
"assistant_surfaces": [text_dict(r) for r in texts],
}
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
path.write_text(
json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
def main() -> int:
@@ -1016,22 +1100,46 @@ def main() -> int:
"tail = only H or marker after parsed boundary"
),
)
ap.add_argument("--provider", default=None, help="restrict tool/message rows to a provider")
ap.add_argument("--model", default=None, help="restrict tool/message rows to a model")
ap.add_argument(
"--provider", default=None, help="restrict tool/message rows to a provider"
)
ap.add_argument(
"--model", default=None, help="restrict tool/message rows to a model"
)
ap.add_argument("--tool", default=None, help="restrict tool-call rows to one tool")
ap.add_argument("--include-assistant", action="store_true", help="also scan assistant text/thinking surfaces")
ap.add_argument("--include-nullish", action="store_true", help="treat adjacent null/undefined as signal N")
ap.add_argument("--no-fence-context", action="store_true", help="do not exempt Markdown fenced blocks")
ap.add_argument("--legacy-loose-tail", action="store_true", help="model old raw-payload edit inputs as tail-sanitizable at first marker line")
ap.add_argument(
"--include-assistant",
action="store_true",
help="also scan assistant text/thinking surfaces",
)
ap.add_argument(
"--include-nullish",
action="store_true",
help="treat adjacent null/undefined as signal N",
)
ap.add_argument(
"--no-fence-context",
action="store_true",
help="do not exempt Markdown fenced blocks",
)
ap.add_argument(
"--legacy-loose-tail",
action="store_true",
help="model old raw-payload edit inputs as tail-sanitizable at first marker line",
)
ap.add_argument("--show", type=int, default=8, help="examples per action group")
ap.add_argument("--json-out", type=Path, default=None, help="write machine-readable report")
ap.add_argument(
"--json-out", type=Path, default=None, help="write machine-readable report"
)
args = ap.parse_args()
conn = open_ro(args.db)
print("=== harmony leak backtest ===")
print(f"db: {args.db}")
print(f"strategy: {args.strategy}")
print(f"fences: {'ignored for action' if not args.no_fence_context else 'scanned as active text'}")
print(
f"fences: {'ignored for action' if not args.no_fence_context else 'scanned as active text'}"
)
if args.legacy_loose_tail:
print("legacy: loose tail mode")
if args.provider:
scripts/session-stats/optimize_read_config.py
+284 -103
View File
@@ -23,6 +23,7 @@ Output:
scripts/session-stats/out/read-config-sweep.png
console table with the recommended config + savings
"""
from __future__ import annotations
import argparse
@@ -64,19 +65,18 @@ ROUNDTRIP_OVERHEAD = 200
# Selector parser (reuses the same rules as analyze_selector_reads.py).
_RANGE_RE = re.compile(r"^(\d+)(?:([-+])(\d+))?$")
_FOOTER_RE = re.compile(
r"\[Showing lines (\d+)-(\d+) of (\d+)\."
)
_FOOTER_RE = re.compile(r"\[Showing lines (\d+)-(\d+) of (\d+)\.")
_TRUNCATED_RE = re.compile(r"\[Output truncated")
# --------------------------------------------------------------------------- #
# Selector → intent
class Intent(NamedTuple):
kind: str # 'bare' | 'range' | 'raw' | 'conflicts' | 'other'
start: int | None # requested start line (1-indexed) — only meaningful for 'range'
end: int | None # requested end line (1-indexed, inclusive) — None = open-ended
kind: str # 'bare' | 'range' | 'raw' | 'conflicts' | 'other'
start: int | None # requested start line (1-indexed) — only meaningful for 'range'
end: int | None # requested end line (1-indexed, inclusive) — None = open-ended
def parse_selector(path: str) -> tuple[str, Intent]:
@@ -119,7 +119,12 @@ def parse_args(arg_json: str | None) -> tuple[str | None, Intent]:
# Legacy offset/limit treated as an explicit range.
offset = obj.get("offset")
limit = obj.get("limit")
if isinstance(offset, int) and isinstance(limit, int) and offset >= 1 and limit >= 1:
if (
isinstance(offset, int)
and isinstance(limit, int)
and offset >= 1
and limit >= 1
):
return path, Intent("range", offset, offset + limit - 1)
if isinstance(offset, int) and offset >= 1:
return path, Intent("range", offset, None)
@@ -129,6 +134,7 @@ def parse_args(arg_json: str | None) -> tuple[str | None, Intent]:
# --------------------------------------------------------------------------- #
# Footer parser → (returned_start, returned_end, file_total_lines)
def parse_footer(tail: str | None) -> tuple[int | None, int | None, int | None, bool]:
"""Returns (returned_a, returned_b, file_total, was_byte_truncated)."""
if not tail:
@@ -136,13 +142,18 @@ def parse_footer(tail: str | None) -> tuple[int | None, int | None, int | None,
m = _FOOTER_RE.search(tail)
if not m:
return None, None, None, bool(_TRUNCATED_RE.search(tail))
return (int(m.group(1)), int(m.group(2)), int(m.group(3)),
bool(_TRUNCATED_RE.search(tail)))
return (
int(m.group(1)),
int(m.group(2)),
int(m.group(3)),
bool(_TRUNCATED_RE.search(tail)),
)
# --------------------------------------------------------------------------- #
# Coverage utilities
def merge(ivs: list[tuple[int, int]]) -> list[tuple[int, int]]:
if not ivs:
return []
@@ -186,15 +197,16 @@ def subtract(s: int, e: int, ivs: list[tuple[int, int]]) -> list[tuple[int, int]
# --------------------------------------------------------------------------- #
# Data model
class ReadCall(NamedTuple):
seq: int
intent: Intent
base: str
actual_a: int | None # what came back: lines [actual_a, actual_b]
actual_a: int | None # what came back: lines [actual_a, actual_b]
actual_b: int | None
file_total: int | None # from footer
tokens: int # observed result tokens
was_truncated: bool # [Output truncated marker present
file_total: int | None # from footer
tokens: int # observed result tokens
was_truncated: bool # [Output truncated marker present
def fetch_reads(conn: sqlite3.Connection, since_ms: int) -> list[tuple[str, ReadCall]]:
@@ -223,16 +235,30 @@ def fetch_reads(conn: sqlite3.Connection, since_ms: int) -> list[tuple[str, Read
if not base or base.endswith("/") or "://" in base:
continue
actual_a, actual_b, file_total, was_trunc = parse_footer(tail)
out.append((session, ReadCall(seq, intent, base, actual_a, actual_b,
file_total, int(tokens), was_trunc)))
out.append(
(
session,
ReadCall(
seq,
intent,
base,
actual_a,
actual_b,
file_total,
int(tokens),
was_trunc,
),
)
)
return out
# --------------------------------------------------------------------------- #
# Per-file aggregates
class FileStats(NamedTuple):
size_lines: int # best-effort estimate
size_lines: int # best-effort estimate
tokens_per_line: float
bytes_per_line: float # only when we can derive (currently we can't, so fallback)
@@ -271,11 +297,7 @@ def aggregate_files(reads: list[tuple[str, ReadCall]]) -> dict[str, FileStats]:
by_file_tok_lines[rc.base].append((rc.tokens, n))
out: dict[str, FileStats] = {}
files = (
set(by_file_total_lines)
| set(by_file_max_end)
| set(by_file_tok_lines)
)
files = set(by_file_total_lines) | set(by_file_max_end) | set(by_file_tok_lines)
for f in files:
size = by_file_total_lines.get(f) or by_file_max_end.get(f, 1)
tok_lines = by_file_tok_lines.get(f, [])
@@ -285,15 +307,19 @@ def aggregate_files(reads: list[tuple[str, ReadCall]]) -> dict[str, FileStats]:
tpl = tot_tok / max(tot_ln, 1)
else:
tpl = FALLBACK_TPL
out[f] = FileStats(size_lines=size, tokens_per_line=tpl,
bytes_per_line=max(8.0, tpl * 4.0))
out[f] = FileStats(
size_lines=size, tokens_per_line=tpl, bytes_per_line=max(8.0, tpl * 4.0)
)
return out
# --------------------------------------------------------------------------- #
# Per-pair grouping
def group_pairs(reads: list[tuple[str, ReadCall]]) -> dict[tuple[str, str], list[ReadCall]]:
def group_pairs(
reads: list[tuple[str, ReadCall]],
) -> dict[tuple[str, str], list[ReadCall]]:
by_pair: dict[tuple[str, str], list[ReadCall]] = defaultdict(list)
for session, rc in reads:
by_pair[(session, rc.base)].append(rc)
@@ -304,15 +330,18 @@ def group_pairs(reads: list[tuple[str, ReadCall]]) -> dict[tuple[str, str], list
# --------------------------------------------------------------------------- #
# Simulator
class Config(NamedTuple):
default_page: int # lines returned for a bare path read
line_cap: int # absolute max lines per read
byte_cap: int # max bytes per read (modelled as line cap via bytes_per_line)
default_page: int # lines returned for a bare path read
line_cap: int # absolute max lines per read
byte_cap: int # max bytes per read (modelled as line cap via bytes_per_line)
summarize_min: int # min file size (lines) for summarizer to fire on bare reads
# (-1 disables summarizer; 0 = always)
# (-1 disables summarizer; 0 = always)
def effective_returned(rc: ReadCall, fs: FileStats, cfg: Config) -> tuple[int, int] | None:
def effective_returned(
rc: ReadCall, fs: FileStats, cfg: Config
) -> tuple[int, int] | None:
"""Range the tool actually returns for one call under cfg.
Honours intent (what the agent asked for), then applies (default_page,
@@ -330,7 +359,9 @@ def effective_returned(rc: ReadCall, fs: FileStats, cfg: Config) -> tuple[int, i
start, end_intent = 1, cfg.default_page
elif intent.kind == "range":
start = intent.start or 1
end_intent = intent.end if intent.end is not None else (start + cfg.default_page - 1)
end_intent = (
intent.end if intent.end is not None else (start + cfg.default_page - 1)
)
elif intent.kind == "raw":
start, end_intent = 1, size
else:
@@ -343,11 +374,17 @@ def effective_returned(rc: ReadCall, fs: FileStats, cfg: Config) -> tuple[int, i
return (start, end)
def cost_of_chunk(start: int, end: int, fs: FileStats, intent_kind: str, cfg: Config) -> float:
def cost_of_chunk(
start: int, end: int, fs: FileStats, intent_kind: str, cfg: Config
) -> float:
"""Estimated result tokens for returning [start, end] of this file."""
span = max(end - start + 1, 0)
raw = span * fs.tokens_per_line
if intent_kind == "bare" and cfg.summarize_min >= 0 and fs.size_lines >= cfg.summarize_min:
if (
intent_kind == "bare"
and cfg.summarize_min >= 0
and fs.size_lines >= cfg.summarize_min
):
# Calibrated from observed post-deploy summary-eligible reads:
# tokens/line collapses to ~0.35× the verbatim rate.
return raw * 0.35
@@ -387,7 +424,11 @@ def replay_pair(reads: list[ReadCall], fs: FileStats, cfg: Config) -> tuple[floa
total = 0.0
kept = 0
for rc in reads:
if rc.actual_a is not None and rc.actual_b is not None and rc.actual_b >= rc.actual_a:
if (
rc.actual_a is not None
and rc.actual_b is not None
and rc.actual_b >= rc.actual_a
):
observed_needed.append((rc.actual_a, rc.actual_b))
ret = effective_returned(rc, fs, cfg)
if ret is None:
@@ -415,12 +456,15 @@ def replay_pair(reads: list[ReadCall], fs: FileStats, cfg: Config) -> tuple[floa
return total, kept
def simulate(by_pair: dict, files: dict[str, FileStats], cfg: Config) -> tuple[float, int]:
def simulate(
by_pair: dict, files: dict[str, FileStats], cfg: Config
) -> tuple[float, int]:
grand = 0.0
kept = 0
for (_session, base), reads in by_pair.items():
fs = files.get(base) or FileStats(size_lines=1, tokens_per_line=FALLBACK_TPL,
bytes_per_line=FALLBACK_BPL)
fs = files.get(base) or FileStats(
size_lines=1, tokens_per_line=FALLBACK_TPL, bytes_per_line=FALLBACK_BPL
)
t, k = replay_pair(reads, fs, cfg)
grand += t
kept += k
@@ -436,6 +480,7 @@ def baseline_observed(reads: list[tuple[str, ReadCall]]) -> tuple[int, int]:
# --------------------------------------------------------------------------- #
# Sweep + report
def sweep(by_pair: dict, files: dict[str, FileStats]) -> dict:
defaults = [200, 300, 400, 500, 700, 1000, 1500, 2000, 3000]
line_caps = [500, 1000, 1500, 2000, 3000, 5000]
@@ -445,8 +490,9 @@ def sweep(by_pair: dict, files: dict[str, FileStats]) -> dict:
grid_calls = np.zeros((len(defaults), len(line_caps)), dtype=np.int64)
for i, D in enumerate(defaults):
for j, L in enumerate(line_caps):
cfg = Config(default_page=D, line_cap=L,
byte_cap=CURRENT_BYTE_CAP, summarize_min=0)
cfg = Config(
default_page=D, line_cap=L, byte_cap=CURRENT_BYTE_CAP, summarize_min=0
)
t, k = simulate(by_pair, files, cfg)
grid_tokens[i, j] = t
grid_calls[i, j] = k
@@ -459,27 +505,46 @@ def sweep(by_pair: dict, files: dict[str, FileStats]) -> dict:
# Sweep summarize_min at best (D, L).
sm_tokens = []
for sm in summary_thresholds:
cfg = Config(default_page=best_DL[0], line_cap=best_DL[1],
byte_cap=CURRENT_BYTE_CAP, summarize_min=sm)
cfg = Config(
default_page=best_DL[0],
line_cap=best_DL[1],
byte_cap=CURRENT_BYTE_CAP,
summarize_min=sm,
)
t, k = simulate(by_pair, files, cfg)
sm_tokens.append((sm, t, k))
best_sm = min(sm_tokens, key=lambda x: x[1])
# Sweep byte_cap at best (D, L, summarize_min).
byte_caps = [16 * 1024, 32 * 1024, 50 * 1024, 75 * 1024, 100 * 1024,
150 * 1024, 200 * 1024]
byte_caps = [
16 * 1024,
32 * 1024,
50 * 1024,
75 * 1024,
100 * 1024,
150 * 1024,
200 * 1024,
]
bc_tokens = []
for bc in byte_caps:
cfg = Config(default_page=best_DL[0], line_cap=best_DL[1],
byte_cap=bc, summarize_min=best_sm[0])
cfg = Config(
default_page=best_DL[0],
line_cap=best_DL[1],
byte_cap=bc,
summarize_min=best_sm[0],
)
t, k = simulate(by_pair, files, cfg)
bc_tokens.append((bc, t, k))
best_bc = min(bc_tokens, key=lambda x: x[1])
# Final combined config (D, L, summarize_min, byte_cap) — should be the
# global minimum given the order of dimensions.
final_cfg = Config(default_page=best_DL[0], line_cap=best_DL[1],
byte_cap=best_bc[0], summarize_min=best_sm[0])
final_cfg = Config(
default_page=best_DL[0],
line_cap=best_DL[1],
byte_cap=best_bc[0],
summarize_min=best_sm[0],
)
final_tokens, final_calls = simulate(by_pair, files, final_cfg)
return {
@@ -501,6 +566,7 @@ def sweep(by_pair: dict, files: dict[str, FileStats]) -> dict:
# --------------------------------------------------------------------------- #
# Plotting
def plot(result: dict, baseline_sim: float, observed: int, out_path: Path) -> None:
OUT_DIR.mkdir(parents=True, exist_ok=True)
plt.rcParams.update({"figure.dpi": 110, "font.size": 10})
@@ -511,8 +577,14 @@ def plot(result: dict, baseline_sim: float, observed: int, out_path: Path) -> No
ax = axes[0, 0]
grid = result["grid_tokens"]
rel = grid / baseline_sim
im = ax.imshow(rel, cmap="RdYlGn_r", aspect="auto", origin="lower",
vmin=max(0.6, rel.min()), vmax=min(1.4, rel.max() + 0.02))
im = ax.imshow(
rel,
cmap="RdYlGn_r",
aspect="auto",
origin="lower",
vmin=max(0.6, rel.min()),
vmax=min(1.4, rel.max() + 0.02),
)
ax.set_xticks(range(len(result["line_caps"])))
ax.set_xticklabels(result["line_caps"])
ax.set_yticks(range(len(result["defaults"])))
@@ -522,21 +594,53 @@ def plot(result: dict, baseline_sim: float, observed: int, out_path: Path) -> No
ax.set_title("simulated read tokens / baseline\n(green = cheaper, red = more)")
for i in range(grid.shape[0]):
for j in range(grid.shape[1]):
ax.text(j, i, f"{rel[i,j]:.2f}", ha="center", va="center",
color="black", fontsize=8)
ax.text(
j,
i,
f"{rel[i, j]:.2f}",
ha="center",
va="center",
color="black",
fontsize=8,
)
fig.colorbar(im, ax=ax, fraction=0.05)
# Highlight current and best.
cur_i = result["defaults"].index(CURRENT_DEFAULT) if CURRENT_DEFAULT in result["defaults"] else None
cur_j = result["line_caps"].index(CURRENT_LINE_CAP) if CURRENT_LINE_CAP in result["line_caps"] else None
cur_i = (
result["defaults"].index(CURRENT_DEFAULT)
if CURRENT_DEFAULT in result["defaults"]
else None
)
cur_j = (
result["line_caps"].index(CURRENT_LINE_CAP)
if CURRENT_LINE_CAP in result["line_caps"]
else None
)
if cur_i is not None and cur_j is not None:
ax.add_patch(mpatches.Rectangle((cur_j - 0.5, cur_i - 0.5), 1, 1,
fill=False, edgecolor="#1d4ed8",
linewidth=2.4, label="current"))
ax.add_patch(
mpatches.Rectangle(
(cur_j - 0.5, cur_i - 0.5),
1,
1,
fill=False,
edgecolor="#1d4ed8",
linewidth=2.4,
label="current",
)
)
best_i = result["defaults"].index(result["best_DL"][0])
best_j = result["line_caps"].index(result["best_DL"][1])
ax.add_patch(mpatches.Rectangle((best_j - 0.5, best_i - 0.5), 1, 1,
fill=False, edgecolor="#000",
linewidth=2.4, linestyle="--", label="optimum"))
ax.add_patch(
mpatches.Rectangle(
(best_j - 0.5, best_i - 0.5),
1,
1,
fill=False,
edgecolor="#000",
linewidth=2.4,
linestyle="--",
label="optimum",
)
)
ax.legend(loc="upper right", frameon=True, fontsize=9)
# Default-page line (at best line cap).
@@ -546,9 +650,17 @@ def plot(result: dict, baseline_sim: float, observed: int, out_path: Path) -> No
col = grid[:, j] / baseline_sim
ax.plot(result["defaults"], col, marker="o", linewidth=1.8, color="#0f766e")
ax.axhline(1.0, color="#9ca3af", linestyle="--", linewidth=1.0)
ax.axvline(CURRENT_DEFAULT, color="#1d4ed8", linestyle=":", linewidth=1.2, label="current default")
ax.axvline(
CURRENT_DEFAULT,
color="#1d4ed8",
linestyle=":",
linewidth=1.2,
label="current default",
)
best_D = result["best_DL"][0]
ax.axvline(best_D, color="#000", linestyle="--", linewidth=1.4, label=f"optimum D={best_D}")
ax.axvline(
best_D, color="#000", linestyle="--", linewidth=1.4, label=f"optimum D={best_D}"
)
ax.set_xscale("log")
ax.set_xlabel("default page (D) — log scale")
ax.set_ylabel("simulated tokens / baseline")
@@ -559,17 +671,33 @@ def plot(result: dict, baseline_sim: float, observed: int, out_path: Path) -> No
# Summarizer threshold sweep.
ax = axes[1, 0]
sm_data = result["summary_sweep"]
xs = [str("off") if sm == -1 else ("always" if sm == 0 else f"≥{sm}") for sm, _, _ in sm_data]
xs = [
str("off") if sm == -1 else ("always" if sm == 0 else f"≥{sm}")
for sm, _, _ in sm_data
]
ys = [t / baseline_sim for _, t, _ in sm_data]
bars = ax.bar(xs, ys, color=["#dc2626" if y > 1 else "#0f766e" for y in ys],
edgecolor="#111", linewidth=0.5)
bars = ax.bar(
xs,
ys,
color=["#dc2626" if y > 1 else "#0f766e" for y in ys],
edgecolor="#111",
linewidth=0.5,
)
for bar, y in zip(bars, ys):
ax.text(bar.get_x() + bar.get_width() / 2, y + 0.005,
f"{(y - 1) * 100:+.1f}%", ha="center", va="bottom", fontsize=9)
ax.text(
bar.get_x() + bar.get_width() / 2,
y + 0.005,
f"{(y - 1) * 100:+.1f}%",
ha="center",
va="bottom",
fontsize=9,
)
ax.axhline(1.0, color="#9ca3af", linestyle="--", linewidth=1.0)
ax.set_ylabel("simulated tokens / baseline")
ax.set_xlabel("summarize files ≥ N lines")
ax.set_title(f"summarizer threshold sweep (D={result['best_DL'][0]}, L={result['best_DL'][1]})")
ax.set_title(
f"summarizer threshold sweep (D={result['best_DL'][0]}, L={result['best_DL'][1]})"
)
ax.grid(True, axis="y", alpha=0.25, linestyle="--")
# Byte cap sweep.
@@ -579,11 +707,21 @@ def plot(result: dict, baseline_sim: float, observed: int, out_path: Path) -> No
ys = [t / baseline_sim for _, t, _ in bc_data]
ax.plot(xs_kb, ys, marker="o", linewidth=1.8, color="#7c3aed")
ax.axhline(1.0, color="#9ca3af", linestyle="--", linewidth=1.0)
ax.axvline(CURRENT_BYTE_CAP / 1024, color="#1d4ed8", linestyle=":",
linewidth=1.2, label="current byte cap")
ax.axvline(
CURRENT_BYTE_CAP / 1024,
color="#1d4ed8",
linestyle=":",
linewidth=1.2,
label="current byte cap",
)
best_bc_kb = result["best_byte_cap"][0] // 1024
ax.axvline(best_bc_kb, color="#000", linestyle="--", linewidth=1.4,
label=f"optimum {best_bc_kb} KB")
ax.axvline(
best_bc_kb,
color="#000",
linestyle="--",
linewidth=1.4,
label=f"optimum {best_bc_kb} KB",
)
ax.set_xlabel("byte cap (KB)")
ax.set_ylabel("simulated tokens / baseline")
ax.set_title("sensitivity to byte cap")
@@ -593,7 +731,8 @@ def plot(result: dict, baseline_sim: float, observed: int, out_path: Path) -> No
fig.suptitle(
f"read tool config sweep — observed read spend {observed:,}, "
f"simulator baseline {baseline_sim:,.0f}",
fontsize=12, y=1.02,
fontsize=12,
y=1.02,
)
fig.tight_layout()
fig.savefig(out_path, bbox_inches="tight")
@@ -603,14 +742,20 @@ def plot(result: dict, baseline_sim: float, observed: int, out_path: Path) -> No
# --------------------------------------------------------------------------- #
# Report
def fmt_pct(x: float) -> str:
if x >= 0:
return f"+{x*100:.1f}%"
return f"{x*100:.1f}%"
return f"+{x * 100:.1f}%"
return f"{x * 100:.1f}%"
def report(result: dict, baseline_sim: float, baseline_calls: int,
observed: int, observed_calls: int) -> None:
def report(
result: dict,
baseline_sim: float,
baseline_calls: int,
observed: int,
observed_calls: int,
) -> None:
defaults = result["defaults"]
line_caps = result["line_caps"]
grid = result["grid_tokens"]
@@ -618,8 +763,10 @@ def report(result: dict, baseline_sim: float, baseline_calls: int,
print(f"\nbaseline (current config: D={CURRENT_DEFAULT}, L={CURRENT_LINE_CAP}):")
print(f" observed result tokens = {observed:>13,} (truth)")
print(f" simulator under baseline = {baseline_sim:>13,.0f} "
f"({fmt_pct((baseline_sim - observed) / observed)} vs observed)")
print(
f" simulator under baseline = {baseline_sim:>13,.0f} "
f"({fmt_pct((baseline_sim - observed) / observed)} vs observed)"
)
print(f" observed read calls = {observed_calls:>13,}")
print(f" simulator calls (baseline) = {baseline_calls:>11,}")
@@ -628,51 +775,77 @@ def report(result: dict, baseline_sim: float, baseline_calls: int,
header = " D \\ L " + " ".join(f"{L:>6}" for L in line_caps)
print(header)
for i, D in enumerate(defaults):
row = " ".join(f"{grid[i,j]/baseline_sim:>6.2f}" for j in range(len(line_caps)))
row = " ".join(
f"{grid[i, j] / baseline_sim:>6.2f}" for j in range(len(line_caps))
)
print(f" D={D:<6} {row}")
print(f"\nbest (D, L) = {result['best_DL']} → "
f"{grid[defaults.index(result['best_DL'][0]), line_caps.index(result['best_DL'][1])]:,.0f} tokens"
f" ({fmt_pct(grid.min()/baseline_sim - 1)})")
print(
f"\nbest (D, L) = {result['best_DL']} → "
f"{grid[defaults.index(result['best_DL'][0]), line_caps.index(result['best_DL'][1])]:,.0f} tokens"
f" ({fmt_pct(grid.min() / baseline_sim - 1)})"
)
# Summarizer threshold sweep at best (D, L).
print(f"\nsummarizer threshold sweep at best (D, L) = {result['best_DL']}:")
print(f" {'min_file_lines':<16} {'tokens':>12} {'vs baseline':>12}")
for sm, t, k in result["summary_sweep"]:
label = "off" if sm == -1 else ("always" if sm == 0 else f">={sm}")
print(f" {label:<16} {t:>12,.0f} {fmt_pct(t/baseline_sim - 1):>12}")
print(f"\nbest summarize_min = {result['best_summary'][0]} → "
f"{result['best_summary'][1]:,.0f} tokens "
f"({fmt_pct(result['best_summary'][1]/baseline_sim - 1)})")
print(f" {label:<16} {t:>12,.0f} {fmt_pct(t / baseline_sim - 1):>12}")
print(
f"\nbest summarize_min = {result['best_summary'][0]} → "
f"{result['best_summary'][1]:,.0f} tokens "
f"({fmt_pct(result['best_summary'][1] / baseline_sim - 1)})"
)
# Byte cap sweep at (best D, L, summarize_min).
print(f"\nbyte cap sweep at best (D, L, summarize_min):")
print(f" {'byte_cap':<10} {'tokens':>12} {'vs baseline':>12}")
for bc, t, k in result["byte_cap_sweep"]:
print(f" {bc//1024:>4} KB {t:>12,.0f} {fmt_pct(t/baseline_sim - 1):>12}")
print(f"\nbest byte_cap = {result['best_byte_cap'][0]//1024} KB → "
f"{result['best_byte_cap'][1]:,.0f} tokens "
f"({fmt_pct(result['best_byte_cap'][1]/baseline_sim - 1)})")
print(
f" {bc // 1024:>4} KB {t:>12,.0f} {fmt_pct(t / baseline_sim - 1):>12}"
)
print(
f"\nbest byte_cap = {result['best_byte_cap'][0] // 1024} KB → "
f"{result['best_byte_cap'][1]:,.0f} tokens "
f"({fmt_pct(result['best_byte_cap'][1] / baseline_sim - 1)})"
)
# Final recommendation.
cfg = result["final_cfg"]
print("\n" + "=" * 64)
print(" RECOMMENDED CONFIG")
print("=" * 64)
print(f" read.defaultLimit {cfg.default_page} lines (current: {CURRENT_DEFAULT})")
print(
f" read.defaultLimit {cfg.default_page} lines (current: {CURRENT_DEFAULT})"
)
print(f" read.lineCap {cfg.line_cap} lines (current: {CURRENT_LINE_CAP})")
print(f" read.byteCap {cfg.byte_cap//1024} KB (current: {CURRENT_BYTE_CAP//1024} KB)")
sm_label = "off" if cfg.summarize_min == -1 else (
"always" if cfg.summarize_min == 0 else f"only files ≥ {cfg.summarize_min} lines")
print(
f" read.byteCap {cfg.byte_cap // 1024} KB (current: {CURRENT_BYTE_CAP // 1024} KB)"
)
sm_label = (
"off"
if cfg.summarize_min == -1
else (
"always"
if cfg.summarize_min == 0
else f"only files ≥ {cfg.summarize_min} lines"
)
)
print(f" read.summarizer {sm_label}")
print(f" simulated savings {fmt_pct(result['final_tokens']/baseline_sim - 1)} "
f"({baseline_sim - result['final_tokens']:,.0f} fewer tokens / window)")
print(f" calls {result['final_calls']:,} "
f"(baseline sim: {baseline_calls:,})")
print(
f" simulated savings {fmt_pct(result['final_tokens'] / baseline_sim - 1)} "
f"({baseline_sim - result['final_tokens']:,.0f} fewer tokens / window)"
)
print(
f" calls {result['final_calls']:,} "
f"(baseline sim: {baseline_calls:,})"
)
# --------------------------------------------------------------------------- #
# Entry
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__.splitlines()[1])
ap.add_argument("--since", default=DEFAULT_SINCE)
@@ -694,10 +867,14 @@ def main() -> int:
sizes = np.array([f.size_lines for f in files.values()], dtype=np.int64)
tpls = np.array([f.tokens_per_line for f in files.values()], dtype=float)
print(f" {len(files):,} distinct files")
print(f" file size p50={int(np.percentile(sizes,50))} "
f"p90={int(np.percentile(sizes,90))} max={int(sizes.max())}")
print(f" tokens/line p50={np.percentile(tpls,50):.2f} "
f"p90={np.percentile(tpls,90):.2f} max={tpls.max():.2f}")
print(
f" file size p50={int(np.percentile(sizes, 50))} "
f"p90={int(np.percentile(sizes, 90))} max={int(sizes.max())}"
)
print(
f" tokens/line p50={np.percentile(tpls, 50):.2f} "
f"p90={np.percentile(tpls, 90):.2f} max={tpls.max():.2f}"
)
# Per-pair.
by_pair = group_pairs(reads)
@@ -705,8 +882,12 @@ def main() -> int:
# Baseline simulation.
print("\nsimulating baseline...")
baseline_cfg = Config(default_page=CURRENT_DEFAULT, line_cap=CURRENT_LINE_CAP,
byte_cap=CURRENT_BYTE_CAP, summarize_min=0)
baseline_cfg = Config(
default_page=CURRENT_DEFAULT,
line_cap=CURRENT_LINE_CAP,
byte_cap=CURRENT_BYTE_CAP,
summarize_min=0,
)
baseline_sim, baseline_calls = simulate(by_pair, files, baseline_cfg)
observed, observed_calls = baseline_observed(reads)
scripts/session-stats/plot_read_summarizer.py
+86 -28
View File
@@ -15,6 +15,7 @@ thinking + user messages. That removes the "I worked harder that day" effect.
Outputs to scripts/session-stats/out/read-summarizer-*.png.
"""
from __future__ import annotations
import argparse
@@ -44,6 +45,7 @@ COHORT_COLORS = {
# --------------------------------------------------------------------------- #
# Classification
def has_selector(path: str) -> bool:
"""True iff `path` carries a read selector (`:50-200`, `:raw`, ...)."""
if not path:
@@ -76,6 +78,7 @@ def cohort_of(arg_json: str | None) -> str | None:
# --------------------------------------------------------------------------- #
# Data
def fetch_read_calls(conn) -> dict[str, dict[str, np.ndarray]]:
sql = """
SELECT c.timestamp,
@@ -97,7 +100,10 @@ def fetch_read_calls(conn) -> dict[str, dict[str, np.ndarray]]:
out: dict[str, dict[str, np.ndarray]] = {}
for c, rows in by.items():
if not rows:
out[c] = {"ts": np.array([], dtype=np.int64), "tok": np.array([], dtype=np.int64)}
out[c] = {
"ts": np.array([], dtype=np.int64),
"tok": np.array([], dtype=np.int64),
}
continue
ts = np.fromiter((r[0] for r in rows), dtype=np.int64, count=len(rows))
tok = np.fromiter((r[1] for r in rows), dtype=np.int64, count=len(rows))
@@ -152,7 +158,9 @@ def daily_sum(ts_ms: np.ndarray, tok: np.ndarray, day_axis: np.ndarray) -> np.nd
return out
def daily_percentile(ts_ms: np.ndarray, tok: np.ndarray, q: float) -> tuple[np.ndarray, np.ndarray]:
def daily_percentile(
ts_ms: np.ndarray, tok: np.ndarray, q: float
) -> tuple[np.ndarray, np.ndarray]:
if ts_ms.size == 0:
return np.array([]), np.array([])
day_idx = ts_ms // DAY_MS
@@ -164,7 +172,9 @@ def daily_percentile(ts_ms: np.ndarray, tok: np.ndarray, q: float) -> tuple[np.n
lo, hi = order[i], order[i + 1]
if hi > lo:
pct[i] = np.percentile(tok[lo:hi], q)
dates = np.array([datetime.fromtimestamp(int(d) * DAY_MS / 1000, tz=timezone.utc) for d in days])
dates = np.array(
[datetime.fromtimestamp(int(d) * DAY_MS / 1000, tz=timezone.utc) for d in days]
)
return dates, pct
@@ -183,9 +193,10 @@ def smooth_nan(y: np.ndarray, w: int) -> np.ndarray:
# --------------------------------------------------------------------------- #
# Plot helpers
def thousands(x: float, _p=0) -> str:
if x >= 1000:
return f"{x/1000:.1f}k"
return f"{x / 1000:.1f}k"
return f"{x:.0f}"
@@ -195,11 +206,20 @@ def style_time(ax: plt.Axes, deploy: datetime) -> None:
ax.grid(True, alpha=0.25, linestyle="--")
ax.axvline(deploy, color="#dc2626", linestyle="--", linewidth=1.2, alpha=0.8)
y1 = ax.get_ylim()[1] if ax.get_ylim()[1] > 0 else 1
ax.text(deploy, y1, " summarizer\n deploy", color="#dc2626",
va="top", ha="left", fontsize=9)
ax.text(
deploy,
y1,
" summarizer\n deploy",
color="#dc2626",
va="top",
ha="left",
fontsize=9,
)
def panel_share_stacked(ax: plt.Axes, reads, denom_dates, denom, deploy: datetime) -> None:
def panel_share_stacked(
ax: plt.Axes, reads, denom_dates, denom, deploy: datetime
) -> None:
"""Stacked area: per-day read-cohort share of total tokens."""
series = []
labels = []
@@ -212,11 +232,13 @@ def panel_share_stacked(ax: plt.Axes, reads, denom_dates, denom, deploy: datetim
series.append(smooth_nan(share, 7))
labels.append(cohort)
colors.append(color)
x = np.array([datetime.fromtimestamp(int(d) / 1000, tz=timezone.utc) for d in denom_dates])
x = np.array(
[datetime.fromtimestamp(int(d) / 1000, tz=timezone.utc) for d in denom_dates]
)
ax.stackplot(x, series, labels=labels, colors=colors, alpha=0.85)
ax.set_title("read share of daily token spend (7d MA)")
ax.set_ylabel("share of all tokens that day")
ax.yaxis.set_major_formatter(plt.FuncFormatter(lambda v, _: f"{v*100:.0f}%"))
ax.yaxis.set_major_formatter(plt.FuncFormatter(lambda v, _: f"{v * 100:.0f}%"))
ax.set_ylim(0, None)
ax.legend(loc="upper left", frameon=False)
style_time(ax, deploy)
@@ -224,7 +246,9 @@ def panel_share_stacked(ax: plt.Axes, reads, denom_dates, denom, deploy: datetim
def panel_share_line(ax: plt.Axes, reads, denom_dates, denom, deploy: datetime) -> None:
"""Lines: each cohort's share, plus the combined total."""
x = np.array([datetime.fromtimestamp(int(d) / 1000, tz=timezone.utc) for d in denom_dates])
x = np.array(
[datetime.fromtimestamp(int(d) / 1000, tz=timezone.utc) for d in denom_dates]
)
total = np.zeros(denom_dates.size, dtype=np.int64)
for cohort, color in COHORT_COLORS.items():
d = reads[cohort]
@@ -235,22 +259,37 @@ def panel_share_line(ax: plt.Axes, reads, denom_dates, denom, deploy: datetime)
ax.plot(x, smooth_nan(share, 7), label=cohort, color=color, linewidth=1.7)
with np.errstate(divide="ignore", invalid="ignore"):
combined = np.where(denom > 0, total / denom, 0.0)
ax.plot(x, smooth_nan(combined, 7), label="all reads", color="#111111", linewidth=2.2, linestyle="-")
ax.plot(
x,
smooth_nan(combined, 7),
label="all reads",
color="#111111",
linewidth=2.2,
linestyle="-",
)
ax.set_title("read share by cohort (7d MA)")
ax.set_ylabel("share of daily tokens")
ax.yaxis.set_major_formatter(plt.FuncFormatter(lambda v, _: f"{v*100:.0f}%"))
ax.yaxis.set_major_formatter(plt.FuncFormatter(lambda v, _: f"{v * 100:.0f}%"))
ax.set_ylim(0, None)
ax.legend(loc="upper left", frameon=False)
style_time(ax, deploy)
def panel_per_call(ax: plt.Axes, reads, deploy: datetime, q: float, label_q: str) -> None:
def panel_per_call(
ax: plt.Axes, reads, deploy: datetime, q: float, label_q: str
) -> None:
for cohort, color in COHORT_COLORS.items():
d = reads[cohort]
if d["ts"].size == 0:
continue
dates, pct = daily_percentile(d["ts"], d["tok"], q)
ax.plot(dates, smooth_nan(pct, 7), label=f"{cohort} ({label_q})", color=color, linewidth=1.9)
ax.plot(
dates,
smooth_nan(pct, 7),
label=f"{cohort} ({label_q})",
color=color,
linewidth=1.9,
)
ax.set_title(f"daily {label_q} tokens per read call (7d MA)")
ax.set_ylabel("tokens / call")
ax.set_yscale("log")
@@ -262,14 +301,19 @@ def panel_per_call(ax: plt.Axes, reads, deploy: datetime, q: float, label_q: str
# --------------------------------------------------------------------------- #
# Stats
def share_stats(reads, denom_dates, denom, deploy_ms: int) -> None:
pre_mask = denom_dates < deploy_ms
post_mask = denom_dates >= deploy_ms
pre_total = int(denom[pre_mask].sum())
post_total = int(denom[post_mask].sum())
print(f"\nshare-of-day (pre vs post deploy):")
print(f" denominator pre = {pre_total:>14,} tokens across {int(pre_mask.sum())} days")
print(f" denominator post = {post_total:>14,} tokens across {int(post_mask.sum())} days")
print(
f" denominator pre = {pre_total:>14,} tokens across {int(pre_mask.sum())} days"
)
print(
f" denominator post = {post_total:>14,} tokens across {int(post_mask.sum())} days"
)
print(f" {'cohort':<22} {'pre share':>10} {'post share':>11} {'delta':>10}")
grand_pre = 0
grand_post = 0
@@ -282,17 +326,23 @@ def share_stats(reads, denom_dates, denom, deploy_ms: int) -> None:
grand_post += post
pre_share = pre / pre_total if pre_total else 0
post_share = post / post_total if post_total else 0
print(f" {cohort:<22} {pre_share*100:>9.2f}% {post_share*100:>10.2f}% "
f"{(post_share-pre_share)*100:>+9.2f}pp")
print(
f" {cohort:<22} {pre_share * 100:>9.2f}% {post_share * 100:>10.2f}% "
f"{(post_share - pre_share) * 100:>+9.2f}pp"
)
pre_share = grand_pre / pre_total if pre_total else 0
post_share = grand_post / post_total if post_total else 0
print(f" {'all reads':<22} {pre_share*100:>9.2f}% {post_share*100:>10.2f}% "
f"{(post_share-pre_share)*100:>+9.2f}pp")
print(
f" {'all reads':<22} {pre_share * 100:>9.2f}% {post_share * 100:>10.2f}% "
f"{(post_share - pre_share) * 100:>+9.2f}pp"
)
def per_call_stats(reads, deploy_ms: int) -> None:
print(f"\nper-call stats (pre vs post deploy):")
print(f" {'cohort':<22} {'window':<6} {'n':>9} {'p50':>7} {'p90':>7} {'mean':>8}")
print(
f" {'cohort':<22} {'window':<6} {'n':>9} {'p50':>7} {'p90':>7} {'mean':>8}"
)
for cohort in COHORT_COLORS:
d = reads[cohort]
if d["ts"].size == 0:
@@ -302,19 +352,25 @@ def per_call_stats(reads, deploy_ms: int) -> None:
for name, arr in (("pre", pre), ("post", post)):
if arr.size == 0:
continue
print(f" {cohort:<22} {name:<6} {arr.size:>9,} "
f"{int(np.percentile(arr,50)):>7,} "
f"{int(np.percentile(arr,90)):>7,} "
f"{int(arr.mean()):>8,}")
print(
f" {cohort:<22} {name:<6} {arr.size:>9,} "
f"{int(np.percentile(arr, 50)):>7,} "
f"{int(np.percentile(arr, 90)):>7,} "
f"{int(arr.mean()):>8,}"
)
# --------------------------------------------------------------------------- #
# Entry
def main() -> int:
ap = argparse.ArgumentParser(description="read summarizer impact analysis")
ap.add_argument("--deploy", default=DEFAULT_DEPLOY,
help=f"deploy date YYYY-MM-DD (default {DEFAULT_DEPLOY})")
ap.add_argument(
"--deploy",
default=DEFAULT_DEPLOY,
help=f"deploy date YYYY-MM-DD (default {DEFAULT_DEPLOY})",
)
args = ap.parse_args()
deploy = datetime.strptime(args.deploy, "%Y-%m-%d").replace(tzinfo=timezone.utc)
@@ -343,7 +399,9 @@ def main() -> int:
panel_share_line(axes[0, 1], reads, denom_dates, denom, deploy)
panel_per_call(axes[1, 0], reads, deploy, q=50, label_q="p50")
panel_per_call(axes[1, 1], reads, deploy, q=90, label_q="p90")
fig.suptitle(f"read summarizer impact — deploy = {args.deploy}", fontsize=13, y=0.995)
fig.suptitle(
f"read summarizer impact — deploy = {args.deploy}", fontsize=13, y=0.995
)
fig.tight_layout()
p = OUT_DIR / "read-summarizer.png"
fig.savefig(p, bbox_inches="tight")
scripts/session-stats/plot_tools.py
+55 -19
View File
@@ -15,6 +15,7 @@ N by total tokens (default 10); override with --top N or --tools a,b,c.
Output: scripts/session-stats/out/tool-trends.png + standalone panels.
"""
from __future__ import annotations
import argparse
@@ -39,9 +40,18 @@ TOOL_ALIAS = {"grep": "search"}
# 10-class qualitative palette (tab10) — distinct hues for line + area work.
PALETTE = [
"#1f77b4", "#d62728", "#2ca02c", "#ff7f0e", "#9467bd",
"#8c564b", "#17becf", "#e377c2", "#bcbd22", "#7f7f7f",
"#393b79", "#637939",
"#1f77b4",
"#d62728",
"#2ca02c",
"#ff7f0e",
"#9467bd",
"#8c564b",
"#17becf",
"#e377c2",
"#bcbd22",
"#7f7f7f",
"#393b79",
"#637939",
]
@@ -56,6 +66,7 @@ def normalize_case_sql(col: str) -> str:
# --------------------------------------------------------------------------- #
# Data access
def _connect() -> sqlite3.Connection:
if not DB_PATH.exists():
sys.exit(f"db missing: {DB_PATH}")
@@ -148,7 +159,10 @@ def fetch_per_call(conn: sqlite3.Connection, tools: list[str]) -> dict[str, dict
out: dict[str, dict] = {}
for t, rows in by_tool.items():
if not rows:
out[t] = {"ts": np.array([], dtype=np.int64), "tok": np.array([], dtype=np.int64)}
out[t] = {
"ts": np.array([], dtype=np.int64),
"tok": np.array([], dtype=np.int64),
}
continue
ts = np.fromiter((r[0] for r in rows), dtype=np.int64, count=len(rows))
tok = np.fromiter((r[1] for r in rows), dtype=np.int64, count=len(rows))
@@ -160,6 +174,7 @@ def fetch_per_call(conn: sqlite3.Connection, tools: list[str]) -> dict[str, dict
# --------------------------------------------------------------------------- #
# Helpers
def smooth(y: np.ndarray, w: int = 7) -> np.ndarray:
if w <= 1 or len(y) < w:
return y.astype(float)
@@ -208,7 +223,10 @@ def weekly_median(ts_ms: np.ndarray, tok: np.ndarray) -> tuple[np.ndarray, np.nd
if hi > lo:
p50[i] = np.percentile(tok[lo:hi], 50)
week_dates = np.array(
[datetime.fromtimestamp(int(w) * WEEK_MS / 1000, tz=timezone.utc) for w in weeks]
[
datetime.fromtimestamp(int(w) * WEEK_MS / 1000, tz=timezone.utc)
for w in weeks
]
)
return week_dates, p50
@@ -216,10 +234,15 @@ def weekly_median(ts_ms: np.ndarray, tok: np.ndarray) -> tuple[np.ndarray, np.nd
# --------------------------------------------------------------------------- #
# Panels
def panel_total_tokens(ax: plt.Axes, daily: dict, tools: list[str], colors: dict) -> None:
def panel_total_tokens(
ax: plt.Axes, daily: dict, tools: list[str], colors: dict
) -> None:
dates = daily["dates"]
series = [smooth(daily[t]["args"] + daily[t]["results"]) for t in tools]
ax.stackplot(dates, series, labels=tools, colors=[colors[t] for t in tools], alpha=0.9)
ax.stackplot(
dates, series, labels=tools, colors=[colors[t] for t in tools], alpha=0.9
)
ax.set_title("Daily token volume (args + results, 7d MA)")
ax.set_ylabel("tokens / day")
ax.yaxis.set_major_formatter(plt.FuncFormatter(millions))
@@ -227,17 +250,23 @@ def panel_total_tokens(ax: plt.Axes, daily: dict, tools: list[str], colors: dict
style_time_axis(ax)
def panel_call_counts(ax: plt.Axes, daily: dict, tools: list[str], colors: dict) -> None:
def panel_call_counts(
ax: plt.Axes, daily: dict, tools: list[str], colors: dict
) -> None:
dates = daily["dates"]
for t in tools:
ax.plot(dates, smooth(daily[t]["calls"]), label=t, color=colors[t], linewidth=1.6)
ax.plot(
dates, smooth(daily[t]["calls"]), label=t, color=colors[t], linewidth=1.6
)
ax.set_title("Daily call count (7d MA)")
ax.set_ylabel("calls / day")
ax.legend(loc="upper left", frameon=False, ncol=2, fontsize=9)
style_time_axis(ax)
def panel_mean_per_call(ax: plt.Axes, daily: dict, tools: list[str], colors: dict) -> None:
def panel_mean_per_call(
ax: plt.Axes, daily: dict, tools: list[str], colors: dict
) -> None:
dates = daily["dates"]
for t in tools:
totals = daily[t]["args"] + daily[t]["results"]
@@ -265,7 +294,9 @@ def panel_cumulative(ax: plt.Axes, daily: dict, tools: list[str], colors: dict)
style_time_axis(ax)
def panel_weekly_median(ax: plt.Axes, per_call: dict, tools: list[str], colors: dict) -> None:
def panel_weekly_median(
ax: plt.Axes, per_call: dict, tools: list[str], colors: dict
) -> None:
for t in tools:
w, p50 = weekly_median(per_call[t]["ts"], per_call[t]["tok"])
if w.size == 0:
@@ -279,8 +310,12 @@ def panel_weekly_median(ax: plt.Axes, per_call: dict, tools: list[str], colors:
style_time_axis(ax)
def panel_histogram(ax: plt.Axes, per_call: dict, tools: list[str], colors: dict) -> None:
all_tok = np.concatenate([per_call[t]["tok"] for t in tools if per_call[t]["tok"].size])
def panel_histogram(
ax: plt.Axes, per_call: dict, tools: list[str], colors: dict
) -> None:
all_tok = np.concatenate(
[per_call[t]["tok"] for t in tools if per_call[t]["tok"].size]
)
if all_tok.size == 0:
return
hi = max(all_tok.max(), 10)
@@ -312,6 +347,7 @@ def panel_histogram(ax: plt.Axes, per_call: dict, tools: list[str], colors: dict
# --------------------------------------------------------------------------- #
# Entry
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__.splitlines()[1])
ap.add_argument("--top", type=int, default=10, help="top N tools by total tokens")
@@ -364,12 +400,12 @@ def main() -> int:
print(f"wrote {combined}")
panels: tuple[tuple[str, Callable, dict], ...] = (
("daily-tokens", panel_total_tokens, daily),
("daily-calls", panel_call_counts, daily),
("tokens-per-call", panel_mean_per_call, daily),
("cumulative-tokens", panel_cumulative, daily),
("per-call-median", panel_weekly_median, per_call),
("per-call-histogram", panel_histogram, per_call),
("daily-tokens", panel_total_tokens, daily),
("daily-calls", panel_call_counts, daily),
("tokens-per-call", panel_mean_per_call, daily),
("cumulative-tokens", panel_cumulative, daily),
("per-call-median", panel_weekly_median, per_call),
("per-call-histogram", panel_histogram, per_call),
)
for name, fn, src in panels:
f2, ax = plt.subplots(figsize=(11, 5))
scripts/session-stats/read_optimizer.py
+192 -42
View File
@@ -20,6 +20,7 @@ truncations, and a Pareto frontier.
Output:
scripts/session-stats/out/read-optimizer.png
"""
from __future__ import annotations
import argparse
@@ -51,12 +52,54 @@ READ_MAX_COLUMN = 768
_RANGE_RE = re.compile(r"^(\d+)(?:([-+])(\d+))?$")
TEXT_EXTS = {
".ts", ".tsx", ".js", ".jsx", ".mts", ".cts", ".mjs", ".cjs",
".rs", ".go", ".py", ".rb", ".java", ".kt", ".kts", ".c", ".cc",
".cpp", ".h", ".hpp", ".cs", ".swift", ".php", ".lua", ".sh",
".bash", ".zsh", ".fish", ".md", ".txt", ".json", ".jsonc", ".json5",
".yaml", ".yml", ".toml", ".xml", ".html", ".css", ".scss", ".sql",
".adoc", ".typ", ".rsx", ".vue", ".svelte", ".dockerfile", "",
".ts",
".tsx",
".js",
".jsx",
".mts",
".cts",
".mjs",
".cjs",
".rs",
".go",
".py",
".rb",
".java",
".kt",
".kts",
".c",
".cc",
".cpp",
".h",
".hpp",
".cs",
".swift",
".php",
".lua",
".sh",
".bash",
".zsh",
".fish",
".md",
".txt",
".json",
".jsonc",
".json5",
".yaml",
".yml",
".toml",
".xml",
".html",
".css",
".scss",
".sql",
".adoc",
".typ",
".rsx",
".vue",
".svelte",
".dockerfile",
"",
}
@@ -65,7 +108,7 @@ class ReadCall:
session: str
file: str
seq: int
kind: str # explicit | open | default | raw | conflicts | other
kind: str # explicit | open | default | raw | conflicts | other
start: int | None
end: int | None
arg_tokens: int
@@ -179,7 +222,12 @@ def parse_call(row) -> ReadCall | None:
if kind == "default":
offset = obj.get("offset")
limit = obj.get("limit")
if isinstance(offset, int) and offset >= 1 and isinstance(limit, int) and limit >= 1:
if (
isinstance(offset, int)
and offset >= 1
and isinstance(limit, int)
and limit >= 1
):
kind = "explicit"
start = offset
end = offset + limit - 1
@@ -258,7 +306,9 @@ def is_covered(intervals: list[tuple[int, int]], target: tuple[int, int]) -> boo
return False
def add_interval(intervals: list[tuple[int, int]], item: tuple[int, int]) -> list[tuple[int, int]]:
def add_interval(
intervals: list[tuple[int, int]], item: tuple[int, int]
) -> list[tuple[int, int]]:
s, e = item
out: list[tuple[int, int]] = []
placed = False
@@ -278,7 +328,9 @@ def add_interval(intervals: list[tuple[int, int]], item: tuple[int, int]) -> lis
return out
def estimate_cost(call: ReadCall, delivered: tuple[int, int]) -> tuple[float, bool, bool]:
def estimate_cost(
call: ReadCall, delivered: tuple[int, int]
) -> tuple[float, bool, bool]:
lines = max(0, delivered[1] - delivered[0] + 1)
line_tokens = call.token_per_line * lines
# Approximate byte cap. The implementation scales byte cap as
@@ -289,10 +341,16 @@ def estimate_cost(call: ReadCall, delivered: tuple[int, int]) -> tuple[float, bo
bytes_limited = approx_bytes > byte_budget
if bytes_limited:
line_tokens = byte_budget / 4
return call.arg_tokens + line_tokens, call.kind == "explicit" and lines >= call.config_max_lines if False else False, bytes_limited
return (
call.arg_tokens + line_tokens,
call.kind == "explicit" and lines >= call.config_max_lines if False else False,
bytes_limited,
)
def load_reads(conn: sqlite3.Connection, since_ms: int) -> dict[tuple[str, str], list[ReadCall]]:
def load_reads(
conn: sqlite3.Connection, since_ms: int
) -> dict[tuple[str, str], list[ReadCall]]:
sql = """
SELECT c.session_file, c.seq, c.arg_json,
COALESCE(c.arg_tokens,0), COALESCE(r.result_tokens,0)
@@ -372,7 +430,9 @@ def replay(groups: dict[tuple[str, str], list[ReadCall]], cfg: Config) -> Replay
lines = delivered[1] - delivered[0] + 1
if lines >= cfg.max_lines and call.kind == "explicit":
# Candidate max cap would truncate this explicit request.
requested_len = max(1, (call.end or call.start or 1) - (call.start or 1) + 1)
requested_len = max(
1, (call.end or call.start or 1) - (call.start or 1) + 1
)
if requested_len + cfg.leading + cfg.trailing > cfg.max_lines:
trunc += 1
line_tokens = call.token_per_line * lines
@@ -443,10 +503,18 @@ def candidate_grid(args) -> list[Config]:
return out
def pareto(results: list[ReplayResult], max_truncations: int, max_regret_tokens: float = math.inf) -> list[ReplayResult]:
def pareto(
results: list[ReplayResult],
max_truncations: int,
max_regret_tokens: float = math.inf,
) -> list[ReplayResult]:
# Frontier over (tokens lower, calls lower), excluding configs that truncate
# more explicit requests than today's cap.
clean = [r for r in results if r.truncations <= max_truncations and r.tokens <= max_regret_tokens]
clean = [
r
for r in results
if r.truncations <= max_truncations and r.tokens <= max_regret_tokens
]
clean.sort(key=lambda r: (r.tokens, r.calls))
frontier: list[ReplayResult] = []
best_calls = math.inf
@@ -457,13 +525,16 @@ def pareto(results: list[ReplayResult], max_truncations: int, max_regret_tokens:
return frontier
def choose_recommended(results: list[ReplayResult], current: ReplayResult) -> ReplayResult:
def choose_recommended(
results: list[ReplayResult], current: ReplayResult
) -> ReplayResult:
# Objective: minimize tokens plus a small penalty for still needing calls,
# while requiring no *additional* explicit-request truncations and at least
# current first-call coverage. One avoided read call is valued at ~250
# tokens of ergonomics.
viable = [
r for r in results
r
for r in results
if r.truncations <= current.truncations
and r.first_cover_rate >= current.first_cover_rate
and r.tokens <= current.tokens * 1.02
@@ -472,7 +543,14 @@ def choose_recommended(results: list[ReplayResult], current: ReplayResult) -> Re
viable = [r for r in results if r.truncations <= current.truncations]
if not viable:
viable = results
return min(viable, key=lambda r: r.tokens + 250 * r.calls + 100_000 * max(0, r.truncations - current.truncations))
return min(
viable,
key=lambda r: (
r.tokens
+ 250 * r.calls
+ 100_000 * max(0, r.truncations - current.truncations)
),
)
def print_result(prefix: str, r: ReplayResult, baseline: ReplayResult) -> None:
@@ -480,15 +558,17 @@ def print_result(prefix: str, r: ReplayResult, baseline: ReplayResult) -> None:
dcalls = r.calls - baseline.calls
print(
f"{prefix:<14} {r.config.label():<22} "
f"tokens={r.tokens/1e6:8.2f}M ({dtok/baseline.tokens*100:+6.2f}%) "
f"tokens={r.tokens / 1e6:8.2f}M ({dtok / baseline.tokens * 100:+6.2f}%) "
f"calls={r.calls:7,} ({dcalls:+7,}) "
f"skipped={r.skipped_calls:6,} "
f"first-cover={r.first_cover_rate*100:5.1f}% "
f"first-cover={r.first_cover_rate * 100:5.1f}% "
f"trunc={r.truncations:4,}"
)
def plot(results: list[ReplayResult], current: ReplayResult, recommended: ReplayResult) -> Path:
def plot(
results: list[ReplayResult], current: ReplayResult, recommended: ReplayResult
) -> Path:
OUT_DIR.mkdir(parents=True, exist_ok=True)
plt.rcParams.update({"figure.dpi": 110, "font.size": 10})
fig, axes = plt.subplots(2, 2, figsize=(15, 9))
@@ -499,9 +579,25 @@ def plot(results: list[ReplayResult], current: ReplayResult, recommended: Replay
sizes = np.array([20 + min(80, r.config.trailing * 5) for r in results])
ax = axes[0, 0]
sc = ax.scatter(xs, ys, c=colors, s=sizes, cmap="viridis", alpha=0.65, edgecolors="none")
ax.scatter([current.calls], [current.tokens / 1e6], marker="*", s=180, color="#111", label="current")
ax.scatter([recommended.calls], [recommended.tokens / 1e6], marker="*", s=180, color="#dc2626", label="recommended")
sc = ax.scatter(
xs, ys, c=colors, s=sizes, cmap="viridis", alpha=0.65, edgecolors="none"
)
ax.scatter(
[current.calls],
[current.tokens / 1e6],
marker="*",
s=180,
color="#111",
label="current",
)
ax.scatter(
[recommended.calls],
[recommended.tokens / 1e6],
marker="*",
s=180,
color="#dc2626",
label="recommended",
)
ax.set_xlabel("paid read calls after replay")
ax.set_ylabel("estimated read tokens (M)")
ax.set_title("candidate trade-off: tokens vs follow-up calls")
@@ -513,10 +609,34 @@ def plot(results: list[ReplayResult], current: ReplayResult, recommended: Replay
ax = axes[0, 1]
frontier = pareto(results, current.truncations)
frontier.sort(key=lambda r: r.calls)
ax.plot([r.calls for r in frontier], [r.tokens / 1e6 for r in frontier], color="#2563eb", linewidth=2)
ax.scatter([r.calls for r in frontier], [r.tokens / 1e6 for r in frontier], color="#2563eb", s=20)
ax.scatter([current.calls], [current.tokens / 1e6], marker="*", s=180, color="#111", label="current")
ax.scatter([recommended.calls], [recommended.tokens / 1e6], marker="*", s=180, color="#dc2626", label="recommended")
ax.plot(
[r.calls for r in frontier],
[r.tokens / 1e6 for r in frontier],
color="#2563eb",
linewidth=2,
)
ax.scatter(
[r.calls for r in frontier],
[r.tokens / 1e6 for r in frontier],
color="#2563eb",
s=20,
)
ax.scatter(
[current.calls],
[current.tokens / 1e6],
marker="*",
s=180,
color="#111",
label="current",
)
ax.scatter(
[recommended.calls],
[recommended.tokens / 1e6],
marker="*",
s=180,
color="#dc2626",
label="recommended",
)
ax.set_xlabel("paid read calls")
ax.set_ylabel("estimated read tokens (M)")
ax.set_title("Pareto frontier (no extra explicit truncations)")
@@ -526,20 +646,31 @@ def plot(results: list[ReplayResult], current: ReplayResult, recommended: Replay
ax = axes[1, 0]
by_default: dict[int, list[ReplayResult]] = defaultdict(list)
for r in results:
if r.truncations <= current.truncations and r.config.leading == recommended.config.leading and r.config.trailing == recommended.config.trailing:
if (
r.truncations <= current.truncations
and r.config.leading == recommended.config.leading
and r.config.trailing == recommended.config.trailing
):
by_default[r.config.default].append(r)
defaults = sorted(by_default)
vals = [min(v, key=lambda r: r.tokens).tokens / 1e6 for v in by_default.values()]
ax.bar([str(d) for d in defaults], vals, color="#16a34a")
ax.axhline(current.tokens / 1e6, color="#111", linestyle="--", linewidth=1, label="current")
ax.axhline(
current.tokens / 1e6, color="#111", linestyle="--", linewidth=1, label="current"
)
ax.set_xlabel("defaultLimit")
ax.set_ylabel("best tokens (M)")
ax.set_title(f"defaultLimit sensitivity (L={recommended.config.leading}, T={recommended.config.trailing})")
ax.set_title(
f"defaultLimit sensitivity (L={recommended.config.leading}, T={recommended.config.trailing})"
)
ax.legend(frameon=False)
ax.grid(True, axis="y", alpha=0.25, linestyle="--")
ax = axes[1, 1]
top = sorted([r for r in results if r.truncations <= current.truncations], key=lambda r: r.tokens + 250 * r.calls)[:12]
top = sorted(
[r for r in results if r.truncations <= current.truncations],
key=lambda r: r.tokens + 250 * r.calls,
)[:12]
labels = [r.config.label() for r in top]
token_delta = [(r.tokens - current.tokens) / current.tokens * 100 for r in top]
call_delta = [(r.calls - current.calls) / current.calls * 100 for r in top]
@@ -564,7 +695,9 @@ def plot(results: list[ReplayResult], current: ReplayResult, recommended: Replay
def main() -> int:
ap = argparse.ArgumentParser(description="read configuration optimizer")
ap.add_argument("--since", default=DEFAULT_SINCE, help=f"YYYY-MM-DD (default {DEFAULT_SINCE})")
ap.add_argument(
"--since", default=DEFAULT_SINCE, help=f"YYYY-MM-DD (default {DEFAULT_SINCE})"
)
ap.add_argument("--defaults", default="100,150,200,250,300,400,500,700,1000")
ap.add_argument("--max-lines", default="500,750,1000,1500,2000,3000")
ap.add_argument("--leading", default="0,3,5,10,20")
@@ -581,12 +714,19 @@ def main() -> int:
groups = load_reads(conn, since_ms)
conn.close()
total_calls = sum(len(v) for v in groups.values())
print(f"loaded {total_calls:,} read calls across {len(groups):,} (session,file) groups since {args.since}")
print(
f"loaded {total_calls:,} read calls across {len(groups):,} (session,file) groups since {args.since}"
)
current = replay(groups, Config(CURRENT_DEFAULT, CURRENT_MAX_LINES, CURRENT_LEADING, CURRENT_TRAILING))
current = replay(
groups,
Config(CURRENT_DEFAULT, CURRENT_MAX_LINES, CURRENT_LEADING, CURRENT_TRAILING),
)
configs = candidate_grid(args)
# Ensure current is present even if user overrides grid.
cur_cfg = Config(CURRENT_DEFAULT, CURRENT_MAX_LINES, CURRENT_LEADING, CURRENT_TRAILING)
cur_cfg = Config(
CURRENT_DEFAULT, CURRENT_MAX_LINES, CURRENT_LEADING, CURRENT_TRAILING
)
if cur_cfg not in configs:
configs.append(cur_cfg)
print(f"evaluating {len(configs):,} candidate configs")
@@ -598,22 +738,32 @@ def main() -> int:
print_result("recommended", recommended, current)
allowed = [r for r in results if r.truncations <= current.truncations]
print(f"\nTop token-minimizing configs (truncations <= current {current.truncations:,}):")
print(
f"\nTop token-minimizing configs (truncations <= current {current.truncations:,}):"
)
for i, r in enumerate(sorted(allowed, key=lambda r: r.tokens)[: args.top], 1):
print_result(f"#{i}", r, current)
print(f"\nTop balanced configs (tokens + 250 tokens/read-call objective, truncations <= current {current.truncations:,}):")
for i, r in enumerate(sorted(allowed, key=lambda r: r.tokens + 250 * r.calls)[: args.top], 1):
print(
f"\nTop balanced configs (tokens + 250 tokens/read-call objective, truncations <= current {current.truncations:,}):"
)
for i, r in enumerate(
sorted(allowed, key=lambda r: r.tokens + 250 * r.calls)[: args.top], 1
):
print_result(f"#{i}", r, current)
no_call_increase = [r for r in allowed if r.calls <= current.calls]
print(f"\nBest configs with calls <= current (truncations <= current {current.truncations:,}):")
for i, r in enumerate(sorted(no_call_increase, key=lambda r: r.tokens)[: args.top], 1):
print(
f"\nBest configs with calls <= current (truncations <= current {current.truncations:,}):"
)
for i, r in enumerate(
sorted(no_call_increase, key=lambda r: r.tokens)[: args.top], 1
):
print_result(f"#{i}", r, current)
print("\nRecommended breakdown:")
print(f" selector groups : {recommended.selector_groups:,}")
print(f" selector first-cover : {recommended.first_cover_rate*100:.1f}%")
print(f" selector first-cover : {recommended.first_cover_rate * 100:.1f}%")
print(f" selector skipped calls : {recommended.selector_skipped:,}")
print(f" default skipped calls : {recommended.default_skipped:,}")
print(f" raw/unmodelled calls : {recommended.raw_calls:,}")
scripts/session-stats/sync.py
+114 -42
View File
@@ -251,8 +251,12 @@ def batch_count_tokens(strings: list[str]) -> list[int]:
_HEADER_NEW_RE = re.compile(r"^¶+\s*([^\s#¶]+)(?:#\S+)?\s*$")
# Verb-based v4 (current) ops; body rows are `+TEXT` on the following lines.
_VERB_REPLACE_RE = re.compile(r"^\s*replace\s+([1-9][0-9]*)(?:\s*(?:\.\.|-|…)\s*([1-9][0-9]*))?\s*:?\s*$")
_VERB_DELETE_RE = re.compile(r"^\s*delete\s+([1-9][0-9]*)(?:\s*(?:\.\.|-|…)\s*([1-9][0-9]*))?\s*$")
_VERB_REPLACE_RE = re.compile(
r"^\s*replace\s+([1-9][0-9]*)(?:\s*(?:\.\.|-|…)\s*([1-9][0-9]*))?\s*:?\s*$"
)
_VERB_DELETE_RE = re.compile(
r"^\s*delete\s+([1-9][0-9]*)(?:\s*(?:\.\.|-|…)\s*([1-9][0-9]*))?\s*$"
)
_VERB_INSERT_RE = re.compile(
r"^\s*insert\s+(?:(?P<pos>before|after)\s+(?P<anchor>[1-9][0-9]*)|(?P<edge>head|tail))\s*:?\s*$"
)
@@ -324,9 +328,9 @@ class EditSection:
def parse_hashline_input(input_str: str) -> list[EditSection]:
sections: list[EditSection] = []
cur: EditSection | None = None
cur_format: str | None = None # "hash" (¶) | "legacy" (§)
cur_format: str | None = None # "hash" (¶) | "legacy" (§)
cur_grammar: str | None = None # within "hash": None | "verb" | "sigil"
open_idx: int | None = None # current open payload block in cur
open_idx: int | None = None # current open payload block in cur
def open_new(s: EditSection) -> int:
s.payload_blocks.append([])
@@ -581,6 +585,7 @@ def extract_warnings(text: str) -> list[str]:
# --------------------------------------------------------------------------- #
# JSONL parsing
def parse_iso_ms(s: str | None) -> int:
if not s:
return 0
@@ -588,6 +593,7 @@ def parse_iso_ms(s: str | None) -> int:
if s.endswith("Z"):
s = s[:-1] + "+00:00"
from datetime import datetime
return int(datetime.fromisoformat(s).timestamp() * 1000)
except Exception:
return 0
@@ -633,10 +639,12 @@ class SessionRecords:
tool_results: list[list] = field(default_factory=list)
assistant_msgs: list[list] = field(default_factory=list)
user_msgs: list[list] = field(default_factory=list)
edit_calls: list[tuple] = field(default_factory=list) # initial stub on toolCall
edit_call_results: list[tuple] = field(default_factory=list) # success+warnings on toolResult
edit_sections: list[tuple] = field(default_factory=list) # one row per section
pending_tokens: list[tuple] = field(default_factory=list) # (row, field_idx, text)
edit_calls: list[tuple] = field(default_factory=list) # initial stub on toolCall
edit_call_results: list[tuple] = field(
default_factory=list
) # success+warnings on toolResult
edit_sections: list[tuple] = field(default_factory=list) # one row per section
pending_tokens: list[tuple] = field(default_factory=list) # (row, field_idx, text)
starting_seq: int = 0
full_rebuild: bool = False
starting_offset: int = 0
@@ -759,11 +767,21 @@ def _ingest_assistant(rec, path, seq, entry_id, ts, msg, content) -> None:
elif isinstance(arg_obj, str):
arg_json = arg_obj
else:
arg_json = json.dumps(arg_obj, separators=(",", ":"), ensure_ascii=False)
arg_json = json.dumps(
arg_obj, separators=(",", ":"), ensure_ascii=False
)
row = [
sf, seq, entry_id, call_id,
tool_name, raw_name, ts, model, provider,
arg_json, 0,
sf,
seq,
entry_id,
call_id,
tool_name,
raw_name,
ts,
model,
provider,
arg_json,
0,
]
rec.tool_calls.append(row)
if arg_json:
@@ -785,8 +803,16 @@ def _ingest_assistant(rec, path, seq, entry_id, ts, msg, content) -> None:
thinking_tokens_slot = 0
if text_blob or thinking_blob:
row = [
sf, seq, entry_id, ts, model, provider,
text_blob, thinking_blob, text_tokens_slot, thinking_tokens_slot,
sf,
seq,
entry_id,
ts,
model,
provider,
text_blob,
thinking_blob,
text_tokens_slot,
thinking_tokens_slot,
]
rec.assistant_msgs.append(row)
if text_blob:
@@ -816,11 +842,11 @@ def _ingest_edit_call(rec, sf, seq, ts, call_id, arg_obj, arg_json) -> None:
raw_input_len = len(input_str.encode("utf-8"))
# Stub call row (success + warnings come from toolResult later).
rec.edit_calls.append(
(sf, call_id, seq, ts, raw_input_len, EDIT_PARSER_VERSION)
)
rec.edit_calls.append((sf, call_id, seq, ts, raw_input_len, EDIT_PARSER_VERSION))
if not any(line.startswith(("", "§")) for line in input_str.lstrip("\ufeff").splitlines()):
if not any(
line.startswith(("", "§")) for line in input_str.lstrip("\ufeff").splitlines()
):
# Vim-mode or other shape — no sections to record.
return
@@ -849,12 +875,22 @@ def _ingest_edit_call(rec, sf, seq, ts, call_id, arg_obj, arg_json) -> None:
rec.edit_sections.append(
(
sf, call_id, seq, idx, sec.target_file,
sec.op_count, sec.deleted_lines, sec.payload_count, sec.change_size,
sec.min_line, sec.max_line,
sf,
call_id,
seq,
idx,
sec.target_file,
sec.op_count,
sec.deleted_lines,
sec.payload_count,
sec.change_size,
sec.min_line,
sec.max_line,
json.dumps(sec.payload_blocks, ensure_ascii=False),
json.dumps(sec.op_anchors, ensure_ascii=False),
longest_repeat_len, repeat_block_idx, sample,
longest_repeat_len,
repeat_block_idx,
sample,
json.dumps(dups, ensure_ascii=False),
EDIT_PARSER_VERSION,
)
@@ -893,6 +929,7 @@ def _ingest_user(rec, path, seq, entry_id, ts, content) -> None:
# --------------------------------------------------------------------------- #
# DB
def open_db() -> sqlite3.Connection:
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
conn = sqlite3.connect(DB_PATH, isolation_level=None, check_same_thread=False)
@@ -905,7 +942,9 @@ def open_db() -> sqlite3.Connection:
return conn
def existing_state(conn: sqlite3.Connection) -> dict[str, tuple[int, int, int, int, int]]:
def existing_state(
conn: sqlite3.Connection,
) -> dict[str, tuple[int, int, int, int, int]]:
"""{session_file: (mtime, size, byte_offset, line_count, parser_version)}"""
rows = conn.execute(
"SELECT session_file, mtime, size, byte_offset, line_count, parser_version "
@@ -921,9 +960,12 @@ def write_records(conn: sqlite3.Connection, rec: SessionRecords, now_ms: int) ->
try:
if rec.full_rebuild:
for tbl in (
"ss_tool_calls", "ss_tool_results",
"ss_assistant_msgs", "ss_user_msgs",
"ss_edit_calls", "ss_edit_sections",
"ss_tool_calls",
"ss_tool_results",
"ss_assistant_msgs",
"ss_user_msgs",
"ss_edit_calls",
"ss_edit_sections",
):
cur.execute(f"DELETE FROM {tbl} WHERE session_file = ?", (sf,))
@@ -979,8 +1021,10 @@ def write_records(conn: sqlite3.Connection, rec: SessionRecords, now_ms: int) ->
"ON CONFLICT(session_file, call_id) DO UPDATE SET "
" success=excluded.success, warnings=excluded.warnings, "
" parser_version=excluded.parser_version",
[(sf_, cid, succ, warn, EDIT_PARSER_VERSION)
for (sf_, cid, succ, warn) in rec.edit_call_results],
[
(sf_, cid, succ, warn, EDIT_PARSER_VERSION)
for (sf_, cid, succ, warn) in rec.edit_call_results
],
)
if rec.edit_sections:
cur.executemany(
@@ -1021,10 +1065,24 @@ def write_records(conn: sqlite3.Connection, rec: SessionRecords, now_ms: int) ->
" schema_version=excluded.schema_version, "
" parser_version=excluded.parser_version",
(
sf, m["folder"], m["is_subagent"], m["parent_session"], m["subagent_label"],
m["started_at"], m["title"], m["cwd"], m["session_uuid"], m["version"],
rec.file_mtime, rec.file_size, rec.final_offset, rec.final_line_count,
now_ms, TOKENIZER_NAME, SCHEMA_VERSION, EDIT_PARSER_VERSION,
sf,
m["folder"],
m["is_subagent"],
m["parent_session"],
m["subagent_label"],
m["started_at"],
m["title"],
m["cwd"],
m["session_uuid"],
m["version"],
rec.file_mtime,
rec.file_size,
rec.final_offset,
rec.final_line_count,
now_ms,
TOKENIZER_NAME,
SCHEMA_VERSION,
EDIT_PARSER_VERSION,
),
)
cur.execute("COMMIT")
@@ -1036,6 +1094,7 @@ def write_records(conn: sqlite3.Connection, rec: SessionRecords, now_ms: int) ->
# --------------------------------------------------------------------------- #
# Driver
def discover_sessions(root: Path, limit: int | None) -> list[Path]:
if not root.exists():
return []
@@ -1077,18 +1136,27 @@ def decide_action(
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--workers", type=int, default=min(16, (os.cpu_count() or 4) * 2))
ap.add_argument("--limit", type=int, default=0,
help="only sync the N most-recent files (0 = all)")
ap.add_argument("--full", action="store_true",
help="ignore stored state, re-ingest every file from scratch")
ap.add_argument(
"--limit",
type=int,
default=0,
help="only sync the N most-recent files (0 = all)",
)
ap.add_argument(
"--full",
action="store_true",
help="ignore stored state, re-ingest every file from scratch",
)
ap.add_argument("--root", default=str(SESSIONS_ROOT))
args = ap.parse_args()
root = Path(args.root).expanduser()
print(f"-> sessions root: {root}", file=sys.stderr)
print(f"-> db: {DB_PATH}", file=sys.stderr)
print(f"-> parser_version={EDIT_PARSER_VERSION} schema_version={SCHEMA_VERSION}",
file=sys.stderr)
print(
f"-> parser_version={EDIT_PARSER_VERSION} schema_version={SCHEMA_VERSION}",
file=sys.stderr,
)
conn = open_db()
state = existing_state(conn)
@@ -1123,14 +1191,18 @@ def main() -> int:
try:
write_records(conn, rec, now_ms)
except Exception as e:
print(f"!! write failed for {rec.session_meta['session_file']}: {e}",
file=sys.stderr)
print(
f"!! write failed for {rec.session_meta['session_file']}: {e}",
file=sys.stderr,
)
n += 1
now = time.monotonic()
if now - last_log >= 1.0:
rate = n / max(now - t0, 1e-6)
print(f" wrote {n}/{len(work)} files ({rate:.1f} files/s)",
file=sys.stderr)
print(
f" wrote {n}/{len(work)} files ({rate:.1f} files/s)",
file=sys.stderr,
)
last_log = now
rate = n / max(time.monotonic() - t0, 1e-6)
print(f"-> wrote {n} files total ({rate:.1f} files/s)", file=sys.stderr)
scripts/tool_io.py
+11 -20
View File
@@ -134,7 +134,6 @@ class ReservoirSample[T]:
self.items[index] = item
def list_recent_session_files(config: ToolIOConfig) -> list[Path]:
min_mtime = time.time() - config.since_days * 24 * 60 * 60
candidates: list[tuple[float, Path]] = []
@@ -150,7 +149,6 @@ def list_recent_session_files(config: ToolIOConfig) -> list[Path]:
return [entry[1] for entry in candidates[: config.max_files]]
def iter_tool_invocations(
tool_names: str | Iterable[str],
config: ToolIOConfig | None = None,
@@ -183,7 +181,11 @@ def iter_tool_invocations(
continue
tool_name = _as_string(payload.get("name"))
tool_call_id = _as_string(payload.get("id"))
if tool_name is None or tool_call_id is None or tool_name not in wanted:
if (
tool_name is None
or tool_call_id is None
or tool_name not in wanted
):
continue
arguments = _as_record(payload.get("arguments")) or {}
pending[tool_call_id] = ToolCall(
@@ -230,43 +232,39 @@ def iter_tool_invocations(
return
def iter_results(stream: Iterable[ToolInvocation]) -> Iterator[ToolInvocation]:
for invocation in stream:
if invocation.has_result:
yield invocation
def iter_failed(stream: Iterable[ToolInvocation]) -> Iterator[ToolInvocation]:
for invocation in stream:
if invocation.is_error:
yield invocation
def iter_successful(stream: Iterable[ToolInvocation]) -> Iterator[ToolInvocation]:
for invocation in stream:
if invocation.has_result and not invocation.is_error:
yield invocation
def iter_with_diff(stream: Iterable[ToolInvocation]) -> Iterator[ToolInvocation]:
for invocation in stream:
if invocation.diff:
yield invocation
def iter_paths(stream: Iterable[ToolInvocation], *paths: str) -> Iterator[ToolInvocation]:
def iter_paths(
stream: Iterable[ToolInvocation], *paths: str
) -> Iterator[ToolInvocation]:
wanted = set(paths)
for invocation in stream:
if invocation.path_hint in wanted:
yield invocation
def take(stream: Iterable[ToolInvocation], limit: int) -> Iterator[ToolInvocation]:
if limit <= 0:
return
@@ -278,15 +276,15 @@ def take(stream: Iterable[ToolInvocation], limit: int) -> Iterator[ToolInvocatio
remaining -= 1
def sample_reservoir[T](stream: Iterable[T], size: int, seed: int | None = None) -> list[T]:
def sample_reservoir[T](
stream: Iterable[T], size: int, seed: int | None = None
) -> list[T]:
sample: ReservoirSample[T] = ReservoirSample(size=size, rng=random.Random(seed))
for item in stream:
sample.add(item)
return sample.items
def extract_result_text(message: dict[str, Any] | None) -> str:
if message is None:
return ""
@@ -305,7 +303,6 @@ def extract_result_text(message: dict[str, Any] | None) -> str:
return ""
def extract_path(arguments: dict[str, Any]) -> str:
for key in ("path", "file", "move"):
value = arguments.get(key)
@@ -314,7 +311,6 @@ def extract_path(arguments: dict[str, Any]) -> str:
return ""
def _iter_session_entries(session_file: Path) -> Iterator[dict[str, Any]]:
with session_file.open("r", encoding="utf-8") as handle:
for line in handle:
@@ -330,7 +326,6 @@ def _iter_session_entries(session_file: Path) -> Iterator[dict[str, Any]]:
yield payload
def _extract_thinking(content: list[Any]) -> str | None:
for item in content:
payload = _as_record(item)
@@ -344,7 +339,6 @@ def _extract_thinking(content: list[Any]) -> str | None:
return None
def resolve_tool_names(*names_or_groups: str) -> tuple[str, ...]:
ordered: list[str] = []
seen: set[str] = set()
@@ -367,20 +361,17 @@ def _normalize_tool_names(tool_names: str | Iterable[str]) -> set[str]:
return set(ordered)
def _event_weight(invocation: ToolInvocation, limit_mode: LimitMode) -> int:
if limit_mode == "calls":
return 1
return 2 if invocation.has_result else 1
def _as_record(value: Any) -> dict[str, Any] | None:
if not isinstance(value, dict):
return None
return value
def _as_string(value: Any) -> str | None:
return value if isinstance(value, str) else None