From c39f7e0e20b9bf88575c3f2920f687864c869468 Mon Sep 17 00:00:00 2001 From: can1357 Date: Thu, 30 Apr 2026 01:54:05 +0200 Subject: [PATCH] feat(scripts/session-stats): added session-stats edit/tools subcommands --- .gitignore | 4 + package.json | 3 + scripts/analyze-edit-formats/README.md | 48 -- scripts/analyze-edit-formats/go.mod | 3 - scripts/analyze-edit-formats/main.go | 651 ------------------------- scripts/dump-edit-history.ts | 409 ---------------- scripts/session-stats/Cargo.toml | 29 ++ scripts/session-stats/README.md | 82 ++++ scripts/session-stats/src/cmd_edits.rs | 639 ++++++++++++++++++++++++ scripts/session-stats/src/cmd_tools.rs | 387 +++++++++++++++ scripts/session-stats/src/common.rs | 256 ++++++++++ scripts/session-stats/src/main.rs | 58 +++ 12 files changed, 1458 insertions(+), 1111 deletions(-) delete mode 100644 scripts/analyze-edit-formats/README.md delete mode 100644 scripts/analyze-edit-formats/go.mod delete mode 100644 scripts/analyze-edit-formats/main.go delete mode 100755 scripts/dump-edit-history.ts create mode 100644 scripts/session-stats/Cargo.toml create mode 100644 scripts/session-stats/README.md create mode 100644 scripts/session-stats/src/cmd_edits.rs create mode 100644 scripts/session-stats/src/cmd_tools.rs create mode 100644 scripts/session-stats/src/common.rs create mode 100644 scripts/session-stats/src/main.rs diff --git a/.gitignore b/.gitignore index 88e6ad49d..fd1901cfd 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,7 @@ pi-*.html packages/coding-agent/src/internal-urls/docs-index.generated.ts /runs/ python/omp-rpc/src/omp_rpc.egg-info/ + +scripts/session-stats/Cargo.lock + +scripts/session-stats/edit-analysis.csv diff --git a/package.json b/package.json index ee0422357..1392fc3f3 100644 --- a/package.json +++ b/package.json @@ -117,6 +117,9 @@ "ci:release:publish": "bun scripts/ci-release-publish.ts", "bench:gen-fixtures": "bun --cwd=packages/typescript-edit-benchmark run src/generate.ts --typescript-dir /tmp/typescript-source --count-per-type 8", "bench:edit": "bun --cwd=packages/typescript-edit-benchmark run start", + "stats:run": "cargo run --release --manifest-path scripts/session-stats/Cargo.toml --", + "stats:edits": "cargo run --release --manifest-path scripts/session-stats/Cargo.toml -- edits", + "stats:tools": "cargo run --release --manifest-path scripts/session-stats/Cargo.toml -- tools", "prepublishOnly": "bun run check", "prepare": "bun --cwd=packages/coding-agent run generate-docs-index", "publish": "bun run prepublishOnly && npm publish -ws --access public", diff --git a/scripts/analyze-edit-formats/README.md b/scripts/analyze-edit-formats/README.md deleted file mode 100644 index 374b689f3..000000000 --- a/scripts/analyze-edit-formats/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# analyze-edit-formats - -Audits how agents have used the `edit` / `ast_edit` / `write` tools across -historical session JSONLs in `~/.omp/agent/sessions/`. - -For each tool call we: - -- detect the **argument-schema family** in use (the edit tool has shipped many - shapes over time: `oldText/newText`, `op+pos+end+lines`, `loc+content`, - `loc+splice/pre/post/sed`, etc.); -- record the locator shape and verb combination (for the current - `loc+splice/pre/post/sed` schema); -- pair the call with its `toolResult` and classify the outcome - (`success` / `truncated` / `aborted` / `fail:anchor-stale` / - `fail:no-match` / `fail:parse` / `fail:no-enclosing-block` / …). - -Output is a markdown-ish report on stdout plus per-call CSV at -`/tmp/edit-analysis/edits.csv` (or your CWD if you set that up differently). - -## Usage - -```sh -# Scan every session jsonl on disk (slow — ~25k files). -go run ./scripts/analyze-edit-formats - -# Scan only files whose path contains the given date prefix(es). -go run ./scripts/analyze-edit-formats 2026-04-28 -go run ./scripts/analyze-edit-formats 2026-04-27 2026-04-28 -``` - -The walk root is `~/.omp/agent/sessions/`. Sub-session files (subagent -trajectories nested under `/-.jsonl`) are picked up -automatically. - -## Why Go - -The session corpus is large (>25k files, >100k edit calls). Go iterates the -JSONL stream with negligible memory overhead and finishes in ~90s. The same -analysis in Bun/TS works but is noticeably slower for ad-hoc runs. - -## What it's good for - -- Comparing reliability across edit-tool argument schemas before changing the - current one. -- Spotting which verb / locator shapes have outsized failure rates so the - prompt can warn against them. -- Sanity-checking that a new edit-tool design isn't regressing the - failure-mode mix versus the previous design. diff --git a/scripts/analyze-edit-formats/go.mod b/scripts/analyze-edit-formats/go.mod deleted file mode 100644 index 70e146993..000000000 --- a/scripts/analyze-edit-formats/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module github.com/oh-my-pi/scripts/analyze-edit-formats - -go 1.26.2 diff --git a/scripts/analyze-edit-formats/main.go b/scripts/analyze-edit-formats/main.go deleted file mode 100644 index 3713d3f01..000000000 --- a/scripts/analyze-edit-formats/main.go +++ /dev/null @@ -1,651 +0,0 @@ -// Analyzes how agents use the `edit` tool across today's session jsonl files -// in ~/.omp/agent/sessions/. -// -// For every edit-family toolCall (edit, ast_edit, write) we: -// - record what shape of arguments was supplied (loc, splice/pre/post/sed, -// bracket form of locator, line vs file targeted, etc.) -// - pair it with its toolResult and classify the result as success or as a -// specific failure category (anchor stale, anchor unknown, no enclosing -// block, parse error, ssr no match, etc.). -// -// Output is a markdown-ish report on stdout plus a CSV of every edit attempt -// to ./edit-analysis.csv (override with $EDIT_ANALYSIS_CSV). -package main - -import ( - "bufio" - "encoding/csv" - "encoding/json" - "fmt" - "os" - "path/filepath" - "regexp" - "sort" - "strings" -) - -type rawMessage struct { - Type string `json:"type"` - Message json.RawMessage `json:"message"` -} - -type message struct { - Role string `json:"role"` - Content json.RawMessage `json:"content"` - ToolName string `json:"toolName"` - ToolCallID string `json:"toolCallId"` -} - -type contentItem struct { - Type string `json:"type"` - Text string `json:"text"` - Name string `json:"name"` - ID string `json:"id"` - Arguments json.RawMessage `json:"arguments"` -} - -type editEntry struct { - File string - CallID string - ToolName string - NumEdits int - Verbs []string // splice/pre/post/sed per sub-edit - LocShapes []string // bare/bracket-(/bracket-[/bracket-tail/bracket-head/$pre/$post/$sed - HasNewFile bool - HasGlob bool - HasOps bool - Format string // edit-tool argument schema family - ResultRaw string - Status string // "success" or failure category -} - -// matchDate accepts files whose path contains any of the supplied date -// prefixes (e.g. "2026-04-28"). With no flags it accepts every .jsonl. -var dateFilters []string - -func matchDate(p string) bool { - if len(dateFilters) == 0 { - return true - } - for _, d := range dateFilters { - if strings.Contains(p, d) { - return true - } - } - return false -} - -func main() { - dateFilters = os.Args[1:] - root, err := os.UserHomeDir() - must(err) - base := filepath.Join(root, ".omp", "agent", "sessions") - - var files []string - must(filepath.Walk(base, func(p string, info os.FileInfo, err error) error { - if err != nil { - return nil - } - if info.IsDir() { - return nil - } - if !strings.HasSuffix(p, ".jsonl") { - return nil - } - if !matchDate(p) { - return nil - } - files = append(files, p) - return nil - })) - sort.Strings(files) - fmt.Fprintf(os.Stderr, "loaded %d session files for today\n", len(files)) - - var entries []editEntry - for _, f := range files { - entries = append(entries, processFile(f)...) - } - - report(entries) - writeCSV(entries) -} - -func processFile(path string) []editEntry { - fh, err := os.Open(path) - if err != nil { - fmt.Fprintln(os.Stderr, "open:", err) - return nil - } - defer fh.Close() - - calls := map[string]*editEntry{} - var order []string - - sc := bufio.NewScanner(fh) - sc.Buffer(make([]byte, 0, 64*1024), 64*1024*1024) - for sc.Scan() { - var rm rawMessage - if err := json.Unmarshal(sc.Bytes(), &rm); err != nil { - continue - } - if rm.Type != "message" { - continue - } - var m message - if err := json.Unmarshal(rm.Message, &m); err != nil { - continue - } - var items []contentItem - if err := json.Unmarshal(m.Content, &items); err != nil { - continue - } - - switch m.Role { - case "assistant": - for _, it := range items { - if it.Type != "toolCall" { - continue - } - if !isEditTool(it.Name) { - continue - } - e := classifyArgs(it.Name, it.Arguments) - e.File = path - e.CallID = it.ID - e.ToolName = it.Name - calls[it.ID] = &e - order = append(order, it.ID) - } - case "toolResult": - if !isEditTool(m.ToolName) { - continue - } - e, ok := calls[m.ToolCallID] - if !ok { - // orphan result, skip - continue - } - text := joinText(items) - e.ResultRaw = text - e.Status = classifyResult(m.ToolName, text) - } - } - - out := make([]editEntry, 0, len(order)) - for _, id := range order { - if e, ok := calls[id]; ok { - out = append(out, *e) - } - } - return out -} - -func isEditTool(name string) bool { - switch strings.ToLower(name) { - case "edit", "ast_edit", "write": - return true - } - return false -} - -func joinText(items []contentItem) string { - var b strings.Builder - for _, it := range items { - if it.Type == "text" { - b.WriteString(it.Text) - } - } - return b.String() -} - -// ---- argument classification ---- - -type editOp struct { - Loc string `json:"loc"` - Splice json.RawMessage `json:"splice"` - Pre json.RawMessage `json:"pre"` - Post json.RawMessage `json:"post"` - Sed json.RawMessage `json:"sed"` -} - -type editArgs struct { - Path string `json:"path"` - Edits []editOp `json:"edits"` - - // ast_edit - Ops []json.RawMessage `json:"ops"` - - // Write - Content *string `json:"content,omitempty"` -} - -var anchorBare = regexp.MustCompile(`^[a-zA-Z]?\d+[a-z]{2}$`) -var anchorWithFile = regexp.MustCompile(`^[^:]+:\d+[a-z]{2}$`) - -func classifyArgs(name string, raw json.RawMessage) editEntry { - e := editEntry{} - e.Format = detectFormat(name, raw) - switch strings.ToLower(name) { - case "edit": - var a editArgs - _ = json.Unmarshal(raw, &a) - e.NumEdits = len(a.Edits) - for _, op := range a.Edits { - e.LocShapes = append(e.LocShapes, locShape(op.Loc)) - verbs := []string{} - if !isNullOrEmpty(op.Splice) { - verbs = append(verbs, "splice") - } - if !isNullOrEmpty(op.Pre) { - verbs = append(verbs, "pre") - } - if !isNullOrEmpty(op.Post) { - verbs = append(verbs, "post") - } - if !isNullOrEmpty(op.Sed) { - verbs = append(verbs, "sed") - } - if len(verbs) == 0 { - verbs = append(verbs, "none") - } - e.Verbs = append(e.Verbs, strings.Join(verbs, "+")) - } - case "ast_edit": - var a editArgs - _ = json.Unmarshal(raw, &a) - e.HasOps = len(a.Ops) > 0 - e.NumEdits = len(a.Ops) - if strings.ContainsAny(a.Path, "*?,") { - e.HasGlob = true - } - case "write": - e.HasNewFile = true - e.NumEdits = 1 - e.Verbs = []string{"write"} - } - return e -} - -// detectFormat figures out which edit-tool argument schema is in use by -// looking at the top-level argument keys and (for `edit`) the keys of the -// first sub-edit. Older sessions used many incompatible schemas. -func detectFormat(name string, raw json.RawMessage) string { - switch strings.ToLower(name) { - case "write": - return "write" - case "ast_edit": - return "ast_edit" - } - var top map[string]json.RawMessage - if err := json.Unmarshal(raw, &top); err != nil { - return "unknown" - } - has := func(k string) bool { _, ok := top[k]; return ok } - switch { - case has("oldText") && has("newText"): - return "oldText/newText" - case has("old_text") && has("new_text"): - return "old_text/new_text" - case has("diff") && has("op"): - return "diff+op" - case has("diff") && has("operation"): - return "diff+operation" - case has("diff"): - return "diff" - case has("replace") || has("insert"): - return "replace/insert" - } - if edits, ok := top["edits"]; ok { - var list []map[string]json.RawMessage - if err := json.Unmarshal(edits, &list); err == nil && len(list) > 0 { - first := list[0] - fh := func(k string) bool { _, ok := first[k]; return ok } - switch { - case fh("loc") && (fh("splice") || fh("pre") || fh("post") || fh("sed")): - return "loc+splice/pre/post/sed" - case fh("loc") && fh("content"): - return "loc+content" - case fh("set_line"): - return "set_line" - case fh("insert_after"): - return "insert_after" - case fh("op") && fh("pos") && fh("end") && fh("lines"): - return "op+pos+end+lines" - case fh("op") && fh("pos") && fh("lines"): - return "op+pos+lines" - case fh("op") && fh("sel") && fh("content"): - return "op+sel+content" - case fh("all") && (fh("new_text") || fh("old_text")): - return "per-edit:old_text/new_text" - } - keys := make([]string, 0, len(first)) - for k := range first { - keys = append(keys, k) - } - sort.Strings(keys) - return "edits[" + strings.Join(keys, ",") + "]" - } - } - keys := make([]string, 0, len(top)) - for k := range top { - keys = append(keys, k) - } - sort.Strings(keys) - return strings.Join(keys, ",") -} - -func isNullOrEmpty(b json.RawMessage) bool { - s := strings.TrimSpace(string(b)) - return s == "" || s == "null" -} - -func locShape(loc string) string { - if loc == "" { - return "empty" - } - if loc == "$" { - return "$file" - } - // strip optional file: prefix - rest := loc - if i := strings.LastIndex(loc, ":"); i >= 0 && !strings.HasPrefix(loc, "$") { - rest = loc[i+1:] - } - switch { - case strings.HasPrefix(rest, "(") && strings.HasSuffix(rest, ")"): - return "bracket-(body)" - case strings.HasPrefix(rest, "[") && strings.HasSuffix(rest, "]"): - return "bracket-[block]" - case strings.HasPrefix(rest, "(") || strings.HasPrefix(rest, "["): - return "bracket-tail" - case strings.HasSuffix(rest, ")") || strings.HasSuffix(rest, "]"): - return "bracket-head" - case anchorBare.MatchString(rest): - return "bare-anchor" - } - return "other" -} - -// ---- result classification ---- - -var ( - reAnchorStale = regexp.MustCompile(`(?i)(Edit rejected:.*line[s]? .* changed since the last read|line[s]? ha(s|ve) changed since last read)`) - reAnchorMissing = regexp.MustCompile(`(?i)anchor .* (not found|unknown|missing)|loc requires the full anchor`) - reNoEnclosing = regexp.MustCompile(`(?i)No enclosing .* block`) - reParseError = regexp.MustCompile(`(?i)parse|syntax error|unbalanced|unexpected token`) - reSSRNoMatch = regexp.MustCompile(`(?i)0 matches|no replacements|no match found|No replacements made|Failed to find expected lines`) - reFileNotRead = regexp.MustCompile(`(?i)must be read first|has not been read|not yet read`) - reFileChanged = regexp.MustCompile(`(?i)file has been (modified|changed) externally`) - rePermDenied = regexp.MustCompile(`(?i)permission denied|not allowed`) - reGenericRejected = regexp.MustCompile(`(?i)\b(rejected|failed|error|invalid)\b`) - reTruncated = regexp.MustCompile(`(?i)\[Output truncated`) - reAborted = regexp.MustCompile(`(?i)Tool execution was aborted|Request was aborted|cancelled|canceled by user`) - reSuccess = regexp.MustCompile(`(?i)^(Updated|Successfully (wrote|replaced|edited|deleted|inserted)|Replaced|Applied|Deleted|Created|Wrote|edit applied|Edited|Inserted|OK\b)`) -) - -func classifyResult(tool, text string) string { - t := strings.TrimSpace(text) - if t == "" { - return "empty" - } - first := strings.SplitN(t, "\n", 2)[0] - switch { - case reTruncated.MatchString(first): - return "truncated" - case reAborted.MatchString(t): - return "aborted" - case reSuccess.MatchString(first): - return "success" - case reAnchorStale.MatchString(t): - return "fail:anchor-stale" - case reNoEnclosing.MatchString(t): - return "fail:no-enclosing-block" - case reAnchorMissing.MatchString(t): - return "fail:anchor-missing" - case reParseError.MatchString(t): - return "fail:parse" - case reSSRNoMatch.MatchString(t): - return "fail:no-match" - case reFileNotRead.MatchString(t): - return "fail:file-not-read" - case reFileChanged.MatchString(t): - return "fail:file-changed" - case rePermDenied.MatchString(t): - return "fail:perm" - case reGenericRejected.MatchString(first): - return "fail:other" - } - return "unknown" -} - -// ---- reporting ---- - -func report(entries []editEntry) { - if len(entries) == 0 { - fmt.Println("no edit-family tool calls found in today's sessions") - return - } - - byTool := map[string]int{} - byFormat := map[string]int{} - statusByFormat := map[string]map[string]int{} - statusByTool := map[string]map[string]int{} - verbCount := map[string]int{} - locCount := map[string]int{} - failsByVerb := map[string]map[string]int{} - failsByLoc := map[string]map[string]int{} - - for _, e := range entries { - byTool[e.ToolName]++ - if statusByTool[e.ToolName] == nil { - statusByTool[e.ToolName] = map[string]int{} - } - statusByTool[e.ToolName][e.Status]++ - byFormat[e.Format]++ - if statusByFormat[e.Format] == nil { - statusByFormat[e.Format] = map[string]int{} - } - statusByFormat[e.Format][e.Status]++ - - for _, v := range e.Verbs { - verbCount[v]++ - if failsByVerb[v] == nil { - failsByVerb[v] = map[string]int{} - } - failsByVerb[v][e.Status]++ - } - for _, l := range e.LocShapes { - locCount[l]++ - if failsByLoc[l] == nil { - failsByLoc[l] = map[string]int{} - } - failsByLoc[l][e.Status]++ - } - } - - fmt.Println("# Edit-tool usage in today's sessions") - fmt.Printf("\nTotal tool calls: %d (across %d sessions)\n", - len(entries), countSessions(entries)) - - fmt.Println("\n## By tool") - printSorted(byTool) - - fmt.Println("\n## Outcome by tool") - tools := keys(byTool) - sort.Strings(tools) - for _, t := range tools { - fmt.Printf("\n %s (%d calls):\n", t, byTool[t]) - printSortedIndent(statusByTool[t], " ") - } - - fmt.Println("\n## edit verb distribution (per sub-edit)") - printSorted(verbCount) - - fmt.Println("\n## edit locator shape distribution") - printSorted(locCount) - - fmt.Println("\n## Failure rate per verb shape") - for _, v := range sortedKeys(verbCount) { - total, failed := 0, 0 - for status, n := range failsByVerb[v] { - total += n - if strings.HasPrefix(status, "fail") { - failed += n - } - } - fmt.Printf(" %-20s %d/%d failed (%.0f%%)\n", v, failed, total, pct(failed, total)) - } - - fmt.Println("\n## Failure rate per locator shape") - for _, l := range sortedKeys(locCount) { - total, failed := 0, 0 - for status, n := range failsByLoc[l] { - total += n - if strings.HasPrefix(status, "fail") { - failed += n - } - } - fmt.Printf(" %-20s %d/%d failed (%.0f%%)\n", l, failed, total, pct(failed, total)) - } - - fmt.Println("\n## edit-tool argument-format usage") - printSorted(byFormat) - - fmt.Println("\n## Failure rate per argument format") - for _, fname := range sortedKeys(byFormat) { - total, failed := 0, 0 - for status, n := range statusByFormat[fname] { - total += n - if strings.HasPrefix(status, "fail") { - failed += n - } - } - fmt.Printf(" %-32s %6d/%-6d failed (%.0f%%)\n", fname, failed, total, pct(failed, total)) - } - - fmt.Println("\n## Failure breakdown per top format") - cap := 0 - for _, fname := range sortedKeys(byFormat) { - if cap >= 8 { - break - } - cap++ - fmt.Printf("\n %s (%d total)\n", fname, byFormat[fname]) - printSortedIndent(statusByFormat[fname], " ") - } - - fmt.Println("\n## Sample failed edits") - shown := 0 - for _, e := range entries { - if !strings.HasPrefix(e.Status, "fail") { - continue - } - fmt.Printf("\n— %s [%s] verbs=%v loc=%v\n result: %s\n", - e.ToolName, e.Status, e.Verbs, e.LocShapes, - truncate(strings.SplitN(e.ResultRaw, "\n\n", 2)[0], 220)) - shown++ - if shown >= 8 { - break - } - } -} - -func countSessions(es []editEntry) int { - s := map[string]struct{}{} - for _, e := range es { - s[e.File] = struct{}{} - } - return len(s) -} - -func printSorted(m map[string]int) { - for _, k := range sortedKeys(m) { - fmt.Printf(" %-25s %d\n", k, m[k]) - } -} - -func printSortedIndent(m map[string]int, indent string) { - for _, k := range sortedKeys(m) { - fmt.Printf("%s%-25s %d\n", indent, k, m[k]) - } -} - -func sortedKeys(m map[string]int) []string { - type kv struct { - k string - v int - } - pairs := make([]kv, 0, len(m)) - for k, v := range m { - pairs = append(pairs, kv{k, v}) - } - sort.Slice(pairs, func(i, j int) bool { - if pairs[i].v != pairs[j].v { - return pairs[i].v > pairs[j].v - } - return pairs[i].k < pairs[j].k - }) - out := make([]string, len(pairs)) - for i, p := range pairs { - out[i] = p.k - } - return out -} - -func keys(m map[string]int) []string { - out := make([]string, 0, len(m)) - for k := range m { - out = append(out, k) - } - return out -} - -func pct(a, b int) float64 { - if b == 0 { - return 0 - } - return 100 * float64(a) / float64(b) -} - -func truncate(s string, n int) string { - s = strings.ReplaceAll(s, "\n", " | ") - if len(s) <= n { - return s - } - return s[:n] + "…" -} - -func writeCSV(entries []editEntry) { - csvPath := os.Getenv("EDIT_ANALYSIS_CSV") - if csvPath == "" { - csvPath = "edit-analysis.csv" - } - f, err := os.Create(csvPath) - if err != nil { - fmt.Fprintln(os.Stderr, "csv:", err) - return - } - defer f.Close() - w := csv.NewWriter(f) - defer w.Flush() - _ = w.Write([]string{"session", "tool", "status", "num_edits", "verbs", "loc_shapes", "result_first_line"}) - for _, e := range entries { - first := strings.SplitN(e.ResultRaw, "\n", 2)[0] - _ = w.Write([]string{ - filepath.Base(e.File), - e.ToolName, - e.Status, - fmt.Sprintf("%d", e.NumEdits), - strings.Join(e.Verbs, ","), - strings.Join(e.LocShapes, ","), - truncate(first, 200), - }) - } -} - -func must(err error) { - if err != nil { - fmt.Fprintln(os.Stderr, "fatal:", err) - os.Exit(1) - } -} diff --git a/scripts/dump-edit-history.ts b/scripts/dump-edit-history.ts deleted file mode 100755 index 1a9f2421a..000000000 --- a/scripts/dump-edit-history.ts +++ /dev/null @@ -1,409 +0,0 @@ -#!/usr/bin/env bun -/** - * Dump and analyze edit tool attempts from session JSONL files. - * - * Usage: - * bun scripts/dump-edit-history.ts [options] - * - * Options: - * --failures Show only failed attempts - * --successes Show only successful attempts - * --json Output as JSON - * --stats Show statistics only - * --context Include thinking context before each attempt - * --compact Compact output (no diff content) - */ - -import { Glob } from "bun"; - -import { basename } from "node:path"; - -// ═══════════════════════════════════════════════════════════════════════════ -// Types -// ═══════════════════════════════════════════════════════════════════════════ - -interface Message { - type: string; - id?: string; - message?: { - role?: string; - content?: Array<{ - type: string; - name?: string; - id?: string; - arguments?: Record; - text?: string; - thinking?: string; - }>; - toolCallId?: string; - isError?: boolean; - }; -} - -interface EditAttempt { - id: string; - path: string; - op: string; - diff: string; - isError: boolean; - resultText: string; - errorType?: string; - thinkingContext?: string; -} - -interface SessionResult { - file: string; - attempts: EditAttempt[]; -} - -// ═══════════════════════════════════════════════════════════════════════════ -// Parsing -// ═══════════════════════════════════════════════════════════════════════════ - -function classifyError(resultText: string): string { - if (resultText.includes("Failed to find context")) return "context-not-found"; - if (resultText.includes("matches for context")) return "ambiguous-context"; - if (resultText.includes("Unexpected line in hunk")) return "parse-error"; - if (resultText.includes("Failed to find expected lines")) return "lines-not-found"; - if (resultText.includes("File not found")) return "file-not-found"; - if (resultText.includes("occurrences")) return "ambiguous-match"; - return "unknown"; -} - -async function extractEditAttempts(sessionPath: string): Promise { - const content = await Bun.file(sessionPath).bytes(); - const messages = Bun.JSONL.parse(content) as Message[]; - - const editAttempts: EditAttempt[] = []; - - for (let i = 0; i < messages.length; i++) { - const msg = messages[i]; - if (msg.type !== "message") continue; - - const msgContent = msg.message?.content; - if (!Array.isArray(msgContent)) continue; - - // Extract thinking from this message - const thinking = msgContent.find((c) => c.type === "thinking")?.thinking; - - for (const item of msgContent) { - if (item.type === "toolCall" && item.name === "edit") { - const toolId = item.id!; - const args = item.arguments as { path?: string; op?: string; diff?: string }; - - // Find result - let result: Message["message"] | null = null; - for (let j = i + 1; j < messages.length; j++) { - const resultMsg = messages[j]; - if (resultMsg.type === "message" && resultMsg.message?.role === "toolResult") { - if (resultMsg.message.toolCallId === toolId) { - result = resultMsg.message; - break; - } - } - } - - const resultContent = result?.content; - const resultText = - Array.isArray(resultContent) && resultContent[0]?.type === "text" - ? (resultContent[0].text ?? "") - : ""; - - const isError = result?.isError ?? false; - - editAttempts.push({ - id: toolId, - path: args.path ?? "", - op: args.op ?? "update", - diff: args.diff ?? "", - isError, - resultText, - errorType: isError ? classifyError(resultText) : undefined, - thinkingContext: thinking, - }); - } - } - } - - return editAttempts; -} - -// ═══════════════════════════════════════════════════════════════════════════ -// Formatting -// ═══════════════════════════════════════════════════════════════════════════ - -const colors = { - reset: "\x1b[0m", - bold: "\x1b[1m", - dim: "\x1b[2m", - red: "\x1b[31m", - green: "\x1b[32m", - yellow: "\x1b[33m", - blue: "\x1b[34m", - magenta: "\x1b[35m", - cyan: "\x1b[36m", -}; - -function colorize(text: string, color: keyof typeof colors): string { - return `${colors[color]}${text}${colors.reset}`; -} - -function formatDiff(diff: string): string { - return diff - .split("\n") - .map((line) => { - if (line.startsWith("+")) return colorize(line, "green"); - if (line.startsWith("-")) return colorize(line, "red"); - if (line.startsWith("@@")) return colorize(line, "cyan"); - return colorize(line, "dim"); - }) - .join("\n"); -} - -function formatAttempt(attempt: EditAttempt, index: number, options: Options): string { - const status = attempt.isError - ? `${colorize("✗ FAILED", "red")} ${colorize(`[${attempt.errorType}]`, "yellow")}` - : colorize("✓ SUCCESS", "green"); - - const lines: string[] = [ - ``, - `${colorize(`### Attempt ${index}`, "bold")}: ${status}`, - `${colorize("Path:", "dim")} ${attempt.path}`, - `${colorize("Operation:", "dim")} ${attempt.op}`, - ]; - - if (options.context && attempt.thinkingContext) { - const truncated = - attempt.thinkingContext.length > 300 - ? `${attempt.thinkingContext.slice(0, 300)}…` - : attempt.thinkingContext; - lines.push(`${colorize("Thinking:", "dim")} ${truncated}`); - } - - if (!options.compact) { - lines.push(`${colorize("Diff:", "dim")}`); - lines.push(formatDiff(attempt.diff)); - } - - lines.push(``); - const resultPreview = attempt.resultText.slice(0, 200); - const truncatedResult = attempt.resultText.length > 200 ? `${resultPreview}…` : resultPreview; - lines.push(`${colorize("Result:", "dim")} ${truncatedResult}`); - lines.push(colorize("-".repeat(80), "dim")); - - return lines.join("\n"); -} - -function formatStats(results: SessionResult[]): string { - const allAttempts = results.flatMap((r) => r.attempts); - const failed = allAttempts.filter((a) => a.isError); - const succeeded = allAttempts.filter((a) => !a.isError); - - // Group failures by error type - const errorGroups: Record = {}; - for (const attempt of failed) { - const type = attempt.errorType ?? "unknown"; - if (!errorGroups[type]) errorGroups[type] = []; - errorGroups[type].push(attempt); - } - - const lines: string[] = [ - ``, - colorize("═".repeat(60), "dim"), - colorize(" Statistics", "bold"), - colorize("═".repeat(60), "dim"), - ``, - `Total attempts: ${colorize(String(allAttempts.length), "bold")}`, - ` ${colorize("✓", "green")} Succeeded: ${succeeded.length}`, - ` ${colorize("✗", "red")} Failed: ${failed.length}`, - ``, - ]; - - if (Object.keys(errorGroups).length > 0) { - lines.push(colorize("Failures by type:", "bold")); - for (const [type, attempts] of Object.entries(errorGroups).sort((a, b) => b[1].length - a[1].length)) { - lines.push(` ${colorize(type, "yellow")}: ${attempts.length}`); - // Show example paths - const uniquePaths = [...new Set(attempts.map((a) => a.path))].slice(0, 3); - for (const p of uniquePaths) { - lines.push(` ${colorize("→", "dim")} ${p}`); - } - } - } - - // Show unique @@ contexts that failed - const failedContexts = failed - .map((a) => { - const match = a.diff.match(/^@@\s*(.+)$/m); - return match?.[1]?.trim(); - }) - .filter(Boolean); - - if (failedContexts.length > 0) { - lines.push(``); - lines.push(colorize("Failed @@ contexts:", "bold")); - const uniqueContexts = [...new Set(failedContexts)].slice(0, 10); - for (const ctx of uniqueContexts) { - lines.push(` ${colorize("@@", "cyan")} ${ctx}`); - } - } - - return lines.join("\n"); -} - -function formatJson(results: SessionResult[]): string { - return JSON.stringify( - results.map((r) => ({ - file: r.file, - attempts: r.attempts.map((a) => ({ - path: a.path, - op: a.op, - diff: a.diff, - isError: a.isError, - errorType: a.errorType, - result: a.resultText, - })), - })), - null, - 2, - ); -} - -// ═══════════════════════════════════════════════════════════════════════════ -// Main -// ═══════════════════════════════════════════════════════════════════════════ - -interface Options { - failures: boolean; - successes: boolean; - json: boolean; - stats: boolean; - context: boolean; - compact: boolean; -} - -function parseArgs(): { paths: string[]; options: Options } { - const args = process.argv.slice(2); - const options: Options = { - failures: false, - successes: false, - json: false, - stats: false, - context: false, - compact: false, - }; - const paths: string[] = []; - - for (const arg of args) { - if (arg === "--failures") options.failures = true; - else if (arg === "--successes") options.successes = true; - else if (arg === "--json") options.json = true; - else if (arg === "--stats") options.stats = true; - else if (arg === "--context") options.context = true; - else if (arg === "--compact") options.compact = true; - else if (!arg.startsWith("-")) paths.push(arg); - } - - return { paths, options }; -} - -async function expandGlobs(patterns: string[]): Promise { - const files: string[] = []; - for (const pattern of patterns) { - if (pattern.includes("*")) { - const glob = new Glob(pattern); - for await (const file of glob.scan({ absolute: true })) { - files.push(file); - } - } else { - try { - await Bun.file(pattern).text(); - files.push(pattern); - } catch (err) { - const error = err as NodeJS.ErrnoException; - if (typeof err === "object" && err !== null && "code" in err ) continue; - if (error.code === "EISDIR" || error.code === "EACCES" || error.code === "EPERM" || error.code === "ENOENT") continue; - throw err; - } - } - } - return files; -} - -async function main() { - const { paths, options } = parseArgs(); - - if (paths.length === 0) { - console.error(`Usage: bun scripts/dump-edit-history.ts [options] - -Options: - --failures Show only failed attempts - --successes Show only successful attempts - --json Output as JSON - --stats Show statistics only - --context Include thinking context before each attempt - --compact Compact output (no diff content) - -Examples: - bun scripts/dump-edit-history.ts session.jsonl - bun scripts/dump-edit-history.ts ~/.omp/agent/sessions/**/*.jsonl --stats - bun scripts/dump-edit-history.ts session.jsonl --failures --compact`); - process.exit(1); - } - - const files = await expandGlobs(paths); - if (files.length === 0) { - console.error("No matching files found"); - process.exit(1); - } - - const results: SessionResult[] = []; - - for (const file of files) { - try { - let attempts = await extractEditAttempts(file); - - // Filter - if (options.failures) attempts = attempts.filter((a) => a.isError); - if (options.successes) attempts = attempts.filter((a) => !a.isError); - - if (attempts.length > 0) { - results.push({ file, attempts }); - } - } catch (e) { - console.error(`Error processing ${file}: ${e}`); - } - } - - // Output - if (options.json) { - console.log(formatJson(results)); - return; - } - - if (options.stats) { - console.log(formatStats(results)); - return; - } - - for (const result of results) { - if (results.length > 1) { - console.log(`\n${colorize("═".repeat(80), "cyan")}`); - console.log(colorize(` ${basename(result.file)}`, "bold")); - console.log(colorize("═".repeat(80), "cyan")); - } - - console.log(`Found ${result.attempts.length} edit attempt(s)`); - - for (let i = 0; i < result.attempts.length; i++) { - console.log(formatAttempt(result.attempts[i], i + 1, options)); - } - } - - // Always show summary - const total = results.reduce((sum, r) => sum + r.attempts.length, 0); - const failed = results.reduce((sum, r) => sum + r.attempts.filter((a) => a.isError).length, 0); - console.log(`\n${colorize("Summary:", "bold")} ${total - failed} succeeded, ${failed} failed`); -} - -main(); diff --git a/scripts/session-stats/Cargo.toml b/scripts/session-stats/Cargo.toml new file mode 100644 index 000000000..f0c801db2 --- /dev/null +++ b/scripts/session-stats/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "session-stats" +version = "0.1.0" +edition = "2024" +publish = false + +# Standalone crate: do not inherit from the parent workspace. +[workspace] + +[[bin]] +name = "session-stats" +path = "src/main.rs" + +[dependencies] +anyhow = "1" +csv = "1" +dirs = "5" +rayon = "1.10" +regex = "1" +serde = { version = "1", features = ["derive"] } +serde_json = { version = "1", features = ["raw_value"] } +tiktoken-rs = "0.7" +walkdir = "2" + +[profile.release] +opt-level = 3 +lto = "thin" +codegen-units = 1 +strip = true diff --git a/scripts/session-stats/README.md b/scripts/session-stats/README.md new file mode 100644 index 000000000..255e23c1a --- /dev/null +++ b/scripts/session-stats/README.md @@ -0,0 +1,82 @@ +# session-stats + +Ad-hoc analyses over the local agent session corpus +(`~/.omp/agent/sessions/`). Single Rust binary with subcommands. + +## Subcommands + +### `edits` — edit-tool reliability audit + +Audits how agents have used the `edit` / `ast_edit` / `write` tools. + +For each call we: + +- detect the **argument-schema family** in use (the edit tool has shipped many + shapes over time: `oldText/newText`, `op+pos+end+lines`, `loc+content`, + `loc+splice/pre/post/sed`, etc.); +- record the locator shape and verb combination (for the current schema); +- pair the call with its `toolResult` and classify the outcome + (`success` / `truncated` / `aborted` / `fail:anchor-stale` / + `fail:no-match` / `fail:parse` / `fail:no-enclosing-block` / …). + +Output: markdown-ish report on stdout plus per-call CSV at `$EDIT_ANALYSIS_CSV` +(default `./edit-analysis.csv`). + +### `tools` — per-tool token budget + +Aggregates token usage across the most-recent N sessions. Buckets: + +- `tool ARGS` — assistant tool-call argument JSON +- `tool RESULTS` — tool result content text +- `assistant THINKING` — assistant `thinking` blocks +- `assistant TEXT` — assistant prose +- `user TEXT` — user-authored text content + +Token counting uses **`o200k_base`** via `tiktoken-rs` (the GPT-4o / GPT-5 +family BPE — well-defined offline and within ~5-10% of Claude's own counts in +aggregate across English/code). + +Output: grand totals + per-tool breakdown sorted by total (arg+res) tokens. +Optional CSV at `$TOOL_USAGE_CSV`. + +## Usage + +```sh +# Edit audit on the most-recent sessions. +cargo run --release --manifest-path scripts/session-stats/Cargo.toml -- edits + +# Edit audit on the 200 most-recent sessions. +cargo run --release --manifest-path scripts/session-stats/Cargo.toml -- edits -n 200 + +# Edit audit on a specific date. +cargo run --release --manifest-path scripts/session-stats/Cargo.toml -- edits 2026-04-28 + +# Tool token budget on the 1000 most-recent sessions. +cargo run --release --manifest-path scripts/session-stats/Cargo.toml -- tools -n 1000 + +# Tool token budget on every jsonl on disk. +cargo run --release --manifest-path scripts/session-stats/Cargo.toml -- tools -n 0 + +# Dump per-tool CSV alongside the report. +TOOL_USAGE_CSV=tools.csv \ + cargo run --release --manifest-path scripts/session-stats/Cargo.toml -- tools -n 200 +``` + +The walk root is `~/.omp/agent/sessions/`. Subagent jsonls +(`/-.jsonl`) count as their own session and are included +in the recency window independently. + +## Layout + +``` +scripts/session-stats/ + Cargo.toml + src/ + main.rs # subcommand dispatch + common.rs # shared JSONL shapes, walk, tokenizer, formatting helpers + cmd_edits.rs # edits subcommand + cmd_tools.rs # tools subcommand +``` + +The crate is a standalone Cargo project (it carries its own `[workspace]` +declaration) so it does not perturb the main workspace's lockfile. diff --git a/scripts/session-stats/src/cmd_edits.rs b/scripts/session-stats/src/cmd_edits.rs new file mode 100644 index 000000000..e96791081 --- /dev/null +++ b/scripts/session-stats/src/cmd_edits.rs @@ -0,0 +1,639 @@ +//! `edits` subcommand — audits how agents have used the edit / ast_edit / +//! write tools across session jsonl files. +//! +//! For every edit-family toolCall we record: +//! - which argument-schema family is in use (the edit tool has shipped many +//! shapes over time: oldText/newText, op+pos+end+lines, loc+content, +//! loc+splice/pre/post/sed, etc.); +//! - the locator shape and verb combination (for the current +//! loc+splice/pre/post/sed schema); +//! then pair the call with its toolResult and classify success / failure +//! category (anchor-stale, no-match, parse, etc.). +//! +//! Output: markdown-ish report on stdout plus a per-call CSV at +//! `$EDIT_ANALYSIS_CSV` (default `./edit-analysis.csv`). + +use crate::common::*; +use anyhow::{Context, Result, bail}; +use regex::Regex; +use serde::Deserialize; +use serde_json::Value; +use serde_json::value::RawValue; +use std::collections::HashMap; +use std::fs::File; +use std::io::{BufRead, BufReader}; +use std::path::Path; +use std::sync::LazyLock; + +#[derive(Default, Clone)] +struct EditEntry { + file: String, + call_id: String, + tool_name: String, + num_edits: i64, + /// splice/pre/post/sed per sub-edit + verbs: Vec, + /// bare-anchor / bracket-(body) / ... + loc_shapes: Vec, + /// edit-tool argument schema family + format: String, + result_raw: String, + /// "success" / "fail:..." / etc. + status: String, +} + +#[derive(Deserialize, Default)] +struct EditOp { + #[serde(default)] + loc: String, + #[serde(default)] + splice: Option>, + #[serde(default)] + pre: Option>, + #[serde(default)] + post: Option>, + #[serde(default)] + sed: Option>, +} + +#[derive(Deserialize, Default)] +struct EditArgs { + #[serde(default)] + edits: Vec, + #[serde(default)] + ops: Vec>, +} + +pub fn run(args: Vec) -> Result<()> { + let mut limit: usize = 100_000; + let mut workers: usize = 0; + let mut date_filters: Vec = Vec::new(); + + let mut iter = args.into_iter(); + while let Some(a) = iter.next() { + match a.as_str() { + "-n" => { + limit = iter + .next() + .context("-n requires a value")? + .parse() + .context("-n value")?; + } + "-j" => { + workers = iter + .next() + .context("-j requires a value")? + .parse() + .context("-j value")?; + } + "-h" | "--help" => { + eprintln!( + "usage: session-stats edits [-n N] [-j workers] [date prefix ...]" + ); + return Ok(()); + } + other if other.starts_with('-') => bail!("unknown flag: {other}"), + other => date_filters.push(other.to_string()), + } + } + + let files = collect_sessions(&WalkOpts { + date_filters, + limit_most_recent: limit, + })?; + eprintln!("scanning {} session files", files.len()); + + let mut entries: Vec = parallel_collect(&files, workers, 5_000, |p| { + Some(process_file(p)) + }) + .into_iter() + .flatten() + .collect(); + + // Stable ordering for sample output. + entries.sort_by(|a, b| a.file.cmp(&b.file)); + + report_edits(&entries); + write_csv(&entries)?; + Ok(()) +} + +fn process_file(path: &Path) -> Vec { + let f = match File::open(path) { + Ok(f) => f, + Err(e) => { + eprintln!("open {}: {e}", path.display()); + return Vec::new(); + } + }; + let reader = BufReader::with_capacity(64 * 1024, f); + let path_str = path.to_string_lossy().into_owned(); + + let mut calls: HashMap = HashMap::new(); + let mut order: Vec = Vec::new(); + + for line in reader.lines() { + let Ok(line) = line else { continue }; + if line.is_empty() { + continue; + } + let Ok(ev) = serde_json::from_str::(&line) else { + continue; + }; + if ev.kind != "message" { + continue; + } + let Some(msg_raw) = ev.message else { continue }; + let Ok(m) = serde_json::from_str::(msg_raw.get()) else { + continue; + }; + let Some(content_raw) = m.content else { continue }; + let items = parse_content(&content_raw); + + match m.role.as_str() { + "assistant" => { + for it in items { + if it.kind != "toolCall" || !is_edit_tool(&it.name) { + continue; + } + let raw = it.arguments.as_deref(); + let mut e = classify_edit_args(&it.name, raw); + e.file.clone_from(&path_str); + e.call_id.clone_from(&it.id); + e.tool_name.clone_from(&it.name); + let id = it.id.clone(); + calls.insert(id.clone(), e); + order.push(id); + } + } + "toolResult" => { + if !is_edit_tool(&m.tool_name) { + continue; + } + let Some(e) = calls.get_mut(&m.tool_call_id) else { + continue; + }; + let text = join_text(&items); + e.status = classify_edit_result(&text); + e.result_raw = text; + } + _ => {} + } + } + + let mut out: Vec = Vec::with_capacity(order.len()); + for id in order { + if let Some(e) = calls.remove(&id) { + out.push(e); + } + } + out +} + +fn is_edit_tool(name: &str) -> bool { + matches!( + name.to_ascii_lowercase().as_str(), + "edit" | "ast_edit" | "write" + ) +} + +// ---- argument classification ---- + +static ANCHOR_BARE: LazyLock = + LazyLock::new(|| Regex::new(r"^[a-zA-Z]?[0-9]+[a-z]{2}$").expect("anchor_bare")); + +fn classify_edit_args(name: &str, raw: Option<&RawValue>) -> EditEntry { + let mut e = EditEntry { + format: detect_edit_format(name, raw), + ..EditEntry::default() + }; + let lname = name.to_ascii_lowercase(); + match lname.as_str() { + "edit" => { + let a: EditArgs = raw + .and_then(|r| serde_json::from_str(r.get()).ok()) + .unwrap_or_default(); + e.num_edits = a.edits.len() as i64; + for op in &a.edits { + e.loc_shapes.push(loc_shape(&op.loc)); + let mut verbs: Vec<&str> = Vec::new(); + if !is_null_or_empty(op.splice.as_deref()) { + verbs.push("splice"); + } + if !is_null_or_empty(op.pre.as_deref()) { + verbs.push("pre"); + } + if !is_null_or_empty(op.post.as_deref()) { + verbs.push("post"); + } + if !is_null_or_empty(op.sed.as_deref()) { + verbs.push("sed"); + } + if verbs.is_empty() { + verbs.push("none"); + } + e.verbs.push(verbs.join("+")); + } + } + "ast_edit" => { + let a: EditArgs = raw + .and_then(|r| serde_json::from_str(r.get()).ok()) + .unwrap_or_default(); + e.num_edits = a.ops.len() as i64; + } + "write" => { + e.num_edits = 1; + e.verbs.push("write".to_string()); + } + _ => {} + } + e +} + +/// Looks at top-level argument keys (and the first sub-edit for the `edit` +/// tool) to identify which schema is in use. Older sessions used many +/// incompatible schemas. +fn detect_edit_format(name: &str, raw: Option<&RawValue>) -> String { + match name.to_ascii_lowercase().as_str() { + "write" => return "write".to_string(), + "ast_edit" => return "ast_edit".to_string(), + _ => {} + } + let Some(raw) = raw else { + return "unknown".to_string(); + }; + let top: HashMap = match serde_json::from_str(raw.get()) { + Ok(v) => v, + Err(_) => return "unknown".to_string(), + }; + let has = |k: &str| top.contains_key(k); + + if has("oldText") && has("newText") { + return "oldText/newText".to_string(); + } + if has("old_text") && has("new_text") { + return "old_text/new_text".to_string(); + } + if has("diff") && has("op") { + return "diff+op".to_string(); + } + if has("diff") && has("operation") { + return "diff+operation".to_string(); + } + if has("diff") { + return "diff".to_string(); + } + if has("replace") || has("insert") { + return "replace/insert".to_string(); + } + + if let Some(edits_val) = top.get("edits") + && let Some(arr) = edits_val.as_array() + && let Some(first) = arr.first().and_then(Value::as_object) + { + let fh = |k: &str| first.contains_key(k); + if fh("loc") && (fh("splice") || fh("pre") || fh("post") || fh("sed")) { + return "loc+splice/pre/post/sed".to_string(); + } + if fh("loc") && fh("content") { + return "loc+content".to_string(); + } + if fh("set_line") { + return "set_line".to_string(); + } + if fh("insert_after") { + return "insert_after".to_string(); + } + if fh("op") && fh("pos") && fh("end") && fh("lines") { + return "op+pos+end+lines".to_string(); + } + if fh("op") && fh("pos") && fh("lines") { + return "op+pos+lines".to_string(); + } + if fh("op") && fh("sel") && fh("content") { + return "op+sel+content".to_string(); + } + if fh("all") && (fh("new_text") || fh("old_text")) { + return "per-edit:old_text/new_text".to_string(); + } + let mut keys: Vec<&str> = first.keys().map(String::as_str).collect(); + keys.sort_unstable(); + return format!("edits[{}]", keys.join(",")); + } + + let mut keys: Vec<&str> = top.keys().map(String::as_str).collect(); + keys.sort_unstable(); + keys.join(",") +} + +fn is_null_or_empty(b: Option<&RawValue>) -> bool { + let Some(b) = b else { return true }; + let s = b.get().trim(); + s.is_empty() || s == "null" +} + +fn loc_shape(loc: &str) -> String { + if loc.is_empty() { + return "empty".to_string(); + } + if loc == "$" { + return "$file".to_string(); + } + let rest = if let Some(i) = loc.rfind(':') + && !loc.starts_with('$') + { + &loc[i + 1..] + } else { + loc + }; + if rest.starts_with('(') && rest.ends_with(')') { + return "bracket-(body)".to_string(); + } + if rest.starts_with('[') && rest.ends_with(']') { + return "bracket-[block]".to_string(); + } + if rest.starts_with('(') || rest.starts_with('[') { + return "bracket-tail".to_string(); + } + if rest.ends_with(')') || rest.ends_with(']') { + return "bracket-head".to_string(); + } + if ANCHOR_BARE.is_match(rest) { + return "bare-anchor".to_string(); + } + "other".to_string() +} + +// ---- result classification ---- + +macro_rules! re { + ($pat:expr) => { + LazyLock::new(|| Regex::new($pat).expect("compile result regex")) + }; +} + +static RE_ANCHOR_STALE: LazyLock = re!( + r"(?i)(Edit rejected:.*line[s]? .* changed since the last read|line[s]? ha(s|ve) changed since last read)" +); +static RE_ANCHOR_MISSING: LazyLock = + re!(r"(?i)anchor .* (not found|unknown|missing)|loc requires the full anchor"); +static RE_NO_ENCLOSING: LazyLock = re!(r"(?i)No enclosing .* block"); +static RE_PARSE_ERROR: LazyLock = + re!(r"(?i)parse|syntax error|unbalanced|unexpected token"); +static RE_SSR_NO_MATCH: LazyLock = re!( + r"(?i)0 matches|no replacements|no match found|No replacements made|Failed to find expected lines" +); +static RE_FILE_NOT_READ: LazyLock = + re!(r"(?i)must be read first|has not been read|not yet read"); +static RE_FILE_CHANGED: LazyLock = + re!(r"(?i)file has been (modified|changed) externally"); +static RE_PERM_DENIED: LazyLock = re!(r"(?i)permission denied|not allowed"); +static RE_GENERIC_REJECTED: LazyLock = + re!(r"(?i)\b(rejected|failed|error|invalid)\b"); +static RE_TRUNCATED: LazyLock = re!(r"(?i)\[Output truncated"); +static RE_ABORTED: LazyLock = re!( + r"(?i)Tool execution was aborted|Request was aborted|cancelled|canceled by user" +); +static RE_SUCCESS: LazyLock = re!( + r"(?i)^(Updated|Successfully (wrote|replaced|edited|deleted|inserted)|Replaced|Applied|Deleted|Created|Wrote|edit applied|Edited|Inserted|OK\b)" +); + +fn classify_edit_result(text: &str) -> String { + let t = text.trim(); + if t.is_empty() { + return "empty".to_string(); + } + let first = t.split_once('\n').map_or(t, |(a, _)| a); + + if RE_TRUNCATED.is_match(first) { + return "truncated".to_string(); + } + if RE_ABORTED.is_match(t) { + return "aborted".to_string(); + } + if RE_SUCCESS.is_match(first) { + return "success".to_string(); + } + if RE_ANCHOR_STALE.is_match(t) { + return "fail:anchor-stale".to_string(); + } + if RE_NO_ENCLOSING.is_match(t) { + return "fail:no-enclosing-block".to_string(); + } + if RE_ANCHOR_MISSING.is_match(t) { + return "fail:anchor-missing".to_string(); + } + if RE_PARSE_ERROR.is_match(t) { + return "fail:parse".to_string(); + } + if RE_SSR_NO_MATCH.is_match(t) { + return "fail:no-match".to_string(); + } + if RE_FILE_NOT_READ.is_match(t) { + return "fail:file-not-read".to_string(); + } + if RE_FILE_CHANGED.is_match(t) { + return "fail:file-changed".to_string(); + } + if RE_PERM_DENIED.is_match(t) { + return "fail:perm".to_string(); + } + if RE_GENERIC_REJECTED.is_match(first) { + return "fail:other".to_string(); + } + "unknown".to_string() +} + +// ---- reporting ---- + +fn report_edits(entries: &[EditEntry]) { + if entries.is_empty() { + println!("no edit-family tool calls found in matched sessions"); + return; + } + + let mut by_tool: HashMap = HashMap::new(); + let mut by_format: HashMap = HashMap::new(); + let mut status_by_format: HashMap> = HashMap::new(); + let mut status_by_tool: HashMap> = HashMap::new(); + let mut verb_count: HashMap = HashMap::new(); + let mut loc_count: HashMap = HashMap::new(); + let mut fails_by_verb: HashMap> = HashMap::new(); + let mut fails_by_loc: HashMap> = HashMap::new(); + + for e in entries { + *by_tool.entry(e.tool_name.clone()).or_insert(0) += 1; + *status_by_tool + .entry(e.tool_name.clone()) + .or_default() + .entry(e.status.clone()) + .or_insert(0) += 1; + *by_format.entry(e.format.clone()).or_insert(0) += 1; + *status_by_format + .entry(e.format.clone()) + .or_default() + .entry(e.status.clone()) + .or_insert(0) += 1; + for v in &e.verbs { + *verb_count.entry(v.clone()).or_insert(0) += 1; + *fails_by_verb + .entry(v.clone()) + .or_default() + .entry(e.status.clone()) + .or_insert(0) += 1; + } + for l in &e.loc_shapes { + *loc_count.entry(l.clone()).or_insert(0) += 1; + *fails_by_loc + .entry(l.clone()) + .or_default() + .entry(e.status.clone()) + .or_insert(0) += 1; + } + } + + println!("# Edit-tool usage"); + println!( + "\nTotal tool calls: {} (across {} sessions)", + entries.len(), + count_edit_sessions(entries) + ); + + println!("\n## By tool"); + print_sorted(&by_tool); + + println!("\n## Outcome by tool"); + let mut tools: Vec<&String> = by_tool.keys().collect(); + tools.sort(); + for t in tools { + println!("\n {t} ({} calls):", by_tool[t.as_str()]); + if let Some(m) = status_by_tool.get(t.as_str()) { + print_sorted_indent(m, " "); + } + } + + println!("\n## edit verb distribution (per sub-edit)"); + print_sorted(&verb_count); + + println!("\n## edit locator shape distribution"); + print_sorted(&loc_count); + + println!("\n## Failure rate per verb shape"); + for v in sorted_by_count(&verb_count) { + let (total, failed) = fail_totals(fails_by_verb.get(v.as_str())); + println!( + " {v:<20} {failed}/{total} failed ({:.0}%)", + pct(failed, total) + ); + } + + println!("\n## Failure rate per locator shape"); + for l in sorted_by_count(&loc_count) { + let (total, failed) = fail_totals(fails_by_loc.get(l.as_str())); + println!( + " {l:<20} {failed}/{total} failed ({:.0}%)", + pct(failed, total) + ); + } + + println!("\n## edit-tool argument-format usage"); + print_sorted(&by_format); + + println!("\n## Failure rate per argument format"); + for fname in sorted_by_count(&by_format) { + let (total, failed) = fail_totals(status_by_format.get(fname.as_str())); + println!( + " {fname:<32} {failed:>6}/{total:<6} failed ({:.0}%)", + pct(failed, total) + ); + } + + println!("\n## Failure breakdown per top format"); + for fname in sorted_by_count(&by_format).into_iter().take(8) { + println!("\n {fname} ({} total)", by_format[fname.as_str()]); + if let Some(m) = status_by_format.get(fname.as_str()) { + print_sorted_indent(m, " "); + } + } + + println!("\n## Sample failed edits"); + let mut shown = 0; + for e in entries { + if !e.status.starts_with("fail") { + continue; + } + let first = e + .result_raw + .split_once("\n\n") + .map_or(e.result_raw.as_str(), |(a, _)| a); + println!( + "\n— {} [{}] verbs={:?} loc={:?}\n result: {}", + e.tool_name, + e.status, + e.verbs, + e.loc_shapes, + truncate_line(first, 220) + ); + shown += 1; + if shown >= 8 { + break; + } + } +} + +fn fail_totals(m: Option<&HashMap>) -> (i64, i64) { + let Some(m) = m else { return (0, 0) }; + let mut total = 0i64; + let mut failed = 0i64; + for (status, n) in m { + total += n; + if status.starts_with("fail") { + failed += n; + } + } + (total, failed) +} + +fn count_edit_sessions(entries: &[EditEntry]) -> usize { + let mut s: std::collections::HashSet<&str> = std::collections::HashSet::new(); + for e in entries { + s.insert(&e.file); + } + s.len() +} + +fn write_csv(entries: &[EditEntry]) -> Result<()> { + let path = std::env::var("EDIT_ANALYSIS_CSV").unwrap_or_else(|_| "edit-analysis.csv".to_string()); + let f = File::create(&path).with_context(|| format!("create {path}"))?; + let mut w = csv::Writer::from_writer(f); + w.write_record([ + "session", + "tool", + "status", + "num_edits", + "verbs", + "loc_shapes", + "result_first_line", + ])?; + for e in entries { + let first = e + .result_raw + .split_once('\n') + .map_or(e.result_raw.as_str(), |(a, _)| a); + let session = Path::new(&e.file) + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or(&e.file); + w.write_record([ + session, + &e.tool_name, + &e.status, + &e.num_edits.to_string(), + &e.verbs.join(","), + &e.loc_shapes.join(","), + &truncate_line(first, 200), + ])?; + } + w.flush()?; + Ok(()) +} diff --git a/scripts/session-stats/src/cmd_tools.rs b/scripts/session-stats/src/cmd_tools.rs new file mode 100644 index 000000000..e514a6375 --- /dev/null +++ b/scripts/session-stats/src/cmd_tools.rs @@ -0,0 +1,387 @@ +//! `tools` subcommand — per-tool token totals across the most-recent N session +//! jsonl files. +//! +//! Token counting uses o200k_base via tiktoken-rs (the GPT-4o / GPT-5 family +//! tokenizer). It is not Claude's own BPE, but it is well-defined offline and +//! within ~5-10% across English/code in aggregate. +//! +//! Buckets: +//! tool ARGS — assistant tool-call argument JSON +//! tool RESULTS — tool result content text +//! assistant THINKING — assistant `thinking` blocks +//! assistant TEXT — assistant prose +//! user TEXT — user-authored text content +//! +//! Output: grand totals + per-tool breakdown sorted by total (arg+res) tokens. +//! Optional CSV at `$TOOL_USAGE_CSV`. + +use crate::common::*; +use anyhow::{Context, Result, bail}; +use std::collections::HashMap; +use std::fs::File; +use std::io::{BufRead, BufReader}; +use std::path::Path; + +#[derive(Default, Clone)] +struct ToolAgg { + calls: i64, + results: i64, + arg_tok: i64, + res_tok: i64, +} + +#[derive(Default, Clone)] +struct SessionTotals { + arg_tok: i64, + res_tok: i64, + thinking_tok: i64, + text_tok: i64, + user_tok: i64, + n_calls: i64, + n_results: i64, +} + +struct FileResult { + totals: SessionTotals, + tools: HashMap, +} + +pub fn run(args: Vec) -> Result<()> { + let mut limit: usize = 100_000; + let mut workers: usize = 0; + + let mut iter = args.into_iter(); + while let Some(a) = iter.next() { + match a.as_str() { + "-n" => { + limit = iter + .next() + .context("-n requires a value")? + .parse() + .context("-n value")?; + } + "-j" => { + workers = iter + .next() + .context("-j requires a value")? + .parse() + .context("-j value")?; + } + "-h" | "--help" => { + eprintln!( + "usage: session-stats tools [-n N] [-j workers]\n\ + \n\ + Aggregates per-tool token usage across the most-recent N session\n\ + jsonl files (default 100000). Tokenizer: o200k_base." + ); + return Ok(()); + } + other => bail!("unknown flag: {other}"), + } + } + + let files = collect_sessions(&WalkOpts { + date_filters: Vec::new(), + limit_most_recent: limit, + })?; + eprintln!( + "scanning {} session files (tokenizer: o200k_base)", + files.len() + ); + + let results = parallel_collect(&files, workers, 5_000, process_file); + + let sessions = results.len(); + let mut grand = SessionTotals::default(); + let mut tools: HashMap = HashMap::new(); + for r in results { + grand.arg_tok += r.totals.arg_tok; + grand.res_tok += r.totals.res_tok; + grand.thinking_tok += r.totals.thinking_tok; + grand.text_tok += r.totals.text_tok; + grand.user_tok += r.totals.user_tok; + grand.n_calls += r.totals.n_calls; + grand.n_results += r.totals.n_results; + for (name, t) in r.tools { + let dst = tools.entry(name).or_default(); + dst.calls += t.calls; + dst.results += t.results; + dst.arg_tok += t.arg_tok; + dst.res_tok += t.res_tok; + } + } + + print_grand(&grand, sessions); + println!(); + print_table(&tools); + write_csv(&tools)?; + Ok(()) +} + +fn process_file(path: &Path) -> Option { + let f = match File::open(path) { + Ok(f) => f, + Err(e) => { + eprintln!("open {}: {e}", path.display()); + return None; + } + }; + let reader = BufReader::with_capacity(64 * 1024, f); + + let mut totals = SessionTotals::default(); + let mut tools: HashMap = HashMap::new(); + // Pending arg attribution: when a result arrives we credit the tool listed + // here; otherwise we fall back to message.toolName on the result event. + let mut pending: HashMap = HashMap::new(); + + for line in reader.lines() { + let Ok(line) = line else { continue }; + if line.is_empty() { + continue; + } + let Ok(ev) = serde_json::from_str::(&line) else { + continue; + }; + if ev.kind != "message" { + continue; + } + let Some(msg_raw) = ev.message else { continue }; + let Ok(m) = serde_json::from_str::(msg_raw.get()) else { + continue; + }; + let Some(content_raw) = m.content else { continue }; + let items = parse_content(&content_raw); + + match m.role.as_str() { + "assistant" => { + for it in items { + match it.kind.as_str() { + "toolCall" => { + let name = normalize_tool(&it.name); + let args_str = it.arguments.as_deref().map(RawValue::get).unwrap_or(""); + let tok = count_tokens(args_str) as i64; + totals.arg_tok += tok; + totals.n_calls += 1; + let t = tools.entry(name.clone()).or_default(); + t.calls += 1; + t.arg_tok += tok; + pending.insert(it.id, name); + } + "thinking" => { + totals.thinking_tok += count_tokens(&it.thinking) as i64; + } + "text" => { + totals.text_tok += count_tokens(&it.text) as i64; + } + _ => {} + } + } + } + "toolResult" => { + let text = join_text(&items); + let tok = count_tokens(&text) as i64; + totals.res_tok += tok; + totals.n_results += 1; + let name = pending + .remove(&m.tool_call_id) + .unwrap_or_else(|| normalize_tool(&m.tool_name)); + let t = tools.entry(name).or_default(); + t.results += 1; + t.res_tok += tok; + } + "user" => { + for it in items { + if it.kind == "text" { + totals.user_tok += count_tokens(&it.text) as i64; + } + } + } + _ => {} + } + } + + Some(FileResult { totals, tools }) +} + +use serde_json::value::RawValue; + +fn normalize_tool(name: &str) -> String { + if name.is_empty() { + "".to_string() + } else { + name.to_string() + } +} + +// ---- reporting ---- + +fn print_grand(g: &SessionTotals, sessions: usize) { + let total = g.arg_tok + g.res_tok + g.thinking_tok + g.text_tok + g.user_tok; + let share = |n: i64| pct(n, total); + println!("=== Grand totals across {sessions} sessions ==="); + println!( + "tool call ARGS: {:>10} tok ({:>5.1}%)", + commas(g.arg_tok), + share(g.arg_tok) + ); + println!( + "tool RESULTS: {:>10} tok ({:>5.1}%)", + commas(g.res_tok), + share(g.res_tok) + ); + println!( + "assistant THINKING: {:>10} tok ({:>5.1}%)", + commas(g.thinking_tok), + share(g.thinking_tok) + ); + println!( + "assistant TEXT: {:>10} tok ({:>5.1}%)", + commas(g.text_tok), + share(g.text_tok) + ); + println!( + "user TEXT: {:>10} tok ({:>5.1}%)", + commas(g.user_tok), + share(g.user_tok) + ); + println!(" ---------------"); + println!("TOTAL: {:>10} tok", commas(total)); + println!(); + println!( + "tool calls: {}, tool results: {}", + commas(g.n_calls), + commas(g.n_results) + ); + if g.n_calls > 0 { + println!( + "avg arg tokens / call: {:.1}", + g.arg_tok as f64 / g.n_calls as f64 + ); + } + if g.n_results > 0 { + println!( + "avg result tokens / call: {:.1}", + g.res_tok as f64 / g.n_results as f64 + ); + } + if g.arg_tok > 0 { + println!( + "ratio result / arg: {:.2}x", + g.res_tok as f64 / g.arg_tok as f64 + ); + } +} + +struct ToolRow { + name: String, + calls: i64, + arg_tok: i64, + res_tok: i64, + total: i64, + avg_arg: f64, + avg_res: f64, + res_o_arg: f64, +} + +fn print_table(tools: &HashMap) { + let mut rows: Vec = tools + .iter() + .filter_map(|(name, t)| { + if t.calls == 0 && t.results == 0 { + return None; + } + let mut r = ToolRow { + name: name.clone(), + calls: t.calls, + arg_tok: t.arg_tok, + res_tok: t.res_tok, + total: t.arg_tok + t.res_tok, + avg_arg: 0.0, + avg_res: 0.0, + res_o_arg: 0.0, + }; + if t.calls > 0 { + r.avg_arg = t.arg_tok as f64 / t.calls as f64; + r.avg_res = t.res_tok as f64 / t.calls as f64; + } + if t.arg_tok > 0 { + r.res_o_arg = t.res_tok as f64 / t.arg_tok as f64; + } + Some(r) + }) + .collect(); + rows.sort_by(|a, b| b.total.cmp(&a.total)); + + println!( + "{:<22} {:>6} {:>10} {:>10} {:>10} {:>8} {:>8} {:>8}", + "tool", "calls", "arg_tok", "res_tok", "total", "avg_arg", "avg_res", "res/arg" + ); + println!("{}", "-".repeat(100)); + + const TOP: usize = 25; + let shown = TOP.min(rows.len()); + for r in &rows[..shown] { + println!( + "{:<22} {:>6} {:>10} {:>10} {:>10} {:>8.1} {:>8.1} {:>8.2}", + r.name, + commas(r.calls), + commas(r.arg_tok), + commas(r.res_tok), + commas(r.total), + r.avg_arg, + r.avg_res, + r.res_o_arg + ); + } + if rows.len() > TOP { + let (mut sc, mut sa, mut sr) = (0i64, 0i64, 0i64); + for r in &rows[TOP..] { + sc += r.calls; + sa += r.arg_tok; + sr += r.res_tok; + } + println!( + "{:<22} {:>6} {:>10} {:>10} {:>10}", + format!("({} others)", rows.len() - TOP), + commas(sc), + commas(sa), + commas(sr), + commas(sa + sr), + ); + } +} + +fn write_csv(tools: &HashMap) -> Result<()> { + let path = std::env::var("TOOL_USAGE_CSV").unwrap_or_default(); + if path.is_empty() { + return Ok(()); + } + let f = File::create(&path).with_context(|| format!("create {path}"))?; + let mut w = csv::Writer::from_writer(f); + w.write_record(["tool", "calls", "results", "arg_tok", "res_tok", "total"])?; + let mut names: Vec<&String> = tools.keys().collect(); + names.sort_by(|a, b| { + let ai = { + let t = &tools[a.as_str()]; + t.arg_tok + t.res_tok + }; + let aj = { + let t = &tools[b.as_str()]; + t.arg_tok + t.res_tok + }; + aj.cmp(&ai) + }); + for n in names { + let t = &tools[n.as_str()]; + w.write_record([ + n.as_str(), + &t.calls.to_string(), + &t.results.to_string(), + &t.arg_tok.to_string(), + &t.res_tok.to_string(), + &(t.arg_tok + t.res_tok).to_string(), + ])?; + } + w.flush()?; + Ok(()) +} diff --git a/scripts/session-stats/src/common.rs b/scripts/session-stats/src/common.rs new file mode 100644 index 000000000..a5279f5a3 --- /dev/null +++ b/scripts/session-stats/src/common.rs @@ -0,0 +1,256 @@ +//! Shared JSONL shapes, walk helpers, tokenizer, and formatting helpers. + +use anyhow::{Context, Result}; +use rayon::prelude::*; +use serde::Deserialize; +use serde_json::value::RawValue; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::LazyLock; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::SystemTime; +use tiktoken_rs::CoreBPE; +use walkdir::WalkDir; + +// ---- jsonl shapes ---- + +#[derive(Deserialize)] +pub struct RawEvent { + #[serde(rename = "type", default)] + pub kind: String, + #[serde(default)] + pub message: Option>, +} + +#[derive(Deserialize)] +pub struct Message { + #[serde(default)] + pub role: String, + #[serde(default)] + pub content: Option>, + #[serde(default, rename = "toolName")] + pub tool_name: String, + #[serde(default, rename = "toolCallId")] + pub tool_call_id: String, +} + +#[derive(Deserialize)] +pub struct ContentItem { + #[serde(rename = "type", default)] + pub kind: String, + #[serde(default)] + pub text: String, + #[serde(default)] + pub thinking: String, + #[serde(default)] + pub name: String, + #[serde(default)] + pub id: String, + #[serde(default)] + pub arguments: Option>, +} + +// ---- session walking ---- + +pub fn sessions_root() -> Result { + let home = dirs::home_dir().context("could not resolve home directory")?; + Ok(home.join(".omp").join("agent").join("sessions")) +} + +pub struct WalkOpts { + /// Keeps only paths containing any of these substrings (e.g. "2026-04-28"). + /// Empty means accept all. + pub date_filters: Vec, + /// Keeps only the N most-recently-modified files (after the date filter). + /// 0 means no limit. + pub limit_most_recent: usize, +} + +/// Walks the sessions root and returns the matching `.jsonl` paths. +/// With `limit_most_recent > 0` the result is sorted by mtime descending and +/// truncated to N entries; otherwise it's lexically sorted. +pub fn collect_sessions(opts: &WalkOpts) -> Result> { + let base = sessions_root()?; + let need_mtime = opts.limit_most_recent > 0; + + let mut all: Vec<(PathBuf, SystemTime)> = Vec::new(); + for entry in WalkDir::new(&base).into_iter().filter_map(Result::ok) { + if !entry.file_type().is_file() { + continue; + } + let p = entry.path(); + if p.extension().and_then(|e| e.to_str()) != Some("jsonl") { + continue; + } + let path_str = p.to_string_lossy(); + if !match_date(&path_str, &opts.date_filters) { + continue; + } + let mt = if need_mtime { + entry + .metadata() + .ok() + .and_then(|m| m.modified().ok()) + .unwrap_or(SystemTime::UNIX_EPOCH) + } else { + SystemTime::UNIX_EPOCH + }; + all.push((p.to_path_buf(), mt)); + } + + if need_mtime { + all.sort_by(|a, b| b.1.cmp(&a.1)); + all.truncate(opts.limit_most_recent); + } else { + all.sort_by(|a, b| a.0.cmp(&b.0)); + } + Ok(all.into_iter().map(|(p, _)| p).collect()) +} + +fn match_date(p: &str, filters: &[String]) -> bool { + filters.is_empty() || filters.iter().any(|d| p.contains(d)) +} + +// ---- content helpers ---- + +pub fn parse_content(raw: &RawValue) -> Vec { + serde_json::from_str(raw.get()).unwrap_or_default() +} + +/// Concatenates all `text` items in a content array. +pub fn join_text(items: &[ContentItem]) -> String { + let mut out = String::new(); + for it in items { + if it.kind == "text" { + out.push_str(&it.text); + } + } + out +} + +// ---- tokenizer (o200k_base) ---- + +static BPE: LazyLock = + LazyLock::new(|| tiktoken_rs::o200k_base().expect("load o200k_base BPE")); + +/// Counts tokens for `s` using the o200k_base BPE (GPT-4o / GPT-5 family). +/// Uses the ordinary encoder so embedded `<|...|>` sequences in tool args do +/// not trigger special-token handling. +pub fn count_tokens(s: &str) -> usize { + if s.is_empty() { + return 0; + } + BPE.encode_ordinary(s).len() +} + +// ---- formatting helpers ---- + +/// Formats an integer with thousand separators. +pub fn commas(n: i64) -> String { + let neg = n < 0; + let mag = if neg { (n as i128).unsigned_abs() } else { n as u128 }; + let s = mag.to_string(); + let bytes = s.as_bytes(); + let mut out = String::with_capacity(bytes.len() + bytes.len() / 3 + 1); + if neg { + out.push('-'); + } + let pre = bytes.len() % 3; + if pre > 0 { + out.push_str(&s[..pre]); + if bytes.len() > pre { + out.push(','); + } + } + let mut i = pre; + while i + 3 <= bytes.len() { + out.push_str(&s[i..i + 3]); + if i + 3 < bytes.len() { + out.push(','); + } + i += 3; + } + out +} + +pub fn pct(a: i64, b: i64) -> f64 { + if b == 0 { + 0.0 + } else { + 100.0 * a as f64 / b as f64 + } +} + +/// Truncates a string to at most `n` chars, replacing newlines with " | ". +/// Adds an ellipsis when truncation occurs. +pub fn truncate_line(s: &str, n: usize) -> String { + let s = s.replace('\n', " | "); + if s.chars().count() <= n { + return s; + } + let mut out: String = s.chars().take(n).collect(); + out.push('…'); + out +} + +/// Returns map keys sorted by descending value, ties broken alphabetically. +pub fn sorted_by_count(m: &HashMap) -> Vec<&String> { + let mut keys: Vec<&String> = m.keys().collect(); + keys.sort_by(|a, b| { + let av = m.get(a.as_str()).copied().unwrap_or(0); + let bv = m.get(b.as_str()).copied().unwrap_or(0); + bv.cmp(&av).then_with(|| a.cmp(b)) + }); + keys +} + +pub fn print_sorted(m: &HashMap) { + print_sorted_indent(m, " "); +} + +pub fn print_sorted_indent(m: &HashMap, indent: &str) { + for k in sorted_by_count(m) { + println!("{indent}{k:<25} {}", m[k.as_str()]); + } +} + +// ---- parallel processing ---- + +/// Runs `handle(path)` in parallel across rayon workers and collects the +/// non-`None` results into a Vec. Logs progress every `progress_every` files +/// (set 0 to silence). +pub fn parallel_collect( + paths: &[PathBuf], + workers: usize, + progress_every: u64, + handle: H, +) -> Vec +where + R: Send, + H: Fn(&Path) -> Option + Sync, +{ + let total = paths.len(); + let done = AtomicU64::new(0); + + let pool = { + let mut b = rayon::ThreadPoolBuilder::new(); + if workers > 0 { + b = b.num_threads(workers); + } + b.build().expect("rayon thread pool") + }; + + pool.install(|| { + paths + .par_iter() + .filter_map(|p| { + let r = handle(p); + let n = done.fetch_add(1, Ordering::Relaxed) + 1; + if progress_every > 0 && n % progress_every == 0 { + eprintln!(" processed {n}/{total}"); + } + r + }) + .collect() + }) +} diff --git a/scripts/session-stats/src/main.rs b/scripts/session-stats/src/main.rs new file mode 100644 index 000000000..49092b6e9 --- /dev/null +++ b/scripts/session-stats/src/main.rs @@ -0,0 +1,58 @@ +//! session-stats: ad-hoc analyses over the local agent session corpus +//! (`~/.omp/agent/sessions/`). +//! +//! Subcommands: +//! +//! edits [-n N] [date ...] audit edit/ast_edit/write tool usage by argument schema +//! tools [-n N] per-tool token totals across the most-recent N sessions +//! +//! Run with no subcommand for help. + +mod cmd_edits; +mod cmd_tools; +mod common; + +use std::process::ExitCode; + +fn usage() { + eprintln!( + "usage: session-stats [args...] + + edits [-n N] [date prefix ...] + audit edit-tool usage across N most-recent + sessions (default 100000). Optional date filters + (e.g. 2026-04-28) further narrow the set. + tools [-n N] per-tool token totals across the N most-recent + session jsonl files (default 100000). + +Token counting uses the o200k_base tokenizer (the GPT-4o / Claude-adjacent BPE). +Walk root: ~/.omp/agent/sessions/" + ); +} + +fn main() -> ExitCode { + let mut args = std::env::args().skip(1); + let Some(cmd) = args.next() else { + usage(); + return ExitCode::from(2); + }; + let rest: Vec = args.collect(); + let result = match cmd.as_str() { + "edits" => cmd_edits::run(rest), + "tools" => cmd_tools::run(rest), + "-h" | "--help" | "help" => { + usage(); + return ExitCode::SUCCESS; + } + other => { + eprintln!("unknown subcommand {other:?}\n"); + usage(); + return ExitCode::from(2); + } + }; + if let Err(err) = result { + eprintln!("fatal: {err:#}"); + return ExitCode::FAILURE; + } + ExitCode::SUCCESS +}