diff --git a/crates/pi-natives/src/lib.rs b/crates/pi-natives/src/lib.rs index db6deb58f..5ef602144 100644 --- a/crates/pi-natives/src/lib.rs +++ b/crates/pi-natives/src/lib.rs @@ -51,3 +51,4 @@ pub mod task; pub mod text; pub mod tokens; pub(crate) mod utils; +pub mod workspace; diff --git a/crates/pi-natives/src/workspace.rs b/crates/pi-natives/src/workspace.rs new file mode 100644 index 000000000..174d2fb3d --- /dev/null +++ b/crates/pi-natives/src/workspace.rs @@ -0,0 +1,386 @@ +//! Workspace discovery for startup context. +//! +//! Walks a project tree once and returns the bounded entries needed to render +//! the workspace tree plus directory-scoped AGENTS.md files. AGENTS.md files +//! are checked directly in every traversed directory so a file-level gitignore +//! rule cannot hide them, while ignored directories are still pruned by the +//! walker. + +use std::{ + collections::HashSet, + path::{Path, PathBuf}, + sync::{Arc, LazyLock}, +}; + +use ignore::{DirEntry, ParallelVisitor, ParallelVisitorBuilder, WalkBuilder, WalkState}; +use napi::bindgen_prelude::*; +use napi_derive::napi; +use parking_lot::Mutex; + +use crate::{ + fs_cache::{self, FileType, GlobMatch}, + task, +}; + +const AGENTS_MD_FILENAME: &str = "AGENTS.md"; +const AGENTS_MD_MIN_DEPTH: usize = 1; +const AGENTS_MD_MAX_DEPTH: usize = 4; +const AGENTS_MD_LIMIT: usize = 200; +const MAX_ENTRIES: usize = 100_000; + +/// Directory names pruned during traversal. The TypeScript caller no longer has +/// to plumb this list through; it lives here so a single source of truth +/// governs what counts as a non-source directory in startup scans. +const EXCLUDED_DIRS: &[&str] = &[ + "node_modules", + ".git", + ".next", + "dist", + "build", + "target", + ".venv", + ".cache", + ".turbo", + ".parcel-cache", + "coverage", +]; + +static EXCLUDED_DIR_SET: LazyLock> = + LazyLock::new(|| EXCLUDED_DIRS.iter().copied().collect()); + +/// Input options for `listWorkspace`, the single-pass workspace startup scan. +#[napi(object)] +pub struct ListWorkspaceOptions<'env> { + /// Directory to scan. + pub path: String, + /// Maximum depth for returned tree entries. Root children are depth 1. + pub max_depth: u32, + /// Include hidden files and directories. Default: false. + pub hidden: Option, + /// Respect .gitignore files. Default: true. + pub gitignore: Option, + /// Also surface AGENTS.md files in directories at depth 1..=4, even when + /// gitignore would otherwise hide the file. Walks deeper than `maxDepth` + /// to find them. Default: false. + pub collect_agents_md: Option, + /// Timeout in milliseconds for the operation. + pub timeout_ms: Option, + /// Abort signal for cancelling the operation. + pub signal: Option>, +} + +/// Result payload returned by a workspace scan. +#[napi(object)] +pub struct ListWorkspaceResult { + /// Entries within `maxDepth`, with mtime and regular-file size metadata. + pub entries: Vec, + /// Directory-scoped AGENTS.md files within depth 1..=4 (capped at 200). + /// Always empty when `collectAgentsMd` is false. + pub agents_md_files: Vec, + /// True when any output cap was hit. + pub truncated: bool, +} + +struct WorkspaceConfig { + root: PathBuf, + max_depth: usize, + walk_max_depth: usize, + include_hidden: bool, + use_gitignore: bool, + collect_agents_md: bool, +} + +fn build_workspace_walker(config: &WorkspaceConfig) -> WalkBuilder { + let mut builder = WalkBuilder::new(&config.root); + builder + .hidden(!config.include_hidden) + .follow_links(false) + .sort_by_file_path(|a, b| a.cmp(b)) + .max_depth(Some(config.walk_max_depth)) + .filter_entry(|entry| { + let name = entry.file_name().to_str().unwrap_or_default(); + if name == ".DS_Store" { + return false; + } + if entry + .file_type() + .is_some_and(|file_type| file_type.is_dir()) + && EXCLUDED_DIR_SET.contains(name) + { + return false; + } + true + }); + + if config.use_gitignore { + builder + .git_ignore(true) + .git_exclude(true) + .git_global(true) + .ignore(true) + .parents(true) + // Honor .gitignore even when the directory isn't a git repo, + // matching what users expect from a plain directory listing. + .require_git(false); + } else { + builder + .git_ignore(false) + .git_exclude(false) + .git_global(false) + .ignore(false) + .parents(false); + } + + builder +} + +fn glob_match_from_path(root: &Path, path: &Path) -> Option { + let relative = fs_cache::normalize_relative_path(root, path); + if relative.is_empty() { + return None; + } + let (file_type, mtime, size) = fs_cache::classify_file_type(path)?; + Some(GlobMatch { + path: relative.into_owned(), + file_type, + mtime, + size: size.map(|value| value as f64), + }) +} + +fn glob_match_from_entry(root: &Path, entry: &DirEntry) -> Option { + glob_match_from_path(root, entry.path()) +} + +fn is_file_or_file_symlink(path: &Path, file_type: FileType) -> bool { + match file_type { + FileType::File => true, + FileType::Symlink => std::fs::metadata(path).is_ok_and(|metadata| metadata.is_file()), + FileType::Dir => false, + } +} + +fn collect_agents_md_in_directory( + config: &WorkspaceConfig, + directory: &Path, + directory_depth: usize, + entries: &mut Vec, + agents_md_files: &mut Vec, +) { + if !config.collect_agents_md { + return; + } + let candidate = directory.join(AGENTS_MD_FILENAME); + let Some(entry) = glob_match_from_path(&config.root, &candidate) else { + return; + }; + if !is_file_or_file_symlink(&candidate, entry.file_type) { + return; + } + let tree_depth = directory_depth + 1; + if tree_depth <= config.max_depth { + entries.push(entry.clone()); + } + // AGENTS.md directory depth: root AGENTS.md is depth 0, child dir AGENTS.md + // is depth 1, and so on. We only surface files in depth 1..=4. + if (AGENTS_MD_MIN_DEPTH..=AGENTS_MD_MAX_DEPTH).contains(&directory_depth) { + agents_md_files.push(entry.path); + } +} + +struct WorkspaceVisitor<'a> { + config: &'a WorkspaceConfig, + ct: &'a task::CancelToken, + entries: Vec, + agents_md_files: Vec, + shared_entries: Arc>>>, + shared_agents_md_files: Arc>>>, + error: Arc>>, + visited: usize, +} + +impl Drop for WorkspaceVisitor<'_> { + fn drop(&mut self) { + if !self.entries.is_empty() { + let entries = std::mem::take(&mut self.entries); + self.shared_entries.lock().push(entries); + } + if !self.agents_md_files.is_empty() { + let agents_md_files = std::mem::take(&mut self.agents_md_files); + self.shared_agents_md_files.lock().push(agents_md_files); + } + } +} + +impl ParallelVisitor for WorkspaceVisitor<'_> { + fn visit(&mut self, entry: std::result::Result) -> WalkState { + if self.visited == 0 || self.visited >= 128 { + self.visited = 0; + if let Err(err) = self.ct.heartbeat() { + *self.error.lock() = Some(err.to_string()); + return WalkState::Quit; + } + } + self.visited += 1; + + let Ok(entry) = entry else { + return WalkState::Continue; + }; + let entry_depth = entry.depth(); + if entry + .file_type() + .is_some_and(|file_type| file_type.is_dir()) + { + collect_agents_md_in_directory( + self.config, + entry.path(), + entry_depth, + &mut self.entries, + &mut self.agents_md_files, + ); + } + if entry_depth <= self.config.max_depth + && let Some(entry) = glob_match_from_entry(&self.config.root, &entry) + { + self.entries.push(entry); + } + WalkState::Continue + } +} + +struct WorkspaceVisitorBuilder<'a> { + config: &'a WorkspaceConfig, + ct: &'a task::CancelToken, + shared_entries: Arc>>>, + shared_agents_md_files: Arc>>>, + error: Arc>>, +} + +impl<'a> ParallelVisitorBuilder<'a> for WorkspaceVisitorBuilder<'a> { + fn build(&mut self) -> Box { + Box::new(WorkspaceVisitor { + config: self.config, + ct: self.ct, + entries: Vec::new(), + agents_md_files: Vec::new(), + shared_entries: Arc::clone(&self.shared_entries), + shared_agents_md_files: Arc::clone(&self.shared_agents_md_files), + error: Arc::clone(&self.error), + visited: 0, + }) + } +} + +fn sort_dedup_entries(entries: &mut Vec) { + entries.sort_unstable_by(|a, b| a.path.cmp(&b.path)); + entries.dedup_by(|a, b| a.path == b.path); +} + +fn sort_dedup_paths(paths: &mut Vec) { + paths.sort_unstable(); + paths.dedup(); +} + +fn run_list_workspace( + config: WorkspaceConfig, + ct: task::CancelToken, +) -> Result { + let mut root_entries = Vec::new(); + let mut root_agents_md_files = Vec::new(); + collect_agents_md_in_directory( + &config, + &config.root, + 0, + &mut root_entries, + &mut root_agents_md_files, + ); + + let mut builder = build_workspace_walker(&config); + let workers = fs_cache::grep_workers(); + if workers > 0 { + builder.threads(workers); + } + + let shared_entries = Arc::new(Mutex::new(Vec::new())); + let shared_agents_md_files = Arc::new(Mutex::new(Vec::new())); + let error = Arc::new(Mutex::new(None)); + let mut visitor_builder = WorkspaceVisitorBuilder { + config: &config, + ct: &ct, + shared_entries: Arc::clone(&shared_entries), + shared_agents_md_files: Arc::clone(&shared_agents_md_files), + error: Arc::clone(&error), + }; + + ct.heartbeat()?; + builder.build_parallel().visit(&mut visitor_builder); + + let walk_error = error.lock().take(); + if let Some(error) = walk_error { + return Err(Error::from_reason(error)); + } + + let mut entries: Vec = shared_entries.lock().drain(..).flatten().collect(); + entries.extend(root_entries); + sort_dedup_entries(&mut entries); + + let mut agents_md_files: Vec = + shared_agents_md_files.lock().drain(..).flatten().collect(); + agents_md_files.extend(root_agents_md_files); + sort_dedup_paths(&mut agents_md_files); + + let entries_truncated = entries.len() > MAX_ENTRIES; + if entries_truncated { + entries.truncate(MAX_ENTRIES); + } + let agents_md_truncated = agents_md_files.len() > AGENTS_MD_LIMIT; + if agents_md_truncated { + agents_md_files.truncate(AGENTS_MD_LIMIT); + } + + Ok(ListWorkspaceResult { + entries, + agents_md_files, + truncated: entries_truncated || agents_md_truncated, + }) +} + +/// Walk the workspace once and return tree entries plus AGENTS.md candidates. +/// +/// File-level ignore rules for AGENTS.md are bypassed by checking each +/// traversed directory directly when `collectAgentsMd` is enabled, but ignored +/// directories are still pruned by the walker and are not searched. +#[napi(js_name = "listWorkspace")] +pub fn list_workspace(options: ListWorkspaceOptions<'_>) -> task::Promise { + let ListWorkspaceOptions { + path, + max_depth, + hidden, + gitignore, + collect_agents_md, + timeout_ms, + signal, + } = options; + + let ct = task::CancelToken::new(timeout_ms, signal); + task::blocking("listWorkspace", ct, move |ct| { + let max_depth = max_depth as usize; + let collect_agents_md = collect_agents_md.unwrap_or(false); + let walk_max_depth = if collect_agents_md { + max_depth.max(AGENTS_MD_MAX_DEPTH) + } else { + max_depth + }; + run_list_workspace( + WorkspaceConfig { + root: fs_cache::resolve_search_path(&path)?, + max_depth, + walk_max_depth, + include_hidden: hidden.unwrap_or(false), + use_gitignore: gitignore.unwrap_or(true), + collect_agents_md, + }, + ct, + ) + }) +} diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index d72f2dd38..f5e1a5fdf 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -1,9 +1,17 @@ # Changelog ## [Unreleased] +### Added + +- Added `agentsMdFiles` to `WorkspaceTree` so AGENTS.md discovery results are returned with the workspace scan output + +### Changed + +- Changed startup workspace discovery to use one native `listWorkspace` walk for both the rendered tree and AGENTS.md directory-context candidates, removing the layered `git ls-files` orchestration and secondary AGENTS.md glob. ### Fixed +- Fixed AGENTS.md context discovery to include AGENTS.md files that are explicitly gitignored while still excluding AGENTS.md files under ignored directories - Fixed task tool renderer spamming `Tool renderer failed: undefined is not an object (evaluating 'args.tasks.length')` warnings while a `task` call was streaming in (the `tasks` array is undefined until the partial JSON parser closes it); the renderer now tolerates an absent `tasks` field and shows `0 agents` until the array arrives ([#985](https://github.com/can1357/oh-my-pi/issues/985)). - Fixed MCP HTTP streamable transport spamming `HTTP SSE stream error: ReadableStream already has a controller` after every JSON-RPC request whose response was returned as `text/event-stream`. The transport used to break out of the SSE iterator once the matching response was captured and then re-open `response.body` for a background drain, but the body had already been piped through a `TransformStream` and could not be re-read. The drain now runs from a single iterator that resolves the response promise inline and continues to dispatch piggybacked notifications on the same stream. diff --git a/packages/coding-agent/src/sdk.ts b/packages/coding-agent/src/sdk.ts index be1874c4e..7d1c1476b 100644 --- a/packages/coding-agent/src/sdk.ts +++ b/packages/coding-agent/src/sdk.ts @@ -100,9 +100,7 @@ import { SessionManager } from "./session/session-manager"; import { closeAllConnections } from "./ssh/connection-manager"; import { unmountAll } from "./ssh/sshfs-mount"; import { - type AgentsMdSearch, type BuildSystemPromptResult, - buildAgentsMdSearch, buildSystemPrompt as buildSystemPromptInternal, buildSystemPromptToolMetadata, loadProjectContextFiles as loadContextFilesInternal, @@ -201,8 +199,6 @@ export interface CreateAgentSessionOptions { rules?: Rule[]; /** Context files (AGENTS.md content). Default: discovered walking up from cwd */ contextFiles?: Array<{ path: string; content: string }>; - /** Pre-built AGENTS.md search (skips re-scanning the workspace; passed by parents to subagents). */ - agentsMdSearch?: AgentsMdSearch; /** Pre-built workspace tree (skips re-scanning; passed by parents to subagents). */ workspaceTree?: WorkspaceTree; /** Prompt templates. Default: discovered from cwd/.omp/prompts/ + agentDir/prompts/ */ @@ -691,16 +687,14 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} if (!options.modelRegistry) { modelRegistry.refreshInBackground(); } - // Kick off AGENTS.md filesystem search and workspace tree in parallel — they are the slowest pieces of - // buildSystemPrompt (can be many seconds on large repos) and only need `cwd`, so they overlap with - // everything that follows. Subagents inherit the parent's resolved values via options. - const agentsMdSearchPromise: Promise = options.agentsMdSearch - ? Promise.resolve(options.agentsMdSearch) - : logger.time("buildAgentsMdSearch", buildAgentsMdSearch, cwd); - agentsMdSearchPromise.catch(() => {}); + // Kick off workspace tree discovery early. The native workspace scan returns + // both the rendered-tree input and the AGENTS.md directory-context index, so + // startup does not perform a second recursive filesystem search. Subagents + // inherit the parent's resolved values via options. + const STARTUP_SCAN_DEADLINE_MS = 5000; const workspaceTreePromise: Promise = options.workspaceTree ? Promise.resolve(options.workspaceTree) - : logger.time("buildWorkspaceTree", buildWorkspaceTree, cwd); + : logger.time("buildWorkspaceTree", () => buildWorkspaceTree(cwd, { timeoutMs: STARTUP_SCAN_DEADLINE_MS })); workspaceTreePromise.catch(() => {}); // Independent discoveries that depend only on cwd/agentDir — kicked off in parallel and awaited @@ -898,12 +892,11 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} return { ttsrManager, rulebookRules, alwaysApplyRules }; }); - // Resolve contextFiles up-front (it's needed before tool creation). The agentsMd / workspace tree - // scans are slowest on large repos and we MUST NOT block startup on them — race them against a - // short deadline. On timeout we forward `undefined` to ToolSession; buildSystemPromptInternal will - // re-race them through its own withDeadline path, and subagents will scan independently (still - // cheaper than an unbounded parent hang). Background work continues so caches still warm. - const STARTUP_SCAN_DEADLINE_MS = 5000; + // Resolve contextFiles up-front (it's needed before tool creation). The + // workspace tree scan is slow on large repos and we MUST NOT block startup on + // it. On timeout we forward `undefined` to ToolSession; buildSystemPromptInternal + // will re-race the same promise through its own withDeadline path. Background + // work continues so caches still warm. const raceWithDeadline = (name: string, work: Promise): Promise => Promise.race([ work, @@ -916,9 +909,8 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} return undefined; }), ]); - const [contextFiles, resolvedAgentsMdSearch, resolvedWorkspaceTree] = await Promise.all([ + const [contextFiles, resolvedWorkspaceTree] = await Promise.all([ contextFilesPromise, - raceWithDeadline("buildAgentsMdSearch", agentsMdSearchPromise), raceWithDeadline("buildWorkspaceTree", workspaceTreePromise), ]); @@ -1004,7 +996,6 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} }, skipPythonPreflight: options.skipPythonPreflight, contextFiles, - agentsMdSearch: resolvedAgentsMdSearch, workspaceTree: resolvedWorkspaceTree, skills, eventBus, @@ -1456,7 +1447,6 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} mcpDiscoveryServerSummaries: discoverableToolSummary.servers.map(formatDiscoverableMCPToolServerSummary), eagerTasks, secretsEnabled, - agentsMdSearch: agentsMdSearchPromise, workspaceTree: workspaceTreePromise, }); diff --git a/packages/coding-agent/src/system-prompt.ts b/packages/coding-agent/src/system-prompt.ts index 17a3e3590..e55e0d5f9 100644 --- a/packages/coding-agent/src/system-prompt.ts +++ b/packages/coding-agent/src/system-prompt.ts @@ -4,7 +4,6 @@ import * as os from "node:os"; import type { AgentTool } from "@oh-my-pi/pi-agent-core"; -import { FileType, glob } from "@oh-my-pi/pi-natives"; import { $env, getGpuCachePath, getProjectDir, hasFsCode, isEnoent, logger, prompt } from "@oh-my-pi/pi-utils"; import { $ } from "bun"; import { contextFileCapability } from "./capability/context-file"; @@ -15,7 +14,7 @@ import { loadSkills, type Skill } from "./extensibility/skills"; import customSystemPromptTemplate from "./prompts/system/custom-system-prompt.md" with { type: "text" }; import projectPromptTemplate from "./prompts/system/project-prompt.md" with { type: "text" }; import systemPromptTemplate from "./prompts/system/system-prompt.md" with { type: "text" }; -import { buildWorkspaceTree, type WorkspaceTree } from "./workspace-tree"; +import { AGENTS_MD_LIMIT, buildWorkspaceTree, type WorkspaceTree } from "./workspace-tree"; interface AlwaysApplyRule { name: string; @@ -84,58 +83,7 @@ function parseWmicTable(output: string, header: string): string | null { return filtered[0] ?? null; } -const AGENTS_MD_MIN_DEPTH = 1; -const AGENTS_MD_MAX_DEPTH = 4; -const AGENTS_MD_LIMIT = 200; const SYSTEM_PROMPT_PREP_TIMEOUT_MS = 5000; -const AGENTS_MD_EXCLUDED_DIRS = new Set(["node_modules", ".git"]); - -export interface AgentsMdSearch { - scopePath: string; - limit: number; - pattern: string; - files: string[]; -} - -async function listAgentsMdFiles(root: string, limit: number): Promise { - try { - const result = await glob({ - pattern: "**/AGENTS.md", - path: root, - fileType: FileType.File, - recursive: true, - hidden: false, - gitignore: true, - maxResults: limit * 4, - cache: true, - }); - const files: string[] = []; - for (const m of result.matches) { - const rel = m.path.replace(/\\/g, "/"); - if (!rel?.endsWith("AGENTS.md")) continue; - const segments = rel.split("/"); - const depth = segments.length - 1; - if (depth < AGENTS_MD_MIN_DEPTH || depth > AGENTS_MD_MAX_DEPTH) continue; - const dirSegments = segments.slice(0, -1); - if (dirSegments.some(seg => AGENTS_MD_EXCLUDED_DIRS.has(seg) || seg.startsWith("."))) continue; - files.push(rel); - if (files.length >= limit) break; - } - return Array.from(new Set(files)).sort().slice(0, limit); - } catch { - return []; - } -} - -export async function buildAgentsMdSearch(cwd: string): Promise { - const files = await listAgentsMdFiles(cwd, AGENTS_MD_LIMIT); - return { - scopePath: ".", - limit: AGENTS_MD_LIMIT, - pattern: `AGENTS.md depth ${AGENTS_MD_MIN_DEPTH}-${AGENTS_MD_MAX_DEPTH}`, - files, - }; -} async function getGpuModel(): Promise { switch (process.platform) { @@ -409,8 +357,6 @@ export interface BuildSystemPromptOptions { alwaysApplyRules?: AlwaysApplyRule[]; /** Whether secret obfuscation is active. When true, explains the redaction format in the prompt. */ secretsEnabled?: boolean; - /** Pre-loaded AGENTS.md search (skips discovery if provided). May be a Promise to allow early kick-off. */ - agentsMdSearch?: AgentsMdSearch | Promise; /** Pre-loaded workspace tree (skips discovery if provided). May be a Promise to allow early kick-off. */ workspaceTree?: WorkspaceTree | Promise; } @@ -444,7 +390,6 @@ export async function buildSystemPrompt(options: BuildSystemPromptOptions = {}): mcpDiscoveryServerSummaries = [], eagerTasks = false, secretsEnabled = false, - agentsMdSearch: providedAgentsMdSearch, workspaceTree: providedWorkspaceTree, } = options; const resolvedCwd = cwd ?? getProjectDir(); @@ -454,18 +399,13 @@ export async function buildSystemPrompt(options: BuildSystemPromptOptions = {}): resolvedAppendPrompt: undefined as string | undefined, systemPromptCustomization: null as string | null, contextFiles: dedupeExactContextFiles(providedContextFiles ?? []), - agentsMdSearch: { - scopePath: ".", - limit: AGENTS_MD_LIMIT, - pattern: `AGENTS.md depth ${AGENTS_MD_MIN_DEPTH}-${AGENTS_MD_MAX_DEPTH}`, - files: [] as string[], - } satisfies AgentsMdSearch, skills: providedSkills ?? ([] as Skill[]), workspaceTree: { rootPath: resolvedCwd, rendered: "", truncated: false, totalLines: 0, + agentsMdFiles: [], } satisfies WorkspaceTree, }; @@ -503,14 +443,12 @@ export async function buildSystemPrompt(options: BuildSystemPromptOptions = {}): const contextFilesPromise = providedContextFiles ? Promise.resolve(providedContextFiles) : logger.time("loadProjectContextFiles", loadProjectContextFiles, { cwd: resolvedCwd }); - const agentsMdSearchPromise = - providedAgentsMdSearch !== undefined - ? Promise.resolve(providedAgentsMdSearch) - : logger.time("buildAgentsMdSearch", buildAgentsMdSearch, resolvedCwd); const workspaceTreePromise = providedWorkspaceTree !== undefined ? Promise.resolve(providedWorkspaceTree) - : logger.time("buildWorkspaceTree", buildWorkspaceTree, resolvedCwd); + : logger.time("buildWorkspaceTree", () => + buildWorkspaceTree(resolvedCwd, { timeoutMs: SYSTEM_PROMPT_PREP_TIMEOUT_MS }), + ); const skillsPromise: Promise = providedSkills !== undefined ? Promise.resolve(providedSkills) @@ -518,33 +456,30 @@ export async function buildSystemPrompt(options: BuildSystemPromptOptions = {}): ? loadSkills({ ...skillsSettings, cwd: resolvedCwd }).then(result => result.skills) : Promise.resolve([]); - const [ - resolvedCustomPrompt, - resolvedAppendPrompt, - systemPromptCustomization, - contextFiles, - agentsMdSearch, - skills, - workspaceTree, - ] = await Promise.all([ - withDeadline( - "customPrompt", - resolvePromptInput(customPrompt, "system prompt"), - prepDefaults.resolvedCustomPrompt, - ), - withDeadline( - "appendSystemPrompt", - resolvePromptInput(appendSystemPrompt, "append system prompt"), - prepDefaults.resolvedAppendPrompt, - ), - withDeadline("loadSystemPromptFiles", systemPromptCustomizationPromise, prepDefaults.systemPromptCustomization), - withDeadline("loadProjectContextFiles", contextFilesPromise, prepDefaults.contextFiles).then( - dedupeExactContextFiles, - ), - withDeadline("buildAgentsMdSearch", agentsMdSearchPromise, prepDefaults.agentsMdSearch), - withDeadline("loadSkills", skillsPromise, prepDefaults.skills), - withDeadline("buildWorkspaceTree", workspaceTreePromise, prepDefaults.workspaceTree), - ]); + const [resolvedCustomPrompt, resolvedAppendPrompt, systemPromptCustomization, contextFiles, skills, workspaceTree] = + await Promise.all([ + withDeadline( + "customPrompt", + resolvePromptInput(customPrompt, "system prompt"), + prepDefaults.resolvedCustomPrompt, + ), + withDeadline( + "appendSystemPrompt", + resolvePromptInput(appendSystemPrompt, "append system prompt"), + prepDefaults.resolvedAppendPrompt, + ), + withDeadline( + "loadSystemPromptFiles", + systemPromptCustomizationPromise, + prepDefaults.systemPromptCustomization, + ), + withDeadline("loadProjectContextFiles", contextFilesPromise, prepDefaults.contextFiles).then( + dedupeExactContextFiles, + ), + withDeadline("loadSkills", skillsPromise, prepDefaults.skills), + withDeadline("buildWorkspaceTree", workspaceTreePromise, prepDefaults.workspaceTree), + ]); + const agentsMdFiles = Array.from(new Set(workspaceTree.agentsMdFiles)).sort().slice(0, AGENTS_MD_LIMIT); if (timedOut.length > 0) { logger.warn("System prompt preparation steps timed out; using minimal fallback for those steps", { @@ -617,7 +552,7 @@ export async function buildSystemPrompt(options: BuildSystemPromptOptions = {}): toolRefs, environment, contextFiles, - agentsMdSearch, + agentsMdSearch: { files: agentsMdFiles }, workspaceTree, skills: filteredSkills, rules: rules ?? [], diff --git a/packages/coding-agent/src/task/executor.ts b/packages/coding-agent/src/task/executor.ts index 7659542a1..762033205 100644 --- a/packages/coding-agent/src/task/executor.ts +++ b/packages/coding-agent/src/task/executor.ts @@ -28,7 +28,6 @@ import { createAgentSession, discoverAuthStorage } from "../sdk"; import type { AgentSession, AgentSessionEvent } from "../session/agent-session"; import type { AuthStorage } from "../session/auth-storage"; import { SessionManager } from "../session/session-manager"; -import type { AgentsMdSearch } from "../system-prompt"; import { type ContextFileEntry, truncateTail } from "../tools"; import { jtdToJsonSchema, normalizeSchema } from "../tools/jtd-to-json-schema"; import { ToolAbortError } from "../tools/tool-errors"; @@ -165,7 +164,6 @@ export interface ExecutorOptions { contextFiles?: ContextFileEntry[]; skills?: Skill[]; promptTemplates?: PromptTemplate[]; - agentsMdSearch?: AgentsMdSearch; workspaceTree?: WorkspaceTree; mcpManager?: MCPManager; authStorage?: AuthStorage; @@ -995,7 +993,6 @@ export async function runSubprocess(options: ExecutorOptions): Promise [ prompt.render(subagentSystemPromptTemplate, { diff --git a/packages/coding-agent/src/task/index.ts b/packages/coding-agent/src/task/index.ts index 73058adf3..9d8cdf76e 100644 --- a/packages/coding-agent/src/task/index.ts +++ b/packages/coding-agent/src/task/index.ts @@ -866,7 +866,6 @@ export class TaskTool implements AgentTool { mcpManager: this.session.mcpManager, contextFiles, skills: availableSkills, - agentsMdSearch: this.session.agentsMdSearch, workspaceTree: this.session.workspaceTree, promptTemplates, localProtocolOptions, @@ -924,7 +923,6 @@ export class TaskTool implements AgentTool { mcpManager: this.session.mcpManager, contextFiles, skills: availableSkills, - agentsMdSearch: this.session.agentsMdSearch, workspaceTree: this.session.workspaceTree, promptTemplates, localProtocolOptions, diff --git a/packages/coding-agent/src/tools/index.ts b/packages/coding-agent/src/tools/index.ts index 294f1dd20..2a29c605c 100644 --- a/packages/coding-agent/src/tools/index.ts +++ b/packages/coding-agent/src/tools/index.ts @@ -14,7 +14,6 @@ import type { PlanModeState } from "../plan-mode/state"; import type { AgentRegistry } from "../registry/agent-registry"; import type { CustomMessage } from "../session/messages"; import type { ToolChoiceQueue } from "../session/tool-choice-queue"; -import type { AgentsMdSearch } from "../system-prompt"; import { TaskTool } from "../task"; import type { AgentOutputManager } from "../task/output-manager"; import type { DiscoverableTool, DiscoverableToolSearchIndex } from "../tool-discovery/tool-index"; @@ -122,8 +121,6 @@ export interface ToolSession { skipPythonPreflight?: boolean; /** Pre-loaded context files (AGENTS.md, etc) */ contextFiles?: ContextFileEntry[]; - /** Pre-loaded AGENTS.md search (forwarded to subagents to skip re-scanning) */ - agentsMdSearch?: AgentsMdSearch; /** Pre-loaded workspace tree (forwarded to subagents to skip re-scanning) */ workspaceTree?: WorkspaceTree; /** Pre-loaded skills */ diff --git a/packages/coding-agent/src/tools/read.ts b/packages/coding-agent/src/tools/read.ts index 4ca020140..b3c6e1199 100644 --- a/packages/coding-agent/src/tools/read.ts +++ b/packages/coding-agent/src/tools/read.ts @@ -73,20 +73,6 @@ const PROSE_SUMMARY_EXTENSIONS = new Set([".md", ".txt"]); // Remote mount path prefix (sshfs mounts) - skip fuzzy matching to avoid hangs const REMOTE_MOUNT_PREFIX = getRemoteDir() + path.sep; -const READ_DIRECTORY_EXCLUDED_DIRS = new Set([ - "node_modules", - ".git", - ".next", - "dist", - "build", - "target", - ".venv", - ".cache", - ".turbo", - ".parcel-cache", - "coverage", -]); - function isRemoteMountPath(absolutePath: string): boolean { return absolutePath.startsWith(REMOTE_MOUNT_PREFIX); } @@ -1581,15 +1567,9 @@ export class ReadTool implements AgentTool { try { tree = await buildDirectoryTree(absolutePath, { maxDepth: READ_DIRECTORY_MAX_DEPTH, - directoryEntryLimit: READ_DIRECTORY_CHILD_LIMIT, - rootEntryLimit: null, + perDirLimit: READ_DIRECTORY_CHILD_LIMIT, + rootLimit: null, lineCap: limit ?? null, - lineCapProtectedDepth: 1, - hidden: true, - gitignore: false, - cache: true, - excludedDirectoryNames: READ_DIRECTORY_EXCLUDED_DIRS, - rootLabel: ".", }); } catch (error) { const message = error instanceof Error ? error.message : String(error); diff --git a/packages/coding-agent/src/workspace-tree.ts b/packages/coding-agent/src/workspace-tree.ts index 3a5333036..0bd890019 100644 --- a/packages/coding-agent/src/workspace-tree.ts +++ b/packages/coding-agent/src/workspace-tree.ts @@ -1,7 +1,19 @@ -import * as fs from "node:fs/promises"; import * as path from "node:path"; -import { glob } from "@oh-my-pi/pi-natives"; -import { $which, formatAge, formatBytes, logger } from "@oh-my-pi/pi-utils"; +import { FileType, type GlobMatch, listWorkspace } from "@oh-my-pi/pi-natives"; +import { formatAge, formatBytes } from "@oh-my-pi/pi-utils"; + +/** Defaults for the workspace tree shown in the system prompt. */ +const WORKSPACE_DEFAULTS = { + maxDepth: 3, + perDirLimit: 12, + lineCap: 120, +} as const; + +/** + * Hard cap on AGENTS.md files surfaced by `buildWorkspaceTree`. Mirrors the + * native cap so the system-prompt builder does not need a second pass. + */ +export const AGENTS_MD_LIMIT = 200; export interface DirectoryTree { rootPath: string; @@ -10,360 +22,250 @@ export interface DirectoryTree { totalLines: number; } -export interface WorkspaceTree extends DirectoryTree {} - -export interface DirectoryTreeOptions { - /** Directory depth below the root to include. Root itself is depth 0. */ - maxDepth?: number; - /** Per-directory child cap. Use null to disable per-directory truncation. */ - directoryEntryLimit?: number | null; - /** Optional root child cap. Defaults to directoryEntryLimit; use null to keep all root children. */ - rootEntryLimit?: number | null; - /** Hard rendered line cap. Use null to disable line-cap pruning. */ - lineCap?: number | null; - /** Depth at or above which line-cap pruning is forbidden. Root is 0, root children are 1. */ - lineCapProtectedDepth?: number; - /** Entry names to skip before stat/render. */ - excludedNames?: ReadonlySet | readonly string[]; - /** Directory names to skip before traversal. */ - excludedDirectoryNames?: ReadonlySet | readonly string[]; - /** Include hidden files and directories. */ - hidden?: boolean; - /** Respect .gitignore while listing children. */ - gitignore?: boolean; - /** Use native glob shared cache. */ - cache?: boolean; - /** Rendered label for the root line. */ - rootLabel?: string; - /** - * Pre-built map of `parentRelativePath` → child name set used in place of - * native directory listing. When provided, the tree builder consults this - * map for child enumeration instead of `glob` / `readdir`. Stat calls per - * displayed node are still performed for mtime/size/dir-ness. - */ - childIndex?: ReadonlyMap>; +export interface WorkspaceTree extends DirectoryTree { + /** AGENTS.md files beneath the root whose rules may apply to subdirectories. */ + agentsMdFiles: string[]; } -const WORKSPACE_TREE_MAX_DEPTH = 3; -const WORKSPACE_TREE_DIR_LIMIT = 12; -const WORKSPACE_TREE_LINE_CAP = 120; -const WORKSPACE_TREE_EXCLUDED_DIRS = new Set([ - "node_modules", - ".git", - ".next", - "dist", - "build", - "target", - ".venv", - ".cache", - ".turbo", - ".parcel-cache", - "coverage", -]); +export interface BuildDirectoryTreeOptions { + /** Directory depth below the root to include. Root itself is depth 0. Default: 1. */ + maxDepth?: number; + /** Per-directory child cap. `null` disables the cap. Default: `null`. */ + perDirLimit?: number | null; + /** Optional override for the root level. Defaults to `perDirLimit`. */ + rootLimit?: number | null; + /** Hard rendered line cap. `null` disables. Default: `null`. */ + lineCap?: number | null; +} -const DIRECTORY_TREE_EXCLUDED_NAMES = new Set([".DS_Store"]); +export interface BuildWorkspaceTreeOptions { + /** Abort the native workspace scan after this many milliseconds. */ + timeoutMs?: number; +} -const GLOB_SPECIAL_CHARS = new Set(["!", "(", ")", "*", "?", "[", "]", "{", "}", "\\"]); +/** + * Build a generic directory tree using a single native scan. Hidden files are + * shown, .gitignore is not consulted, and the standard non-source directories + * (`node_modules`, `.git`, build outputs, caches…) are pruned by the native + * walker. Used by the read tool's directory-listing path. + */ +export async function buildDirectoryTree(cwd: string, options: BuildDirectoryTreeOptions = {}): Promise { + const rootPath = path.resolve(cwd); + const maxDepth = options.maxDepth ?? 1; + const perDirLimit = options.perDirLimit === undefined ? null : options.perDirLimit; + const rootLimit = options.rootLimit === undefined ? perDirLimit : options.rootLimit; -interface DirectoryTreeNode { + let entries: readonly GlobMatch[]; + let nativeTruncated: boolean; + try { + const result = await listWorkspace({ + path: rootPath, + maxDepth, + hidden: true, + gitignore: false, + }); + entries = result.entries; + nativeTruncated = result.truncated; + } catch { + return emptyTree(rootPath); + } + + return assembleTree(rootPath, entries, { + perDirLimit, + rootLimit, + lineCap: options.lineCap === undefined ? null : options.lineCap, + nativeTruncated, + }); +} + +/** + * Build the workspace tree shown in the system prompt. Returns the rendered + * tree plus the AGENTS.md files surfaced by the same native walk so callers + * never need to do a second filesystem scan. + */ +export async function buildWorkspaceTree(cwd: string, options: BuildWorkspaceTreeOptions = {}): Promise { + const rootPath = path.resolve(cwd); + try { + const result = await listWorkspace({ + path: rootPath, + maxDepth: WORKSPACE_DEFAULTS.maxDepth, + hidden: false, + gitignore: true, + collectAgentsMd: true, + timeoutMs: options.timeoutMs, + }); + const tree = assembleTree(rootPath, result.entries, { + perDirLimit: WORKSPACE_DEFAULTS.perDirLimit, + rootLimit: WORKSPACE_DEFAULTS.perDirLimit, + lineCap: WORKSPACE_DEFAULTS.lineCap, + nativeTruncated: result.truncated, + }); + return { ...tree, agentsMdFiles: result.agentsMdFiles }; + } catch { + return { ...emptyTree(rootPath), agentsMdFiles: [] }; + } +} + +// ─── internals ────────────────────────────────────────────────────────────── + +interface Node { name: string; - relativePath: string; - depth: number; - isDirectory: boolean; + isDir: boolean; mtimeMs: number; size: number; - children: DirectoryTreeNode[]; - droppedChildCount: number; + depth: number; + children: Node[]; + /** When > 0, `children` is laid out as `[recent…, oldest]`. */ + droppedCount: number; } -interface ResolvedDirectoryTreeOptions { - maxDepth: number; - directoryEntryLimit: number | null; - rootEntryLimit: number | null; - lineCap: number | null; - lineCapProtectedDepth: number; - excludedDirectoryNames: ReadonlySet; - excludedNames: ReadonlySet; - hidden: boolean; - gitignore: boolean; - cache: boolean; - rootLabel: string; - childIndex: ReadonlyMap> | null; -} - -interface RenderLine { +interface RenderedLine { label: string; depth: number; + isRoot: boolean; size?: string; age?: string; - isRoot?: boolean; } -function emptyWorkspaceTree(rootPath: string): WorkspaceTree { - return { - rootPath, - rendered: "", - truncated: false, - totalLines: 0, - }; +interface AssembleOptions { + perDirLimit: number | null; + rootLimit: number | null; + lineCap: number | null; + nativeTruncated: boolean; } -function resolveDirectoryTreeOptions(options: DirectoryTreeOptions): ResolvedDirectoryTreeOptions { - const directoryEntryLimit = options.directoryEntryLimit === undefined ? null : options.directoryEntryLimit; - const rootEntryLimit = options.rootEntryLimit === undefined ? directoryEntryLimit : options.rootEntryLimit; - const excludedDirectoryNames = - options.excludedDirectoryNames instanceof Set - ? options.excludedDirectoryNames - : new Set(options.excludedDirectoryNames ?? []); - const providedExcludedNames = - options.excludedNames instanceof Set ? options.excludedNames : new Set(options.excludedNames ?? []); - const excludedNames = new Set([...DIRECTORY_TREE_EXCLUDED_NAMES, ...providedExcludedNames]); - return { - maxDepth: options.maxDepth ?? 1, - directoryEntryLimit, - rootEntryLimit, - lineCap: options.lineCap === undefined ? null : options.lineCap, - lineCapProtectedDepth: options.lineCapProtectedDepth ?? 0, - excludedDirectoryNames, - excludedNames, - hidden: options.hidden ?? true, - gitignore: options.gitignore ?? false, - cache: options.cache ?? true, - rootLabel: options.rootLabel ?? ".", - childIndex: options.childIndex ?? null, - }; -} - -function compareByRecency(a: DirectoryTreeNode, b: DirectoryTreeNode): number { - const mtimeCompare = b.mtimeMs - a.mtimeMs; - if (mtimeCompare !== 0) return mtimeCompare; - return a.name.localeCompare(b.name); -} - -function childRelativePath(parentRelativePath: string, name: string): string { - return parentRelativePath ? `${parentRelativePath}/${name}` : name; -} - -function escapeGlobSegment(segment: string): string { - return Array.from(segment, char => (GLOB_SPECIAL_CHARS.has(char) ? `\\${char}` : char)).join(""); -} - -function directChildPattern(parentRelativePath: string): string { - if (!parentRelativePath) return "*"; - return `${parentRelativePath.split("/").map(escapeGlobSegment).join("/")}/*`; -} - -function matchChildName(parentRelativePath: string, matchPath: string): string | null { - if (!parentRelativePath) return matchPath.includes("/") ? null : matchPath; - const prefix = `${parentRelativePath}/`; - if (!matchPath.startsWith(prefix)) return null; - const name = matchPath.slice(prefix.length); - return name.includes("/") ? null : name; -} - -async function listDirectChildNames( - rootPath: string, - parent: DirectoryTreeNode, - options: ResolvedDirectoryTreeOptions, -): Promise { - if (options.childIndex) { - const names = options.childIndex.get(parent.relativePath); - return names ? Array.from(names) : []; - } - if (!options.gitignore) { - const directoryPath = parent.relativePath ? path.join(rootPath, parent.relativePath) : rootPath; - return await fs.readdir(directoryPath); - } - - const result = await glob({ - pattern: directChildPattern(parent.relativePath), - path: rootPath, - recursive: false, - hidden: options.hidden, - gitignore: true, - cache: options.cache, - }); - - return result.matches - .map(match => matchChildName(parent.relativePath, match.path)) - .filter((name): name is string => name !== null); -} - -async function listDirectoryTreeChildren( - rootPath: string, - parent: DirectoryTreeNode, - options: ResolvedDirectoryTreeOptions, -): Promise { - const childNames = await listDirectChildNames(rootPath, parent, options); - - const children = await Promise.all( - childNames.map(async (name): Promise => { - if (options.excludedNames.has(name)) return null; - if (!options.hidden && name.startsWith(".")) return null; - const relativePath = childRelativePath(parent.relativePath, name); - const absolutePath = path.join(rootPath, relativePath); - try { - const stat = await Bun.file(absolutePath).stat(); - const isDirectory = stat.isDirectory(); - if (isDirectory && options.excludedDirectoryNames.has(name)) return null; - return { - name, - relativePath, - depth: parent.depth + 1, - isDirectory, - mtimeMs: stat.mtimeMs, - size: stat.size, - children: [], - droppedChildCount: 0, - } satisfies DirectoryTreeNode; - } catch { - return null; - } - }), - ); - - return children.filter((child): child is DirectoryTreeNode => child !== null).sort(compareByRecency); -} - -function entryLimitForNode(node: DirectoryTreeNode, options: ResolvedDirectoryTreeOptions): number | null { - return node.depth === 0 ? options.rootEntryLimit : options.directoryEntryLimit; -} - -function applyDirectoryLimit( - node: DirectoryTreeNode, - children: DirectoryTreeNode[], - options: ResolvedDirectoryTreeOptions, -): { visibleChildren: DirectoryTreeNode[]; droppedCount: number } { - const entryLimit = entryLimitForNode(node, options); - if (entryLimit === null || children.length <= entryLimit) { - return { visibleChildren: children, droppedCount: 0 }; - } - if (entryLimit <= 1) { - return { - visibleChildren: children.slice(0, Math.max(0, entryLimit)), - droppedCount: children.length - entryLimit, +function assembleTree(rootPath: string, entries: readonly GlobMatch[], opts: AssembleOptions): DirectoryTree { + // Bucket entries by parent path. The native walker may yield siblings in + // any order across worker threads, so we group by string key and sort once + // per directory below. + const byParent = new Map(); + for (const entry of entries) { + const slash = entry.path.lastIndexOf("/"); + const name = slash === -1 ? entry.path : entry.path.slice(slash + 1); + const parentPath = slash === -1 ? "" : entry.path.slice(0, slash); + const node: Node = { + name, + isDir: entry.fileType === FileType.Dir, + mtimeMs: entry.mtime ?? 0, + size: entry.size ?? 0, + depth: parentPath ? parentPath.split("/").length + 1 : 1, + children: [], + droppedCount: 0, }; + const bucket = byParent.get(parentPath); + if (bucket) bucket.push(node); + else byParent.set(parentPath, [node]); } - const recentChildren = children.slice(0, entryLimit - 1); - const oldestChild = children[children.length - 1]; - return { - visibleChildren: oldestChild ? [...recentChildren, oldestChild] : recentChildren, - droppedCount: children.length - entryLimit, - }; -} - -async function collectDirectoryTree( - rootPath: string, - options: ResolvedDirectoryTreeOptions, -): Promise<{ root: DirectoryTreeNode; truncated: boolean }> { - const rootStat = await Bun.file(rootPath).stat(); - const root: DirectoryTreeNode = { - name: options.rootLabel, - relativePath: "", + const root: Node = { + name: ".", + isDir: true, + mtimeMs: 0, + size: 0, depth: 0, - isDirectory: true, - mtimeMs: rootStat.mtimeMs, - size: rootStat.size, children: [], - droppedChildCount: 0, + droppedCount: 0, }; - let truncated = false; - const queue: DirectoryTreeNode[] = [root]; - let cursor = 0; - - while (cursor < queue.length) { - const parent = queue[cursor]; - cursor += 1; - if (!parent || parent.depth >= options.maxDepth) continue; - - const children = await listDirectoryTreeChildren(rootPath, parent, options); - const limited = applyDirectoryLimit(parent, children, options); - parent.children = limited.visibleChildren; - parent.droppedChildCount = limited.droppedCount; - if (limited.droppedCount > 0) truncated = true; - - for (const child of parent.children) { - if (child.isDirectory) queue.push(child); + let truncated = opts.nativeTruncated; + const stack: Array<{ node: Node; relPath: string }> = [{ node: root, relPath: "" }]; + while (stack.length > 0) { + const { node, relPath } = stack.pop()!; + const all = (byParent.get(relPath) ?? []).slice().sort(byRecency); + const limit = node.depth === 0 ? opts.rootLimit : opts.perDirLimit; + if (limit !== null && all.length > limit) { + node.children = limit <= 1 ? all.slice(0, Math.max(0, limit)) : [...all.slice(0, limit - 1), all.at(-1)!]; + node.droppedCount = all.length - limit; + truncated = true; + } else { + node.children = all; + } + for (const child of node.children) { + if (!child.isDir) continue; + stack.push({ node: child, relPath: relPath ? `${relPath}/${child.name}` : child.name }); } } - return { root, truncated }; + const rawLines: RenderedLine[] = []; + renderNode(root, Date.now(), rawLines); + const { lines, elidedCount } = applyLineCap(rawLines, opts.lineCap); + + return { + rootPath, + rendered: formatLines(lines), + truncated: truncated || elidedCount > 0, + totalLines: lines.length, + }; } -function formatNodeAge(nowMs: number, mtimeMs: number): string { - const ageSeconds = Math.max(0, Math.floor((nowMs - mtimeMs) / 1000)); - return formatAge(ageSeconds); +function byRecency(a: Node, b: Node): number { + return b.mtimeMs - a.mtimeMs || a.name.localeCompare(b.name); } -function pushNodeLine(lines: RenderLine[], node: DirectoryTreeNode, nowMs: number): void { +function renderNode(node: Node, nowMs: number, out: RenderedLine[]): void { if (node.depth === 0) { - lines.push({ label: node.name, depth: 0, isRoot: true }); + out.push({ label: node.name, depth: 0, isRoot: true }); + } else { + const indent = " ".repeat(node.depth); + const suffix = node.isDir ? "/" : ""; + out.push({ + label: `${indent}- ${node.name}${suffix}`, + depth: node.depth, + isRoot: false, + size: node.isDir ? undefined : formatBytes(node.size), + age: formatAge(Math.max(0, Math.floor((nowMs - node.mtimeMs) / 1000))), + }); + } + + if (node.droppedCount === 0) { + for (const child of node.children) renderNode(child, nowMs, out); return; } - const indent = " ".repeat(node.depth); - const suffix = node.isDirectory ? "/" : ""; - lines.push({ - label: `${indent}- ${node.name}${suffix}`, - depth: node.depth, - size: node.isDirectory ? undefined : formatBytes(node.size), - age: formatNodeAge(nowMs, node.mtimeMs), - }); -} - -function pushDroppedChildrenLine(lines: RenderLine[], parent: DirectoryTreeNode): void { - if (parent.droppedChildCount <= 0) return; - const childDepth = parent.depth + 1; - const indent = " ".repeat(childDepth); - lines.push({ - label: `${indent}- … ${parent.droppedChildCount} more`, + // Layout: recent children, then "… N more" marker, then the oldest child. + const recent = node.children.slice(0, -1); + const oldest = node.children.at(-1); + for (const child of recent) renderNode(child, nowMs, out); + const childDepth = node.depth + 1; + out.push({ + label: `${" ".repeat(childDepth)}- … ${node.droppedCount} more`, depth: childDepth, + isRoot: false, }); + if (oldest) renderNode(oldest, nowMs, out); } -function collectRenderLines(node: DirectoryTreeNode, nowMs: number, lines: RenderLine[]): void { - pushNodeLine(lines, node, nowMs); - - if (node.droppedChildCount > 0) { - const recentChildren = node.children.slice(0, -1); - const oldestChild = node.children[node.children.length - 1]; - for (const child of recentChildren) collectRenderLines(child, nowMs, lines); - pushDroppedChildrenLine(lines, node); - if (oldestChild && !recentChildren.includes(oldestChild)) collectRenderLines(oldestChild, nowMs, lines); - return; - } - - for (const child of node.children) collectRenderLines(child, nowMs, lines); -} - +/** + * Cap the rendered tree at `lineCap` lines by removing the deepest trailing + * entries first. Root and root children (depth ≤ 1) are always preserved so + * the structural overview stays intact. + */ function applyLineCap( - lines: RenderLine[], - options: ResolvedDirectoryTreeOptions, -): { lines: RenderLine[]; elidedCount: number } { - if (options.lineCap === null || lines.length <= options.lineCap) return { lines, elidedCount: 0 }; + lines: readonly RenderedLine[], + lineCap: number | null, +): { lines: RenderedLine[]; elidedCount: number } { + if (lineCap === null || lines.length <= lineCap) return { lines: [...lines], elidedCount: 0 }; - const targetLineCount = Math.max(1, options.lineCap - 1); - const removeCount = lines.length - targetLineCount; + const PROTECTED_DEPTH = 1; + const target = Math.max(1, lineCap - 1); const removable = lines .map((line, index) => ({ line, index })) - .filter(item => !item.line.isRoot && item.line.depth > options.lineCapProtectedDepth) + .filter(({ line }) => !line.isRoot && line.depth > PROTECTED_DEPTH) .sort((a, b) => b.line.depth - a.line.depth || b.index - a.index) - .slice(0, removeCount); - if (removable.length === 0) return { lines, elidedCount: 0 }; + .slice(0, lines.length - target); + if (removable.length === 0) return { lines: [...lines], elidedCount: 0 }; - const removedIndexes = new Set(removable.map(item => item.index)); - const cappedLines = lines.filter((_, index) => !removedIndexes.has(index)); - cappedLines.push({ + const removed = new Set(removable.map(item => item.index)); + const kept = lines.filter((_, index) => !removed.has(index)); + kept.push({ label: `… (${removable.length} lines elided beyond depth/cap)`, depth: 0, + isRoot: false, }); - - return { lines: cappedLines, elidedCount: removable.length }; + return { lines: kept, elidedCount: removable.length }; } -function renderLines(lines: RenderLine[]): string { +function formatLines(lines: readonly RenderedLine[]): string { const maxLabelLength = lines.reduce((max, line) => Math.max(max, line.label.length), 0); return lines .map(line => { @@ -374,113 +276,11 @@ function renderLines(lines: RenderLine[]): string { .join("\n"); } -export async function buildDirectoryTree(rootPath: string, options: DirectoryTreeOptions = {}): Promise { - const resolvedRootPath = path.resolve(rootPath); - const resolvedOptions = resolveDirectoryTreeOptions(options); - const nowMs = Date.now(); - const { root, truncated: directoryTruncated } = await collectDirectoryTree(resolvedRootPath, resolvedOptions); - const lines: RenderLine[] = []; - collectRenderLines(root, nowMs, lines); - const { lines: cappedLines, elidedCount } = applyLineCap(lines, resolvedOptions); +function emptyTree(rootPath: string): DirectoryTree { return { - rootPath: resolvedRootPath, - rendered: renderLines(cappedLines), - truncated: directoryTruncated || elidedCount > 0, - totalLines: cappedLines.length, + rootPath, + rendered: "", + truncated: false, + totalLines: 0, }; } - -/** - * Build a `parentRelativePath` → child name index from a flat list of POSIX - * paths. Intermediate directory components are inferred from path segments; - * the index covers every ancestor directory implied by the input. - */ -function buildChildIndexFromPaths(paths: readonly string[]): Map> { - const index = new Map>(); - const ensure = (parent: string): Set => { - let bucket = index.get(parent); - if (!bucket) { - bucket = new Set(); - index.set(parent, bucket); - } - return bucket; - }; - for (const raw of paths) { - if (!raw) continue; - const normalized = raw.replace(/\\/g, "/"); - const parts = normalized.split("/").filter(segment => segment.length > 0); - if (parts.length === 0) continue; - for (let i = 0; i < parts.length; i += 1) { - const parent = parts.slice(0, i).join("/"); - const segment = parts[i]; - if (segment !== undefined) ensure(parent).add(segment); - } - } - return index; -} - -const GIT_LS_FILES_TIMEOUT_MS = 3000; - -/** - * List tracked + untracked-not-ignored files at `rootPath` via `git ls-files`. - * Returns `null` when git is unavailable, the directory is not inside a - * worktree, or the call fails / times out — caller falls back to native - * directory listing. - */ -async function tryListGitFiles(rootPath: string): Promise { - const gitPath = $which("git"); - if (!gitPath) return null; - const signal = AbortSignal.timeout(GIT_LS_FILES_TIMEOUT_MS); - try { - const child = Bun.spawn([gitPath, "ls-files", "--cached", "--others", "--exclude-standard", "-z"], { - cwd: rootPath, - stdout: "pipe", - stderr: "pipe", - stdin: "ignore", - signal, - }); - const [stdout, exitCode] = await Promise.all([ - new Response(child.stdout as ReadableStream).text(), - child.exited, - ]); - if (exitCode !== 0) return null; - if (!stdout) return []; - // `-z` separates entries with NUL; trailing NUL after final entry. - return stdout.split("\0").filter(entry => entry.length > 0); - } catch (error) { - logger.debug("git ls-files failed; falling back to native directory listing", { - rootPath, - error: error instanceof Error ? error.message : String(error), - }); - return null; - } -} - -export async function buildWorkspaceTree(cwd: string): Promise { - const rootPath = path.resolve(cwd); - const baseOptions = { - maxDepth: WORKSPACE_TREE_MAX_DEPTH, - directoryEntryLimit: WORKSPACE_TREE_DIR_LIMIT, - lineCap: WORKSPACE_TREE_LINE_CAP, - excludedDirectoryNames: WORKSPACE_TREE_EXCLUDED_DIRS, - hidden: false, - cache: true, - rootLabel: ".", - } satisfies DirectoryTreeOptions; - - try { - const gitFiles = await tryListGitFiles(rootPath); - if (gitFiles !== null) { - // Git already applied gitignore + tracking semantics: bypass native - // recursive scan and feed the index directly to the tree builder. - return await buildDirectoryTree(rootPath, { - ...baseOptions, - gitignore: false, - childIndex: buildChildIndexFromPaths(gitFiles), - }); - } - return await buildDirectoryTree(rootPath, { ...baseOptions, gitignore: true }); - } catch { - return emptyWorkspaceTree(rootPath); - } -} diff --git a/packages/coding-agent/test/system-prompt-templates.test.ts b/packages/coding-agent/test/system-prompt-templates.test.ts index 8cb046ff2..e999d59f3 100644 --- a/packages/coding-agent/test/system-prompt-templates.test.ts +++ b/packages/coding-agent/test/system-prompt-templates.test.ts @@ -211,17 +211,12 @@ describe("system Handlebars prompt templates", () => { skills: [], rules: [], toolNames: ["read"], - agentsMdSearch: { - scopePath: ".", - limit: 200, - pattern: "AGENTS.md depth 1-4", - files: ["packages/coding-agent/AGENTS.md"], - }, workspaceTree: { rootPath: dir, rendered: ".\n - src/ 1m", truncated: true, totalLines: 2, + agentsMdFiles: ["packages/coding-agent/AGENTS.md"], }, }); diff --git a/packages/coding-agent/test/workspace-tree.test.ts b/packages/coding-agent/test/workspace-tree.test.ts index 8b07997a2..861f47603 100644 --- a/packages/coding-agent/test/workspace-tree.test.ts +++ b/packages/coding-agent/test/workspace-tree.test.ts @@ -3,7 +3,6 @@ import * as fs from "node:fs/promises"; import * as os from "node:os"; import * as path from "node:path"; import { buildDirectoryTree, buildWorkspaceTree } from "@oh-my-pi/pi-coding-agent/workspace-tree"; -import { $ } from "bun"; const tempDirs: string[] = []; @@ -141,10 +140,8 @@ describe("buildWorkspaceTree", () => { const tree = await buildDirectoryTree(cwd, { maxDepth: 2, - directoryEntryLimit: 12, - rootEntryLimit: null, - hidden: true, - gitignore: false, + perDirLimit: 12, + rootLimit: null, }); expect(tree.truncated).toBe(true); @@ -156,36 +153,45 @@ describe("buildWorkspaceTree", () => { expect(tree.rendered).toContain("child-00.txt"); expect(tree.rendered).toContain("… 1 more"); }); -}); -describe("buildWorkspaceTree (git-backed listing)", () => { - afterEach(async () => { - await Promise.all(tempDirs.splice(0).map(dir => fs.rm(dir, { recursive: true, force: true }))); - }); - - it("derives the tree from `git ls-files` and skips gitignored entries even without explicit excludes", async () => { + it("returns AGENTS.md files at directory depths one through four", async () => { const cwd = await makeTempDir(); - // `git init` produces a real worktree so tryListGitFiles activates. - await $`git init -q --initial-branch=main`.cwd(cwd).quiet().nothrow(); - await $`git config user.email test@example.com`.cwd(cwd).quiet().nothrow(); - await $`git config user.name test`.cwd(cwd).quiet().nothrow(); - const base = Date.now() - 60_000; - await Bun.write(path.join(cwd, ".gitignore"), "secret.txt\nbuild-output/\n"); - await writeFileWithMtime(path.join(cwd, "kept.txt"), "kept", base + 5_000); - await writeFileWithMtime(path.join(cwd, "secret.txt"), "secret", base + 4_000); - await touchDirWithMtime(path.join(cwd, "build-output"), base + 3_000); - await writeFileWithMtime(path.join(cwd, "build-output", "artifact.bin"), "x", base + 3_000); - await touchDirWithMtime(path.join(cwd, "src"), base + 2_000); - await writeFileWithMtime(path.join(cwd, "src", "main.ts"), "main", base + 2_000); + await Bun.write(path.join(cwd, "AGENTS.md"), "root rules"); + await Bun.write(path.join(cwd, "one", "AGENTS.md"), "depth one rules"); + await Bun.write(path.join(cwd, "one", "two", "AGENTS.md"), "depth two rules"); + await Bun.write(path.join(cwd, "one", "two", "three", "AGENTS.md"), "depth three rules"); + await Bun.write(path.join(cwd, "one", "two", "three", "four", "AGENTS.md"), "depth four rules"); + await Bun.write(path.join(cwd, "one", "two", "three", "four", "five", "AGENTS.md"), "too deep"); + + const tree = await buildWorkspaceTree(cwd); + + expect(tree.agentsMdFiles).toEqual([ + "one/AGENTS.md", + "one/two/AGENTS.md", + "one/two/three/AGENTS.md", + "one/two/three/four/AGENTS.md", + ]); + }); + + it("surfaces gitignored AGENTS.md files but not AGENTS.md under ignored directories", async () => { + const cwd = await makeTempDir(); + await Bun.write(path.join(cwd, ".gitignore"), "src/AGENTS.md\nignored-dir/\nnode_modules/\n.git/\n.hidden/\n"); + await Bun.write(path.join(cwd, "src", "AGENTS.md"), "src rules"); + await Bun.write(path.join(cwd, "src", "main.ts"), "source"); + await Bun.write(path.join(cwd, "ignored-dir", "AGENTS.md"), "ignored dir rules"); + await Bun.write(path.join(cwd, "node_modules", "pkg", "AGENTS.md"), "ignored dependency rules"); + await Bun.write(path.join(cwd, ".git", "AGENTS.md"), "ignored git rules"); + await Bun.write(path.join(cwd, ".hidden", "AGENTS.md"), "ignored hidden rules"); const tree = await buildWorkspaceTree(cwd); - expect(tree.rendered).toContain("kept.txt"); expect(tree.rendered).toContain("src/"); + expect(tree.rendered).toContain("AGENTS.md"); expect(tree.rendered).toContain("main.ts"); - expect(tree.rendered).not.toContain("secret.txt"); - expect(tree.rendered).not.toContain("build-output"); - expect(tree.rendered).not.toContain("artifact.bin"); + expect(tree.rendered).not.toContain("ignored-dir"); + expect(tree.rendered).not.toContain("node_modules"); + expect(tree.rendered).not.toContain(".hidden"); + expect(tree.agentsMdFiles).toEqual(["src/AGENTS.md"]); }); }); diff --git a/packages/natives/CHANGELOG.md b/packages/natives/CHANGELOG.md index 9113c4ad9..1f50712fe 100644 --- a/packages/natives/CHANGELOG.md +++ b/packages/natives/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Added + +- Added `listWorkspace`, a native single-pass workspace walker that returns bounded tree entries and AGENTS.md directory-context candidates together. + ## [14.7.1] - 2026-05-06 ### Added diff --git a/packages/natives/native/index.d.ts b/packages/natives/native/index.d.ts index ce22cd71c..839b98f39 100644 --- a/packages/natives/native/index.d.ts +++ b/packages/natives/native/index.d.ts @@ -768,6 +768,50 @@ export declare enum KeyEventType { Release = 3 } +/** + * Walk the workspace once and return tree entries plus AGENTS.md candidates. + * + * File-level ignore rules for AGENTS.md are bypassed by checking each + * traversed directory directly when `collectAgentsMd` is enabled, but ignored + * directories are still pruned by the walker and are not searched. + */ +export declare function listWorkspace(options: ListWorkspaceOptions): Promise + +/** Input options for `listWorkspace`, the single-pass workspace startup scan. */ +export interface ListWorkspaceOptions { + /** Directory to scan. */ + path: string + /** Maximum depth for returned tree entries. Root children are depth 1. */ + maxDepth: number + /** Include hidden files and directories. Default: false. */ + hidden?: boolean + /** Respect .gitignore files. Default: true. */ + gitignore?: boolean + /** + * Also surface AGENTS.md files in directories at depth 1..=4, even when + * gitignore would otherwise hide the file. Walks deeper than `maxDepth` + * to find them. Default: false. + */ + collectAgentsMd?: boolean + /** Timeout in milliseconds for the operation. */ + timeoutMs?: number + /** Abort signal for cancelling the operation. */ + signal?: unknown +} + +/** Result payload returned by a workspace scan. */ +export interface ListWorkspaceResult { + /** Entries within `maxDepth`, with mtime and regular-file size metadata. */ + entries: Array + /** + * Directory-scoped AGENTS.md files within depth 1..=4 (capped at 200). + * Always empty when `collectAgentsMd` is false. + */ + agentsMdFiles: Array + /** True when any output cap was hit. */ + truncated: boolean +} + /** * System UI appearance reported by native macOS APIs (`detectMacOSAppearance` * and observer). diff --git a/packages/natives/native/index.js b/packages/natives/native/index.js index 271fc96b2..5edea39c3 100644 --- a/packages/natives/native/index.js +++ b/packages/natives/native/index.js @@ -231,6 +231,7 @@ export const hasMatch = nativeBindings.hasMatch; export const highlightCode = nativeBindings.highlightCode; export const htmlToMarkdown = nativeBindings.htmlToMarkdown; export const invalidateFsScanCache = nativeBindings.invalidateFsScanCache; +export const listWorkspace = nativeBindings.listWorkspace; export const matchesKey = nativeBindings.matchesKey; export const matchesKittySequence = nativeBindings.matchesKittySequence; export const matchesLegacySequence = nativeBindings.matchesLegacySequence; diff --git a/packages/natives/test/native.test.ts b/packages/natives/test/native.test.ts index 523980532..b67bf3325 100644 --- a/packages/natives/test/native.test.ts +++ b/packages/natives/test/native.test.ts @@ -12,6 +12,7 @@ import { grep, htmlToMarkdown, invalidateFsScanCache, + listWorkspace, MacOSPowerAssertion, PtySession, sanitizeText, @@ -421,6 +422,43 @@ describe("pi-natives", () => { }); }); + describe("listWorkspace", () => { + it("returns tree entries and gitignored AGENTS.md files outside ignored directories", async () => { + const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "natives-workspace-")); + try { + await fs.writeFile(path.join(workspaceDir, ".gitignore"), "ignored.txt\nsrc/AGENTS.md\nignored-dir/\n"); + await fs.writeFile(path.join(workspaceDir, "kept.ts"), "export const kept = true;\n"); + await fs.writeFile(path.join(workspaceDir, "ignored.txt"), "ignored\n"); + await fs.mkdir(path.join(workspaceDir, "src"), { recursive: true }); + await fs.writeFile(path.join(workspaceDir, "src", "AGENTS.md"), "src rules\n"); + await fs.writeFile(path.join(workspaceDir, "src", "main.ts"), "export const main = true;\n"); + await fs.mkdir(path.join(workspaceDir, "ignored-dir"), { recursive: true }); + await fs.writeFile(path.join(workspaceDir, "ignored-dir", "AGENTS.md"), "ignored rules\n"); + + const result = await listWorkspace({ + path: workspaceDir, + maxDepth: 3, + gitignore: true, + hidden: false, + collectAgentsMd: true, + }); + const entryPaths = result.entries.map(entry => entry.path); + + expect(result.truncated).toBe(false); + expect(entryPaths).toContain("kept.ts"); + expect(entryPaths).toContain("src"); + expect(entryPaths).toContain("src/AGENTS.md"); + expect(entryPaths).toContain("src/main.ts"); + expect(entryPaths).not.toContain("ignored.txt"); + expect(entryPaths).not.toContain("ignored-dir"); + expect(entryPaths).not.toContain("ignored-dir/AGENTS.md"); + expect(result.agentsMdFiles).toEqual(["src/AGENTS.md"]); + } finally { + await fs.rm(workspaceDir, { recursive: true, force: true }); + } + }); + }); + describe("text tab width", () => { it("uses default tab width and supports explicit overrides", () => { expect(visibleWidth("a\tb", 3)).toBe(5);