feat: added optional cache parameter to glob/grep/fuzzyFind and improved fs cache handling
- Added optional `cache` parameter to `glob()`, `grep()`, and `fuzzyFind()` functions to enable shared filesystem scan caching. - Added new `invalidateFsScanCache()` function to allow manual invalidation of filesystem scan cache entries. - Improved filesystem path canonicalization in cache invalidation logic to handle relative paths and non-existent paths more robustly. - Fixed variable shadowing bug in grep's `match_ext` method where extension parameter was compared against itself. - Added early return optimization in glob when `maxResults` is zero to avoid unnecessary filesystem scanning.
This commit is contained in:
+23
-14
@@ -25,6 +25,8 @@ pub struct FuzzyFindOptions<'env> {
|
||||
pub hidden: Option<bool>,
|
||||
/// Respect .gitignore (default: true).
|
||||
pub gitignore: Option<bool>,
|
||||
/// Enable shared filesystem scan cache (default: false).
|
||||
pub cache: Option<bool>,
|
||||
/// Maximum number of matches to return (default: 100).
|
||||
#[napi(js_name = "maxResults")]
|
||||
pub max_results: Option<u32>,
|
||||
@@ -156,6 +158,7 @@ struct FuzzyFindConfig {
|
||||
hidden: Option<bool>,
|
||||
gitignore: Option<bool>,
|
||||
max_results: Option<u32>,
|
||||
cache: Option<bool>,
|
||||
}
|
||||
|
||||
fn clamp_u32(value: u64) -> u32 {
|
||||
@@ -177,18 +180,24 @@ fn fuzzy_find_sync(config: FuzzyFindConfig, ct: task::CancelToken) -> Result<Fuz
|
||||
return Ok(FuzzyFindResult { matches: Vec::new(), total_matches: 0 });
|
||||
}
|
||||
|
||||
let scan = fs_cache::get_or_scan(&root, include_hidden, respect_gitignore, &ct)?;
|
||||
let mut scored = score_entries(&scan.entries, &query_lower, &normalized_query, &ct)?;
|
||||
|
||||
// Empty-result recheck: if the query was non-trivial but produced zero matches
|
||||
// from a cached scan that's old enough, force one rescan before giving up.
|
||||
if scored.is_empty()
|
||||
&& !query_lower.is_empty()
|
||||
&& scan.cache_age_ms >= fs_cache::empty_recheck_ms()
|
||||
{
|
||||
let fresh = fs_cache::force_rescan(&root, include_hidden, respect_gitignore, &ct)?;
|
||||
scored = score_entries(&fresh, &query_lower, &normalized_query, &ct)?;
|
||||
}
|
||||
let use_cache = config.cache.unwrap_or(false);
|
||||
let mut scored = if use_cache {
|
||||
let scan = fs_cache::get_or_scan(&root, include_hidden, respect_gitignore, &ct)?;
|
||||
let mut scored = score_entries(&scan.entries, &query_lower, &normalized_query, &ct)?;
|
||||
// Empty-result recheck: if the query was non-trivial but produced zero matches
|
||||
// from a cached scan that's old enough, force one rescan before giving up.
|
||||
if scored.is_empty()
|
||||
&& !query_lower.is_empty()
|
||||
&& scan.cache_age_ms >= fs_cache::empty_recheck_ms()
|
||||
{
|
||||
let fresh = fs_cache::force_rescan(&root, include_hidden, respect_gitignore, true, &ct)?;
|
||||
scored = score_entries(&fresh, &query_lower, &normalized_query, &ct)?;
|
||||
}
|
||||
scored
|
||||
} else {
|
||||
let fresh = fs_cache::force_rescan(&root, include_hidden, respect_gitignore, false, &ct)?;
|
||||
score_entries(&fresh, &query_lower, &normalized_query, &ct)?
|
||||
};
|
||||
|
||||
scored.sort_by(|a, b| b.score.cmp(&a.score).then_with(|| a.path.cmp(&b.path)));
|
||||
let total_matches = clamp_u32(scored.len() as u64);
|
||||
@@ -235,9 +244,9 @@ fn score_entries(
|
||||
/// Matching file and directory entries sorted by match quality.
|
||||
#[napi(js_name = "fuzzyFind")]
|
||||
pub fn fuzzy_find(options: FuzzyFindOptions<'_>) -> task::Async<FuzzyFindResult> {
|
||||
let FuzzyFindOptions { query, path, hidden, gitignore, max_results, timeout_ms, signal } =
|
||||
let FuzzyFindOptions { query, path, hidden, gitignore, cache, max_results, timeout_ms, signal } =
|
||||
options;
|
||||
let ct = task::CancelToken::new(timeout_ms, signal);
|
||||
let config = FuzzyFindConfig { query, path, hidden, gitignore, max_results };
|
||||
let config = FuzzyFindConfig { query, path, hidden, gitignore, max_results, cache };
|
||||
task::blocking("fuzzy_find", ct, move |ct| fuzzy_find_sync(config, ct))
|
||||
}
|
||||
|
||||
@@ -147,7 +147,7 @@ pub fn resolve_search_path(path: &str) -> Result<PathBuf> {
|
||||
if !metadata.is_dir() {
|
||||
return Err(Error::from_reason("Search path must be a directory".to_string()));
|
||||
}
|
||||
Ok(root)
|
||||
Ok(std::fs::canonicalize(&root).unwrap_or(root))
|
||||
}
|
||||
|
||||
/// Normalize a filesystem path to a forward-slash relative string.
|
||||
@@ -324,20 +324,24 @@ pub fn get_or_scan(
|
||||
/// Force a fresh scan, replacing any existing cache entry.
|
||||
///
|
||||
/// Use when a cached query produced zero matches and the cache was old enough
|
||||
/// to warrant a recheck.
|
||||
/// to warrant a recheck. When `store` is false, the fresh scan result is
|
||||
/// returned without repopulating the cache.
|
||||
pub fn force_rescan(
|
||||
root: &Path,
|
||||
include_hidden: bool,
|
||||
use_gitignore: bool,
|
||||
store: bool,
|
||||
ct: &task::CancelToken,
|
||||
) -> Result<Vec<GlobMatch>> {
|
||||
let key = CacheKey { root: root.to_path_buf(), include_hidden, use_gitignore };
|
||||
FS_CACHE.remove(&key);
|
||||
|
||||
let entries = collect_entries(root, include_hidden, use_gitignore, ct)?;
|
||||
let now = Instant::now();
|
||||
FS_CACHE.insert(key, CacheEntry { created_at: now, entries: entries.clone() });
|
||||
evict_oldest();
|
||||
if store {
|
||||
let now = Instant::now();
|
||||
FS_CACHE.insert(key, CacheEntry { created_at: now, entries: entries.clone() });
|
||||
evict_oldest();
|
||||
}
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
@@ -375,7 +379,26 @@ pub fn invalidate_all() {
|
||||
#[napi(js_name = "invalidateFsScanCache")]
|
||||
pub fn invalidate_fs_scan_cache(path: Option<String>) {
|
||||
match path {
|
||||
Some(p) => invalidate_path(Path::new(&p)),
|
||||
Some(p) => {
|
||||
let candidate = PathBuf::from(&p);
|
||||
let absolute = if candidate.is_absolute() {
|
||||
candidate
|
||||
} else if let Ok(cwd) = std::env::current_dir() {
|
||||
cwd.join(candidate)
|
||||
} else {
|
||||
PathBuf::from(&p)
|
||||
};
|
||||
let target = std::fs::canonicalize(&absolute)
|
||||
.or_else(|_| {
|
||||
absolute
|
||||
.parent()
|
||||
.and_then(|parent| std::fs::canonicalize(parent).ok())
|
||||
.and_then(|parent| absolute.file_name().map(|name| parent.join(name)))
|
||||
.ok_or_else(|| std::io::Error::from(std::io::ErrorKind::NotFound))
|
||||
})
|
||||
.unwrap_or(absolute);
|
||||
invalidate_path(&target);
|
||||
},
|
||||
None => invalidate_all(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,6 +44,8 @@ pub struct GlobOptions<'env> {
|
||||
pub max_results: Option<u32>,
|
||||
/// Respect .gitignore files (default: true).
|
||||
pub gitignore: Option<bool>,
|
||||
/// Enable shared filesystem scan cache (default: false).
|
||||
pub cache: Option<bool>,
|
||||
/// Sort results by mtime (most recent first) before applying limit.
|
||||
#[napi(js_name = "sortByMtime")]
|
||||
pub sort_by_mtime: Option<bool>,
|
||||
@@ -101,6 +103,7 @@ struct GlobConfig {
|
||||
use_gitignore: bool,
|
||||
mentions_node_modules: bool,
|
||||
sort_by_mtime: bool,
|
||||
use_cache: bool,
|
||||
}
|
||||
|
||||
/// Filter and collect matching entries from a pre-scanned list.
|
||||
@@ -152,19 +155,38 @@ fn run_glob(
|
||||
ct: task::CancelToken,
|
||||
) -> Result<GlobResult> {
|
||||
let glob_set = compile_glob(&config.pattern)?;
|
||||
|
||||
let scan =
|
||||
fs_cache::get_or_scan(&config.root, config.include_hidden, config.use_gitignore, &ct)?;
|
||||
let mut matches = filter_entries(&scan.entries, &glob_set, &config, on_match, &ct)?;
|
||||
|
||||
// Empty-result recheck: if we got zero matches from a cached scan that's old
|
||||
// enough, force a rescan and try once more before returning empty.
|
||||
if matches.is_empty() && scan.cache_age_ms >= fs_cache::empty_recheck_ms() {
|
||||
let fresh =
|
||||
fs_cache::force_rescan(&config.root, config.include_hidden, config.use_gitignore, &ct)?;
|
||||
matches = filter_entries(&fresh, &glob_set, &config, on_match, &ct)?;
|
||||
if config.max_results == 0 {
|
||||
return Ok(GlobResult { matches: Vec::new(), total_matches: 0 });
|
||||
}
|
||||
|
||||
let mut matches = if config.use_cache {
|
||||
let scan =
|
||||
fs_cache::get_or_scan(&config.root, config.include_hidden, config.use_gitignore, &ct)?;
|
||||
let mut matches = filter_entries(&scan.entries, &glob_set, &config, on_match, &ct)?;
|
||||
// Empty-result recheck: if we got zero matches from a cached scan that's old
|
||||
// enough, force a rescan and try once more before returning empty.
|
||||
if matches.is_empty() && scan.cache_age_ms >= fs_cache::empty_recheck_ms() {
|
||||
let fresh = fs_cache::force_rescan(
|
||||
&config.root,
|
||||
config.include_hidden,
|
||||
config.use_gitignore,
|
||||
true,
|
||||
&ct,
|
||||
)?;
|
||||
matches = filter_entries(&fresh, &glob_set, &config, on_match, &ct)?;
|
||||
}
|
||||
matches
|
||||
} else {
|
||||
let fresh = fs_cache::force_rescan(
|
||||
&config.root,
|
||||
config.include_hidden,
|
||||
config.use_gitignore,
|
||||
false,
|
||||
&ct,
|
||||
)?;
|
||||
filter_entries(&fresh, &glob_set, &config, on_match, &ct)?
|
||||
};
|
||||
|
||||
if config.sort_by_mtime {
|
||||
// Sorting mode: rank by mtime descending, then apply max-results truncation.
|
||||
matches.sort_by(|a, b| {
|
||||
@@ -207,6 +229,7 @@ pub fn glob(
|
||||
max_results,
|
||||
gitignore,
|
||||
sort_by_mtime,
|
||||
cache,
|
||||
include_node_modules,
|
||||
timeout_ms,
|
||||
signal,
|
||||
@@ -229,6 +252,7 @@ pub fn glob(
|
||||
mentions_node_modules: include_node_modules
|
||||
.unwrap_or_else(|| pattern.contains("node_modules")),
|
||||
sort_by_mtime: sort_by_mtime.unwrap_or(false),
|
||||
use_cache: cache.unwrap_or(false),
|
||||
pattern,
|
||||
},
|
||||
on_match.as_ref(),
|
||||
|
||||
@@ -87,6 +87,8 @@ pub struct GrepOptions<'env> {
|
||||
pub multiline: Option<bool>,
|
||||
/// Include hidden files (default: true).
|
||||
pub hidden: Option<bool>,
|
||||
/// Enable shared filesystem scan cache (default: false).
|
||||
pub cache: Option<bool>,
|
||||
/// Maximum number of matches to return.
|
||||
#[napi(js_name = "maxCount")]
|
||||
pub max_count: Option<u32>,
|
||||
@@ -207,7 +209,7 @@ impl TypeFilter {
|
||||
fn match_ext(&self, ext: &str) -> bool {
|
||||
match self {
|
||||
Self::Known { exts, .. } => exts.iter().any(|e| ext.eq_ignore_ascii_case(e)),
|
||||
Self::Custom(ext) => ext.eq_ignore_ascii_case(ext),
|
||||
Self::Custom(custom_ext) => ext.eq_ignore_ascii_case(custom_ext),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -612,6 +614,7 @@ struct GrepConfig {
|
||||
ignore_case: Option<bool>,
|
||||
multiline: Option<bool>,
|
||||
hidden: Option<bool>,
|
||||
cache: Option<bool>,
|
||||
max_count: Option<u32>,
|
||||
offset: Option<u32>,
|
||||
context_before: Option<u32>,
|
||||
@@ -816,6 +819,7 @@ fn grep_sync(
|
||||
let max_count = options.max_count.map(u64::from);
|
||||
let offset = options.offset.unwrap_or(0) as u64;
|
||||
let include_hidden = options.hidden.unwrap_or(true);
|
||||
let use_cache = options.cache.unwrap_or(false);
|
||||
let glob_set = compile_glob(options.glob.as_deref())?;
|
||||
let type_filter = resolve_type_filter(options.type_filter.as_deref());
|
||||
|
||||
@@ -897,13 +901,19 @@ fn grep_sync(
|
||||
});
|
||||
}
|
||||
|
||||
let scan = fs_cache::get_or_scan(&search_path, include_hidden, true, &ct)?;
|
||||
let mut entries =
|
||||
collect_files(&search_path, &scan.entries, glob_set.as_ref(), type_filter.as_ref());
|
||||
if entries.is_empty() && scan.cache_age_ms >= fs_cache::empty_recheck_ms() {
|
||||
let fresh = fs_cache::force_rescan(&search_path, include_hidden, true, &ct)?;
|
||||
entries = collect_files(&search_path, &fresh, glob_set.as_ref(), type_filter.as_ref());
|
||||
}
|
||||
let entries = if use_cache {
|
||||
let scan = fs_cache::get_or_scan(&search_path, include_hidden, true, &ct)?;
|
||||
let mut entries =
|
||||
collect_files(&search_path, &scan.entries, glob_set.as_ref(), type_filter.as_ref());
|
||||
if entries.is_empty() && scan.cache_age_ms >= fs_cache::empty_recheck_ms() {
|
||||
let fresh = fs_cache::force_rescan(&search_path, include_hidden, true, true, &ct)?;
|
||||
entries = collect_files(&search_path, &fresh, glob_set.as_ref(), type_filter.as_ref());
|
||||
}
|
||||
entries
|
||||
} else {
|
||||
let fresh = fs_cache::force_rescan(&search_path, include_hidden, true, false, &ct)?;
|
||||
collect_files(&search_path, &fresh, glob_set.as_ref(), type_filter.as_ref())
|
||||
};
|
||||
// Check cancellation before heavy work
|
||||
ct.heartbeat()?;
|
||||
if entries.is_empty() {
|
||||
@@ -1093,6 +1103,7 @@ pub fn grep(
|
||||
ignore_case,
|
||||
multiline,
|
||||
hidden,
|
||||
cache,
|
||||
max_count,
|
||||
offset,
|
||||
context_before,
|
||||
@@ -1112,6 +1123,7 @@ pub fn grep(
|
||||
ignore_case,
|
||||
multiline,
|
||||
hidden,
|
||||
cache,
|
||||
max_count,
|
||||
offset,
|
||||
context_before,
|
||||
|
||||
@@ -132,6 +132,7 @@ export class GrepTool implements AgentTool<typeof grepSchema, GrepToolDetails> {
|
||||
ignoreCase,
|
||||
multiline: effectiveMultiline,
|
||||
hidden: true,
|
||||
cache: false,
|
||||
maxCount: effectiveLimit,
|
||||
offset: normalizedOffset > 0 ? normalizedOffset : undefined,
|
||||
contextBefore: normalizedContextBefore,
|
||||
|
||||
@@ -33,12 +33,14 @@ type MentionDiscoveryProfile = {
|
||||
gitignore: boolean;
|
||||
includeNodeModules: boolean;
|
||||
maxResults: number;
|
||||
cache: boolean;
|
||||
};
|
||||
|
||||
function getMentionCandidateDiscoveryProfile(): MentionDiscoveryProfile {
|
||||
return {
|
||||
hidden: true,
|
||||
gitignore: true,
|
||||
cache: true,
|
||||
includeNodeModules: true,
|
||||
maxResults: MAX_RESOLUTION_CANDIDATES,
|
||||
};
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
# Changelog
|
||||
|
||||
## [Unreleased]
|
||||
### Added
|
||||
|
||||
- Added `cache` option to `glob()`, `grep()`, and `fuzzyFind()` to enable shared filesystem scan caching
|
||||
- Added `invalidateFsScanCache()` function to manually invalidate filesystem scan cache entries
|
||||
|
||||
## [11.14.0] - 2026-02-12
|
||||
### Added
|
||||
|
||||
@@ -27,6 +27,8 @@ export interface GlobOptions extends Cancellable {
|
||||
maxResults?: number;
|
||||
/** Respect .gitignore files (default: true). */
|
||||
gitignore?: boolean;
|
||||
/** Enable shared filesystem scan cache (default: false). */
|
||||
cache?: boolean;
|
||||
/** Sort results by mtime (most recent first) before applying limit. */
|
||||
sortByMtime?: boolean;
|
||||
/** Include node_modules entries even when pattern does not mention node_modules. */
|
||||
|
||||
@@ -20,6 +20,8 @@ export interface GrepOptions extends Cancellable {
|
||||
multiline?: boolean;
|
||||
/** Include hidden files (default: true) */
|
||||
hidden?: boolean;
|
||||
/** Enable shared filesystem scan cache (default: false). */
|
||||
cache?: boolean;
|
||||
/** Maximum number of matches to return */
|
||||
maxCount?: number;
|
||||
/** Skip first N matches */
|
||||
@@ -140,6 +142,8 @@ export interface FuzzyFindOptions extends Cancellable {
|
||||
hidden?: boolean;
|
||||
/** Respect .gitignore (default: true). */
|
||||
gitignore?: boolean;
|
||||
/** Enable shared filesystem scan cache (default: false). */
|
||||
cache?: boolean;
|
||||
/** Maximum number of matches to return (default: 100). */
|
||||
maxResults?: number;
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ import { beforeAll, describe, expect, it } from "bun:test";
|
||||
import * as fs from "node:fs/promises";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { FileType, fuzzyFind, type GlobMatch, glob, grep, htmlToMarkdown } from "../src/index";
|
||||
import { FileType, fuzzyFind, type GlobMatch, glob, grep, htmlToMarkdown, invalidateFsScanCache } from "../src/index";
|
||||
|
||||
let testDir: string;
|
||||
|
||||
@@ -82,6 +82,17 @@ describe("pi-natives", () => {
|
||||
|
||||
expect(result.filesWithMatches).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("should treat unknown grep type filter as a strict extension filter", async () => {
|
||||
const result = await grep({
|
||||
pattern: "return",
|
||||
path: testDir,
|
||||
type: "definitelynotatype",
|
||||
});
|
||||
|
||||
expect(result.totalMatches).toBe(0);
|
||||
expect(result.filesWithMatches).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("fuzzyFind", () => {
|
||||
@@ -118,8 +129,43 @@ describe("pi-natives", () => {
|
||||
|
||||
expect(result.totalMatches).toBe(4);
|
||||
});
|
||||
});
|
||||
|
||||
it("should invalidate scan cache when invalidateFsScanCache receives a relative path", async () => {
|
||||
await glob({ pattern: "*.ts", path: testDir, cache: true });
|
||||
const newFile = path.join(testDir, "newly-added.ts");
|
||||
await fs.writeFile(newFile, "export const newer = true;\n");
|
||||
|
||||
const relativePath = path.relative(process.cwd(), newFile);
|
||||
invalidateFsScanCache(relativePath);
|
||||
|
||||
const result = await glob({ pattern: "newly-added.ts", path: testDir, cache: true });
|
||||
expect(result.matches.some(match => match.path === "newly-added.ts")).toBe(true);
|
||||
});
|
||||
|
||||
it("should avoid scan work when maxResults is zero", async () => {
|
||||
const result = await glob({
|
||||
pattern: "**/*",
|
||||
path: testDir,
|
||||
maxResults: 0,
|
||||
});
|
||||
|
||||
expect(result.totalMatches).toBe(0);
|
||||
expect(result.matches).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("should fast-recheck empty cached results when threshold is reached", async () => {
|
||||
const fileName = "cache-empty-recheck-target.txt";
|
||||
const filePath = path.join(testDir, fileName);
|
||||
await fs.rm(filePath, { force: true });
|
||||
invalidateFsScanCache();
|
||||
const first = await glob({ pattern: fileName, path: testDir, hidden: true, gitignore: true, cache: true });
|
||||
expect(first.totalMatches).toBe(0);
|
||||
await fs.writeFile(filePath, "created after empty cached query\n");
|
||||
await Bun.sleep(250);
|
||||
const second = await glob({ pattern: fileName, path: testDir, hidden: true, gitignore: true, cache: true });
|
||||
expect(second.totalMatches).toBe(1);
|
||||
});
|
||||
});
|
||||
describe("htmlToMarkdown", () => {
|
||||
it("should convert basic HTML to markdown", async () => {
|
||||
const html = "<h1>Hello World</h1><p>This is a paragraph.</p>";
|
||||
|
||||
@@ -14,6 +14,7 @@ function buildAutocompleteFuzzyDiscoveryProfile(
|
||||
maxResults: number;
|
||||
hidden: boolean;
|
||||
gitignore: boolean;
|
||||
cache: boolean;
|
||||
} {
|
||||
return {
|
||||
query,
|
||||
@@ -21,6 +22,7 @@ function buildAutocompleteFuzzyDiscoveryProfile(
|
||||
maxResults: 100,
|
||||
hidden: true,
|
||||
gitignore: true,
|
||||
cache: true,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user