feat(cross-cutting): added pi-ast/pi-shell crates and rewired pi-natives

- Added `pi-ast` as a new crate with public `language`, `ops`, and `summary` modules and exported `SupportLang`.
- Added AST summary and rewrite APIs in `pi-ast`, including language inference, overlap-safe edits, and span normalization.
- Added `pi-shell` crate with new `Shell` and cross-platform `Process` APIs, session execution, and cancellation-aware runs.
- Updated `pi-natives` to use shared `pi_ast`/`pi_shell` APIs, switched to local path deps, and moved minimizer assets to `pi-shell`.
This commit is contained in:
can1357
2026-05-11 03:43:58 +02:00
parent 770f175795
commit 41710a0a5f
115 changed files with 5035 additions and 2723 deletions
+11 -3
View File
@@ -37,9 +37,17 @@ incremental = true
strip = false
[profile.dev]
opt-level = 3
lto = "thin"
codegen-units = 16
opt-level = 0
lto = false
codegen-units = 256
incremental = true
debug = "line-tables-only"
split-debuginfo = "unpacked"
# Deps compile optimized once and cache; your own crates stay fast.
[profile.dev.package."*"]
opt-level = 2
debug = false
[workspace.lints.clippy]
# Base Lint Levels
+74
View File
@@ -0,0 +1,74 @@
[package]
name = "pi-ast"
version.workspace = true
edition.workspace = true
license.workspace = true
authors.workspace = true
repository.workspace = true
[lints]
workspace = true
[dependencies]
anyhow = "1.0"
ast-grep-core = { version = "0.39", default-features = false, features = ["tree-sitter"] }
globset = "0.4"
ignore = "0.4"
phf = { version = "0.13", features = ["macros"] }
serde = { version = "1.0", features = ["derive"] }
tree-sitter = "0.25"
tree-sitter-astro = { version = "0.1.1", package = "tree-sitter-astro-next" }
tree-sitter-bash = "0.25"
tree-sitter-c = "0.24"
tree-sitter-clojure = "0.1"
tree-sitter-cmake = "0.7.1"
tree-sitter-c-sharp = "0.23"
tree-sitter-cpp = "0.23"
tree-sitter-dart = "0.2"
tree-sitter-css = "0.25"
tree-sitter-diff = "0.1"
tree-sitter-dockerfile = { version = "0.2.0", package = "tree-sitter-dockerfile-updated" }
tree-sitter-elixir = "0.3"
tree-sitter-erlang = "0.16.0"
tree-sitter-go = "0.25"
tree-sitter-graphql = "0.1.0"
tree-sitter-haskell = "0.23"
tree-sitter-hcl = "1.1"
tree-sitter-html = "0.23"
tree-sitter-ini = "1.4.0"
tree-sitter-java = "0.23"
tree-sitter-javascript = "0.25"
tree-sitter-json = "0.24"
tree-sitter-just = "0.2.0"
tree-sitter-julia = "0.23"
tree-sitter-kotlin = { version = "0.4", package = "tree-sitter-kotlin-sg" }
tree-sitter-lua = "0.5"
tree-sitter-make = "1.1"
tree-sitter-md = "0.5"
tree-sitter-nix = "0.3"
tree-sitter-objc = "3.0"
tree-sitter-ocaml = "0.24.2"
tree-sitter-odin = "1.3"
tree-sitter-perl = { version = "0.1.0", package = "tree-sitter-perl-next" }
tree-sitter-php = "0.24"
tree-sitter-powershell = "0.26.4"
tree-sitter-proto = "0.4.0"
tree-sitter-python = "0.25"
tree-sitter-r = "1.2.0"
tree-sitter-regex = "0.25"
tree-sitter-ruby = "0.23"
tree-sitter-rust = "0.24"
tree-sitter-scala = "0.26"
tree-sitter-solidity = "1.2"
tree-sitter-sql = { version = "0.3.11", package = "tree-sitter-sequel" }
tree-sitter-starlark = "1.3"
tree-sitter-svelte = { version = "0.1.1", package = "tree-sitter-svelte-next" }
tree-sitter-swift = "0.7"
tree-sitter-toml-ng = "0.7"
tree-sitter-tlaplus = "1.5"
tree-sitter-typescript = "0.23"
tree-sitter-verilog = "1.0"
tree-sitter-vue = { version = "0.1.0", package = "tree-sitter-vue-next" }
tree-sitter-xml = "0.7"
tree-sitter-yaml = "0.7"
tree-sitter-zig = "1.1"
@@ -411,6 +411,10 @@ impl SupportLang {
LANG_ALIASES.get(lowered.as_str()).copied()
}
pub fn from_path(path: &Path) -> Option<Self> {
from_extension(path)
}
pub fn sorted_aliases() -> &'static [&'static str] {
&SORTED_ALIASES
}
+5
View File
@@ -0,0 +1,5 @@
pub mod language;
pub mod ops;
pub mod summary;
pub use language::SupportLang;
+305
View File
@@ -0,0 +1,305 @@
use std::path::{Path, PathBuf};
use anyhow::{Result, anyhow};
use ast_grep_core::{
MatchStrictness, Position,
matcher::{Pattern, PatternError},
source::Edit,
tree_sitter::{LanguageExt, StrDoc},
};
use globset::{Glob, GlobSet, GlobSetBuilder};
use ignore::WalkBuilder;
use crate::language::SupportLang;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum AstMatchStrictness {
Cst,
Smart,
Ast,
Relaxed,
Signature,
Template,
}
impl From<AstMatchStrictness> for MatchStrictness {
fn from(value: AstMatchStrictness) -> Self {
match value {
AstMatchStrictness::Cst => Self::Cst,
AstMatchStrictness::Smart => Self::Smart,
AstMatchStrictness::Ast => Self::Ast,
AstMatchStrictness::Relaxed => Self::Relaxed,
AstMatchStrictness::Signature => Self::Signature,
AstMatchStrictness::Template => Self::Template,
}
}
}
#[derive(Debug, Clone)]
pub struct AstMatch {
pub line: usize,
pub column: usize,
pub end_line: usize,
pub end_column: usize,
pub byte_start: usize,
pub byte_end: usize,
pub text: String,
}
#[derive(Debug, Clone)]
pub struct MatchedFile {
pub absolute_path: PathBuf,
pub relative_path: String,
}
#[derive(Debug, Clone)]
pub struct CompiledRewrite {
pub out: String,
pub patterns: Vec<Pattern>,
}
#[must_use]
pub fn resolve_strictness(value: Option<AstMatchStrictness>) -> MatchStrictness {
value.map_or(MatchStrictness::Smart, Into::into)
}
#[must_use]
pub fn supported_lang_list() -> String {
SupportLang::sorted_aliases().join(", ")
}
pub fn resolve_supported_lang(value: &str) -> Result<SupportLang> {
SupportLang::from_alias(value).ok_or_else(|| {
anyhow!("Unsupported language '{value}'. Supported: {}", supported_lang_list())
})
}
pub fn resolve_language(lang: Option<&str>, file_path: &Path) -> Result<SupportLang> {
if let Some(lang) = lang.map(str::trim).filter(|lang| !lang.is_empty()) {
return resolve_supported_lang(lang);
}
SupportLang::from_path(file_path).ok_or_else(|| {
anyhow!(
"Unable to infer language from file extension: {}. Specify `lang` explicitly.",
file_path.display()
)
})
}
#[must_use]
pub fn is_supported_file(file_path: &Path, explicit_lang: Option<&str>) -> bool {
if explicit_lang.is_some() {
return true;
}
resolve_language(None, file_path).is_ok()
}
pub fn compile_pattern(
pattern: &str,
selector: Option<&str>,
strictness: &MatchStrictness,
lang: SupportLang,
) -> Result<Pattern> {
let mut compiled = if let Some(selector) = selector.map(str::trim).filter(|s| !s.is_empty()) {
Pattern::contextual(pattern, selector, lang)
} else {
Pattern::try_new(pattern, lang)
}
.map_err(|err| anyhow!("Invalid pattern: {err}"))?;
compiled.strictness = strictness.clone();
Ok(compiled)
}
pub fn compile_search_patterns(
pattern: &str,
language: SupportLang,
) -> Result<Vec<Pattern>, PatternError> {
let mut compiled = vec![Pattern::try_new(pattern, language)?];
if language == SupportLang::Rust {
let trimmed = pattern.trim_end();
if let Some(contextual) = compile_rust_contextual_pattern(trimmed) {
compiled.push(contextual);
}
}
Ok(compiled)
}
pub fn compile_rewrite_rules(
rules: &[(String, String)],
language: SupportLang,
) -> Result<Vec<CompiledRewrite>, (usize, PatternError)> {
rules
.iter()
.enumerate()
.map(|(index, (pattern, out))| {
compile_search_patterns(pattern, language)
.map(|patterns| CompiledRewrite { out: out.clone(), patterns })
.map_err(|error| (index, error))
})
.collect()
}
#[must_use]
pub fn collect_matches(source: &str, language: SupportLang, patterns: &[Pattern]) -> Vec<AstMatch> {
let ast = language.ast_grep(source);
let mut matches = Vec::new();
for pattern in patterns {
for matched in ast.root().find_all(pattern.clone()) {
let start = matched.start_pos();
let end = matched.end_pos();
let range = matched.range();
matches.push(AstMatch {
line: start.line() + 1,
column: char_column(start, matched.get_node()) + 1,
end_line: end.line() + 1,
end_column: char_column(end, matched.get_node()) + 1,
byte_start: range.start,
byte_end: range.end,
text: matched.text().into_owned(),
});
}
}
matches
}
pub fn rewrite_source(
source: &str,
language: SupportLang,
ops: &[CompiledRewrite],
) -> Result<(String, u32), String> {
let mut ast = language.ast_grep(source);
let mut replacements = 0_u32;
for op in ops {
for pattern in &op.patterns {
let edits = ast.root().replace_all(pattern.clone(), op.out.as_str());
if edits.is_empty() {
continue;
}
replacements = replacements.saturating_add(edits.len() as u32);
let updated =
apply_edits(ast.root().text().as_ref(), &edits).map_err(|error| error.to_string())?;
ast = language.ast_grep(updated);
}
}
Ok((ast.root().text().into_owned(), replacements))
}
pub fn apply_edits(content: &str, edits: &[Edit<String>]) -> Result<String> {
let mut sorted: Vec<&Edit<String>> = edits.iter().collect();
sorted.sort_by_key(|edit| edit.position);
let mut prev_end = 0usize;
for edit in &sorted {
if edit.position < prev_end {
return Err(anyhow!(
"Overlapping replacements detected; refine pattern to avoid ambiguous edits"
));
}
prev_end = edit.position.saturating_add(edit.deleted_length);
}
let mut output = content.to_string();
for edit in sorted.into_iter().rev() {
let start = edit.position;
let end = edit.position.saturating_add(edit.deleted_length);
if end > output.len() || start > end {
return Err(anyhow!("Computed edit range is out of bounds"));
}
let replacement = String::from_utf8(edit.inserted_text.clone())
.map_err(|err| anyhow!("Replacement text is not valid UTF-8: {err}"))?;
output.replace_range(start..end, &replacement);
}
Ok(output)
}
pub fn collect_matched_files(
cwd: &Path,
patterns: &[String],
) -> Result<Vec<MatchedFile>, std::io::Error> {
let globset = build_globset(patterns)?;
let mut builder = WalkBuilder::new(cwd);
builder
.hidden(false)
.git_ignore(true)
.git_global(true)
.git_exclude(true);
let mut files = Vec::new();
for entry in builder.build() {
let entry = match entry {
Ok(entry) => entry,
Err(error) => return Err(std::io::Error::other(error)),
};
if !entry.file_type().is_some_and(|ft| ft.is_file()) {
continue;
}
let absolute_path = entry.into_path();
let relative_path = absolute_path
.strip_prefix(cwd)
.unwrap_or(&absolute_path)
.to_string_lossy()
.replace('\\', "/");
if globset.is_match(&relative_path)
|| patterns.iter().any(|pattern| pattern == &relative_path)
{
files.push(MatchedFile { absolute_path, relative_path });
}
}
files.sort_unstable_by(|left, right| left.relative_path.cmp(&right.relative_path));
Ok(files)
}
fn build_globset(patterns: &[String]) -> Result<GlobSet, std::io::Error> {
let mut builder = GlobSetBuilder::new();
for pattern in patterns {
if has_glob_syntax(pattern) {
let glob = Glob::new(pattern).map_err(|error| {
std::io::Error::new(
std::io::ErrorKind::InvalidInput,
format!("invalid glob `{pattern}`: {error}"),
)
})?;
builder.add(glob);
}
}
builder.build().map_err(std::io::Error::other)
}
#[must_use]
pub fn has_glob_syntax(pattern: &str) -> bool {
pattern.contains('*') || pattern.contains('?') || pattern.contains('[')
}
fn char_column(position: Position, node: &ast_grep_core::Node<'_, StrDoc<SupportLang>>) -> usize {
position.column(node)
}
fn compile_rust_contextual_pattern(pattern: &str) -> Option<Pattern> {
let language = SupportLang::Rust;
let context = format!("fn __rwp_wrapper() {{ {pattern}; }}");
let ast = language.ast_grep(&context);
let selector = ast.root().find("expression_statement")?;
Pattern::contextual(pattern, selector.kind().as_ref(), language).ok()
}
#[cfg(test)]
mod tests {
use ast_grep_core::source::Edit;
use super::{SupportLang, apply_edits, compile_search_patterns};
#[test]
fn compile_search_patterns_compiles_rust_patterns() {
let patterns = compile_search_patterns("foo($$$ARGS)", SupportLang::Rust)
.expect("rust pattern should compile");
assert!(!patterns.is_empty());
}
#[test]
fn apply_edits_rejects_overlaps() {
let source = "abcdef";
let edits = vec![
Edit::<String> { position: 1, deleted_length: 3, inserted_text: b"x".to_vec() },
Edit::<String> { position: 2, deleted_length: 1, inserted_text: b"y".to_vec() },
];
assert!(apply_edits(source, &edits).is_err());
}
}
File diff suppressed because it is too large Load Diff
+2 -60
View File
@@ -18,13 +18,10 @@ tokio = { version = "1", features = ["full"] }
tokio-util = { version = "0.7", features = ["full"] }
napi = { version = "3", features = ["napi10", "tokio_rt", "tokio_time"] }
napi-derive = "3"
brush-core = { version = "0.5.0", path = "../brush-core-vendored" }
brush-builtins = { version = "0.2.0", path = "../brush-builtins-vendored" }
brush-parser = "0.3"
pi-shell = { path = "../pi-shell" }
parking_lot = "0.12.5"
dashmap = "6.1"
clap = { version = "4", features = ["derive"] }
os_pipe = "1"
portable-pty = "0.9"
grep-regex = "0.1"
grep-searcher = "0.1"
@@ -35,62 +32,7 @@ rayon = "1.12"
ast-grep-core = { version = "0.39", default-features = false, features = [
"tree-sitter",
] }
tree-sitter = "0.25"
tree-sitter-astro = { version = "0.1.1", package = "tree-sitter-astro-next" }
tree-sitter-bash = "0.25"
tree-sitter-c = "0.24"
tree-sitter-clojure = "0.1"
tree-sitter-cmake = "0.7.1"
tree-sitter-c-sharp = "0.23"
tree-sitter-cpp = "0.23"
tree-sitter-dart = "0.2"
tree-sitter-css = "0.25"
tree-sitter-diff = "0.1"
tree-sitter-dockerfile = { version = "0.2.0", package = "tree-sitter-dockerfile-updated" }
tree-sitter-elixir = "0.3"
tree-sitter-erlang = "0.16.0"
tree-sitter-go = "0.25"
tree-sitter-graphql = "0.1.0"
tree-sitter-haskell = "0.23"
tree-sitter-hcl = "1.1"
tree-sitter-html = "0.23"
tree-sitter-ini = "1.4.0"
tree-sitter-java = "0.23"
tree-sitter-javascript = "0.25"
tree-sitter-json = "0.24"
tree-sitter-just = "0.2.0"
tree-sitter-julia = "0.23"
tree-sitter-kotlin = { version = "0.4", package = "tree-sitter-kotlin-sg" }
tree-sitter-lua = "0.5"
tree-sitter-make = "1.1"
tree-sitter-md = "0.5"
tree-sitter-nix = "0.3"
tree-sitter-objc = "3.0"
tree-sitter-ocaml = "0.24.2"
tree-sitter-odin = "1.3"
tree-sitter-perl = { version = "0.1.0", package = "tree-sitter-perl-next" }
tree-sitter-php = "0.24"
tree-sitter-powershell = "0.26.4"
tree-sitter-proto = "0.4.0"
tree-sitter-python = "0.25"
tree-sitter-r = "1.2.0"
tree-sitter-regex = "0.25"
tree-sitter-ruby = "0.23"
tree-sitter-rust = "0.24"
tree-sitter-scala = "0.26"
tree-sitter-solidity = "1.2"
tree-sitter-sql = { version = "0.3.11", package = "tree-sitter-sequel" }
tree-sitter-starlark = "1.3"
tree-sitter-svelte = { version = "0.1.1", package = "tree-sitter-svelte-next" }
tree-sitter-swift = "0.7"
tree-sitter-toml-ng = "0.7"
tree-sitter-tlaplus = "1.5"
tree-sitter-typescript = "0.23"
tree-sitter-verilog = "1.0"
tree-sitter-vue = { version = "0.1.0", package = "tree-sitter-vue-next" }
tree-sitter-xml = "0.7"
tree-sitter-yaml = "0.7"
tree-sitter-zig = "1.1"
pi-ast = { path = "../pi-ast" }
inferno = { version = "0.12", default-features = false }
image = { version = "0.25", default-features = false, features = [
"png",
+12 -61
View File
@@ -5,13 +5,15 @@ use std::{
path::{Path, PathBuf},
};
use ast_grep_core::{
Language, MatchStrictness, matcher::Pattern, source::Edit, tree_sitter::LanguageExt,
};
use ast_grep_core::{MatchStrictness, matcher::Pattern, source::Edit, tree_sitter::LanguageExt};
use napi::bindgen_prelude::*;
use napi_derive::napi;
use pi_ast::{
SupportLang,
ops::{self as shared_ops},
};
use crate::{fs_cache, glob_util, language::SupportLang, task};
use crate::{fs_cache, glob_util, task};
const DEFAULT_FIND_LIMIT: u32 = 50;
@@ -227,29 +229,12 @@ fn to_u32(value: usize) -> u32 {
value.min(u32::MAX as usize) as u32
}
fn supported_lang_list() -> String {
SupportLang::sorted_aliases().join(", ")
}
fn resolve_supported_lang(value: &str) -> Result<SupportLang> {
SupportLang::from_alias(value).ok_or_else(|| {
Error::from_reason(format!(
"Unsupported language '{value}'. Supported: {}",
supported_lang_list()
))
})
shared_ops::resolve_supported_lang(value).map_err(|err| Error::from_reason(err.to_string()))
}
fn resolve_language(lang: Option<&str>, file_path: &Path) -> Result<SupportLang> {
if let Some(lang) = lang.map(str::trim).filter(|lang| !lang.is_empty()) {
return resolve_supported_lang(lang);
}
SupportLang::from_path(file_path).ok_or_else(|| {
Error::from_reason(format!(
"Unable to infer language from file extension: {}. Specify `lang` explicitly.",
file_path.display()
))
})
shared_ops::resolve_language(lang, file_path).map_err(|err| Error::from_reason(err.to_string()))
}
/// Returns true if the file's extension resolves to a supported language.
@@ -257,10 +242,7 @@ fn resolve_language(lang: Option<&str>, file_path: &Path) -> Result<SupportLang>
/// (the user chose to treat them as that language). When `lang` is None,
/// only files with recognizable code extensions are included.
fn is_supported_file(file_path: &Path, explicit_lang: Option<&str>) -> bool {
if explicit_lang.is_some() {
return true;
}
resolve_language(None, file_path).is_ok()
shared_ops::is_supported_file(file_path, explicit_lang)
}
fn infer_single_replace_lang(
@@ -414,43 +396,12 @@ fn compile_pattern(
strictness: &MatchStrictness,
lang: SupportLang,
) -> Result<Pattern> {
let mut compiled = if let Some(selector) = selector.map(str::trim).filter(|s| !s.is_empty()) {
Pattern::contextual(pattern, selector, lang)
} else {
Pattern::try_new(pattern, lang)
}
.map_err(|err| Error::from_reason(format!("Invalid pattern: {err}")))?;
compiled.strictness = strictness.clone();
Ok(compiled)
shared_ops::compile_pattern(pattern, selector, strictness, lang)
.map_err(|err| Error::from_reason(err.to_string()))
}
fn apply_edits(content: &str, edits: &[Edit<String>]) -> Result<String> {
let mut sorted: Vec<&Edit<String>> = edits.iter().collect();
sorted.sort_by_key(|edit| edit.position);
let mut prev_end = 0usize;
for edit in &sorted {
if edit.position < prev_end {
return Err(Error::from_reason(
"Overlapping replacements detected; refine pattern to avoid ambiguous edits"
.to_string(),
));
}
prev_end = edit.position.saturating_add(edit.deleted_length);
}
let mut output = content.to_string();
for edit in sorted.into_iter().rev() {
let start = edit.position;
let end = edit.position.saturating_add(edit.deleted_length);
if end > output.len() || start > end {
return Err(Error::from_reason("Computed edit range is out of bounds".to_string()));
}
let replacement = String::from_utf8(edit.inserted_text.clone()).map_err(|err| {
Error::from_reason(format!("Replacement text is not valid UTF-8: {err}"))
})?;
output.replace_range(start..end, &replacement);
}
Ok(output)
shared_ops::apply_edits(content, edits).map_err(|err| Error::from_reason(err.to_string()))
}
fn normalize_pattern_list(patterns: Option<Vec<String>>) -> Result<Vec<String>> {
+1 -1
View File
@@ -37,7 +37,7 @@ pub mod highlight;
pub mod html;
pub mod image;
pub mod keys;
pub mod language;
pub use pi_ast::language;
pub mod power;
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+38 -128
View File
@@ -27,17 +27,10 @@
//! }
//! ```
use std::{
future::Future,
sync::{
Arc, Weak,
atomic::{AtomicU8, Ordering},
},
time::{Duration, Instant},
};
use std::future::Future;
use napi::{Env, Error, Result, Task, bindgen_prelude::*};
use tokio::sync::Notify;
use pi_shell::cancel as core_cancel;
use crate::prof::profile_region;
@@ -47,55 +40,31 @@ use crate::prof::profile_region;
/// Reason for task abortion.
#[derive(Debug, Clone, Copy)]
#[repr(u8)]
pub enum AbortReason {
Unknown = 1,
Timeout = 2,
Signal = 3,
User = 4,
Unknown,
Timeout,
Signal,
User,
}
impl TryFrom<u8> for AbortReason {
type Error = ();
fn try_from(value: u8) -> std::result::Result<Self, ()> {
impl From<core_cancel::AbortReason> for AbortReason {
fn from(value: core_cancel::AbortReason) -> Self {
match value {
0 => Err(()),
2 => Ok(Self::Timeout),
3 => Ok(Self::Signal),
4 => Ok(Self::User),
_ => Ok(Self::Unknown),
core_cancel::AbortReason::Unknown => Self::Unknown,
core_cancel::AbortReason::Timeout => Self::Timeout,
core_cancel::AbortReason::Signal => Self::Signal,
core_cancel::AbortReason::User => Self::User,
}
}
}
#[derive(Default)]
struct Flag {
reason: AtomicU8,
notifier: Notify,
}
impl Flag {
fn cause(&self) -> Option<AbortReason> {
self.reason.load(Ordering::Relaxed).try_into().ok()
}
async fn wait(&self) -> AbortReason {
if let Some(reason) = self.cause() {
return reason;
}
let notifier = self.notifier.notified();
if let Some(reason) = self.cause() {
return reason;
}
notifier.await;
self.cause().unwrap_or(AbortReason::Unknown)
}
fn abort(&self, reason: AbortReason) {
let old = self.reason.swap(reason as u8, Ordering::SeqCst);
if old == 0 {
self.notifier.notify_waiters();
impl From<AbortReason> for core_cancel::AbortReason {
fn from(value: AbortReason) -> Self {
match value {
AbortReason::Unknown => Self::Unknown,
AbortReason::Timeout => Self::Timeout,
AbortReason::Signal => Self::Signal,
AbortReason::User => Self::User,
}
}
}
@@ -106,8 +75,7 @@ impl Flag {
/// cancellation requests from timeouts or abort signals.
#[derive(Clone, Default)]
pub struct CancelToken {
deadline: Option<Instant>,
flag: Option<Arc<Flag>>,
core: core_cancel::CancelToken,
}
impl From<()> for CancelToken {
@@ -119,21 +87,10 @@ impl From<()> for CancelToken {
impl CancelToken {
/// Create a new cancel token from optional timeout and abort signal.
pub fn new(timeout_ms: Option<u32>, signal: Option<Unknown>) -> Self {
let mut result = Self::default();
if let Some(signal) = signal.and_then(|s| AbortSignal::from_unknown(s).ok()) {
let flag = Arc::new(Flag::default());
signal.on_abort({
let weak = Arc::downgrade(&flag);
move || {
if let Some(flag) = weak.upgrade() {
flag.abort(AbortReason::Signal);
}
}
});
result.flag = Some(flag);
}
if let Some(timeout_ms) = timeout_ms {
result.deadline = Some(Instant::now() + Duration::from_millis(timeout_ms as u64));
let mut result = Self { core: core_cancel::CancelToken::new(timeout_ms) };
if let Some(signal) = signal.and_then(|value| AbortSignal::from_unknown(value).ok()) {
let abort_token = result.emplace_abort_token();
signal.on_abort(move || abort_token.abort(AbortReason::Signal));
}
result
}
@@ -143,92 +100,45 @@ impl CancelToken {
/// Returns `Ok(())` if work should continue, or an error if cancelled.
/// Call this periodically in long-running loops.
pub fn heartbeat(&self) -> Result<()> {
if let Some(flag) = &self.flag
&& let Some(reason) = flag.cause()
{
return Err(Error::from_reason(format!("Aborted: {reason:?}")));
}
if let Some(deadline) = self.deadline
&& deadline < Instant::now()
{
return Err(Error::from_reason("Aborted: Timeout"));
}
Ok(())
self
.core
.heartbeat()
.map_err(|err| Error::from_reason(err.to_string()))
}
/// Wait for the cancel token to be aborted.
pub async fn wait(&self) -> AbortReason {
let flag = self.flag.as_ref();
if let Some(flag) = flag.and_then(|f| f.cause()) {
return flag;
}
let fflag = async {
let Some(flag) = self.flag.as_ref() else {
return std::future::pending().await;
};
flag.wait().await
};
let fttl = async {
let Some(ttl) = self.deadline else {
return std::future::pending().await;
};
tokio::time::sleep_until(ttl.into()).await;
AbortReason::Timeout
};
let fuser = async {
if tokio::signal::ctrl_c().await.is_err() {
return std::future::pending().await;
}
AbortReason::User
};
tokio::select! {
reason = fflag => reason,
reason = fttl => reason,
reason = fuser => reason,
}
self.core.wait().await.into()
}
/// Get an abort token for external cancellation.
pub fn abort_token(&self) -> AbortToken {
AbortToken(self.flag.as_ref().map(Arc::downgrade))
AbortToken(self.core.abort_token())
}
/// Emplaces a cancel token if there is none, returns the abort token.
pub fn emplace_abort_token(&mut self) -> AbortToken {
AbortToken(Some(Arc::downgrade(self.flag.get_or_insert_default())))
AbortToken(self.core.emplace_abort_token())
}
/// Check if already aborted (non-blocking).
pub fn aborted(&self) -> bool {
if let Some(flag) = &self.flag
&& flag.cause().is_some()
{
return true;
}
if let Some(deadline) = self.deadline
&& deadline < Instant::now()
{
return true;
}
false
self.core.aborted()
}
pub fn into_core(self) -> core_cancel::CancelToken {
self.core
}
}
/// Token for requesting cancellation from outside the task.
#[derive(Clone, Default)]
pub struct AbortToken(Option<Weak<Flag>>);
pub struct AbortToken(core_cancel::AbortToken);
impl AbortToken {
/// Request cancellation of the associated task.
pub fn abort(&self, reason: AbortReason) {
if let Some(flag) = &self.0
&& let Some(flag) = flag.upgrade()
{
flag.abort(reason);
}
self.0.abort(reason.into());
}
}
+37
View File
@@ -0,0 +1,37 @@
[package]
name = "pi-shell"
version.workspace = true
edition.workspace = true
license.workspace = true
authors.workspace = true
repository.workspace = true
[lints]
workspace = true
[dependencies]
anyhow = "1.0"
tokio = { version = "1", features = ["full"] }
tokio-util = { version = "0.7", features = ["full"] }
brush-core = { version = "0.5.0", path = "../brush-core-vendored" }
brush-builtins = { version = "0.2.0", path = "../brush-builtins-vendored" }
brush-parser = "0.3"
clap = { version = "4", features = ["derive"] }
os_pipe = "1"
serde = { version = "1.0", features = ["derive"] }
serde_json = { version = "1.0", features = ["preserve_order"] }
toml = "1.1"
regex = "1"
xxhash-rust = { version = "0.8", features = ["xxh64"] }
[target.'cfg(unix)'.dependencies]
libc = "0.2"
[target.'cfg(windows)'.dependencies]
winreg = "0.56"
windows-sys = { version = "0.61", features = [
"Win32_Foundation",
"Win32_Storage_ProjectedFileSystem",
"Win32_System_Com",
"Win32_System_LibraryLoader",
] }
+62
View File
@@ -0,0 +1,62 @@
use std::{
env,
fmt::Write as _,
fs,
path::{Path, PathBuf},
};
fn main() {
generate_minimizer_builtin_filters();
}
fn generate_minimizer_builtin_filters() {
let manifest_dir = env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR should be set");
let defs_dir = Path::new(&manifest_dir)
.join("src")
.join("minimizer")
.join("defs");
let out_dir = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR should be set"));
let output_path = out_dir.join("builtin_filters.toml");
println!("cargo:rerun-if-changed={}", defs_dir.display());
let mut concatenated =
String::from("# Auto-generated by build.rs -- do not edit.\nschema_version = 1\n\n");
let mut entries: Vec<PathBuf> = Vec::new();
if let Ok(read_dir) = fs::read_dir(&defs_dir) {
for entry in read_dir.flatten() {
let path = entry.path();
if path.extension().and_then(|extension| extension.to_str()) == Some("toml") {
entries.push(path);
}
}
}
entries.sort();
for path in entries {
println!("cargo:rerun-if-changed={}", path.display());
match fs::read_to_string(&path) {
Ok(body) => {
let filename = path
.file_name()
.and_then(|name| name.to_str())
.unwrap_or("unknown");
writeln!(concatenated, "# --- {filename} ---").expect("write to String");
for line in body.lines() {
let trimmed = line.trim_start();
if trimmed.starts_with("schema_version") {
continue;
}
concatenated.push_str(line);
concatenated.push('\n');
}
concatenated.push('\n');
},
Err(error) => panic!("failed to read filter definition {}: {error}", path.display()),
}
}
fs::write(&output_path, concatenated)
.unwrap_or_else(|error| panic!("failed to write {}: {error}", output_path.display()));
}
+161
View File
@@ -0,0 +1,161 @@
use std::{
sync::{
Arc, Weak,
atomic::{AtomicU8, Ordering},
},
time::{Duration, Instant},
};
use anyhow::{Error, Result};
use tokio::sync::Notify;
#[derive(Debug, Clone, Copy)]
#[repr(u8)]
pub enum AbortReason {
Unknown = 1,
Timeout = 2,
Signal = 3,
User = 4,
}
impl TryFrom<u8> for AbortReason {
type Error = ();
fn try_from(value: u8) -> std::result::Result<Self, ()> {
match value {
0 => Err(()),
2 => Ok(Self::Timeout),
3 => Ok(Self::Signal),
4 => Ok(Self::User),
_ => Ok(Self::Unknown),
}
}
}
#[derive(Default)]
struct Flag {
reason: AtomicU8,
notifier: Notify,
}
impl Flag {
fn cause(&self) -> Option<AbortReason> {
self.reason.load(Ordering::Relaxed).try_into().ok()
}
async fn wait(&self) -> AbortReason {
if let Some(reason) = self.cause() {
return reason;
}
let notifier = self.notifier.notified();
if let Some(reason) = self.cause() {
return reason;
}
notifier.await;
self.cause().unwrap_or(AbortReason::Unknown)
}
fn abort(&self, reason: AbortReason) {
let old = self.reason.swap(reason as u8, Ordering::SeqCst);
if old == 0 {
self.notifier.notify_waiters();
}
}
}
#[derive(Clone, Default)]
pub struct CancelToken {
deadline: Option<Instant>,
flag: Option<Arc<Flag>>,
}
impl From<()> for CancelToken {
fn from((): ()) -> Self {
Self::default()
}
}
impl CancelToken {
pub fn new(timeout_ms: Option<u32>) -> Self {
Self::with_timeout(timeout_ms.map(|ms| Duration::from_millis(u64::from(ms))))
}
pub fn with_timeout(timeout: Option<Duration>) -> Self {
Self { deadline: timeout.map(|duration| Instant::now() + duration), flag: None }
}
pub fn heartbeat(&self) -> Result<()> {
if let Some(flag) = &self.flag
&& let Some(reason) = flag.cause()
{
return Err(Error::msg(format!("Aborted: {reason:?}")));
}
if let Some(deadline) = self.deadline
&& deadline < Instant::now()
{
return Err(Error::msg("Aborted: Timeout"));
}
Ok(())
}
pub async fn wait(&self) -> AbortReason {
if let Some(flag) = self.flag.as_ref().and_then(|flag| flag.cause()) {
return flag;
}
let by_flag = async {
let Some(flag) = self.flag.as_ref() else {
return std::future::pending().await;
};
flag.wait().await
};
let by_timeout = async {
let Some(deadline) = self.deadline else {
return std::future::pending().await;
};
tokio::time::sleep_until(deadline.into()).await;
AbortReason::Timeout
};
tokio::select! {
reason = by_flag => reason,
reason = by_timeout => reason,
}
}
pub fn abort_token(&self) -> AbortToken {
AbortToken(self.flag.as_ref().map(Arc::downgrade))
}
pub fn emplace_abort_token(&mut self) -> AbortToken {
AbortToken(Some(Arc::downgrade(self.flag.get_or_insert_default())))
}
pub fn aborted(&self) -> bool {
if let Some(flag) = &self.flag
&& flag.cause().is_some()
{
return true;
}
if let Some(deadline) = self.deadline
&& deadline < Instant::now()
{
return true;
}
false
}
}
#[derive(Clone, Default)]
pub struct AbortToken(Option<Weak<Flag>>);
impl AbortToken {
pub fn abort(&self, reason: AbortReason) {
if let Some(flag) = &self.0
&& let Some(flag) = flag.upgrade()
{
flag.abort(reason);
}
}
}
+12
View File
@@ -0,0 +1,12 @@
pub mod cancel;
pub mod minimizer;
pub mod process;
pub mod shell;
#[cfg(windows)]
pub mod windows;
pub use brush_core::commands::{ChildSessionAction, child_session_action};
pub use shell::{
MinimizerResult, Shell, ShellExecuteOptions, ShellExecuteResult, ShellOptions, ShellRunOptions,
ShellRunResult, execute_shell,
};
@@ -12,15 +12,13 @@ use std::{
sync::Arc,
};
use napi_derive::napi;
use serde::Deserialize;
use crate::shell::minimizer::pipeline::{self, PipelineRegistry, SUPPORTED_SCHEMA_VERSION};
use crate::minimizer::pipeline::{self, PipelineRegistry, SUPPORTED_SCHEMA_VERSION};
const DEFAULT_MAX_CAPTURE_BYTES: u32 = 4 * 1024 * 1024;
/// N-API opt-in handle for the minimizer.
#[napi(object)]
#[derive(Debug, Clone, Default)]
pub struct MinimizerOptions {
/// Master switch. Absent / false = disabled.
@@ -8,7 +8,7 @@ use std::{
},
};
use crate::shell::minimizer::{
use crate::minimizer::{
MinimizerConfig, MinimizerCtx, MinimizerOutput, detect, filters,
pipeline::{self, CompiledPipeline, PipelineRegistry},
plan,
@@ -420,7 +420,7 @@ mod tests {
#[cfg(test)]
mod pipeline_integration_tests {
use super::*;
use crate::shell::minimizer::MinimizerOptions;
use crate::minimizer::MinimizerOptions;
#[test]
fn builtin_filters_parse_and_pass_inline_tests() {
@@ -1,7 +1,7 @@
//! Bun package-manager, test-runner, and tool output filters.
use super::{cpp, generic, js_tools, lint, node_tests, pkg};
use crate::shell::minimizer::{MinimizerCtx, MinimizerOutput, primitives};
use crate::minimizer::{MinimizerCtx, MinimizerOutput, primitives};
const BUN_PACKAGE_SUBCOMMANDS: &[&str] = &[
"install", "i", "add", "update", "up", "upgrade", "remove", "rm", "outdated", "pm", "audit",
@@ -142,7 +142,7 @@ fn is_important(line: &str) -> bool {
#[cfg(test)]
mod tests {
use super::*;
use crate::shell::minimizer::MinimizerConfig;
use crate::minimizer::MinimizerConfig;
fn ctx<'a>(
program: &'a str,
@@ -1,6 +1,6 @@
//! Cargo build/test output filters.
use crate::shell::minimizer::{MinimizerCtx, MinimizerOutput, primitives};
use crate::minimizer::{MinimizerCtx, MinimizerOutput, primitives};
pub fn supports(subcommand: Option<&str>) -> bool {
matches!(
@@ -293,7 +293,7 @@ fn is_general_cargo_noise(line: &str) -> bool {
#[cfg(test)]
mod tests {
use super::*;
use crate::shell::minimizer::MinimizerConfig;
use crate::minimizer::MinimizerConfig;
#[test]
fn strips_compiling_noise() {
@@ -1,6 +1,6 @@
//! Cloud and data command output filters.
use crate::shell::minimizer::{MinimizerCtx, MinimizerOutput, primitives};
use crate::minimizer::{MinimizerCtx, MinimizerOutput, primitives};
const MAX_PSQL_ROWS: usize = 30;
const MAX_LINE_CHARS: usize = 500;
@@ -401,7 +401,7 @@ fn join_lines(lines: Vec<String>) -> String {
#[cfg(test)]
mod tests {
use super::*;
use crate::shell::minimizer::MinimizerConfig;
use crate::minimizer::MinimizerConfig;
fn ctx<'a>(program: &'a str, cfg: &'a MinimizerConfig) -> MinimizerCtx<'a> {
MinimizerCtx { program, subcommand: None, command: program, config: cfg }
@@ -2,7 +2,7 @@
use std::path::Path;
use crate::shell::minimizer::{MinimizerCtx, MinimizerOutput, primitives};
use crate::minimizer::{MinimizerCtx, MinimizerOutput, primitives};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum CppTool {
@@ -249,7 +249,7 @@ fn push_line(out: &mut String, line: &str) {
#[cfg(test)]
mod tests {
use super::*;
use crate::shell::minimizer::MinimizerConfig;
use crate::minimizer::MinimizerConfig;
fn ctx<'a>(
program: &'a str,
@@ -1,6 +1,6 @@
//! Container and cloud command output filters.
use crate::shell::minimizer::{MinimizerCtx, MinimizerOutput, primitives};
use crate::minimizer::{MinimizerCtx, MinimizerOutput, primitives};
pub fn supports(subcommand: Option<&str>) -> bool {
matches!(
@@ -168,7 +168,7 @@ fn head_tail_dedup(input: &str) -> String {
#[cfg(test)]
mod tests {
use super::*;
use crate::shell::minimizer::MinimizerConfig;
use crate::minimizer::MinimizerConfig;
#[test]
fn dedups_repeated_log_lines_before_truncation() {
@@ -1,6 +1,6 @@
//! .NET CLI output filters.
use crate::shell::minimizer::{MinimizerCtx, MinimizerOutput, primitives};
use crate::minimizer::{MinimizerCtx, MinimizerOutput, primitives};
pub fn supports(program: &str, subcommand: Option<&str>) -> bool {
program == "dotnet" && matches!(subcommand, Some("build" | "test" | "restore" | "format"))
@@ -326,7 +326,7 @@ fn contains_diagnostic_signal(lower: &str) -> bool {
#[cfg(test)]
mod tests {
use super::*;
use crate::shell::minimizer::MinimizerConfig;
use crate::minimizer::MinimizerConfig;
#[test]
fn keeps_dotnet_build_diagnostic_and_strips_restore_noise() {
@@ -1,6 +1,6 @@
//! Generic fallback transforms.
use crate::shell::minimizer::{MinimizerCtx, MinimizerOutput, primitives};
use crate::minimizer::{MinimizerCtx, MinimizerOutput, primitives};
pub fn filter(_ctx: &MinimizerCtx<'_>, input: &str, _exit_code: i32) -> MinimizerOutput {
let stripped = primitives::strip_ansi(input);
@@ -1,6 +1,6 @@
//! GitHub CLI output filters.
use crate::shell::minimizer::{MinimizerCtx, MinimizerOutput, primitives};
use crate::minimizer::{MinimizerCtx, MinimizerOutput, primitives};
pub fn supports(subcommand: Option<&str>) -> bool {
matches!(
@@ -157,7 +157,7 @@ fn head_tail_dedup(input: &str) -> String {
#[cfg(test)]
mod tests {
use super::*;
use crate::shell::minimizer::MinimizerConfig;
use crate::minimizer::MinimizerConfig;
fn test_ctx<'a>(
subcommand: Option<&'a str>,
@@ -1,6 +1,6 @@
//! Git output filters.
use crate::shell::minimizer::{MinimizerCtx, MinimizerOutput, primitives};
use crate::minimizer::{MinimizerCtx, MinimizerOutput, primitives};
pub fn supports(subcommand: Option<&str>) -> bool {
matches!(
@@ -370,7 +370,7 @@ fn condense_noisy_output(input: &str) -> String {
#[cfg(test)]
mod tests {
use super::*;
use crate::shell::minimizer::MinimizerConfig;
use crate::minimizer::MinimizerConfig;
fn test_ctx<'a>(
subcommand: Option<&'a str>,
@@ -1,6 +1,6 @@
//! Go toolchain output filters.
use crate::shell::minimizer::{MinimizerCtx, MinimizerOutput, primitives};
use crate::minimizer::{MinimizerCtx, MinimizerOutput, primitives};
pub fn supports(program: &str, subcommand: Option<&str>) -> bool {
match program {
@@ -326,7 +326,7 @@ fn is_golangci_noise(line: &str) -> bool {
#[cfg(test)]
mod tests {
use super::*;
use crate::shell::minimizer::MinimizerConfig;
use crate::minimizer::MinimizerConfig;
#[test]
fn keeps_go_test_failure_from_json_lines() {
@@ -1,7 +1,7 @@
//! Graphite (`gt`) output filters.
use super::git;
use crate::shell::minimizer::{MinimizerCtx, MinimizerOutput, primitives};
use crate::minimizer::{MinimizerCtx, MinimizerOutput, primitives};
const GT_SUBCOMMANDS: &[&str] = &[
"log", "submit", "sync", "restack", "create", "branch", "diff", "show", "add", "push", "pull",
@@ -177,7 +177,7 @@ fn is_low_value_status(line: &str) -> bool {
#[cfg(test)]
mod tests {
use super::*;
use crate::shell::minimizer::MinimizerConfig;
use crate::minimizer::MinimizerConfig;
fn test_ctx<'a>(subcommand: Option<&'a str>, config: &'a MinimizerConfig) -> MinimizerCtx<'a> {
test_ctx_with_command(subcommand, "gt", config)

Some files were not shown because too many files have changed in this diff Show More