smashimo/oh-my-pi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(natives): migrated native bindings from WASM to Node.js N-API with ripgrep-backed grep module

Browse Source 
- Migrated native bindings from WebAssembly (wasm-bindgen) to Node.js N-API architecture for improved performance and platform support.
- Replaced worker pool-based async architecture with synchronous native addon calls, eliminating thread overhead and simplifying the API.
- Added new grep module with ripgrep-backed filesystem search supporting parallel processing, glob filtering, and context line extraction.
- Updated build system to compile Rust crates to platform-specific .node binaries instead of WASM modules.
- Removed WASI implementation and worker pool infrastructure no longer needed with N-API native bindings.
- Updated CI/CD pipeline to build native addons for multiple platforms (Linux x64/arm64, macOS x64/arm64, Windows x64) and include them in releases.
This commit is contained in:
can1357
2026-01-31 18:02:56 +01:00
parent e67e02947f
commit eabbde0c8a
41 changed files with 1858 additions and 5159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci.yml
+44 -19
View File
@@ -33,27 +33,44 @@ jobs:
- name: Check
run: cargo check --workspace
wasm:
runs-on: ubuntu-latest
native:
strategy:
fail-fast: false
matrix:
include:
- os: ubuntu-latest
platform: linux
arch: x64
- os: ubuntu-24.04-arm64
platform: linux
arch: arm64
- os: macos-13
platform: darwin
arch: x64
- os: macos-14
platform: darwin
arch: arm64
- os: windows-latest
platform: win32
arch: x64
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
with:
lfs: true
- uses: dtolnay/rust-toolchain@nightly
with:
targets: wasm32-unknown-unknown
- uses: Swatinem/rust-cache@v2
- name: Install wasm-pack
run: cargo install wasm-pack
- name: Install binaryen (wasm-opt)
run: |
sudo apt-get update
sudo apt-get install -y binaryen
- uses: oven-sh/setup-bun@v2
with:
bun-version: "1.3"
- name: Build WASM
run: bun --cwd=packages/natives run build:wasm
- name: Build native addon
run: bun --cwd=packages/natives run build:native
- name: Upload native addon
uses: actions/upload-artifact@v4
with:
name: pi-natives-${{ matrix.platform }}-${{ matrix.arch }}
path: packages/natives/native/pi_natives.${{ matrix.platform }}-${{ matrix.arch }}.node
if-no-files-found: error
test:
runs-on: ubuntu-latest
@@ -76,6 +93,7 @@ jobs:
sudo ln -s $(which fdfind) /usr/local/bin/fd
sudo ln -sf /usr/bin/convert /usr/local/bin/magick
- run: bun install --frozen-lockfile
- run: bun --cwd=packages/natives run build:native
- run: bun run check
- run: bun run test
- name: CLI smoke test
@@ -86,7 +104,7 @@ jobs:
release:
if: startsWith(github.ref, 'refs/tags/v')
needs: [rust, wasm, test]
needs: [rust, native, test]
runs-on: ubuntu-latest
permissions:
contents: write
@@ -108,18 +126,25 @@ jobs:
path: ~/.bun/install/cache
key: bun-${{ runner.os }}-${{ hashFiles('**/bun.lock') }}
- run: bun install --frozen-lockfile
- name: Download native addons
uses: actions/download-artifact@v4
with:
pattern: pi-natives-*
path: packages/natives/native
merge-multiple: true
- name: Build binaries
run: |
mkdir -p packages/coding-agent/binaries
ENTRYPOINT="./packages/coding-agent/src/cli.ts"
WORKERS="./packages/natives/src/grep/worker.ts ./packages/natives/src/html/worker.ts ./packages/natives/src/image/worker.ts"
DEFINES="--define OMP_COMPILED=true --root ."
bun build --compile $DEFINES --target=bun-darwin-arm64 $ENTRYPOINT $WORKERS --outfile packages/coding-agent/binaries/omp-darwin-arm64
bun build --compile $DEFINES --target=bun-darwin-x64 $ENTRYPOINT $WORKERS --outfile packages/coding-agent/binaries/omp-darwin-x64
bun build --compile $DEFINES --target=bun-linux-x64 $ENTRYPOINT $WORKERS --outfile packages/coding-agent/binaries/omp-linux-x64
bun build --compile $DEFINES --target=bun-linux-arm64 $ENTRYPOINT $WORKERS --outfile packages/coding-agent/binaries/omp-linux-arm64
bun build --compile $DEFINES --target=bun-windows-x64 $ENTRYPOINT $WORKERS --outfile packages/coding-agent/binaries/omp-windows-x64.exe
bun build --compile $DEFINES --target=bun-darwin-arm64 $ENTRYPOINT --outfile packages/coding-agent/binaries/omp-darwin-arm64
bun build --compile $DEFINES --target=bun-darwin-x64 $ENTRYPOINT --outfile packages/coding-agent/binaries/omp-darwin-x64
bun build --compile $DEFINES --target=bun-linux-x64 $ENTRYPOINT --outfile packages/coding-agent/binaries/omp-linux-x64
bun build --compile $DEFINES --target=bun-linux-arm64 $ENTRYPOINT --outfile packages/coding-agent/binaries/omp-linux-arm64
bun build --compile $DEFINES --target=bun-windows-x64 $ENTRYPOINT --outfile packages/coding-agent/binaries/omp-windows-x64.exe
- name: Stage native addons for release
run: cp packages/natives/native/*.node packages/coding-agent/binaries/
- name: Create GitHub Release
uses: softprops/action-gh-release@v2
with:
.gitignore
+2
View File
@@ -46,3 +46,5 @@ __pycache__/
*.cpuprofile
target/
*.node
Cargo.lock
Generated
+189 -88
View File
@@ -85,12 +85,6 @@ dependencies = [
"serde",
]
[[package]]
name = "bumpalo"
version = "3.19.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510"
[[package]]
name = "bytemuck"
version = "1.25.0"
@@ -116,13 +110,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
[[package]]
name = "console_error_panic_hook"
version = "0.1.7"
name = "convert_case"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc"
checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca"
dependencies = [
"cfg-if",
"wasm-bindgen",
"unicode-segmentation",
]
[[package]]
@@ -134,6 +127,47 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
[[package]]
name = "ctor"
version = "0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501"
dependencies = [
"quote",
"syn",
]
[[package]]
name = "either"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
[[package]]
name = "encoding_rs"
version = "0.8.35"
@@ -226,6 +260,19 @@ dependencies = [
"weezl",
]
[[package]]
name = "globset"
version = "0.4.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52dfc19153a48bde0cbd630453615c8151bce3a5adfac7a0aebfbf0a1e1f57e3"
dependencies = [
"aho-corasick",
"bstr",
"log",
"regex-automata",
"regex-syntax",
]
[[package]]
name = "grep-matcher"
version = "0.1.8"
@@ -310,6 +357,22 @@ dependencies = [
"markup5ever",
]
[[package]]
name = "ignore"
version = "0.4.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3d782a365a015e0f5c04902246139249abf769125006fbe7649e2ee88169b4a"
dependencies = [
"crossbeam-deque",
"globset",
"log",
"memchr",
"regex-automata",
"same-file",
"walkdir",
"winapi-util",
]
[[package]]
name = "image"
version = "0.25.9"
@@ -338,22 +401,22 @@ dependencies = [
"quick-error",
]
[[package]]
name = "js-sys"
version = "0.3.85"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3"
dependencies = [
"once_cell",
"wasm-bindgen",
]
[[package]]
name = "libc"
version = "0.2.180"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc"
[[package]]
name = "libloading"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"
dependencies = [
"cfg-if",
"windows-link",
]
[[package]]
name = "lock_api"
version = "0.4.14"
@@ -442,6 +505,64 @@ dependencies = [
"pxfm",
]
[[package]]
name = "napi"
version = "2.16.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "55740c4ae1d8696773c78fdafd5d0e5fe9bc9f1b071c7ba493ba5c413a9184f3"
dependencies = [
"bitflags",
"ctor",
"napi-derive",
"napi-sys",
"once_cell",
"tokio",
]
[[package]]
name = "napi-build"
version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d376940fd5b723c6893cd1ee3f33abbfd86acb1cd1ec079f3ab04a2a3bc4d3b1"
[[package]]
name = "napi-derive"
version = "2.16.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7cbe2585d8ac223f7d34f13701434b9d5f4eb9c332cccce8dee57ea18ab8ab0c"
dependencies = [
"cfg-if",
"convert_case",
"napi-derive-backend",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "napi-derive-backend"
version = "1.0.75"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1639aaa9eeb76e91c6ae66da8ce3e89e921cd3885e99ec85f4abacae72fc91bf"
dependencies = [
"convert_case",
"once_cell",
"proc-macro2",
"quote",
"regex",
"semver",
"syn",
]
[[package]]
name = "napi-sys"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "427802e8ec3a734331fec1035594a210ce1ff4dc5bc1950530920ab717964ea3"
dependencies = [
"libloading",
]
[[package]]
name = "new_debug_unreachable"
version = "1.0.6"
@@ -529,20 +650,28 @@ dependencies = [
name = "pi-natives"
version = "9.4.0"
dependencies = [
"console_error_panic_hook",
"globset",
"grep-matcher",
"grep-regex",
"grep-searcher",
"html-to-markdown-rs",
"ignore",
"image",
"serde",
"serde-wasm-bindgen",
"napi",
"napi-build",
"napi-derive",
"rayon",
"syntect",
"unicode-segmentation",
"unicode-width",
"wasm-bindgen",
]
[[package]]
name = "pin-project-lite"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b"
[[package]]
name = "png"
version = "0.18.0"
@@ -595,6 +724,26 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "rayon"
version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f"
dependencies = [
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
dependencies = [
"crossbeam-deque",
"crossbeam-utils",
]
[[package]]
name = "redox_syscall"
version = "0.5.18"
@@ -633,12 +782,6 @@ version = "0.8.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58"
[[package]]
name = "rustversion"
version = "1.0.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
[[package]]
name = "same-file"
version = "1.0.6"
@@ -654,6 +797,12 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "semver"
version = "1.0.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2"
[[package]]
name = "serde"
version = "1.0.228"
@@ -661,18 +810,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
dependencies = [
"serde_core",
"serde_derive",
]
[[package]]
name = "serde-wasm-bindgen"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b"
dependencies = [
"js-sys",
"serde",
"wasm-bindgen",
]
[[package]]
@@ -797,6 +934,15 @@ dependencies = [
"syn",
]
[[package]]
name = "tokio"
version = "1.49.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86"
dependencies = [
"pin-project-lite",
]
[[package]]
name = "unicode-ident"
version = "1.0.22"
@@ -837,51 +983,6 @@ dependencies = [
"winapi-util",
]
[[package]]
name = "wasm-bindgen"
version = "0.2.108"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566"
dependencies = [
"cfg-if",
"once_cell",
"rustversion",
"wasm-bindgen-macro",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.108"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
]
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.108"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55"
dependencies = [
"bumpalo",
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.108"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12"
dependencies = [
"unicode-ident",
]
[[package]]
name = "web_atoms"
version = "0.2.3"
crates/pi-natives/Cargo.toml
+7 -11
View File
@@ -13,23 +13,19 @@ crate-type = ["cdylib"]
workspace = true
[dependencies]
wasm-bindgen = "0.2"
serde = { version = "1.0", features = ["derive"] }
serde-wasm-bindgen = "0.6"
napi = { version = "2", features = ["napi8", "tokio_rt"] }
napi-derive = "2"
grep-regex = "0.1.14"
grep-searcher = "0.1.16"
grep-matcher = "0.1.8"
globset = "0.4"
ignore = "0.4"
rayon = "1.10"
image = { version = "0.25", default-features = false, features = ["png", "jpeg", "gif", "webp"] }
unicode-segmentation = "1.11"
unicode-width = "0.2"
syntect = { version = "5.3", default-features = false, features = ["default-syntaxes", "default-themes", "regex-fancy"] }
html-to-markdown-rs = { version = "2.24", default-features = false }
# Better panic messages in debug builds
console_error_panic_hook = { version = "0.1.7", optional = true }
[features]
default = ["console_error_panic_hook"]
[package.metadata.wasm-pack.profile.release]
wasm-opt = false
[build-dependencies]
napi-build = "2"
crates/pi-natives/build.rs
+3
View File
@@ -0,0 +1,3 @@
fn main() {
napi_build::setup();
}
crates/pi-natives/src/grep.rs
+886
View File
@@ -0,0 +1,886 @@
//! Ripgrep-backed search exported via N-API.
//!
//! Provides two layers:
//! - `search()` for in-memory content search.
//! - `grep()` for filesystem search with glob/type filtering.
//!
//! The filesystem search matches the previous JS wrapper behavior, including
//! global offsets, optional match limits, and per-file match summaries.
use std::fs::File;
use std::io::{self, Cursor, Read};
use std::path::{Path, PathBuf};
use globset::{Glob, GlobSet, GlobSetBuilder};
use grep_matcher::Matcher;
use grep_regex::RegexMatcherBuilder;
use grep_searcher::{BinaryDetection, Searcher, SearcherBuilder, Sink, SinkContext, SinkContextKind, SinkMatch};
use ignore::WalkBuilder;
use napi::bindgen_prelude::*;
use napi_derive::napi;
use rayon::prelude::*;
const MAX_FILE_BYTES: u64 = 4 * 1024 * 1024;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum OutputMode {
Content,
Count,
}
/// Options for searching file content.
#[napi(object)]
pub struct SearchOptions {
/// Regex pattern to search for.
pub pattern: String,
/// Case-insensitive search.
#[napi(js_name = "ignoreCase")]
pub ignore_case: Option<bool>,
/// Enable multiline matching.
pub multiline: Option<bool>,
/// Maximum number of matches to return.
#[napi(js_name = "maxCount")]
pub max_count: Option<u32>,
/// Skip first N matches.
pub offset: Option<u32>,
/// Lines of context before/after matches.
pub context: Option<u32>,
/// Truncate lines longer than this (characters).
#[napi(js_name = "maxColumns")]
pub max_columns: Option<u32>,
/// Output mode (content or count).
pub mode: Option<String>,
}
/// Options for searching files on disk.
#[napi(object)]
pub struct GrepOptions {
/// Regex pattern to search for.
pub pattern: String,
/// Directory or file to search.
pub path: String,
/// Glob filter for filenames (e.g., "*.ts").
pub glob: Option<String>,
/// Filter by file type (e.g., "js", "py", "rust").
#[napi(js_name = "type")]
pub type_filter: Option<String>,
/// Case-insensitive search.
#[napi(js_name = "ignoreCase")]
pub ignore_case: Option<bool>,
/// Enable multiline matching.
pub multiline: Option<bool>,
/// Include hidden files (default: true).
pub hidden: Option<bool>,
/// Maximum number of matches to return.
#[napi(js_name = "maxCount")]
pub max_count: Option<u32>,
/// Skip first N matches.
pub offset: Option<u32>,
/// Lines of context before/after matches.
pub context: Option<u32>,
/// Truncate lines longer than this (characters).
#[napi(js_name = "maxColumns")]
pub max_columns: Option<u32>,
/// Output mode (content, filesWithMatches, or count).
pub mode: Option<String>,
}
/// A context line (before or after a match).
#[napi(object)]
pub struct ContextLine {
#[napi(js_name = "lineNumber")]
pub line_number: u32,
pub line: String,
}
/// A single match in the content.
#[napi(object)]
pub struct Match {
/// 1-indexed line number.
#[napi(js_name = "lineNumber")]
pub line_number: u32,
/// The matched line content.
pub line: String,
/// Context lines before the match.
#[napi(js_name = "contextBefore")]
pub context_before: Option<Vec<ContextLine>>,
/// Context lines after the match.
#[napi(js_name = "contextAfter")]
pub context_after: Option<Vec<ContextLine>>,
/// Whether the line was truncated.
pub truncated: Option<bool>,
}
/// Result of searching content.
#[napi(object)]
pub struct SearchResult {
/// All matches found.
pub matches: Vec<Match>,
/// Total number of matches (may exceed `matches.len()` due to offset/limit).
#[napi(js_name = "matchCount")]
pub match_count: u32,
/// Whether the limit was reached.
#[napi(js_name = "limitReached")]
pub limit_reached: bool,
/// Error message, if any.
pub error: Option<String>,
}
/// A single match in a grep result.
#[napi(object)]
pub struct GrepMatch {
pub path: String,
#[napi(js_name = "lineNumber")]
pub line_number: u32,
pub line: String,
#[napi(js_name = "contextBefore")]
pub context_before: Option<Vec<ContextLine>>,
#[napi(js_name = "contextAfter")]
pub context_after: Option<Vec<ContextLine>>,
pub truncated: Option<bool>,
#[napi(js_name = "matchCount")]
pub match_count: Option<u32>,
}
/// Result of searching files.
#[napi(object)]
pub struct GrepResult {
pub matches: Vec<GrepMatch>,
#[napi(js_name = "totalMatches")]
pub total_matches: u32,
#[napi(js_name = "filesWithMatches")]
pub files_with_matches: u32,
#[napi(js_name = "filesSearched")]
pub files_searched: u32,
#[napi(js_name = "limitReached")]
pub limit_reached: Option<bool>,
}
struct TypeFilter {
extensions: Vec<String>,
names: Vec<String>,
}
struct MatchCollector {
matches: Vec<CollectedMatch>,
match_count: u64,
collected_count: u64,
max_count: Option<u64>,
offset: u64,
skipped: u64,
limit_reached: bool,
context_before: Vec<ContextLine>,
max_columns: Option<usize>,
collect_matches: bool,
}
struct CollectedMatch {
line_number: u64,
line: String,
context_before: Vec<ContextLine>,
context_after: Vec<ContextLine>,
truncated: bool,
}
struct SearchResultInternal {
matches: Vec<CollectedMatch>,
match_count: u64,
limit_reached: bool,
}
struct FileEntry {
path: PathBuf,
relative_path: String,
}
struct FileSearchResult {
relative_path: String,
matches: Vec<CollectedMatch>,
match_count: u64,
}
impl MatchCollector {
const fn new(max_count: Option<u64>, offset: u64, max_columns: Option<usize>, collect_matches: bool) -> Self {
Self {
matches: Vec::new(),
match_count: 0,
collected_count: 0,
max_count,
offset,
skipped: 0,
limit_reached: false,
context_before: Vec::new(),
max_columns,
collect_matches,
}
}
fn truncate_line(&self, line: &str) -> (String, bool) {
match self.max_columns {
Some(max) if line.len() > max => {
let truncated = format!("{}...", &line[..max.saturating_sub(3)]);
(truncated, true)
},
_ => (line.to_string(), false),
}
}
}
impl Sink for MatchCollector {
type Error = io::Error;
fn matched(
&mut self,
_searcher: &Searcher,
mat: &SinkMatch<'_>,
) -> std::result::Result<bool, Self::Error> {
self.match_count += 1;
// If we already hit the limit, stop now (after-context for previous match was
// collected).
if self.limit_reached {
return Ok(false);
}
if self.skipped < self.offset {
self.skipped += 1;
self.context_before.clear();
return Ok(true);
}
if self.collect_matches {
let raw_line = String::from_utf8_lossy(mat.bytes()).trim_end().to_string();
let (line, truncated) = self.truncate_line(&raw_line);
let line_number = mat.line_number().unwrap_or(0);
self.matches.push(CollectedMatch {
line_number,
line,
context_before: std::mem::take(&mut self.context_before),
context_after: Vec::new(),
truncated,
});
} else {
self.context_before.clear();
}
self.collected_count += 1;
// Mark limit reached but don't stop yet - allow after-context to be collected.
if let Some(max) = self.max_count
&& self.collected_count >= max
{
self.limit_reached = true;
}
Ok(true)
}
fn context(
&mut self,
_searcher: &Searcher,
ctx: &SinkContext<'_>,
) -> std::result::Result<bool, Self::Error> {
if !self.collect_matches {
return Ok(true);
}
let raw_line = String::from_utf8_lossy(ctx.bytes()).trim_end().to_string();
let (line, _) = self.truncate_line(&raw_line);
let line_number = ctx.line_number().unwrap_or(0);
match ctx.kind() {
SinkContextKind::Before => {
self.context_before.push(ContextLine {
line_number: clamp_u32(line_number),
line,
});
},
SinkContextKind::After => {
if let Some(last_match) = self.matches.last_mut() {
last_match.context_after.push(ContextLine {
line_number: clamp_u32(line_number),
line,
});
}
},
SinkContextKind::Other => {},
}
Ok(true)
}
}
fn clamp_u32(value: u64) -> u32 {
value.min(u32::MAX as u64) as u32
}
fn parse_output_mode(mode: Option<&str>) -> OutputMode {
match mode {
Some("count") | Some("filesWithMatches") => OutputMode::Count,
Some("content") | _ => OutputMode::Content,
}
}
fn resolve_search_path(path: &str) -> Result<PathBuf> {
let candidate = PathBuf::from(path);
if candidate.is_absolute() {
return Ok(candidate);
}
let cwd = std::env::current_dir()
.map_err(|err| Error::from_reason(format!("Failed to resolve cwd: {err}")))?;
Ok(cwd.join(candidate))
}
fn build_glob_pattern(glob: &str) -> String {
let normalized = glob.replace('\\', "/");
if normalized.contains('/') || normalized.starts_with("**/") {
normalized
} else {
format!("**/{normalized}")
}
}
fn compile_glob(glob: Option<&str>) -> Result<Option<GlobSet>> {
let Some(glob) = glob.map(str::trim).filter(|value| !value.is_empty()) else {
return Ok(None);
};
let mut builder = GlobSetBuilder::new();
let pattern = build_glob_pattern(glob);
let glob = Glob::new(&pattern)
.map_err(|err| Error::from_reason(format!("Invalid glob pattern: {err}")))?;
builder.add(glob);
builder
.build()
.map(Some)
.map_err(|err| Error::from_reason(format!("Failed to build glob matcher: {err}")))
}
fn resolve_type_filter(type_name: Option<&str>) -> Option<TypeFilter> {
let normalized = type_name
.map(str::trim)
.filter(|value| !value.is_empty())
.map(|value| value.trim_start_matches('.').to_lowercase())?;
let (extensions, names): (Vec<&str>, Vec<&str>) = match normalized.as_str() {
"js" | "javascript" => (vec!["js", "jsx", "mjs", "cjs"], vec![]),
"ts" | "typescript" => (vec!["ts", "tsx", "mts", "cts"], vec![]),
"json" => (vec!["json", "jsonc", "json5"], vec![]),
"yaml" | "yml" => (vec!["yaml", "yml"], vec![]),
"toml" => (vec!["toml"], vec![]),
"md" | "markdown" => (vec!["md", "markdown", "mdx"], vec![]),
"py" | "python" => (vec!["py", "pyi"], vec![]),
"rs" | "rust" => (vec!["rs"], vec![]),
"go" => (vec!["go"], vec![]),
"java" => (vec!["java"], vec![]),
"kt" | "kotlin" => (vec!["kt", "kts"], vec![]),
"c" => (vec!["c", "h"], vec![]),
"cpp" | "cxx" => (vec!["cpp", "cc", "cxx", "hpp", "hxx", "hh"], vec![]),
"cs" | "csharp" => (vec!["cs", "csx"], vec![]),
"php" => (vec!["php", "phtml"], vec![]),
"rb" | "ruby" => (vec!["rb", "rake", "gemspec"], vec![]),
"sh" | "bash" => (vec!["sh", "bash", "zsh"], vec![]),
"zsh" => (vec!["zsh"], vec![]),
"fish" => (vec!["fish"], vec![]),
"html" => (vec!["html", "htm"], vec![]),
"css" => (vec!["css"], vec![]),
"scss" => (vec!["scss"], vec![]),
"sass" => (vec!["sass"], vec![]),
"less" => (vec!["less"], vec![]),
"xml" => (vec!["xml"], vec![]),
"docker" | "dockerfile" => (vec![], vec!["dockerfile"]),
"make" | "makefile" => (vec![], vec!["makefile"]),
_ => (vec![normalized.as_str()], vec![]),
};
Some(TypeFilter {
extensions: extensions.into_iter().map(|value| value.to_string()).collect(),
names: names.into_iter().map(|value| value.to_string()).collect(),
})
}
fn matches_type_filter(path: &Path, filter: &TypeFilter) -> bool {
let base_name = path
.file_name()
.and_then(|name| name.to_str())
.unwrap_or("")
.to_lowercase();
if filter.names.iter().any(|name| name == &base_name) {
return true;
}
let ext = path
.extension()
.and_then(|ext| ext.to_str())
.unwrap_or("")
.to_lowercase();
if ext.is_empty() {
return false;
}
filter.extensions.iter().any(|value| value == &ext)
}
fn read_file_prefix(path: &Path) -> io::Result<Vec<u8>> {
let file = File::open(path)?;
let mut limited = file.take(MAX_FILE_BYTES);
let mut buffer = Vec::new();
limited.read_to_end(&mut buffer)?;
Ok(buffer)
}
fn normalize_relative_path(root: &Path, path: &Path) -> String {
let relative = path.strip_prefix(root).unwrap_or(path);
relative.to_string_lossy().replace('\\', "/")
}
fn build_searcher(context: u32) -> Searcher {
SearcherBuilder::new()
.binary_detection(BinaryDetection::quit(b'\x00'))
.line_number(true)
.before_context(context as usize)
.after_context(context as usize)
.build()
}
fn run_search(
matcher: &grep_regex::RegexMatcher,
content: &[u8],
context: u32,
max_columns: Option<u32>,
mode: OutputMode,
max_count: Option<u64>,
offset: u64,
) -> io::Result<SearchResultInternal> {
let mut searcher = build_searcher(if mode == OutputMode::Content { context } else { 0 });
let mut collector = MatchCollector::new(
max_count,
offset,
max_columns.map(|value| value as usize),
mode == OutputMode::Content,
);
let cursor = Cursor::new(content);
searcher.search_reader(matcher, cursor, &mut collector)?;
Ok(SearchResultInternal {
matches: collector.matches,
match_count: collector.match_count,
limit_reached: collector.limit_reached,
})
}
fn to_public_match(matched: CollectedMatch) -> Match {
let context_before = if matched.context_before.is_empty() {
None
} else {
Some(matched.context_before)
};
let context_after = if matched.context_after.is_empty() {
None
} else {
Some(matched.context_after)
};
Match {
line_number: clamp_u32(matched.line_number),
line: matched.line,
context_before,
context_after,
truncated: if matched.truncated { Some(true) } else { None },
}
}
fn to_grep_match(path: &str, matched: CollectedMatch) -> GrepMatch {
let context_before = if matched.context_before.is_empty() {
None
} else {
Some(matched.context_before)
};
let context_after = if matched.context_after.is_empty() {
None
} else {
Some(matched.context_after)
};
GrepMatch {
path: path.to_string(),
line_number: clamp_u32(matched.line_number),
line: matched.line,
context_before,
context_after,
truncated: if matched.truncated { Some(true) } else { None },
match_count: None,
}
}
fn empty_search_result(error: Option<String>) -> SearchResult {
SearchResult {
matches: Vec::new(),
match_count: 0,
limit_reached: false,
error,
}
}
fn collect_files(
root: &Path,
glob_set: Option<&GlobSet>,
include_hidden: bool,
type_filter: Option<&TypeFilter>,
) -> Vec<FileEntry> {
let mut builder = WalkBuilder::new(root);
builder
.hidden(!include_hidden)
.git_ignore(true)
.git_exclude(true)
.git_global(true)
.ignore(true)
.parents(true)
.follow_links(false)
.sort_by_file_path(|a, b| a.cmp(b));
let mut entries = Vec::new();
for entry in builder.build() {
let entry = match entry {
Ok(entry) => entry,
Err(_) => continue,
};
let file_type = entry.file_type();
if !file_type.map(|ft| ft.is_file()).unwrap_or(false) {
continue;
}
let path = entry.into_path();
if let Some(glob_set) = glob_set {
let relative = path.strip_prefix(root).unwrap_or(&path);
if !glob_set.is_match(relative) {
continue;
}
}
if let Some(filter) = type_filter {
if !matches_type_filter(&path, filter) {
continue;
}
}
entries.push(FileEntry {
path: path.clone(),
relative_path: normalize_relative_path(root, &path),
});
}
entries
}
fn build_matcher(pattern: &str, ignore_case: bool, multiline: bool) -> Result<grep_regex::RegexMatcher> {
RegexMatcherBuilder::new()
.case_insensitive(ignore_case)
.multi_line(multiline)
.build(pattern)
.map_err(|err| Error::from_reason(format!("Regex error: {err}")))
}
fn run_parallel_search(
entries: &[FileEntry],
matcher: &grep_regex::RegexMatcher,
context: u32,
max_columns: Option<u32>,
mode: OutputMode,
) -> Vec<FileSearchResult> {
let mut results: Vec<FileSearchResult> = entries
.par_iter()
.filter_map(|entry| {
let bytes = read_file_prefix(&entry.path).ok()?;
let search = run_search(matcher, &bytes, context, max_columns, mode, None, 0).ok()?;
Some(FileSearchResult {
relative_path: entry.relative_path.clone(),
matches: search.matches,
match_count: search.match_count,
})
})
.collect();
results.sort_by(|a, b| a.relative_path.cmp(&b.relative_path));
results
}
fn run_sequential_search(
entries: &[FileEntry],
matcher: &grep_regex::RegexMatcher,
context: u32,
max_columns: Option<u32>,
mode: OutputMode,
max_count: Option<u64>,
offset: u64,
) -> (Vec<GrepMatch>, u64, u32, u32, bool) {
let mut matches = Vec::new();
let mut total_matches = 0u64;
let mut files_with_matches = 0u32;
let mut files_searched = 0u32;
let mut limit_reached = false;
for entry in entries {
if limit_reached {
break;
}
let bytes = match read_file_prefix(&entry.path) {
Ok(bytes) => bytes,
Err(_) => continue,
};
files_searched = files_searched.saturating_add(1);
if !matcher.is_match(&bytes).unwrap_or(false) {
continue;
}
let file_offset = offset.saturating_sub(total_matches);
let remaining = max_count.map(|max| max.saturating_sub(total_matches));
if let Some(0) = remaining {
limit_reached = true;
break;
}
let search = match run_search(
matcher,
&bytes,
context,
max_columns,
mode,
remaining,
file_offset,
) {
Ok(result) => result,
Err(_) => continue,
};
if search.match_count == 0 {
continue;
}
files_with_matches = files_with_matches.saturating_add(1);
total_matches = total_matches.saturating_add(search.match_count);
match mode {
OutputMode::Content => {
for matched in search.matches {
matches.push(to_grep_match(&entry.relative_path, matched));
}
},
OutputMode::Count => {
matches.push(GrepMatch {
path: entry.relative_path.clone(),
line_number: 0,
line: String::new(),
context_before: None,
context_after: None,
truncated: None,
match_count: Some(clamp_u32(search.match_count)),
});
},
}
if search.limit_reached || max_count.is_some_and(|max| total_matches >= max) {
limit_reached = true;
}
}
(matches, total_matches, files_with_matches, files_searched, limit_reached)
}
/// Search content for a pattern (one-shot, compiles pattern each time).
/// For repeated searches with the same pattern, use [`grep`] with file filters.
#[napi(js_name = "search")]
pub fn search(content: String, options: SearchOptions) -> SearchResult {
let ignore_case = options.ignore_case.unwrap_or(false);
let multiline = options.multiline.unwrap_or(false);
let mode = parse_output_mode(options.mode.as_deref());
let matcher = match build_matcher(&options.pattern, ignore_case, multiline) {
Ok(matcher) => matcher,
Err(err) => return empty_search_result(Some(err.to_string())),
};
let context = options.context.unwrap_or(0);
let max_columns = options.max_columns;
let max_count = options.max_count.map(u64::from);
let offset = options.offset.unwrap_or(0) as u64;
let result = match run_search(&matcher, content.as_bytes(), context, max_columns, mode, max_count, offset) {
Ok(result) => result,
Err(err) => return empty_search_result(Some(err.to_string())),
};
SearchResult {
matches: result.matches.into_iter().map(to_public_match).collect(),
match_count: clamp_u32(result.match_count),
limit_reached: result.limit_reached,
error: None,
}
}
/// Quick check if content matches a pattern.
#[napi(js_name = "has_match")]
pub fn has_match(content: String, pattern: String, ignore_case: bool, multiline: bool) -> Result<bool> {
let matcher = build_matcher(&pattern, ignore_case, multiline)?;
Ok(matcher.is_match(content.as_bytes()).unwrap_or(false))
}
/// Search files for a regex pattern.
#[napi(js_name = "grep")]
pub fn grep(options: GrepOptions) -> Result<GrepResult> {
let search_path = resolve_search_path(&options.path)?;
let metadata = std::fs::metadata(&search_path)
.map_err(|err| Error::from_reason(format!("Path not found: {err}")))?;
let ignore_case = options.ignore_case.unwrap_or(false);
let multiline = options.multiline.unwrap_or(false);
let output_mode = parse_output_mode(options.mode.as_deref());
let matcher = build_matcher(&options.pattern, ignore_case, multiline)?;
let context = if output_mode == OutputMode::Content {
options.context.unwrap_or(0)
} else {
0
};
let max_columns = options.max_columns;
let max_count = options.max_count.map(u64::from);
let offset = options.offset.unwrap_or(0) as u64;
let include_hidden = options.hidden.unwrap_or(true);
let glob_set = compile_glob(options.glob.as_deref())?;
let type_filter = resolve_type_filter(options.type_filter.as_deref());
if metadata.is_file() {
if let Some(filter) = type_filter.as_ref() {
if !matches_type_filter(&search_path, filter) {
return Ok(GrepResult {
matches: Vec::new(),
total_matches: 0,
files_with_matches: 0,
files_searched: 0,
limit_reached: None,
});
}
}
let bytes = match read_file_prefix(&search_path) {
Ok(bytes) => bytes,
Err(_) => {
return Ok(GrepResult {
matches: Vec::new(),
total_matches: 0,
files_with_matches: 0,
files_searched: 0,
limit_reached: None,
});
},
};
let search = run_search(&matcher, &bytes, context, max_columns, output_mode, max_count, offset)
.map_err(|err| Error::from_reason(format!("Search failed: {err}")))?;
if search.match_count == 0 {
return Ok(GrepResult {
matches: Vec::new(),
total_matches: 0,
files_with_matches: 0,
files_searched: 1,
limit_reached: None,
});
}
let path_string = search_path.to_string_lossy().to_string();
let mut matches = Vec::new();
match output_mode {
OutputMode::Content => {
for matched in search.matches {
matches.push(to_grep_match(&path_string, matched));
}
},
OutputMode::Count => {
matches.push(GrepMatch {
path: path_string,
line_number: 0,
line: String::new(),
context_before: None,
context_after: None,
truncated: None,
match_count: Some(clamp_u32(search.match_count)),
});
},
}
let limit_reached = search.limit_reached
|| max_count.is_some_and(|max| search.match_count >= max);
return Ok(GrepResult {
matches,
total_matches: clamp_u32(search.match_count),
files_with_matches: 1,
files_searched: 1,
limit_reached: if limit_reached { Some(true) } else { None },
});
}
let entries = collect_files(&search_path, glob_set.as_ref(), include_hidden, type_filter.as_ref());
if entries.is_empty() {
return Ok(GrepResult {
matches: Vec::new(),
total_matches: 0,
files_with_matches: 0,
files_searched: 0,
limit_reached: None,
});
}
let allow_parallel = max_count.is_none() && offset == 0;
if allow_parallel {
let results = run_parallel_search(&entries, &matcher, context, max_columns, output_mode);
let mut matches = Vec::new();
let mut total_matches = 0u64;
let mut files_with_matches = 0u32;
let files_searched = clamp_u32(results.len() as u64);
for result in results {
if result.match_count == 0 {
continue;
}
files_with_matches = files_with_matches.saturating_add(1);
total_matches = total_matches.saturating_add(result.match_count);
match output_mode {
OutputMode::Content => {
for matched in result.matches {
matches.push(to_grep_match(&result.relative_path, matched));
}
},
OutputMode::Count => {
matches.push(GrepMatch {
path: result.relative_path.clone(),
line_number: 0,
line: String::new(),
context_before: None,
context_after: None,
truncated: None,
match_count: Some(clamp_u32(result.match_count)),
});
},
}
}
return Ok(GrepResult {
matches,
total_matches: clamp_u32(total_matches),
files_with_matches,
files_searched,
limit_reached: None,
});
}
let (matches, total_matches, files_with_matches, files_searched, limit_reached) = run_sequential_search(
&entries,
&matcher,
context,
max_columns,
output_mode,
max_count,
offset,
);
Ok(GrepResult {
matches,
total_matches: clamp_u32(total_matches),
files_with_matches,
files_searched,
limit_reached: if limit_reached { Some(true) } else { None },
})
}
crates/pi-natives/src/highlight.rs
+31 -32
View File
@@ -7,9 +7,9 @@
use std::sync::OnceLock;
use serde::Deserialize;
use napi::bindgen_prelude::*;
use napi_derive::napi;
use syntect::parsing::{ParseState, Scope, ScopeStack, ScopeStackOp, SyntaxReference, SyntaxSet};
use wasm_bindgen::prelude::*;
static SYNTAX_SET: OnceLock<SyntaxSet> = OnceLock::new();
static SCOPE_MATCHERS: OnceLock<ScopeMatchers> = OnceLock::new();
@@ -118,8 +118,8 @@ fn get_scope_matchers() -> &'static ScopeMatchers {
/// Theme colors for syntax highlighting.
/// Each color is an ANSI escape sequence (e.g., "\x1b[38;2;255;0;0m").
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
#[derive(Debug)]
#[napi(object)]
pub struct HighlightColors {
pub comment: String,
pub keyword: String,
@@ -127,13 +127,14 @@ pub struct HighlightColors {
pub variable: String,
pub string: String,
pub number: String,
#[napi(js_name = "type")]
pub r#type: String,
pub operator: String,
pub punctuation: String,
#[serde(default)]
pub inserted: String,
#[serde(default)]
pub deleted: String,
#[napi(js_name = "inserted")]
pub inserted: Option<String>,
#[napi(js_name = "deleted")]
pub deleted: Option<String>,
}
/// Language alias mappings: (aliases, target syntax name).
@@ -333,26 +334,24 @@ fn find_syntax<'a>(ss: &'a SyntaxSet, lang: &str) -> Option<&'a SyntaxReference>
/// # Returns
/// Highlighted code with ANSI color codes, or the original code if highlighting
/// fails.
#[wasm_bindgen]
pub fn highlight_code(code: &str, lang: Option<String>, colors: JsValue) -> String {
let colors: HighlightColors = match serde_wasm_bindgen::from_value(colors) {
Ok(c) => c,
Err(_) => return code.to_string(),
};
#[napi(js_name = "highlight_code")]
pub fn highlight_code(code: String, lang: Option<String>, colors: HighlightColors) -> Result<String> {
let inserted = colors.inserted.as_deref().unwrap_or("");
let deleted = colors.deleted.as_deref().unwrap_or("");
// Color palette as array for quick indexing
let palette = [
&colors.comment, // 0
&colors.keyword, // 1
&colors.function, // 2
&colors.variable, // 3
&colors.string, // 4
&colors.number, // 5
&colors.r#type, // 6
&colors.operator, // 7
&colors.punctuation, // 8
&colors.inserted, // 9
&colors.deleted, // 10
colors.comment.as_str(), // 0
colors.keyword.as_str(), // 1
colors.function.as_str(), // 2
colors.variable.as_str(), // 3
colors.string.as_str(), // 4
colors.number.as_str(), // 5
colors.r#type.as_str(), // 6
colors.operator.as_str(), // 7
colors.punctuation.as_str(), // 8
inserted, // 9
deleted, // 10
];
let ss = get_syntax_set();
@@ -368,7 +367,7 @@ pub fn highlight_code(code: &str, lang: Option<String>, colors: JsValue) -> Stri
let mut scope_stack = ScopeStack::new();
let mut result = String::with_capacity(code.len() * 2);
for line in syntect::util::LinesWithEndings::from(code) {
for line in syntect::util::LinesWithEndings::from(code.as_str()) {
let Ok(ops) = parse_state.parse_line(line, ss) else {
result.push_str(line);
continue;
@@ -422,25 +421,25 @@ pub fn highlight_code(code: &str, lang: Option<String>, colors: JsValue) -> Stri
}
}
result
Ok(result)
}
/// Check if a language is supported for highlighting.
/// Returns true if the language has either direct support or a fallback
/// mapping.
#[wasm_bindgen]
pub fn supports_language(lang: &str) -> bool {
if is_known_alias(lang) {
#[napi(js_name = "supports_language")]
pub fn supports_language(lang: String) -> bool {
if is_known_alias(&lang) {
return true;
}
// Fall back to direct syntax lookup
let ss = get_syntax_set();
find_syntax(ss, lang).is_some()
find_syntax(ss, &lang).is_some()
}
/// Get list of supported languages.
#[wasm_bindgen]
#[napi(js_name = "get_supported_languages")]
pub fn get_supported_languages() -> Vec<String> {
let ss = get_syntax_set();
ss.syntaxes().iter().map(|s| s.name.clone()).collect()
crates/pi-natives/src/html.rs
+16 -22
View File
@@ -1,35 +1,29 @@
//! HTML to Markdown conversion.
use html_to_markdown_rs::{ConversionOptions, PreprocessingOptions, PreprocessingPreset, convert};
use serde::Deserialize;
use wasm_bindgen::prelude::*;
use html_to_markdown_rs::{convert, ConversionOptions, PreprocessingOptions, PreprocessingPreset};
use napi::bindgen_prelude::*;
use napi_derive::napi;
/// Options for HTML to Markdown conversion.
#[derive(Debug, Deserialize, Default)]
#[serde(rename_all = "camelCase")]
#[napi(object)]
#[derive(Debug, Default)]
pub struct HtmlToMarkdownOptions {
/// Remove navigation elements, forms, headers, footers.
#[serde(default)]
pub clean_content: bool,
#[napi(js_name = "cleanContent")]
pub clean_content: Option<bool>,
/// Skip images during conversion.
#[serde(default)]
pub skip_images: bool,
#[napi(js_name = "skipImages")]
pub skip_images: Option<bool>,
}
/// Convert HTML to Markdown.
#[wasm_bindgen]
pub fn html_to_markdown(html: &str, options: JsValue) -> Result<String, JsValue> {
let opts: HtmlToMarkdownOptions = if options.is_undefined() || options.is_null() {
HtmlToMarkdownOptions::default()
} else {
serde_wasm_bindgen::from_value(options)
.map_err(|e| JsValue::from_str(&format!("Invalid options: {e}")))?
};
#[napi(js_name = "html_to_markdown")]
pub fn html_to_markdown(html: String, options: Option<HtmlToMarkdownOptions>) -> Result<String> {
let options = options.unwrap_or_default();
let conversion_opts = ConversionOptions {
skip_images: opts.skip_images,
skip_images: options.skip_images.unwrap_or(false),
preprocessing: PreprocessingOptions {
enabled: opts.clean_content,
enabled: options.clean_content.unwrap_or(false),
preset: PreprocessingPreset::Aggressive,
remove_navigation: true,
remove_forms: true,
@@ -37,6 +31,6 @@ pub fn html_to_markdown(html: &str, options: JsValue) -> Result<String, JsValue>
..Default::default()
};
convert(html, Some(conversion_opts))
.map_err(|e| JsValue::from_str(&format!("Conversion error: {e}")))
convert(&html, Some(conversion_opts))
.map_err(|err| Error::from_reason(format!("Conversion error: {err}")))
}
crates/pi-natives/src/image.rs
+53 -38
View File
@@ -4,16 +4,16 @@
//! - Load image from bytes (PNG, JPEG, WebP, GIF)
//! - Get dimensions
//! - Resize with Lanczos3 filter
//! - Export as PNG or JPEG
//! - Export as PNG, JPEG, WebP, or GIF
use std::io::Cursor;
use image::{DynamicImage, ImageFormat, ImageReader, imageops::FilterType};
use wasm_bindgen::prelude::*;
use image::{imageops::FilterType, DynamicImage, ImageFormat, ImageReader};
use napi::bindgen_prelude::*;
use napi_derive::napi;
/// Sampling filter for resize operations.
#[wasm_bindgen]
#[derive(Clone, Copy)]
#[napi]
pub enum SamplingFilter {
Nearest = 1,
Triangle = 2,
@@ -34,91 +34,106 @@ impl From<SamplingFilter> for FilterType {
}
}
/// Image container for WASM interop.
#[wasm_bindgen]
/// Image container for native interop.
#[napi]
pub struct PhotonImage {
img: DynamicImage,
}
#[wasm_bindgen]
#[napi]
impl PhotonImage {
/// Create a new `PhotonImage` from encoded image bytes (PNG, JPEG, WebP,
/// GIF).
#[wasm_bindgen(js_name = new_from_byteslice)]
pub fn new_from_byteslice(bytes: &[u8]) -> Result<Self, JsValue> {
let reader = ImageReader::new(Cursor::new(bytes))
///
/// # Errors
/// Returns an error if the image format cannot be detected or decoded.
#[napi(factory, js_name = "new_from_byteslice")]
pub fn new_from_byteslice(bytes: Uint8Array) -> Result<Self> {
let reader = ImageReader::new(Cursor::new(bytes.as_ref()))
.with_guessed_format()
.map_err(|e| JsValue::from_str(&format!("Failed to detect image format: {e}")))?;
.map_err(|e| Error::from_reason(format!("Failed to detect image format: {e}")))?;
let img = reader
.decode()
.map_err(|e| JsValue::from_str(&format!("Failed to decode image: {e}")))?;
.map_err(|e| Error::from_reason(format!("Failed to decode image: {e}")))?;
Ok(Self { img })
}
/// Get the width of the image.
#[wasm_bindgen(js_name = get_width)]
#[napi(js_name = "get_width")]
pub fn get_width(&self) -> u32 {
self.img.width()
}
/// Get the height of the image.
#[wasm_bindgen(js_name = get_height)]
#[napi(js_name = "get_height")]
pub fn get_height(&self) -> u32 {
self.img.height()
}
/// Export image as PNG bytes.
#[wasm_bindgen(js_name = get_bytes)]
pub fn get_bytes(&self) -> Result<Vec<u8>, JsValue> {
///
/// # Errors
/// Returns an error if PNG encoding fails.
#[napi(js_name = "get_bytes")]
pub fn get_bytes(&self) -> Result<Uint8Array> {
let mut buffer = Vec::new();
self
.img
.write_to(&mut Cursor::new(&mut buffer), ImageFormat::Png)
.map_err(|e| JsValue::from_str(&format!("Failed to encode PNG: {e}")))?;
Ok(buffer)
.map_err(|e| Error::from_reason(format!("Failed to encode PNG: {e}")))?;
Ok(Uint8Array::from(buffer))
}
/// Export image as JPEG bytes with specified quality (0-100).
#[wasm_bindgen(js_name = get_bytes_jpeg)]
pub fn get_bytes_jpeg(&self, quality: u8) -> Result<Vec<u8>, JsValue> {
///
/// # Errors
/// Returns an error if JPEG encoding fails.
#[napi(js_name = "get_bytes_jpeg")]
pub fn get_bytes_jpeg(&self, quality: u8) -> Result<Uint8Array> {
let mut buffer = Vec::new();
let encoder = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut buffer, quality);
self
.img
.write_with_encoder(encoder)
.map_err(|e| JsValue::from_str(&format!("Failed to encode JPEG: {e}")))?;
Ok(buffer)
.map_err(|e| Error::from_reason(format!("Failed to encode JPEG: {e}")))?;
Ok(Uint8Array::from(buffer))
}
/// Export image as lossless WebP bytes.
#[wasm_bindgen(js_name = get_bytes_webp)]
pub fn get_bytes_webp(&self) -> Result<Vec<u8>, JsValue> {
///
/// # Errors
/// Returns an error if WebP encoding fails.
#[napi(js_name = "get_bytes_webp")]
pub fn get_bytes_webp(&self) -> Result<Uint8Array> {
let mut buffer = Vec::new();
let encoder = image::codecs::webp::WebPEncoder::new_lossless(&mut buffer);
self
.img
.write_with_encoder(encoder)
.map_err(|e| JsValue::from_str(&format!("Failed to encode WebP: {e}")))?;
Ok(buffer)
.map_err(|e| Error::from_reason(format!("Failed to encode WebP: {e}")))?;
Ok(Uint8Array::from(buffer))
}
/// Export image as GIF bytes.
#[wasm_bindgen(js_name = get_bytes_gif)]
pub fn get_bytes_gif(&self) -> Result<Vec<u8>, JsValue> {
///
/// # Errors
/// Returns an error if GIF encoding fails.
#[napi(js_name = "get_bytes_gif")]
pub fn get_bytes_gif(&self) -> Result<Uint8Array> {
let mut buffer = Vec::new();
self
.img
.write_to(&mut Cursor::new(&mut buffer), ImageFormat::Gif)
.map_err(|e| JsValue::from_str(&format!("Failed to encode GIF: {e}")))?;
Ok(buffer)
.map_err(|e| Error::from_reason(format!("Failed to encode GIF: {e}")))?;
Ok(Uint8Array::from(buffer))
}
/// Resize the image to the specified dimensions.
#[napi(js_name = "resize")]
pub fn resize(&self, width: u32, height: u32, filter: SamplingFilter) -> PhotonImage {
let resized = self.img.resize_exact(width, height, filter.into());
PhotonImage { img: resized }
}
}
/// Resize an image to the specified dimensions.
#[wasm_bindgen]
pub fn resize(image: &PhotonImage, width: u32, height: u32, filter: SamplingFilter) -> PhotonImage {
let resized = image.img.resize_exact(width, height, filter.into());
PhotonImage { img: resized }
}
crates/pi-natives/src/lib.rs
+18 -454
View File
@@ -1,460 +1,24 @@
//! WASM module for regex matching using ripgrep's engine.
//! Native utilities exported via N-API for the Oh My Pi toolchain.
//!
//! This module provides pure regex matching - no filesystem access.
//! The JS side handles directory walking and file reading, then passes
//! content here for matching.
//! # Overview
//! High-performance primitives for grep, ANSI-aware text measurement, syntax
//! highlighting, HTML-to-Markdown conversion, and image processing.
//!
//! # Example
//! ```ignore
//! use pi_natives::text::visible_width;
//!
//! let width = visible_width("hello");
//! assert_eq!(width, 5);
//! ```
//!
//! # Architecture
//! ```text
//! JS (packages/natives) -> N-API -> Rust modules (grep/html/highlight/image/text)
//! ```
pub mod grep;
pub mod highlight;
pub mod html;
pub mod image;
pub mod text;
use std::io::{self, Cursor};
use grep_matcher::Matcher;
use grep_regex::RegexMatcherBuilder;
use grep_searcher::{
BinaryDetection, Searcher, SearcherBuilder, Sink, SinkContext, SinkContextKind, SinkMatch,
};
use serde::{Deserialize, Serialize};
use wasm_bindgen::prelude::*;
#[cfg(feature = "console_error_panic_hook")]
fn set_panic_hook() {
console_error_panic_hook::set_once();
}
/// Options for searching file content.
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct SearchOptions {
/// Regex pattern to search for.
pub pattern: String,
/// Case-insensitive search.
#[serde(default)]
pub ignore_case: bool,
/// Enable multiline matching.
#[serde(default)]
pub multiline: bool,
/// Maximum number of matches to return.
#[serde(default)]
pub max_count: Option<u64>,
/// Skip first N matches.
#[serde(default)]
pub offset: Option<u64>,
/// Lines of context before/after matches.
#[serde(default)]
pub context: Option<usize>,
/// Truncate lines longer than this (characters).
#[serde(default)]
pub max_columns: Option<usize>,
/// Output mode (content or count).
#[serde(default)]
pub mode: SearchMode,
}
#[derive(Debug, Deserialize, Default, Clone, Copy, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub enum SearchMode {
#[default]
Content,
Count,
}
/// A context line (before or after a match).
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ContextLine {
pub line_number: u64,
pub line: String,
}
/// A single match in the content.
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Match {
/// 1-indexed line number.
pub line_number: u64,
/// The matched line content.
pub line: String,
/// Context lines before the match.
#[serde(skip_serializing_if = "Vec::is_empty")]
pub context_before: Vec<ContextLine>,
/// Context lines after the match.
#[serde(skip_serializing_if = "Vec::is_empty")]
pub context_after: Vec<ContextLine>,
/// Whether the line was truncated.
#[serde(skip_serializing_if = "std::ops::Not::not")]
pub truncated: bool,
}
/// Result of searching content.
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct SearchResult {
/// All matches found.
pub matches: Vec<Match>,
/// Total number of matches (may exceed `matches.len()` due to offset/limit).
pub match_count: u64,
/// Whether the limit was reached.
#[serde(skip_serializing_if = "std::ops::Not::not")]
pub limit_reached: bool,
}
/// Error result.
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ErrorResult {
pub error: String,
}
struct MatchCollector {
matches: Vec<CollectedMatch>,
match_count: u64,
collected_count: u64,
max_count: Option<u64>,
offset: u64,
skipped: u64,
limit_reached: bool,
context_before: Vec<ContextLine>,
max_columns: Option<usize>,
collect_matches: bool,
}
struct CollectedMatch {
line_number: u64,
line: String,
context_before: Vec<ContextLine>,
context_after: Vec<ContextLine>,
truncated: bool,
}
impl MatchCollector {
const fn new(
max_count: Option<u64>,
offset: u64,
max_columns: Option<usize>,
collect_matches: bool,
) -> Self {
Self {
matches: Vec::new(),
match_count: 0,
collected_count: 0,
max_count,
offset,
skipped: 0,
limit_reached: false,
context_before: Vec::new(),
max_columns,
collect_matches,
}
}
fn truncate_line(&self, line: &str) -> (String, bool) {
match self.max_columns {
Some(max) if line.len() > max => {
let truncated = format!("{}...", &line[..max.saturating_sub(3)]);
(truncated, true)
},
_ => (line.to_string(), false),
}
}
}
impl Sink for MatchCollector {
type Error = io::Error;
fn matched(&mut self, _searcher: &Searcher, mat: &SinkMatch<'_>) -> Result<bool, Self::Error> {
self.match_count += 1;
// If we already hit the limit, stop now (after-context for previous match was
// collected)
if self.limit_reached {
return Ok(false);
}
if self.skipped < self.offset {
self.skipped += 1;
self.context_before.clear();
return Ok(true);
}
if self.collect_matches {
let raw_line = String::from_utf8_lossy(mat.bytes()).trim_end().to_string();
let (line, truncated) = self.truncate_line(&raw_line);
let line_number = mat.line_number().unwrap_or(0);
self.matches.push(CollectedMatch {
line_number,
line,
context_before: std::mem::take(&mut self.context_before),
context_after: Vec::new(),
truncated,
});
} else {
self.context_before.clear();
}
self.collected_count += 1;
// Mark limit reached but don't stop yet - allow after-context to be collected
if let Some(max) = self.max_count
&& self.collected_count >= max
{
self.limit_reached = true;
}
Ok(true)
}
fn context(&mut self, _searcher: &Searcher, ctx: &SinkContext<'_>) -> Result<bool, Self::Error> {
if !self.collect_matches {
return Ok(true);
}
let raw_line = String::from_utf8_lossy(ctx.bytes()).trim_end().to_string();
let (line, _) = self.truncate_line(&raw_line);
let line_number = ctx.line_number().unwrap_or(0);
match ctx.kind() {
SinkContextKind::Before => {
self.context_before.push(ContextLine { line_number, line });
},
SinkContextKind::After => {
if let Some(last_match) = self.matches.last_mut() {
last_match
.context_after
.push(ContextLine { line_number, line });
}
},
SinkContextKind::Other => {},
}
Ok(true)
}
}
/// A compiled regex matcher that can be reused across multiple searches.
#[wasm_bindgen]
pub struct CompiledPattern {
matcher: grep_regex::RegexMatcher,
context: usize,
max_columns: Option<usize>,
mode: SearchMode,
}
#[wasm_bindgen]
impl CompiledPattern {
/// Compile a regex pattern for reuse.
#[wasm_bindgen(constructor)]
pub fn new(options: JsValue) -> Result<Self, JsValue> {
#[cfg(feature = "console_error_panic_hook")]
set_panic_hook();
let opts: SearchOptions = serde_wasm_bindgen::from_value(options)
.map_err(|e| JsValue::from_str(&format!("Invalid options: {e}")))?;
let matcher = RegexMatcherBuilder::new()
.case_insensitive(opts.ignore_case)
.multi_line(opts.multiline)
.build(&opts.pattern)
.map_err(|e| JsValue::from_str(&format!("Regex error: {e}")))?;
Ok(Self {
matcher,
context: opts.context.unwrap_or(0),
max_columns: opts.max_columns,
mode: opts.mode,
})
}
/// Search content using this compiled pattern.
/// Returns matches as a JS object.
pub fn search(&self, content: &str, max_count: Option<u32>, offset: Option<u32>) -> JsValue {
let context_lines = self.context;
let mut searcher = SearcherBuilder::new()
.binary_detection(BinaryDetection::quit(b'\x00'))
.line_number(true)
.before_context(context_lines)
.after_context(context_lines)
.build();
let mut collector = MatchCollector::new(
max_count.map(|n| n as u64),
offset.unwrap_or(0) as u64,
self.max_columns,
self.mode == SearchMode::Content,
);
let cursor = Cursor::new(content.as_bytes());
if let Err(e) = searcher.search_reader(&self.matcher, cursor, &mut collector) {
return serde_wasm_bindgen::to_value(&ErrorResult { error: e.to_string() })
.unwrap_or(JsValue::NULL);
}
let result = SearchResult {
matches: collector
.matches
.into_iter()
.map(|m| Match {
line_number: m.line_number,
line: m.line,
context_before: m.context_before,
context_after: m.context_after,
truncated: m.truncated,
})
.collect(),
match_count: collector.match_count,
limit_reached: collector.limit_reached,
};
serde_wasm_bindgen::to_value(&result).unwrap_or(JsValue::NULL)
}
/// Check if content has any matches (faster than full search).
pub fn has_match(&self, content: &str) -> bool {
self.matcher.is_match(content.as_bytes()).unwrap_or(false)
}
/// Search bytes directly (avoids UTF-16 to UTF-8 conversion).
/// Use with `Bun.mmap()` for best performance.
pub fn search_bytes(
&self,
content: &[u8],
max_count: Option<u32>,
offset: Option<u32>,
) -> JsValue {
let context_lines = self.context;
let mut searcher = SearcherBuilder::new()
.binary_detection(BinaryDetection::quit(b'\x00'))
.line_number(true)
.before_context(context_lines)
.after_context(context_lines)
.build();
let mut collector = MatchCollector::new(
max_count.map(|n| n as u64),
offset.unwrap_or(0) as u64,
self.max_columns,
self.mode == SearchMode::Content,
);
let cursor = Cursor::new(content);
if let Err(e) = searcher.search_reader(&self.matcher, cursor, &mut collector) {
return serde_wasm_bindgen::to_value(&ErrorResult { error: e.to_string() })
.unwrap_or(JsValue::NULL);
}
let result = SearchResult {
matches: collector
.matches
.into_iter()
.map(|m| Match {
line_number: m.line_number,
line: m.line,
context_before: m.context_before,
context_after: m.context_after,
truncated: m.truncated,
})
.collect(),
match_count: collector.match_count,
limit_reached: collector.limit_reached,
};
serde_wasm_bindgen::to_value(&result).unwrap_or(JsValue::NULL)
}
/// Check if bytes have any matches (faster than full search).
pub fn has_match_bytes(&self, content: &[u8]) -> bool {
self.matcher.is_match(content).unwrap_or(false)
}
}
/// Search content for a pattern (one-shot, compiles pattern each time).
/// For repeated searches with the same pattern, use [`CompiledPattern`].
#[wasm_bindgen]
pub fn search(content: &str, options: JsValue) -> JsValue {
#[cfg(feature = "console_error_panic_hook")]
set_panic_hook();
let opts: SearchOptions = match serde_wasm_bindgen::from_value(options) {
Ok(o) => o,
Err(e) => {
return serde_wasm_bindgen::to_value(&ErrorResult {
error: format!("Invalid options: {e}"),
})
.unwrap_or(JsValue::NULL);
},
};
let matcher = match RegexMatcherBuilder::new()
.case_insensitive(opts.ignore_case)
.multi_line(opts.multiline)
.build(&opts.pattern)
{
Ok(m) => m,
Err(e) => {
return serde_wasm_bindgen::to_value(&ErrorResult { error: format!("Regex error: {e}") })
.unwrap_or(JsValue::NULL);
},
};
let context_lines = opts.context.unwrap_or(0);
let mut searcher = SearcherBuilder::new()
.binary_detection(BinaryDetection::quit(b'\x00'))
.line_number(true)
.before_context(context_lines)
.after_context(context_lines)
.build();
let mut collector = MatchCollector::new(
opts.max_count,
opts.offset.unwrap_or(0),
opts.max_columns,
opts.mode == SearchMode::Content,
);
let cursor = Cursor::new(content.as_bytes());
if let Err(e) = searcher.search_reader(&matcher, cursor, &mut collector) {
return serde_wasm_bindgen::to_value(&ErrorResult { error: e.to_string() })
.unwrap_or(JsValue::NULL);
}
let result = SearchResult {
matches: collector
.matches
.into_iter()
.map(|m| Match {
line_number: m.line_number,
line: m.line,
context_before: m.context_before,
context_after: m.context_after,
truncated: m.truncated,
})
.collect(),
match_count: collector.match_count,
limit_reached: collector.limit_reached,
};
serde_wasm_bindgen::to_value(&result).unwrap_or(JsValue::NULL)
}
/// Quick check if content matches a pattern.
#[wasm_bindgen]
pub fn has_match(
content: &str,
pattern: &str,
ignore_case: bool,
multiline: bool,
) -> Result<bool, JsValue> {
#[cfg(feature = "console_error_panic_hook")]
set_panic_hook();
let matcher = RegexMatcherBuilder::new()
.case_insensitive(ignore_case)
.multi_line(multiline)
.build(pattern)
.map_err(|e| JsValue::from_str(&format!("Regex error: {e}")))?;
Ok(matcher.is_match(content.as_bytes()).unwrap_or(false))
}
crates/pi-natives/src/text.rs
+64 -39
View File
@@ -1,26 +1,33 @@
//! ANSI-aware text measurement and slicing utilities.
use serde::Serialize;
use napi_derive::napi;
use unicode_segmentation::UnicodeSegmentation;
use unicode_width::UnicodeWidthStr;
use wasm_bindgen::prelude::*;
const TAB_WIDTH: usize = 3;
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
struct SliceResult {
text: String,
width: usize,
/// Result of slicing a line by visible width.
#[napi(object)]
pub struct SliceResult {
/// The sliced text, including any ANSI codes.
pub text: String,
/// Visible width of the slice in columns.
pub width: u32,
}
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
struct ExtractSegmentsResult {
before: String,
before_width: usize,
after: String,
after_width: usize,
/// Before/after segments extracted around an overlay region.
#[napi(object)]
pub struct ExtractSegmentsResult {
/// Text before the overlay region.
pub before: String,
/// Visible width of `before` in columns.
#[napi(js_name = "beforeWidth")]
pub before_width: u32,
/// Text after the overlay region.
pub after: String,
/// Visible width of `after` in columns.
#[napi(js_name = "afterWidth")]
pub after_width: u32,
}
struct AnsiCodeTracker {
@@ -255,9 +262,11 @@ fn grapheme_width(grapheme: &str) -> usize {
UnicodeWidthStr::width(grapheme)
}
/// Compute the visible width of a string, ignoring ANSI codes.
#[wasm_bindgen]
pub fn visible_width(text: &str) -> usize {
fn clamp_u32(value: usize) -> u32 {
value.min(u32::MAX as usize) as u32
}
fn visible_width_impl(text: &str) -> usize {
if text.is_empty() {
return 0;
}
@@ -286,18 +295,25 @@ pub fn visible_width(text: &str) -> usize {
width
}
/// Compute the visible width of a string, ignoring ANSI codes.
#[napi(js_name = "visible_width")]
pub fn visible_width(text: String) -> u32 {
clamp_u32(visible_width_impl(&text))
}
/// Truncate text to a visible width, preserving ANSI codes.
#[wasm_bindgen]
pub fn truncate_to_width(text: &str, max_width: usize, ellipsis: &str, pad: bool) -> String {
let text_visible_width = visible_width(text);
#[napi(js_name = "truncate_to_width")]
pub fn truncate_to_width(text: String, max_width: u32, ellipsis: String, pad: bool) -> String {
let max_width = max_width as usize;
let text_visible_width = visible_width_impl(&text);
if text_visible_width <= max_width {
if pad {
return format!("{}{}", text, " ".repeat(max_width - text_visible_width));
}
return text.to_string();
return text;
}
let ellipsis_width = visible_width(ellipsis);
let ellipsis_width = visible_width_impl(&ellipsis);
let target_width = max_width.saturating_sub(ellipsis_width);
if target_width == 0 {
return ellipsis.graphemes(true).take(max_width).collect();
@@ -306,13 +322,13 @@ pub fn truncate_to_width(text: &str, max_width: usize, ellipsis: &str, pad: bool
let mut segments: Vec<(bool, &str)> = Vec::new();
let mut i = 0;
while i < text.len() {
if let Some(len) = extract_ansi_code(text, i) {
if let Some(len) = extract_ansi_code(&text, i) {
segments.push((true, &text[i..i + len]));
i += len;
continue;
}
let next_ansi = next_ansi_start(text, i);
let next_ansi = next_ansi_start(&text, i);
let end = next_ansi.unwrap_or(text.len());
for grapheme in text[i..end].graphemes(true) {
segments.push((false, grapheme));
@@ -342,7 +358,7 @@ pub fn truncate_to_width(text: &str, max_width: usize, ellipsis: &str, pad: bool
let mut truncated = format!("{result}\x1b[0m{ellipsis}");
if pad {
let truncated_width = visible_width(&truncated);
let truncated_width = visible_width_impl(&truncated);
if truncated_width < max_width {
truncated.push_str(&" ".repeat(max_width - truncated_width));
}
@@ -402,14 +418,13 @@ fn slice_with_width_impl(line: &str, start_col: usize, length: usize, strict: bo
}
}
SliceResult { text: result, width: result_width }
SliceResult { text: result, width: clamp_u32(result_width) }
}
/// Slice a range of visible columns from a line.
#[wasm_bindgen]
pub fn slice_with_width(line: &str, start_col: usize, length: usize, strict: bool) -> JsValue {
let result = slice_with_width_impl(line, start_col, length, strict);
serde_wasm_bindgen::to_value(&result).unwrap_or(JsValue::NULL)
#[napi(js_name = "slice_with_width")]
pub fn slice_with_width(line: String, start_col: u32, length: u32, strict: bool) -> SliceResult {
slice_with_width_impl(&line, start_col as usize, length as usize, strict)
}
fn extract_segments_impl(
@@ -490,18 +505,28 @@ fn extract_segments_impl(
}
}
ExtractSegmentsResult { before, before_width, after, after_width }
ExtractSegmentsResult {
before,
before_width: clamp_u32(before_width),
after,
after_width: clamp_u32(after_width),
}
}
/// Extract the before/after slices around an overlay region.
#[wasm_bindgen]
#[napi(js_name = "extract_segments")]
pub fn extract_segments(
line: &str,
before_end: usize,
after_start: usize,
after_len: usize,
line: String,
before_end: u32,
after_start: u32,
after_len: u32,
strict_after: bool,
) -> JsValue {
let result = extract_segments_impl(line, before_end, after_start, after_len, strict_after);
serde_wasm_bindgen::to_value(&result).unwrap_or(JsValue::NULL)
) -> ExtractSegmentsResult {
extract_segments_impl(
&line,
before_end as usize,
after_start as usize,
after_len as usize,
strict_after,
)
}
package.json
+1 -1
View File
@@ -22,7 +22,7 @@
"fix": "bun run fix:ts && bun run fix:rs",
"fix:ts": "biome check --write --unsafe . && bun --cwd=packages/coding-agent run format-prompts",
"fix:rs": "cargo clippy --fix --allow-dirty --all-targets --no-deps --allow-staged --broken-code --allow-no-vcs && cargo fmt --all",
"build:wasm": "bun --cwd=packages/natives run build:wasm",
"build:native": "bun --cwd=packages/natives run build:native",
"bench:gen-fixtures": "bun run packages/react-edit-benchmark/generate.ts --react-dir /tmp/react-source --count-per-type 3",
"bench:edit": "bun --cwd=packages/react-edit-benchmark run start",
"version:patch": "npm version patch -ws --no-git-tag-version && bun scripts/sync-versions.ts && rm -rf node_modules packages/*/node_modules bun.lock && bun install",
packages/coding-agent/CHANGELOG.md
+11 -1
View File
@@ -1,12 +1,22 @@
# Changelog
## [Unreleased]
### Added
- Added context-structure template showing required sections (Goal, Constraints, Existing Code, API Contract) with examples of good vs bad context
- Added explicit dependency test: 'Can agent B write correct code without seeing agent A's output?' to determine sequencing
- Added detailed phased execution pattern with four phases (Foundation, Parallel Implementation, Integration, Dependent Layer) and WASM-to-N-API migration example
- Added table of dependency patterns that must be sequential (API creation before bindings, interface definition before implementation, etc.)
- Added phased execution guidance for migrations and refactors to prevent parallel work on dependent layers
- Added example demonstrating phased execution pattern for porting WASM to N-API with sequential foundation, parallel implementation, integration, and dependent layer phases
### Changed
- Restructured Task tool documentation to emphasize context quality and explicit API contracts for subagent success
- Updated task execution guidance to require structured context with Goal, Constraints, Existing Code, and API Contract sections
- Reorganized parallelization rules with explicit dependency patterns and phased execution guidance for migrations
- Clarified that response format requirements must go in schema parameter, never in context descriptions
### Removed
- Removed Nushell language server configuration from LSP defaults
packages/coding-agent/src/prompts/tools/task.md
+182 -119
View File
@@ -1,43 +1,194 @@
# Task
Launch a new agent to handle complex, multi-step tasks autonomously. Each agent type has specific capabilities and tools available to it.
Launch agents to handle complex, multi-step tasks autonomously.
<critical>
This matters. Get it right.
## Context is everything
Subagents can access parent conversation context via a file—they can grep or tail it for details you don't include. Don't repeat information unnecessarily; focus `context` on:
- Task-specific constraints and decisions
- Critical requirements that must not be missed
- Information not easily found in the codebase
Subagents fail when context is vague. They cannot read your mind or infer project conventions. Every task needs:
Use a single Task call with multiple `tasks` entries when parallelizing. Multiple concurrent Task calls bypass coordination.
1. **Goal** - What this accomplishes (one sentence)
2. **Constraints** - Hard requirements, banned approaches, naming conventions
3. **Existing Code** - File paths and function signatures to use as patterns
4. **API Contract** - If the task produces or consumes an interface, spell it out
For code changes, have subagents write files directly with Edit/Write. Do not ask them to return patches for you to apply.
Subagents CAN grep the parent conversation file for supplementary details. They CANNOT grep for:
- Decisions you made but didn't write down
- Conventions that exist only in your head
- Which of 50 possible approaches you want
Agents with `output="structured"` enforce their own schema; the `schema` parameter is ignored for those agents.
**Never describe expected output in `context` or task descriptions.** All response format requirements go in the `schema` parameter. Use structured schemas with typed properties—not `{ "type": "string" }`. Prose like "respond as a bullet list" is prohibited.
**Rule of thumb:** If you'd need to answer a clarifying question for a junior dev to do this task, that information belongs in context.
</critical>
<instruction id="phased-execution">
## Phased execution for migrations and refactors
<context-structure>
## Required context structure
When work has layers where one layer must exist before the next can be built:
Use this template. Sections can be omitted only if truly N/A.
1. **Foundation phase** — Create scaffolding, define interfaces, establish the API shape. Do this yourself or in a single task. Never fan out until the contract is known.
2. **Parallel implementation** — Fan out to independent tasks that all consume the same known interface.
3. **Integration phase** — Wire things together, update build/CI. Do this yourself.
4. **Dependent layer** — Fan out again for work that consumes the previous layer.
```
## Goal
[One sentence: what this task accomplishes]
**Wrong:** Launch "create Rust API" + "update JS bindings to use Rust API" + "update CI for Rust" in parallel. JS bindings need to know the API. CI needs to know what's being built.
## Constraints
- [Hard requirements - MUST/MUST NOT style]
- [API conventions, naming patterns, error handling]
- [What already exists vs what to create]
- [Dependencies: allowed, banned, or "ask first"]
**Right:**
1. Create Rust API scaffold yourself (or single task)
2. Fan out: one task per module to implement against the scaffold
3. Update CI yourself
4. Fan out: one task per JS binding now that Rust exports are known
## Existing Code
Reference files the agent MUST read or use as patterns:
- `path/to/file.ts` - [what pattern it demonstrates]
- `path/to/other.rs` - [what to reuse from it]
The test: Can subagent B write correct code without seeing A's output? If no, they are sequential.
</instruction>
## API Contract (if applicable)
```language
// Exact signatures the agent must implement or consume
fn example(input: Type) -> Result<Output>
```
## Task
{{description}}
## Files
{{files}}
```
### Bad context (agent will fail or guess wrong)
```
N-API migration. Keep highlight sync. Use JsString. No WASM.
Task: {{description}} Files: {{files}}
```
Why it fails:
- No existing code to reference - agent doesn't know your patterns
- No API contract - agent will invent signatures that don't match consumers
- No goal - agent doesn't know what success looks like
- "Keep highlight sync" is meaningless without knowing what highlight is or where it lives
### Good context (agent can act confidently)
```
## Goal
Port grep module from WASM to N-API, matching existing text module patterns.
## Constraints
- Use `#[napi]` attribute macro on all exports (not `#[napi(js_name = "...")]`)
- Return `napi::Result<T>` for fallible ops, never panic
- Use `spawn_blocking` for any operation that touches filesystem or runs >1ms
- Accept `JsString` for string params (NOT JsStringUtf8 - it has lifetime issues)
- Keep all existing function names - TS bindings depend on them
- No new crate dependencies
## Existing Code
- `crates/pi-natives/src/text.rs` - reference N-API pattern: see how `visible_width` uses JsString
- `crates/pi-natives/src/lib.rs` - module registration pattern
- `crates/pi-natives/Cargo.toml` - available dependencies (ignore, regex already present)
## API Contract
Current sync API to convert to async:
```rust
// BEFORE (sync, blocks event loop)
#[napi]
pub fn search(pattern: String, path: String) -> Vec<Match>
// AFTER (async, uses spawn_blocking)
#[napi]
pub async fn search(pattern: JsString, path: JsString, env: Env) -> napi::Result<Vec<Match>>
```
## Task
{{description}}
## Files
{{files}}
```
</context-structure>
<parallelization>
## When to parallelize vs sequence
**The test:** Can agent B write correct code without seeing agent A's output?
- If YES → parallelize
- If NO → sequence (A completes, then B runs with A's output in context)
### Dependency patterns that MUST be sequential
| First | Then | Why |
|-------|------|-----|
| Create Rust API | Update TS bindings | Bindings need to know export names and signatures |
| Define interface/types | Implement consumers | Consumers need the contract |
| Scaffold with signatures | Implement bodies | Implementations need the shape |
| Core module | Dependent modules | Dependents import from core |
### Safe to parallelize
- Independent modules that don't import each other
- Tests for already-implemented code
- Documentation for stable APIs
- Refactors in isolated file scopes
### Phased execution pattern
For migrations/refactors with layers:
**Phase 1 - Foundation (do yourself or single task):**
Create the scaffold, define interfaces, establish API shape. Never fan out until the contract is known.
**Phase 2 - Parallel implementation:**
Fan out to independent tasks that all consume the same known interface. Include the API contract from Phase 1 in every task's context.
**Phase 3 - Integration (do yourself):**
Wire things together, update build/CI, fix any mismatches.
**Phase 4 - Dependent layer:**
Fan out again for work that consumes Phase 2 outputs.
### Example: WASM to N-API migration
**WRONG** (launched together, will fail):
```
tasks: [
{ id: "RustApi", description: "Implement N-API exports" },
{ id: "TsBindings", description: "Update TS to use N-API" }, // ← needs RustApi output!
]
```
**RIGHT** (phased):
```
// Phase 1: You create scaffold with signatures in lib.rs
// Phase 2: Fan out Rust implementation
tasks: [
{ id: "Grep", description: "Implement grep module", args: { files: "src/grep.rs" } },
{ id: "Text", description: "Implement text module", args: { files: "src/text.rs" } },
// Each task gets the API contract you defined in Phase 1
]
// Phase 3: You verify Rust compiles, exports are correct
// Phase 4: Fan out TS bindings (now they know what Rust exports)
tasks: [
{ id: "GrepBindings", description: "Update grep TS", args: { files: "src/grep/index.ts" } },
// Context includes actual export names from Phase 2
]
```
</parallelization>
<parameters>
- `agent`: Agent type for all tasks
- `context`: Template with `{{placeholders}}`. **Must follow the structure above.** Include Goal, Constraints, Existing Code references. Subagents can search parent context for background, but core requirements must be explicit here.
- `isolated`: (optional) Run in git worktree, return patches
- `tasks`: Array of `{id, description, args}`
- `id`: CamelCase identifier (max 32 chars)
- `description`: What the task does (for logging)
- `args`: Object with keys matching `{{placeholders}}` in context
- `skills`: (optional) Skill names to preload
- `schema`: JTD schema for response structure. **Required.** Use typed properties, not `{ "type": "string" }`.
**Schema goes in `schema` parameter. Never describe output format in `context`.**
</parameters>
<agents>
{{#list agents join="\n"}}
@@ -48,99 +199,11 @@ The test: Can subagent B write correct code without seeing A's output? If no, th
{{/list}}
</agents>
<instruction>
This matters. Be thorough.
1. Plan before acting. Define the goal, acceptance criteria, and scope per task.
2. Put shared constraints and decisions in `context`; keep each task request short and unambiguous. **Do not describe response format here.**
3. State whether each task is research-only or should modify files.
4. **Always provide a `schema`** with typed properties. Avoid `{ "type": "string" }`—if data has any structure (list, fields, categories), model it. Plain text is almost never the right choice.
5. Assign distinct file scopes per task to avoid conflicts.
6. Trust the returned data, then verify with tools when correctness matters.
7. For critical constraints, be explicit in `context`. For general background, subagents can search the parent context file themselves.
</instruction>
<parameters>
- `agent`: Agent type to use for all tasks
- `context`: Template with `\{{placeholders}}` for multi-task. Include critical constraints and task-specific decisions. Subagents have access to parent conversation context via a searchable file, so don't repeat everything—focus on what matters. `\{{id}}` and `\{{description}}` are always available.
- `isolated`: (optional) Run each task in its own git worktree and return patches; patches are applied only if all apply cleanly.
- `tasks`: Array of `{id, description, args}` - tasks to run in parallel
- `id`: Short CamelCase identifier (max 32 chars, e.g., "SessionStore", "LspRefactor")
- `description`: Short human-readable description of what the task does
- `args`: Object with keys matching `\{{placeholders}}` in context (always include this, even if empty)
- `skills`: (optional) Array of skill names to preload into this task's system prompt. When set, the skills index section is omitted and the full SKILL.md contents are embedded.
- `schema`: JTD schema defining expected response structure. **Required.** Use objects with typed properties—e.g., `{ "properties": { "items": { "elements": { "type": "string" } } } }` for lists.
</parameters>
<output>
Returns task results for each spawned agent:
- Truncated preview of agent output (use `read agent://<id>` for full content if truncated)
- Summary with line/character counts
- For agents with `schema`: structured JSON accessible via `agent://<id>?q=<query>` or `agent://<id>/<path>`
Results are keyed by task `id` (e.g., "AuthProvider", "AuthApi").
</output>
<example>
user: "Looks good, execute the plan"
assistant: I'll execute the refactoring plan.
assistant: Uses the Task tool:
{
"agent": "task",
"context": "Refactoring the auth module into separate concerns.\n\nPlan:\n1. AuthProvider - Extract React context and provider from src/auth/index.tsx\n2. AuthApi - Extract API calls to src/auth/api.ts, use existing fetchJson helper\n3. AuthTypes - Move types to types.ts, re-export from index\n\nConstraints:\n- Preserve all existing exports from src/auth/index.tsx\n- Use project's fetchJson (src/utils/http.ts), don't use raw fetch\n- No new dependencies\n\nTask: \{{step}}\n\nFiles: \{{files}}",
"schema": {
"properties": {
"summary": { "type": "string" },
"decisions": { "elements": { "type": "string" } },
"concerns": { "elements": { "type": "string" } }
}
},
"tasks": [
{ "id": "AuthProvider", "description": "Extract React context", "args": { "step": "Execute step 1: Extract AuthProvider and AuthContext", "files": "src/auth/index.tsx" } },
{ "id": "AuthApi", "description": "Extract API layer", "args": { "step": "Execute step 2: Extract API calls to api.ts", "files": "src/auth/api.ts" } },
{ "id": "AuthTypes", "description": "Extract types", "args": { "step": "Execute step 3: Move types to types.ts", "files": "src/auth/types.ts" } }
]
}
</example>
<example name="phased-migration">
user: "Port the natives package from WASM to N-API"
assistant: This requires phased execution—JS bindings depend on knowing the Rust API.
Phase 1: I'll create the N-API scaffold myself to establish the API shape.
assistant: [Uses Edit tool to create Cargo.toml with napi-rs, writes lib.rs with module structure and exported function signatures]
Phase 2: Now that the API contract is defined, I'll fan out to implement each module.
assistant: Uses the Task tool:
{
"agent": "task",
"context": "Implementing N-API native module. Scaffold exists at crates/pi-natives with napi-rs setup.\n\nAPI contract (from lib.rs):\n- grep: async fn search(pattern: &str, path: &str, opts: SearchOptions) -> Vec<Match>\n- text: fn visible_width(s: &str) -> usize, fn truncate(s: &str, width: usize) -> String\n- html: fn to_markdown(html: &str) -> String\n- image: fn resize(data: &[u8], width: u32, height: u32) -> Vec<u8>\n\nTask: Implement {{module}} module. File: crates/pi-natives/src/{{module}}.rs",
"schema": { "properties": { "exports": { "elements": { "type": "string" } }, "notes": { "type": "string" } } },
"tasks": [
{ "id": "Grep", "description": "Implement parallel grep with ignore crate", "args": { "module": "grep" } },
{ "id": "Text", "description": "Implement text width/truncate", "args": { "module": "text" } },
{ "id": "Html", "description": "Implement HTML to markdown", "args": { "module": "html" } },
{ "id": "Image", "description": "Implement image resize", "args": { "module": "image" } }
]
}
Phase 3: Rust implementation complete. I'll update CI for native builds myself.
assistant: [Uses Edit tool to update .github/workflows/ci.yml with cross-platform native build matrix]
Phase 4: Now I'll fan out to update JS bindings—the Rust exports are known.
assistant: Uses the Task tool:
{
"agent": "task",
"context": "Update JS bindings to load N-API addon instead of WASM workers.\n\nRust exports (from Phase 2):\n{{exports}}\n\nTask: Update packages/natives/src/{{module}}/index.ts to use native addon. Remove worker.ts usage.",
...
}
</example>
<avoid>
- Describing response format in `context` (e.g., "respond as JSON", "return a bullet list")—use `schema` parameter instead
- Confirmation bias: ask for factual discovery instead of yes/no exploration prompts
- Reading a specific file path → Use Read tool instead
- Finding files by pattern/name → Use Find tool instead
- Searching for a specific class/function definition → Use Grep tool instead
- Searching code within 2-3 specific files → Use Read tool instead
- Tasks unrelated to the agent descriptions above
- Terse context that requires agents to guess conventions
- Launching dependent tasks in parallel (bindings + API, consumer + producer)
- Missing "Existing Code" references - agents need patterns to follow
- Assuming agents know your codebase - they start fresh each time
- Describing output format in context instead of schema
- Single tasks doing too much - prefer focused, file-scoped tasks
</avoid>
packages/natives/README.md
+10 -13
View File
@@ -1,12 +1,12 @@
# @oh-my-pi/pi-natives
Native Rust functionality compiled to WebAssembly via wasm-bindgen.
Native Rust functionality via N-API.
## What's Inside
- **Grep**: Regex-based search powered by ripgrep's engine (WASM handles matching, JS handles I/O + gitignore-aware file walking)
- **Grep**: Regex-based search powered by ripgrep's engine with native file walking and matching
- **Find**: Glob-based file/directory discovery with gitignore support (pure TypeScript via `globPaths`)
- **Image**: Image processing via photon-rs (resize, format conversion)
- **Image**: Image processing via photon-rs (resize, format conversion) exposed through N-API
## Usage
@@ -37,8 +37,8 @@ const pngBytes = await resized.get_bytes();
## Building
```bash
# Build WASM from workspace root (requires Rust + wasm-pack)
bun run build:wasm
# Build native addon from workspace root (requires Rust)
bun run build:native
# Type check
bun run check
@@ -48,16 +48,13 @@ bun run check
```
crates/pi-natives/ # Rust source (workspace member)
src/lib.rs # Grep/search + wasm-bindgen bindings
src/lib.rs # N-API exports
src/image.rs # Image processing (photon-rs)
Cargo.toml # Rust dependencies
wasm/ # Generated WASM output
pi_natives.wasm # Compiled WASM module
pi_natives.js # wasm-bindgen generated JS glue
pi_natives.d.ts # TypeScript definitions
native/ # Native addon binaries
pi_natives.<platform>-<arch>.node
pi_natives.node
src/ # TypeScript wrappers
native.ts # Native addon loader
index.ts # Public API
grep/ # Grep with worker pool
image/ # Async image processing via worker
pool.ts # Generic worker pool infrastructure
```
packages/natives/package.json
+4 -4
View File
@@ -1,7 +1,7 @@
{
"name": "@oh-my-pi/pi-natives",
"version": "9.4.0",
"description": "Native Rust functionality compiled to WebAssembly via wasm-bindgen",
"description": "Native Rust functionality via N-API",
"type": "module",
"main": "./src/index.ts",
"types": "./src/index.ts",
@@ -13,13 +13,13 @@
},
"files": [
"src",
"wasm"
"native"
],
"scripts": {
"build:wasm": "bun scripts/build-wasm.ts",
"build:native": "bun scripts/build-native.ts",
"check": "biome check . && tsgo -p tsconfig.json",
"fix": "biome check --write --unsafe .",
"test": "bun test",
"test": "bun run build:native && bun test",
"bench": "bun bench/grep.ts"
},
"author": "Can Bölük",
packages/natives/scripts/build-native.ts
+75
View File
@@ -0,0 +1,75 @@
import { $ } from "bun";
import * as fs from "node:fs/promises";
import * as path from "node:path";
const repoRoot = path.join(import.meta.dir, "../../..");
const rustDir = path.join(repoRoot, "crates/pi-natives");
const targetRoots = [
process.env.CARGO_TARGET_DIR ? path.resolve(process.env.CARGO_TARGET_DIR) : undefined,
path.join(repoRoot, "target"),
path.join(rustDir, "target"),
].filter((value): value is string => Boolean(value));
const releaseDirs = targetRoots.map(root => path.join(root, "release"));
const nativeDir = path.join(import.meta.dir, "../native");
const platform = process.platform;
const arch = process.arch;
const buildResult = await $`cargo build --release`.cwd(rustDir).nothrow();
if (buildResult.exitCode !== 0) {
const stderrText =
typeof buildResult.stderr === "string"
? buildResult.stderr
: buildResult.stderr?.length
? new TextDecoder().decode(buildResult.stderr)
: "";
throw new Error(
`cargo build --release failed${stderrText ? `:\n${stderrText}` : ""}`,
);
}
const candidateNames = [
"pi_natives.node",
"libpi_natives.so",
"libpi_natives.dylib",
"pi_natives.dll",
"libpi_natives.dll",
];
let sourcePath: string | null = null;
for (const releaseDir of releaseDirs) {
for (const candidate of candidateNames) {
const fullPath = path.join(releaseDir, candidate);
try {
await fs.stat(fullPath);
sourcePath = fullPath;
break;
} catch (err) {
if (err && typeof err === "object" && "code" in err) {
const code = (err as { code?: string }).code;
if (code === "ENOENT") {
continue;
}
}
throw err;
}
}
if (sourcePath) break;
}
if (!sourcePath) {
const locations = releaseDirs.map(dir => `- ${dir}`).join("\n");
throw new Error(`Built library not found. Checked:\n${locations}`);
}
await fs.mkdir(nativeDir, { recursive: true });
const taggedPath = path.join(nativeDir, `pi_natives.${platform}-${arch}.node`);
const fallbackPath = path.join(nativeDir, "pi_natives.node");
const devPath = path.join(path.dirname(sourcePath), "pi_natives.node");
await fs.copyFile(sourcePath, taggedPath);
await fs.copyFile(sourcePath, fallbackPath);
if (sourcePath !== devPath) {
await fs.copyFile(sourcePath, devPath);
}
packages/natives/scripts/build-wasm.ts
-68
View File
@@ -1,68 +0,0 @@
import { $ } from "bun";
import * as path from "node:path";
import * as fs from "node:fs/promises";
const repoRoot = path.join(import.meta.dir, "../../..");
const rustDir = path.join(repoRoot, "crates/pi-natives");
const wasmDir = path.join(import.meta.dir, "../wasm");
await fs.mkdir(wasmDir, { recursive: true });
console.log("Building Rust WASM with wasm-bindgen...");
// Build with wasm-pack (wasm-opt disabled in Cargo.toml)
const buildResult = await $`wasm-pack build --target deno --release --out-dir pkg`
.cwd(rustDir)
.nothrow();
if (buildResult.exitCode !== 0) {
console.error("wasm-pack build failed");
console.error(buildResult.stderr.toString());
process.exit(1);
}
const pkgDir = path.join(rustDir, "pkg");
// Crate name is pi-natives -> output files are pi_natives_*
const wasmSrc = path.join(pkgDir, "pi_natives_bg.wasm");
const wasmDst = path.join(wasmDir, "pi_natives_bg.wasm");
// Optimize with wasm-opt with bulk memory enabled
const optResult = await $`wasm-opt \
--enable-bulk-memory \
--enable-nontrapping-float-to-int \
-O3 \
--strip-debug \
${wasmSrc} \
-o ${wasmDst}`.nothrow();
if (optResult.exitCode !== 0) {
console.log("wasm-opt failed or not available, copying unoptimized build");
await fs.copyFile(wasmSrc, wasmDst);
}
// Copy and patch JS bindings for Bun compiled binary compatibility
// wasm-bindgen uses `new URL('file.wasm', import.meta.url)` which doesn't work
// in Bun's single-file executable. We use asset import + import.meta.resolve() instead.
const jsDst = path.join(wasmDir, "pi_natives.js");
await fs.copyFile(path.join(pkgDir, "pi_natives.js"), jsDst);
let jsContent = await Bun.file(jsDst).text();
jsContent = jsContent.replace(
"/* @ts-self-types=",
'import wasmPath from "./pi_natives_bg.wasm";\n\n/* @ts-self-types='
);
jsContent = jsContent.replace(
"const wasmUrl = new URL('pi_natives_bg.wasm', import.meta.url);",
"const wasmUrl = import.meta.resolve(wasmPath);"
);
await Bun.write(jsDst, jsContent);
// Copy d.ts bindings
await fs.copyFile(path.join(pkgDir, "pi_natives.d.ts"), path.join(wasmDir, "pi_natives.d.ts"));
await fs.copyFile(path.join(pkgDir, "pi_natives_bg.wasm.d.ts"), path.join(wasmDir, "pi_natives_bg.wasm.d.ts"));
const stat = await fs.stat(wasmDst);
console.log(`Done: ${wasmDst} (${(stat.size / 1024).toFixed(0)}KB)`);
// Clean up pkg directory
await fs.rm(pkgDir, { recursive: true, force: true });
packages/natives/src/grep/file-reader.ts
-51
View File
@@ -1,51 +0,0 @@
/**
* Cross-platform file reader for grep.
*
* Uses mmap for files <= 4MB on platforms that support it,
* falls back to reading into a reusable buffer otherwise.
*/
import * as fs from "node:fs/promises";
const MAX_MMAP_SIZE = 4 * 1024 * 1024; // 4MB
export class FileReader {
#buffer: Buffer | null = null;
constructor(private readonly maxSize: number = MAX_MMAP_SIZE) {}
#getBuffer(size: number): Buffer {
if (!this.#buffer) {
this.#buffer = Buffer.allocUnsafe(this.maxSize);
}
return this.#buffer.subarray(0, size);
}
async read(filePath: string): Promise<Uint8Array | null> {
let fileSize: number;
try {
const stat = await fs.stat(filePath);
fileSize = stat.size;
} catch {
return null;
}
// Skip files larger than buffer size (only search first 4MB worth)
const readSize = Math.min(fileSize, this.maxSize);
// Try mmap for small files (fast path on Linux/macOS)
if (fileSize <= this.maxSize) {
try {
return Bun.mmap(filePath);
} catch {
// mmap not supported (Windows) or failed, fall through to read
}
}
// Fall back to reading into buffer
try {
await using handle = await fs.open(filePath, "r");
const buffer = this.#getBuffer(readSize);
const { bytesRead } = await handle.read(buffer, 0, readSize, 0);
return buffer.subarray(0, bytesRead);
} catch {
return null;
}
}
}
packages/natives/src/grep/filters.ts
-77
View File
@@ -1,77 +0,0 @@
import * as path from "node:path";
export interface TypeFilter {
extensions?: string[];
names?: string[];
}
const TYPE_ALIASES: Record<string, TypeFilter> = {
js: { extensions: ["js", "jsx", "mjs", "cjs"] },
javascript: { extensions: ["js", "jsx", "mjs", "cjs"] },
ts: { extensions: ["ts", "tsx", "mts", "cts"] },
typescript: { extensions: ["ts", "tsx", "mts", "cts"] },
json: { extensions: ["json", "jsonc", "json5"] },
yaml: { extensions: ["yaml", "yml"] },
yml: { extensions: ["yaml", "yml"] },
toml: { extensions: ["toml"] },
md: { extensions: ["md", "markdown", "mdx"] },
markdown: { extensions: ["md", "markdown", "mdx"] },
py: { extensions: ["py", "pyi"] },
python: { extensions: ["py", "pyi"] },
rs: { extensions: ["rs"] },
rust: { extensions: ["rs"] },
go: { extensions: ["go"] },
java: { extensions: ["java"] },
kt: { extensions: ["kt", "kts"] },
kotlin: { extensions: ["kt", "kts"] },
c: { extensions: ["c", "h"] },
cpp: { extensions: ["cpp", "cc", "cxx", "hpp", "hxx", "hh"] },
cxx: { extensions: ["cpp", "cc", "cxx", "hpp", "hxx", "hh"] },
cs: { extensions: ["cs", "csx"] },
csharp: { extensions: ["cs", "csx"] },
php: { extensions: ["php", "phtml"] },
rb: { extensions: ["rb", "rake", "gemspec"] },
ruby: { extensions: ["rb", "rake", "gemspec"] },
sh: { extensions: ["sh", "bash", "zsh", "fish"] },
bash: { extensions: ["sh", "bash", "zsh"] },
zsh: { extensions: ["zsh"] },
fish: { extensions: ["fish"] },
html: { extensions: ["html", "htm"] },
css: { extensions: ["css"] },
scss: { extensions: ["scss"] },
sass: { extensions: ["sass"] },
less: { extensions: ["less"] },
xml: { extensions: ["xml"] },
docker: { names: ["dockerfile"] },
dockerfile: { names: ["dockerfile"] },
make: { names: ["makefile"] },
makefile: { names: ["makefile"] },
};
export function buildGlobPattern(glob?: string): string {
const trimmed = glob?.trim();
if (!trimmed) return "**/*";
const normalized = trimmed.replace(/\\/g, "/");
if (normalized.includes("/") || normalized.startsWith("**/")) return normalized;
return `**/${normalized}`;
}
export function resolveTypeFilter(type?: string): TypeFilter | undefined {
if (!type) return undefined;
const trimmed = type.trim();
if (!trimmed) return undefined;
const normalized = trimmed.toLowerCase();
const withoutDot = normalized.startsWith(".") ? normalized.slice(1) : normalized;
return TYPE_ALIASES[withoutDot] ?? { extensions: [withoutDot] };
}
export function matchesTypeFilter(filePath: string, filter?: TypeFilter): boolean {
if (!filter) return true;
const baseName = path.basename(filePath).toLowerCase();
if (filter.names?.some(name => name.toLowerCase() === baseName)) {
return true;
}
const ext = path.extname(baseName).slice(1).toLowerCase();
if (!ext) return false;
return filter.extensions?.includes(ext) ?? false;
}
packages/natives/src/grep/index.ts
+30 -291
View File
@@ -1,266 +1,56 @@
/**
* Native ripgrep wrapper using wasm-bindgen.
*
* JS handles filesystem operations (directory walking, file reading).
* WASM handles pure regex matching using ripgrep's engine.
* Native ripgrep wrapper using N-API.
*/
import * as fs from "node:fs/promises";
import * as path from "node:path";
import { globPaths } from "@oh-my-pi/pi-utils";
import {
CompiledPattern as WasmCompiledPattern,
has_match as wasmHasMatch,
search as wasmSearch,
} from "../../wasm/pi_natives";
import { WorkerPool } from "../pool";
import { resolveWorkerSpecifier } from "../worker-resolver";
import { FileReader } from "./file-reader";
import { buildGlobPattern, matchesTypeFilter, resolveTypeFilter } from "./filters";
import { native } from "../native";
import type {
ContextLine,
GrepMatch,
GrepOptions,
GrepResult,
GrepSummary,
WasmSearchResult,
WorkerRequest,
WorkerResponse,
SearchOptions,
SearchResult,
} from "./types";
export type { ContextLine, GrepMatch, GrepOptions, GrepResult, GrepSummary };
// =============================================================================
// File Walking
// =============================================================================
function filterUndefined<T extends Record<string, unknown>>(obj: T): T {
const result = {} as T;
for (const [key, value] of Object.entries(obj)) {
if (value !== undefined) {
(result as Record<string, unknown>)[key] = value;
}
function notifyMatches(matches: GrepMatch[], onMatch?: (match: GrepMatch) => void): void {
if (!onMatch) return;
for (const match of matches) {
onMatch(match);
}
}
/**
* Search files for a regex pattern.
*/
export async function grep(options: GrepOptions, onMatch?: (match: GrepMatch) => void): Promise<GrepResult> {
const result = native.grep(options);
notifyMatches(result.matches, onMatch);
return result;
}
// =============================================================================
// Grep Implementation
// =============================================================================
const GREP_WORKERS = (() => {
const val = process.env.OMP_GREP_WORKERS;
if (val === undefined) return true;
const n = Number.parseInt(val, 10);
return Number.isNaN(n) || n > 0;
})();
/**
* Search files for a regex pattern (direct, single-threaded).
* Search files for a regex pattern (single-threaded).
*/
async function grepDirect(options: GrepOptions, onMatch?: (match: GrepMatch) => void): Promise<GrepResult> {
const searchPath = path.resolve(options.path);
const outputMode = options.mode ?? "content";
const wasmMode = outputMode === "content" ? "content" : "count";
const stat = await fs.stat(searchPath);
const isFile = stat.isFile();
using compiledPattern = new WasmCompiledPattern(
filterUndefined({
pattern: options.pattern,
ignoreCase: options.ignoreCase,
multiline: options.multiline,
context: options.context,
maxColumns: options.maxColumns,
mode: wasmMode,
}),
);
const typeFilter = resolveTypeFilter(options.type);
const globPattern = buildGlobPattern(options.glob);
const matches: GrepMatch[] = [];
let totalMatches = 0;
let filesWithMatches = 0;
let filesSearched = 0;
let limitReached = false;
const maxCount = options.maxCount;
const globalOffset = options.offset ?? 0;
if (isFile) {
if (typeFilter && !matchesTypeFilter(searchPath, typeFilter)) {
return {
matches,
totalMatches,
filesWithMatches,
filesSearched,
limitReached: limitReached || undefined,
};
}
const fileReader = new FileReader();
const content = await fileReader.read(searchPath);
if (!content) {
return {
matches,
totalMatches,
filesWithMatches,
filesSearched,
limitReached: limitReached || undefined,
};
}
filesSearched = 1;
const result = compiledPattern.search_bytes(
content,
maxCount,
globalOffset > 0 ? globalOffset : undefined,
) as WasmSearchResult;
if (result.error) {
throw new Error(result.error);
}
if (result.matchCount > 0) {
filesWithMatches = 1;
totalMatches = result.matchCount;
if (outputMode === "content") {
for (const m of result.matches) {
const match: GrepMatch = {
path: searchPath,
lineNumber: m.lineNumber,
line: m.line,
contextBefore: m.contextBefore?.length ? m.contextBefore : undefined,
contextAfter: m.contextAfter?.length ? m.contextAfter : undefined,
truncated: m.truncated || undefined,
};
matches.push(match);
onMatch?.(match);
}
} else {
const match: GrepMatch = {
path: searchPath,
lineNumber: 0,
line: "",
matchCount: result.matchCount,
};
matches.push(match);
onMatch?.(match);
}
limitReached = result.limitReached || (maxCount !== undefined && totalMatches >= maxCount);
}
} else {
const paths = await globPaths(globPattern, {
cwd: searchPath,
dot: options.hidden ?? true,
onlyFiles: true,
gitignore: true,
});
const fileReader = new FileReader();
for (const relativePath of paths) {
if (limitReached) break;
if (typeFilter && !matchesTypeFilter(relativePath, typeFilter)) {
continue;
}
const normalizedPath = relativePath.replace(/\\/g, "/");
const fullPath = path.join(searchPath, normalizedPath);
const content = await fileReader.read(fullPath);
if (!content) continue;
filesSearched++;
if (!compiledPattern.has_match_bytes(content)) {
continue;
}
const fileOffset = globalOffset > 0 ? Math.max(globalOffset - totalMatches, 0) : 0;
const remaining = maxCount !== undefined ? Math.max(maxCount - totalMatches, 0) : undefined;
if (remaining === 0) {
limitReached = true;
break;
}
const result = compiledPattern.search_bytes(
content,
remaining,
fileOffset > 0 ? fileOffset : undefined,
) as WasmSearchResult;
if (result.error) {
continue;
}
if (result.matchCount > 0) {
filesWithMatches++;
totalMatches += result.matchCount;
if (outputMode === "content") {
for (const m of result.matches) {
const match: GrepMatch = {
path: normalizedPath,
lineNumber: m.lineNumber,
line: m.line,
contextBefore: m.contextBefore?.length ? m.contextBefore : undefined,
contextAfter: m.contextAfter?.length ? m.contextAfter : undefined,
truncated: m.truncated || undefined,
};
matches.push(match);
onMatch?.(match);
}
} else {
const match: GrepMatch = {
path: normalizedPath,
lineNumber: 0,
line: "",
matchCount: result.matchCount,
};
matches.push(match);
onMatch?.(match);
}
if (result.limitReached || (maxCount !== undefined && totalMatches >= maxCount)) {
limitReached = true;
}
}
}
}
return {
matches,
totalMatches,
filesWithMatches,
filesSearched,
limitReached: limitReached || undefined,
};
export async function grepDirect(options: GrepOptions, onMatch?: (match: GrepMatch) => void): Promise<GrepResult> {
return await grep(options, onMatch);
}
// =============================================================================
// Content Search (lower-level API)
// =============================================================================
/**
* Search files for a regex pattern (compatibility alias).
*/
export async function grepPool(options: GrepOptions): Promise<GrepResult> {
return await grep(options);
}
/**
* Search a single file's content for a pattern.
* Lower-level API for when you already have file content.
*/
export function searchContent(
content: string,
options: {
pattern: string;
ignoreCase?: boolean;
multiline?: boolean;
maxCount?: number;
offset?: number;
context?: number;
maxColumns?: number;
mode?: "content" | "count";
},
): WasmSearchResult {
return wasmSearch(content, filterUndefined(options)) as WasmSearchResult;
export function searchContent(content: string, options: SearchOptions): SearchResult {
return native.search(content, options);
}
/**
@@ -271,59 +61,8 @@ export function hasMatch(
pattern: string,
options?: { ignoreCase?: boolean; multiline?: boolean },
): boolean {
return wasmHasMatch(content, pattern, options?.ignoreCase ?? false, options?.multiline ?? false);
return native.hasMatch(content, pattern, options?.ignoreCase ?? false, options?.multiline ?? false);
}
// =============================================================================
// Public API
// =============================================================================
/**
* Search files for a regex pattern.
*
* Uses worker pool by default. Set `OMP_GREP_WORKERS=0` to disable.
*/
export async function grep(options: GrepOptions, onMatch?: (match: GrepMatch) => void): Promise<GrepResult> {
if (GREP_WORKERS) {
return await grepPoolInternal(options);
}
return await grepDirect(options, onMatch);
}
/**
* Search files using worker pool (always, ignores OMP_GREP_WORKERS).
*/
export async function grepPool(options: GrepOptions): Promise<GrepResult> {
return await grepPoolInternal(options);
}
// =============================================================================
// Worker Pool
// =============================================================================
const pool = new WorkerPool<WorkerRequest, WorkerResponse>({
createWorker: () =>
new Worker(
resolveWorkerSpecifier({
compiled: "./packages/natives/src/grep/worker.ts",
dev: new URL("./worker.ts", import.meta.url),
}),
),
maxWorkers: 4,
idleTimeoutMs: 30_000,
});
async function grepPoolInternal(request: GrepOptions): Promise<GrepResult> {
const response = await pool.request<Extract<WorkerResponse, { type: "result" }>>({
type: "grep",
request,
});
return response.result;
}
/** Terminate all grep workers. */
export function terminate(): void {
pool.terminate();
}
export { grepDirect };
/** Terminate grep resources (no-op for native bindings). */
export function terminate(): void {}
packages/natives/src/grep/types.ts
+29 -21
View File
@@ -52,31 +52,39 @@ export interface GrepResult extends GrepSummary {
matches: GrepMatch[];
}
/** WASM match result from the compiled pattern. */
export interface WasmMatch {
lineNumber: number;
line: string;
contextBefore: ContextLine[];
contextAfter: ContextLine[];
truncated: boolean;
export interface SearchOptions {
/** Regex pattern to search for */
pattern: string;
/** Case-insensitive search */
ignoreCase?: boolean;
/** Enable multiline matching */
multiline?: boolean;
/** Maximum number of matches to return */
maxCount?: number;
/** Skip first N matches */
offset?: number;
/** Lines of context before/after matches */
context?: number;
/** Truncate lines longer than this (characters) */
maxColumns?: number;
/** Output mode */
mode?: "content" | "count";
}
/** WASM search result. */
export interface WasmSearchResult {
matches: WasmMatch[];
export interface SearchMatch {
lineNumber: number;
line: string;
contextBefore?: ContextLine[];
contextAfter?: ContextLine[];
truncated?: boolean;
}
export interface SearchResult {
matches: SearchMatch[];
matchCount: number;
limitReached: boolean;
error?: string;
}
/** Message types from main thread to worker. */
export type WorkerRequest =
| { type: "init"; id: number }
| { type: "grep"; id: number; request: GrepOptions }
| { type: "destroy" };
/** Message types from worker to main thread. */
export type WorkerResponse =
| { type: "ready"; id: number }
| { type: "result"; id: number; result: GrepResult }
| { type: "error"; id: number; error: string };
export type WasmMatch = SearchMatch;
export type WasmSearchResult = SearchResult;
packages/natives/src/grep/worker.ts
-212
View File
@@ -1,212 +0,0 @@
/**
* Worker script for running wasm-bindgen grep.
* Each worker loads its own WASM instance and processes requests.
*/
import * as fs from "node:fs/promises";
import * as path from "node:path";
import { globPaths } from "@oh-my-pi/pi-utils";
import { CompiledPattern } from "../../wasm/pi_natives";
import { FileReader } from "./file-reader";
import { buildGlobPattern, matchesTypeFilter, resolveTypeFilter } from "./filters";
import type { GrepMatch, GrepOptions, GrepResult, WasmSearchResult, WorkerRequest, WorkerResponse } from "./types";
function filterUndefined<T extends Record<string, unknown>>(obj: T): T {
const result = {} as T;
for (const [key, value] of Object.entries(obj)) {
if (value !== undefined) {
(result as Record<string, unknown>)[key] = value;
}
}
return result;
}
async function runGrep(request: GrepOptions): Promise<GrepResult> {
const searchPath = path.resolve(request.path);
const stat = await fs.stat(searchPath);
const isFile = stat.isFile();
using compiledPattern = new CompiledPattern(
filterUndefined({
pattern: request.pattern,
ignoreCase: request.ignoreCase,
multiline: request.multiline,
context: request.context,
maxColumns: request.maxColumns,
mode: request.mode === "content" || !request.mode ? "content" : "count",
}),
);
const matches: GrepMatch[] = [];
let totalMatches = 0;
let filesWithMatches = 0;
let filesSearched = 0;
let limitReached = false;
const maxCount = request.maxCount;
const globalOffset = request.offset ?? 0;
const typeFilter = resolveTypeFilter(request.type);
const globPattern = buildGlobPattern(request.glob);
const fileReader = new FileReader();
if (isFile) {
if (typeFilter && !matchesTypeFilter(searchPath, typeFilter)) {
return {
matches,
totalMatches,
filesWithMatches,
filesSearched,
limitReached: limitReached || undefined,
};
}
const content = await fileReader.read(searchPath);
if (!content) {
return {
matches,
totalMatches,
filesWithMatches,
filesSearched,
limitReached: limitReached || undefined,
};
}
filesSearched = 1;
const result = compiledPattern.search_bytes(
content,
maxCount,
globalOffset > 0 ? globalOffset : undefined,
) as WasmSearchResult;
if (!result.error && result.matchCount > 0) {
filesWithMatches = 1;
totalMatches = result.matchCount;
if (request.mode === "content" || !request.mode) {
for (const m of result.matches) {
matches.push({
path: searchPath,
lineNumber: m.lineNumber,
line: m.line,
contextBefore: m.contextBefore?.length ? m.contextBefore : undefined,
contextAfter: m.contextAfter?.length ? m.contextAfter : undefined,
truncated: m.truncated || undefined,
});
}
} else {
matches.push({
path: searchPath,
lineNumber: 0,
line: "",
matchCount: result.matchCount,
});
}
limitReached = result.limitReached || (maxCount !== undefined && totalMatches >= maxCount);
}
} else {
const paths = await globPaths(globPattern, {
cwd: searchPath,
dot: request.hidden ?? true,
onlyFiles: true,
gitignore: true,
});
for (const relativePath of paths) {
if (limitReached) break;
if (typeFilter && !matchesTypeFilter(relativePath, typeFilter)) {
continue;
}
const normalizedPath = relativePath.replace(/\\/g, "/");
const fullPath = path.join(searchPath, normalizedPath);
const content = await fileReader.read(fullPath);
if (!content) continue;
filesSearched++;
if (!compiledPattern.has_match_bytes(content)) {
continue;
}
const fileOffset = globalOffset > 0 ? Math.max(globalOffset - totalMatches, 0) : 0;
const remaining = maxCount !== undefined ? Math.max(maxCount - totalMatches, 0) : undefined;
if (remaining === 0) {
limitReached = true;
break;
}
const result = compiledPattern.search_bytes(
content,
remaining,
fileOffset > 0 ? fileOffset : undefined,
) as WasmSearchResult;
if (result.error) continue;
if (result.matchCount > 0) {
filesWithMatches++;
totalMatches += result.matchCount;
if (request.mode === "content" || !request.mode) {
for (const m of result.matches) {
matches.push({
path: normalizedPath,
lineNumber: m.lineNumber,
line: m.line,
contextBefore: m.contextBefore?.length ? m.contextBefore : undefined,
contextAfter: m.contextAfter?.length ? m.contextAfter : undefined,
truncated: m.truncated || undefined,
});
}
} else {
matches.push({
path: normalizedPath,
lineNumber: 0,
line: "",
matchCount: result.matchCount,
});
}
if (result.limitReached || (maxCount !== undefined && totalMatches >= maxCount)) {
limitReached = true;
}
}
}
}
return {
matches,
totalMatches,
filesWithMatches,
filesSearched,
limitReached: limitReached || undefined,
};
}
declare const self: Worker;
self.addEventListener("message", async (e: MessageEvent<WorkerRequest>) => {
const msg = e.data;
switch (msg.type) {
case "init":
self.postMessage({ type: "ready", id: msg.id } satisfies WorkerResponse);
break;
case "grep":
try {
const result = await runGrep(msg.request);
self.postMessage({ type: "result", id: msg.id, result } satisfies WorkerResponse);
} catch (err) {
self.postMessage({
type: "error",
id: msg.id,
error: err instanceof Error ? err.message : String(err),
} satisfies WorkerResponse);
}
break;
case "destroy":
break;
}
});
packages/natives/src/highlight/index.ts
+5 -13
View File
@@ -1,16 +1,8 @@
/**
* Syntax highlighting powered by WASM (syntect).
* Syntax highlighting powered by native syntect bindings.
*/
import * as wasm from "../../wasm/pi_natives";
type WasmHighlightExports = typeof wasm & {
highlight_code: (code: string, lang: string | null | undefined, colors: HighlightColors) => string;
supports_language: (lang: string) => boolean;
get_supported_languages: () => string[];
};
const wasmHighlight = wasm as WasmHighlightExports;
import { native } from "../native";
/**
* Theme colors for syntax highlighting.
@@ -41,19 +33,19 @@ export interface HighlightColors {
* @returns Highlighted code as a single string with ANSI color codes
*/
export function highlightCode(code: string, lang: string | undefined, colors: HighlightColors): string {
return wasmHighlight.highlight_code(code, lang, colors);
return native.highlightCode(code, lang, colors);
}
/**
* Check if a language is supported for highlighting.
*/
export function supportsLanguage(lang: string): boolean {
return wasmHighlight.supports_language(lang);
return native.supportsLanguage(lang);
}
/**
* Get list of all supported languages.
*/
export function getSupportedLanguages(): string[] {
return wasmHighlight.get_supported_languages();
return native.getSupportedLanguages();
}
packages/natives/src/html/index.ts
+11 -31
View File
@@ -1,26 +1,16 @@
/**
* HTML to Markdown conversion powered by WASM.
*
* Conversion happens in a worker thread to avoid blocking the main thread.
* HTML to Markdown conversion powered by native bindings.
*/
import { type RequestOptions, WorkerPool } from "../pool";
import { resolveWorkerSpecifier } from "../worker-resolver";
import type { HtmlRequest, HtmlResponse, HtmlToMarkdownOptions } from "./types";
import { native } from "../native";
import type { RequestOptions } from "../request-options";
import type { HtmlToMarkdownOptions } from "./types";
export type { HtmlToMarkdownOptions } from "./types";
const pool = new WorkerPool<HtmlRequest, HtmlResponse>({
createWorker: () =>
new Worker(
resolveWorkerSpecifier({
compiled: "./packages/natives/src/html/worker.ts",
dev: new URL("./worker.ts", import.meta.url),
}),
),
maxWorkers: 2,
idleTimeoutMs: 30_000,
});
function assertRequest(req?: RequestOptions): void {
req?.signal?.throwIfAborted();
}
/**
* Convert HTML to Markdown.
@@ -34,21 +24,11 @@ export async function htmlToMarkdown(
options?: HtmlToMarkdownOptions,
req?: RequestOptions,
): Promise<string> {
const response = await pool.request<Extract<HtmlResponse, { type: "converted" }>>(
{
type: "convert",
html,
options,
},
req,
);
return response.markdown;
assertRequest(req);
return native.htmlToMarkdown(html, options);
}
/**
* Terminate the HTML worker pool.
* Call this when shutting down to clean up resources.
* Terminate HTML resources (no-op for native bindings).
*/
export function terminate(): void {
pool.terminate();
}
export function terminate(): void {}
packages/natives/src/html/types.ts
+1 -16
View File
@@ -1,5 +1,5 @@
/**
* Types for HTML to Markdown worker communication.
* Types for HTML to Markdown conversion.
*/
export interface HtmlToMarkdownOptions {
@@ -8,18 +8,3 @@ export interface HtmlToMarkdownOptions {
/** Skip images during conversion */
skipImages?: boolean;
}
export type HtmlRequest =
| { type: "init"; id: number }
| { type: "destroy" }
| {
type: "convert";
id: number;
html: string;
options?: HtmlToMarkdownOptions;
};
export type HtmlResponse =
| { type: "ready"; id: number }
| { type: "error"; id: number; error: string }
| { type: "converted"; id: number; markdown: string };
packages/natives/src/html/worker.ts
-40
View File
@@ -1,40 +0,0 @@
/**
* Worker for HTML to Markdown conversion.
* Uses WASM for actual conversion.
*/
import { html_to_markdown } from "../../wasm/pi_natives";
import type { HtmlRequest, HtmlResponse } from "./types";
declare const self: Worker;
function respond(msg: HtmlResponse): void {
self.postMessage(msg);
}
self.addEventListener("message", (e: MessageEvent<HtmlRequest>) => {
const msg = e.data;
switch (msg.type) {
case "init":
respond({ type: "ready", id: msg.id });
break;
case "destroy":
break;
case "convert": {
try {
const markdown = html_to_markdown(msg.html, msg.options);
respond({ type: "converted", id: msg.id, markdown });
} catch (err) {
respond({
type: "error",
id: msg.id,
error: err instanceof Error ? err.message : String(err),
});
}
break;
}
}
});
packages/natives/src/image/index.ts
+50 -71
View File
@@ -1,48 +1,55 @@
/**
* Async image processing via worker.
*
* All heavy lifting happens in a worker thread to avoid blocking the main thread.
* Uses transferable ArrayBuffers to avoid copying image data.
* Image processing via native bindings.
*/
import { WorkerPool } from "../pool";
import { resolveWorkerSpecifier } from "../worker-resolver";
import type { ImageRequest, ImageResponse } from "./types";
import { native, type NativePhotonImage } from "../native";
// Re-export the enum for filter selection
export { SamplingFilter } from "../../wasm/pi_natives";
const images = new Map<number, NativePhotonImage>();
let nextHandle = 1;
const pool = new WorkerPool<ImageRequest, ImageResponse>({
createWorker: () =>
new Worker(
resolveWorkerSpecifier({
compiled: "./packages/natives/src/image/worker.ts",
dev: new URL("./worker.ts", import.meta.url),
}),
),
maxWorkers: 1,
idleTimeoutMs: 0, // Keep alive - stateful (image handles)
});
function registerImage(image: NativePhotonImage): number {
const handle = nextHandle++;
images.set(handle, image);
return handle;
}
function getImage(handle: number): NativePhotonImage {
const image = images.get(handle);
if (!image) {
throw new Error("Image already freed");
}
return image;
}
export const SamplingFilter = native.SamplingFilter;
export type SamplingFilter = (typeof SamplingFilter)[keyof typeof SamplingFilter];
/**
* Image handle for async operations.
* Must call free() when done to release WASM memory.
*/
export class PhotonImage {
#handle: number;
#width: number;
#height: number;
#freed = false;
private constructor(handle: number, width: number, height: number) {
private constructor(handle: number) {
this.#handle = handle;
this.#width = width;
this.#height = height;
}
/** @internal */
static _create(handle: number, width: number, height: number): PhotonImage {
return new PhotonImage(handle, width, height);
static _create(handle: number): PhotonImage {
if (!images.has(handle)) {
throw new Error("Invalid image handle");
}
return new PhotonImage(handle);
}
/**
* Load an image from encoded bytes (PNG, JPEG, WebP, GIF).
*/
static async new_from_byteslice(bytes: Uint8Array): Promise<PhotonImage> {
const image = native.PhotonImage.newFromByteslice(bytes);
const handle = registerImage(image);
return new PhotonImage(handle);
}
/** @internal */
@@ -51,56 +58,36 @@ export class PhotonImage {
return this.#handle;
}
/**
* Load an image from encoded bytes (PNG, JPEG, WebP, GIF).
* The bytes are transferred to the worker (zero-copy).
*/
static async new_from_byteslice(bytes: Uint8Array): Promise<PhotonImage> {
const response = await pool.request<Extract<ImageResponse, { type: "loaded" }>>(
{ type: "load", bytes },
{
transfer: [bytes.buffer],
},
);
return new PhotonImage(response.handle, response.width, response.height);
#native(): NativePhotonImage {
if (this.#freed) throw new Error("Image already freed");
return getImage(this.#handle);
}
/** Get image width in pixels. */
get_width(): number {
return this.#width;
return this.#native().getWidth();
}
/** Get image height in pixels. */
get_height(): number {
return this.#height;
return this.#native().getHeight();
}
/** Export as PNG bytes. */
async get_bytes(): Promise<Uint8Array> {
if (this.#freed) throw new Error("Image already freed");
const response = await pool.request<Extract<ImageResponse, { type: "bytes" }>>({
type: "get_png",
handle: this.#handle,
});
return response.bytes;
return this.#native().getBytes();
}
/** Export as JPEG bytes with specified quality (0-100). */
async get_bytes_jpeg(quality: number): Promise<Uint8Array> {
if (this.#freed) throw new Error("Image already freed");
const response = await pool.request<Extract<ImageResponse, { type: "bytes" }>>({
type: "get_jpeg",
handle: this.#handle,
quality,
});
return response.bytes;
return this.#native().getBytesJpeg(quality);
}
/** Release WASM memory. Must be called when done with the image. */
/** Release native resources. */
free() {
if (this.#freed) return;
this.#freed = true;
pool.request({ type: "free", handle: this.#handle }).catch(() => {});
images.delete(this.#handle);
}
/** Alias for free() to support using-declarations. */
@@ -114,21 +101,13 @@ export class PhotonImage {
* Returns a new PhotonImage (original is not modified).
*/
export async function resize(image: PhotonImage, width: number, height: number, filter: number): Promise<PhotonImage> {
const handle = image._getHandle();
const response = await pool.request<Extract<ImageResponse, { type: "resized" }>>({
type: "resize",
handle,
width,
height,
filter,
});
return PhotonImage._create(response.handle, response.width, response.height);
const nativeImage = getImage(image._getHandle());
const resized = nativeImage.resize(width, height, filter);
const handle = registerImage(resized);
return PhotonImage._create(handle);
}
/**
* Terminate the image worker.
* Call this when shutting down to clean up resources.
* Terminate image resources (no-op for native bindings).
*/
export function terminate(): void {
pool.terminate();
}
export function terminate(): void {}
packages/natives/src/image/types.ts
-52
View File
@@ -1,52 +0,0 @@
/**
* Types for image worker communication.
*/
export type ImageRequest =
| { type: "init"; id: number }
| { type: "destroy" }
| {
type: "load";
id: number;
/** Image bytes (transferred, not copied) */
bytes: Uint8Array;
}
| {
type: "resize";
id: number;
/** Handle returned from load */
handle: number;
width: number;
height: number;
filter: number;
}
| {
type: "get_dimensions";
id: number;
handle: number;
}
| {
type: "get_png";
id: number;
handle: number;
}
| {
type: "get_jpeg";
id: number;
handle: number;
quality: number;
}
| {
type: "free";
id: number;
handle: number;
};
export type ImageResponse =
| { type: "ready"; id: number }
| { type: "error"; id: number; error: string }
| { type: "loaded"; id: number; handle: number; width: number; height: number }
| { type: "resized"; id: number; handle: number; width: number; height: number }
| { type: "dimensions"; id: number; width: number; height: number }
| { type: "bytes"; id: number; bytes: Uint8Array }
| { type: "freed"; id: number };
packages/natives/src/image/worker.ts
-152
View File
@@ -1,152 +0,0 @@
/**
* Worker for image processing operations.
* Uses WASM for actual processing, communicates via transferable buffers.
*/
import { PhotonImage, type SamplingFilter, resize as wasmResize } from "../../wasm/pi_natives";
import type { ImageRequest, ImageResponse } from "./types";
declare const self: Worker;
/** Map of handle -> PhotonImage */
const images = new Map<number, PhotonImage>();
let nextHandle = 1;
function respond(msg: ImageResponse, transfer?: ArrayBufferLike[]): void {
if (transfer) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
self.postMessage(msg, transfer as any);
} else {
self.postMessage(msg);
}
}
self.addEventListener("message", (e: MessageEvent<ImageRequest>) => {
const msg = e.data;
switch (msg.type) {
case "init":
respond({ type: "ready", id: msg.id });
break;
case "destroy":
for (const img of images.values()) {
img.free();
}
images.clear();
break;
case "load": {
try {
const img = PhotonImage.new_from_byteslice(msg.bytes);
const handle = nextHandle++;
images.set(handle, img);
respond({
type: "loaded",
id: msg.id,
handle,
width: img.get_width(),
height: img.get_height(),
});
} catch (err) {
respond({
type: "error",
id: msg.id,
error: err instanceof Error ? err.message : String(err),
});
}
break;
}
case "resize": {
const img = images.get(msg.handle);
if (!img) {
respond({ type: "error", id: msg.id, error: "Invalid image handle" });
break;
}
try {
const filter = msg.filter as SamplingFilter;
const resized = wasmResize(img, msg.width, msg.height, filter);
const handle = nextHandle++;
images.set(handle, resized);
respond({
type: "resized",
id: msg.id,
handle,
width: resized.get_width(),
height: resized.get_height(),
});
} catch (err) {
respond({
type: "error",
id: msg.id,
error: err instanceof Error ? err.message : String(err),
});
}
break;
}
case "get_dimensions": {
const img = images.get(msg.handle);
if (!img) {
respond({ type: "error", id: msg.id, error: "Invalid image handle" });
break;
}
respond({
type: "dimensions",
id: msg.id,
width: img.get_width(),
height: img.get_height(),
});
break;
}
case "get_png": {
const img = images.get(msg.handle);
if (!img) {
respond({ type: "error", id: msg.id, error: "Invalid image handle" });
break;
}
try {
const bytes = img.get_bytes();
respond({ type: "bytes", id: msg.id, bytes }, [bytes.buffer]);
} catch (err) {
respond({
type: "error",
id: msg.id,
error: err instanceof Error ? err.message : String(err),
});
}
break;
}
case "get_jpeg": {
const img = images.get(msg.handle);
if (!img) {
respond({ type: "error", id: msg.id, error: "Invalid image handle" });
break;
}
try {
const bytes = img.get_bytes_jpeg(msg.quality);
respond({ type: "bytes", id: msg.id, bytes }, [bytes.buffer]);
} catch (err) {
respond({
type: "error",
id: msg.id,
error: err instanceof Error ? err.message : String(err),
});
}
break;
}
case "free": {
const img = images.get(msg.handle);
if (img) {
img.free();
images.delete(msg.handle);
}
respond({ type: "freed", id: msg.id });
break;
}
}
});
packages/natives/src/index.ts
+2 -12
View File
@@ -1,5 +1,5 @@
/**
* Native utilities powered by WASM.
* Native utilities powered by N-API.
*/
import * as fs from "node:fs/promises";
@@ -24,12 +24,6 @@ export {
terminate,
} from "./grep/index";
// =============================================================================
// WASI implementation
// =============================================================================
export { WASI1, WASIError, WASIExitError, type WASIOptions } from "./wasix";
// =============================================================================
// Find (file discovery)
// =============================================================================
@@ -167,8 +161,4 @@ export {
terminate as terminateHtmlWorker,
} from "./html/index";
// =============================================================================
// Worker Pool (shared infrastructure)
// =============================================================================
export { type BaseRequest, type BaseResponse, type RequestOptions, WorkerPool, type WorkerPoolOptions } from "./pool";
export type { RequestOptions } from "./request-options";
packages/natives/src/native.ts
+123
View File
@@ -0,0 +1,123 @@
import * as path from "node:path";
import { createRequire } from "node:module";
import type { GrepOptions, GrepResult, SearchOptions, SearchResult } from "./grep/types";
import type { HtmlToMarkdownOptions } from "./html/types";
import type { HighlightColors } from "./highlight/index";
import type { ExtractSegmentsResult, SliceWithWidthResult } from "./text/index";
export interface NativePhotonImage {
getWidth(): number;
getHeight(): number;
getBytes(): Uint8Array;
getBytesJpeg(quality: number): Uint8Array;
getBytesWebp(): Uint8Array;
getBytesGif(): Uint8Array;
resize(width: number, height: number, filter: number): NativePhotonImage;
}
export interface NativePhotonImageConstructor {
newFromByteslice(bytes: Uint8Array): NativePhotonImage;
prototype: NativePhotonImage;
}
export interface NativeSamplingFilter {
Nearest: 1;
Triangle: 2;
CatmullRom: 3;
Gaussian: 4;
Lanczos3: 5;
}
export interface NativeBindings {
grep(options: GrepOptions): GrepResult;
search(content: string, options: SearchOptions): SearchResult;
hasMatch(content: string, pattern: string, ignoreCase: boolean, multiline: boolean): boolean;
htmlToMarkdown(html: string, options?: HtmlToMarkdownOptions | null): string;
highlightCode(code: string, lang: string | null | undefined, colors: HighlightColors): string;
supportsLanguage(lang: string): boolean;
getSupportedLanguages(): string[];
SamplingFilter: NativeSamplingFilter;
PhotonImage: NativePhotonImageConstructor;
visibleWidth(text: string): number;
truncateToWidth(text: string, maxWidth: number, ellipsis: string, pad: boolean): string;
sliceWithWidth(line: string, startCol: number, length: number, strict: boolean): SliceWithWidthResult;
extractSegments(
line: string,
beforeEnd: number,
afterStart: number,
afterLen: number,
strictAfter: boolean,
): ExtractSegmentsResult;
}
const require = createRequire(import.meta.url);
const platformTag = `${process.platform}-${process.arch}`;
const nativeDir = path.join(import.meta.dir, "..", "native");
const repoRoot = path.join(import.meta.dir, "..", "..", "..");
const execDir = path.dirname(process.execPath);
const candidates = [
path.join(nativeDir, `pi_natives.${platformTag}.node`),
path.join(nativeDir, "pi_natives.node"),
path.join(execDir, `pi_natives.${platformTag}.node`),
path.join(execDir, "pi_natives.node"),
path.join(repoRoot, "target", "release", "pi_natives.node"),
path.join(repoRoot, "crates", "pi-natives", "target", "release", "pi_natives.node"),
];
function loadNative(): NativeBindings {
const errors: string[] = [];
for (const candidate of candidates) {
try {
const bindings = require(candidate) as NativeBindings;
validateNative(bindings, candidate);
return bindings;
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
errors.push(`${candidate}: ${message}`);
}
}
const details = errors.map(error => `- ${error}`).join("\n");
throw new Error(`Failed to load pi_natives native addon. Tried:\n${details}`);
}
function validateNative(bindings: NativeBindings, source: string): void {
const missing: string[] = [];
const checkFn = (name: keyof NativeBindings) => {
if (typeof bindings[name] !== "function") {
missing.push(name);
}
};
checkFn("grep");
checkFn("search");
checkFn("hasMatch");
checkFn("htmlToMarkdown");
checkFn("highlightCode");
checkFn("supportsLanguage");
checkFn("getSupportedLanguages");
checkFn("visibleWidth");
checkFn("truncateToWidth");
checkFn("sliceWithWidth");
checkFn("extractSegments");
if (!bindings.PhotonImage?.newFromByteslice) {
missing.push("PhotonImage.newFromByteslice");
}
if (!bindings.PhotonImage?.prototype?.resize) {
missing.push("PhotonImage.resize");
}
if (!bindings.SamplingFilter || typeof bindings.SamplingFilter.Lanczos3 !== "number") {
missing.push("SamplingFilter");
}
if (missing.length) {
throw new Error(
`Native addon missing exports (${source}). Missing: ${missing.join(", ")}. ` +
"Rebuild with `bun --cwd=packages/natives run build:native`.",
);
}
}
export const native = loadNative();
packages/natives/src/pool.ts
-362
View File
@@ -1,362 +0,0 @@
/**
* Generic worker pool for WASM-based operations.
*
* Supports both single-worker (maxWorkers: 1) and multi-worker scenarios.
* Workers are lazily created and auto-terminated after idle timeout.
*/
/** Base request type - workers must accept messages with this shape. */
export interface BaseRequest {
type: string;
id?: number;
}
/** Base response type - workers must respond with this shape. */
export interface BaseResponse {
type: string;
id: number;
error?: string;
}
export interface WorkerPoolOptions {
/** URL to the worker script (deprecated: use createWorker for compiled binaries). */
workerUrl?: string | URL;
/** Factory function to create workers. Required for compiled binaries where Bun needs static analysis. */
createWorker?: () => Worker;
/** Maximum number of workers (default: 4). */
maxWorkers?: number;
/** Idle timeout in ms before terminating unused workers (0 = never, default: 30000). */
idleTimeoutMs?: number;
/** Timeout for worker initialization in ms (default: 10000). */
initTimeoutMs?: number;
/** Grace period after request timeout before force-terminating stuck workers (default: 5000). */
stuckGracePeriodMs?: number;
}
export interface RequestOptions {
/** Timeout for this request in ms. After this, the promise rejects but worker gets a grace period. */
timeoutMs?: number;
/** Abort signal for this request. */
signal?: AbortSignal;
/** Transfer list for postMessage. */
transfer?: ArrayBufferLike[];
}
interface PooledWorker {
worker: Worker;
busy: boolean;
lastUsed: number;
currentRequestId: number | null;
}
interface PendingRequest<T> {
resolve: (result: T) => void;
reject: (error: Error) => void;
worker?: PooledWorker;
dispose?: () => void;
}
/**
* A pool of workers that process requests in parallel.
*
* @typeParam TReq - Request message type (must extend BaseRequest)
* @typeParam TRes - Response message type (must extend BaseResponse)
*/
export class WorkerPool<TReq extends BaseRequest, TRes extends BaseResponse> {
readonly #options: {
workerUrl?: string | URL;
createWorker?: () => Worker;
maxWorkers: number;
idleTimeoutMs: number;
initTimeoutMs: number;
stuckGracePeriodMs: number;
};
readonly #pool: PooledWorker[] = [];
readonly #waiters: Array<(worker: PooledWorker) => void> = [];
readonly #pending = new Map<number, PendingRequest<TRes>>();
#nextRequestId = 1;
#idleCheckInterval: ReturnType<typeof setInterval> | null = null;
constructor(options: WorkerPoolOptions) {
if (!options.workerUrl && !options.createWorker) {
throw new Error("WorkerPool requires either workerUrl or createWorker");
}
this.#options = {
workerUrl: options.workerUrl,
createWorker: options.createWorker,
maxWorkers: options.maxWorkers ?? 4,
idleTimeoutMs: options.idleTimeoutMs ?? 30_000,
initTimeoutMs: options.initTimeoutMs ?? 10_000,
stuckGracePeriodMs: options.stuckGracePeriodMs ?? 5_000,
};
}
/**
* Send a request to a worker and wait for the response.
* Workers are acquired from the pool (or created if under limit).
*
* @param msg - Request message
* @param options - Request options (timeout, transfer)
*/
async request<T extends TRes = TRes>(
msg: TReq | (Omit<TReq, "id"> & { id?: number }),
options?: RequestOptions,
): Promise<T> {
const { timeoutMs, signal, transfer } = options ?? {};
signal?.throwIfAborted();
const worker = await this.#acquireWorker();
const id = msg.id ?? this.#nextRequestId++;
const fullMsg = { ...msg, id } as TReq;
const { promise, resolve, reject } = Promise.withResolvers<T>();
const pending: PendingRequest<T> = {
resolve: resolve as (result: TRes) => void,
reject,
worker,
};
this.#pending.set(id, pending as PendingRequest<TRes>);
const onAbort = () => {
this.#handleRequestAbort(id, worker);
};
if (timeoutMs && timeoutMs > 0 && signal) {
const combined = AbortSignal.any([signal, AbortSignal.timeout(timeoutMs)]);
combined.addEventListener("abort", onAbort, { once: true });
pending.dispose = () => combined.removeEventListener("abort", onAbort);
} else if (timeoutMs && timeoutMs > 0) {
const timer = setTimeout(onAbort, timeoutMs);
pending.dispose = () => clearTimeout(timer);
} else if (signal) {
signal.addEventListener("abort", onAbort, { once: true });
pending.dispose = () => signal.removeEventListener("abort", onAbort);
}
worker.currentRequestId = id;
if (transfer) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
worker.worker.postMessage(fullMsg, transfer as any);
} else {
worker.worker.postMessage(fullMsg);
}
return promise;
}
/** Terminate all workers and clear pending requests. */
terminate(): void {
if (this.#idleCheckInterval) {
clearInterval(this.#idleCheckInterval);
this.#idleCheckInterval = null;
}
for (const w of [...this.#pool]) {
this.#removeWorker(w);
}
this.#waiters.length = 0;
for (const pending of this.#pending.values()) {
pending.reject(new Error("Worker pool terminated"));
void pending.dispose?.();
}
this.#pending.clear();
}
#createWorker(): PooledWorker {
const worker = this.#options.createWorker ? this.#options.createWorker() : new Worker(this.#options.workerUrl!);
const pooledWorker: PooledWorker = {
worker,
busy: false,
lastUsed: Date.now(),
currentRequestId: null,
};
worker.onmessage = (e: MessageEvent<TRes>) => {
this.#handleMessage(pooledWorker, e.data);
};
worker.onerror = (e: ErrorEvent) => {
const requestId = pooledWorker.currentRequestId;
if (requestId !== null) {
this.#rejectRequest(requestId, new Error(`Worker error: ${e.message}`));
}
this.#removeWorker(pooledWorker);
};
return pooledWorker;
}
#handleMessage(pooledWorker: PooledWorker, msg: TRes): void {
const pending = this.#pending.get(msg.id);
if (!pending) return;
this.#pending.delete(msg.id);
void pending.dispose?.();
if (msg.type === "error" && "error" in msg) {
pending.reject(new Error(msg.error ?? "Unknown error"));
} else {
pending.resolve(msg);
}
// Release worker back to pool (unless it was the init request)
if (msg.type !== "ready") {
pooledWorker.currentRequestId = null;
this.#releaseWorker(pooledWorker);
}
}
#rejectRequest(id: number, error: Error): void {
const pending = this.#pending.get(id);
if (pending) {
this.#pending.delete(id);
void pending.dispose?.();
pending.reject(error);
}
}
#handleRequestAbort(id: number, worker: PooledWorker): void {
const pending = this.#pending.get(id);
if (!pending) return;
pending.dispose = undefined;
pending.reject(new Error("Request timeout"));
if (this.#options.stuckGracePeriodMs > 0) {
const timer = setTimeout(() => {
this.#terminateStuckWorker(id, worker);
}, this.#options.stuckGracePeriodMs);
pending.dispose = () => {
clearTimeout(timer);
};
}
}
#terminateStuckWorker(id: number, worker: PooledWorker): void {
const pending = this.#pending.get(id);
if (pending) {
this.#pending.delete(id);
void pending.dispose?.();
}
if (worker.currentRequestId !== id) return;
if (!this.#pool.includes(worker)) return;
this.#removeWorker(worker);
if (this.#pool.length === 0 && this.#waiters.length > 0) {
this.#replenishPool();
}
}
async #replenishPool(): Promise<void> {
const worker = this.#createWorker();
worker.busy = true;
this.#pool.push(worker);
try {
await this.#initializeWorker(worker);
this.#releaseWorker(worker);
} catch {
this.#removeWorker(worker);
}
}
#removeWorker(pooledWorker: PooledWorker): void {
const idx = this.#pool.indexOf(pooledWorker);
if (idx !== -1) {
this.#pool.splice(idx, 1);
}
pooledWorker.worker.postMessage({ type: "destroy" } satisfies BaseRequest);
pooledWorker.worker.terminate();
}
#releaseWorker(pooledWorker: PooledWorker): void {
pooledWorker.busy = false;
pooledWorker.lastUsed = Date.now();
if (this.#waiters.length) {
const waiter = this.#waiters.shift()!;
pooledWorker.busy = true;
waiter(pooledWorker);
}
}
#checkIdleWorkers(): void {
if (this.#options.idleTimeoutMs === 0) return;
const now = Date.now();
const toRemove: PooledWorker[] = [];
for (const w of this.#pool) {
if (!w.busy && now - w.lastUsed > this.#options.idleTimeoutMs && !this.#waiters.length) {
toRemove.push(w);
}
}
for (const w of toRemove) {
this.#removeWorker(w);
}
if (this.#pool.length === 0 && this.#idleCheckInterval) {
clearInterval(this.#idleCheckInterval);
this.#idleCheckInterval = null;
}
}
#ensureIdleCheck(): void {
if (this.#options.idleTimeoutMs > 0 && !this.#idleCheckInterval) {
this.#idleCheckInterval = setInterval(() => this.#checkIdleWorkers(), 10_000);
}
}
async #initializeWorker(pooledWorker: PooledWorker): Promise<void> {
const id = this.#nextRequestId++;
const { promise, resolve, reject } = Promise.withResolvers<void>();
const timeout = setTimeout(() => {
this.#rejectRequest(id, new Error("Worker initialization timeout"));
}, this.#options.initTimeoutMs);
this.#pending.set(id, {
resolve: () => resolve(),
reject,
dispose: () => clearTimeout(timeout),
} as PendingRequest<TRes>);
pooledWorker.currentRequestId = id;
pooledWorker.worker.postMessage({ type: "init", id } satisfies BaseRequest);
return promise;
}
async #acquireWorker(): Promise<PooledWorker> {
// Try to find an idle worker
for (const w of this.#pool) {
if (!w.busy) {
w.busy = true;
return w;
}
}
// Create new worker if under limit
if (this.#pool.length < this.#options.maxWorkers) {
const worker = this.#createWorker();
worker.busy = true;
this.#pool.push(worker);
this.#ensureIdleCheck();
await this.#initializeWorker(worker);
return worker;
}
// Wait for a worker to become available
const { promise, resolve } = Promise.withResolvers<PooledWorker>();
this.#waiters.push(w => {
w.busy = true;
resolve(w);
});
return promise;
}
}
packages/natives/src/request-options.ts
+5
View File
@@ -0,0 +1,5 @@
export interface RequestOptions {
timeoutMs?: number;
signal?: AbortSignal;
transfer?: ArrayBufferLike[];
}
packages/natives/src/text/index.ts
+6 -21
View File
@@ -1,8 +1,8 @@
/**
* ANSI-aware text utilities powered by WASM.
* ANSI-aware text utilities powered by native bindings.
*/
import * as wasm from "../../wasm/pi_natives";
import { native } from "../native";
export interface SliceWithWidthResult {
text: string;
@@ -16,38 +16,23 @@ export interface ExtractSegmentsResult {
afterWidth: number;
}
type WasmTextExports = typeof wasm & {
visible_width: (text: string) => number;
truncate_to_width: (text: string, maxWidth: number, ellipsis: string, pad: boolean) => string;
slice_with_width: (line: string, startCol: number, length: number, strict: boolean) => SliceWithWidthResult;
extract_segments: (
line: string,
beforeEnd: number,
afterStart: number,
afterLen: number,
strictAfter: boolean,
) => ExtractSegmentsResult;
};
const wasmText = wasm as WasmTextExports;
/** Compute the visible width of a string, ignoring ANSI codes. */
export function visibleWidth(text: string): number {
return wasmText.visible_width(text);
return native.visibleWidth(text);
}
/**
* Truncate a string to a visible width, preserving ANSI codes.
*/
export function truncateToWidth(text: string, maxWidth: number, ellipsis = "…", pad = false): string {
return wasmText.truncate_to_width(text, maxWidth, ellipsis, pad);
return native.truncateToWidth(text, maxWidth, ellipsis, pad);
}
/**
* Slice a range of visible columns from a line.
*/
export function sliceWithWidth(line: string, startCol: number, length: number, strict = false): SliceWithWidthResult {
return wasmText.slice_with_width(line, startCol, length, strict);
return native.sliceWithWidth(line, startCol, length, strict);
}
/**
@@ -60,5 +45,5 @@ export function extractSegments(
afterLen: number,
strictAfter = false,
): ExtractSegmentsResult {
return wasmText.extract_segments(line, beforeEnd, afterStart, afterLen, strictAfter);
return native.extractSegments(line, beforeEnd, afterStart, afterLen, strictAfter);
}
packages/natives/src/wasix.ts
-1745
View File
File diff suppressed because it is too large Load Diff
packages/natives/src/worker-resolver.ts
-9
View File
@@ -1,9 +0,0 @@
declare const OMP_COMPILED: boolean | undefined;
export function resolveWorkerSpecifier(options: { compiled: string; dev: URL }): string | URL {
if (typeof OMP_COMPILED !== "undefined" && OMP_COMPILED) {
return options.compiled;
}
return options.dev;
}
packages/natives/wasm/pi_natives.d.ts
Vendored
-148
View File
@@ -1,148 +0,0 @@
/* tslint:disable */
/* eslint-disable */
/**
* A compiled regex matcher that can be reused across multiple searches.
*/
export class CompiledPattern {
free(): void;
[Symbol.dispose](): void;
/**
* Check if content has any matches (faster than full search).
*/
has_match(content: string): boolean;
/**
* Check if bytes have any matches (faster than full search).
*/
has_match_bytes(content: Uint8Array): boolean;
/**
* Compile a regex pattern for reuse.
*/
constructor(options: any);
/**
* Search content using this compiled pattern.
* Returns matches as a JS object.
*/
search(content: string, max_count?: number | null, offset?: number | null): any;
/**
* Search bytes directly (avoids UTF-16 to UTF-8 conversion).
* Use with `Bun.mmap()` for best performance.
*/
search_bytes(content: Uint8Array, max_count?: number | null, offset?: number | null): any;
}
/**
* Image container for WASM interop.
*/
export class PhotonImage {
private constructor();
free(): void;
[Symbol.dispose](): void;
/**
* Export image as PNG bytes.
*/
get_bytes(): Uint8Array;
/**
* Export image as GIF bytes.
*/
get_bytes_gif(): Uint8Array;
/**
* Export image as JPEG bytes with specified quality (0-100).
*/
get_bytes_jpeg(quality: number): Uint8Array;
/**
* Export image as lossless WebP bytes.
*/
get_bytes_webp(): Uint8Array;
/**
* Get the height of the image.
*/
get_height(): number;
/**
* Get the width of the image.
*/
get_width(): number;
/**
* Create a new `PhotonImage` from encoded image bytes (PNG, JPEG, WebP,
* GIF).
*/
static new_from_byteslice(bytes: Uint8Array): PhotonImage;
}
/**
* Sampling filter for resize operations.
*/
export enum SamplingFilter {
Nearest = 1,
Triangle = 2,
CatmullRom = 3,
Gaussian = 4,
Lanczos3 = 5,
}
/**
* Extract the before/after slices around an overlay region.
*/
export function extract_segments(line: string, before_end: number, after_start: number, after_len: number, strict_after: boolean): any;
/**
* Get list of supported languages.
*/
export function get_supported_languages(): string[];
/**
* Quick check if content matches a pattern.
*/
export function has_match(content: string, pattern: string, ignore_case: boolean, multiline: boolean): boolean;
/**
* Highlight code and return ANSI-colored lines.
*
* # Arguments
* * `code` - The source code to highlight
* * `lang` - Language identifier (e.g., "rust", "typescript", "python")
* * `colors` - Theme colors as ANSI escape sequences
*
* # Returns
* Highlighted code with ANSI color codes, or the original code if highlighting
* fails.
*/
export function highlight_code(code: string, lang: string | null | undefined, colors: any): string;
/**
* Convert HTML to Markdown.
*/
export function html_to_markdown(html: string, options: any): string;
/**
* Resize an image to the specified dimensions.
*/
export function resize(image: PhotonImage, width: number, height: number, filter: SamplingFilter): PhotonImage;
/**
* Search content for a pattern (one-shot, compiles pattern each time).
* For repeated searches with the same pattern, use [`CompiledPattern`].
*/
export function search(content: string, options: any): any;
/**
* Slice a range of visible columns from a line.
*/
export function slice_with_width(line: string, start_col: number, length: number, strict: boolean): any;
/**
* Check if a language is supported for highlighting.
* Returns true if the language has either direct support or a fallback
* mapping.
*/
export function supports_language(lang: string): boolean;
/**
* Truncate text to a visible width, preserving ANSI codes.
*/
export function truncate_to_width(text: string, max_width: number, ellipsis: string, pad: boolean): string;
/**
* Compute the visible width of a string, ignoring ANSI codes.
*/
export function visible_width(text: string): number;
packages/natives/wasm/pi_natives.js
-891
View File
@@ -1,891 +0,0 @@
import wasmPath from "./pi_natives_bg.wasm";
/* @ts-self-types="./pi_natives.d.ts" */
/**
* A compiled regex matcher that can be reused across multiple searches.
*/
export class CompiledPattern {
__destroy_into_raw() {
const ptr = this.__wbg_ptr;
this.__wbg_ptr = 0;
CompiledPatternFinalization.unregister(this);
return ptr;
}
free() {
const ptr = this.__destroy_into_raw();
wasm.__wbg_compiledpattern_free(ptr, 0);
}
/**
* Check if content has any matches (faster than full search).
* @param {string} content
* @returns {boolean}
*/
has_match(content) {
const ptr0 = passStringToWasm0(content, wasm.__wbindgen_export, wasm.__wbindgen_export2);
const len0 = WASM_VECTOR_LEN;
const ret = wasm.compiledpattern_has_match(this.__wbg_ptr, ptr0, len0);
return ret !== 0;
}
/**
* Check if bytes have any matches (faster than full search).
* @param {Uint8Array} content
* @returns {boolean}
*/
has_match_bytes(content) {
const ptr0 = passArray8ToWasm0(content, wasm.__wbindgen_export);
const len0 = WASM_VECTOR_LEN;
const ret = wasm.compiledpattern_has_match(this.__wbg_ptr, ptr0, len0);
return ret !== 0;
}
/**
* Compile a regex pattern for reuse.
* @param {any} options
*/
constructor(options) {
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
wasm.compiledpattern_new(retptr, addHeapObject(options));
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true);
if (r2) {
throw takeObject(r1);
}
this.__wbg_ptr = r0 >>> 0;
CompiledPatternFinalization.register(this, this.__wbg_ptr, this);
return this;
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
}
}
/**
* Search content using this compiled pattern.
* Returns matches as a JS object.
* @param {string} content
* @param {number | null} [max_count]
* @param {number | null} [offset]
* @returns {any}
*/
search(content, max_count, offset) {
const ptr0 = passStringToWasm0(content, wasm.__wbindgen_export, wasm.__wbindgen_export2);
const len0 = WASM_VECTOR_LEN;
const ret = wasm.compiledpattern_search(this.__wbg_ptr, ptr0, len0, isLikeNone(max_count) ? 0x100000001 : (max_count) >>> 0, isLikeNone(offset) ? 0x100000001 : (offset) >>> 0);
return takeObject(ret);
}
/**
* Search bytes directly (avoids UTF-16 to UTF-8 conversion).
* Use with `Bun.mmap()` for best performance.
* @param {Uint8Array} content
* @param {number | null} [max_count]
* @param {number | null} [offset]
* @returns {any}
*/
search_bytes(content, max_count, offset) {
const ptr0 = passArray8ToWasm0(content, wasm.__wbindgen_export);
const len0 = WASM_VECTOR_LEN;
const ret = wasm.compiledpattern_search(this.__wbg_ptr, ptr0, len0, isLikeNone(max_count) ? 0x100000001 : (max_count) >>> 0, isLikeNone(offset) ? 0x100000001 : (offset) >>> 0);
return takeObject(ret);
}
}
if (Symbol.dispose) CompiledPattern.prototype[Symbol.dispose] = CompiledPattern.prototype.free;
/**
* Image container for WASM interop.
*/
export class PhotonImage {
static __wrap(ptr) {
ptr = ptr >>> 0;
const obj = Object.create(PhotonImage.prototype);
obj.__wbg_ptr = ptr;
PhotonImageFinalization.register(obj, obj.__wbg_ptr, obj);
return obj;
}
__destroy_into_raw() {
const ptr = this.__wbg_ptr;
this.__wbg_ptr = 0;
PhotonImageFinalization.unregister(this);
return ptr;
}
free() {
const ptr = this.__destroy_into_raw();
wasm.__wbg_photonimage_free(ptr, 0);
}
/**
* Export image as PNG bytes.
* @returns {Uint8Array}
*/
get_bytes() {
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
wasm.photonimage_get_bytes(retptr, this.__wbg_ptr);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true);
var r3 = getDataViewMemory0().getInt32(retptr + 4 * 3, true);
if (r3) {
throw takeObject(r2);
}
var v1 = getArrayU8FromWasm0(r0, r1).slice();
wasm.__wbindgen_export3(r0, r1 * 1, 1);
return v1;
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
}
}
/**
* Export image as GIF bytes.
* @returns {Uint8Array}
*/
get_bytes_gif() {
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
wasm.photonimage_get_bytes_gif(retptr, this.__wbg_ptr);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true);
var r3 = getDataViewMemory0().getInt32(retptr + 4 * 3, true);
if (r3) {
throw takeObject(r2);
}
var v1 = getArrayU8FromWasm0(r0, r1).slice();
wasm.__wbindgen_export3(r0, r1 * 1, 1);
return v1;
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
}
}
/**
* Export image as JPEG bytes with specified quality (0-100).
* @param {number} quality
* @returns {Uint8Array}
*/
get_bytes_jpeg(quality) {
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
wasm.photonimage_get_bytes_jpeg(retptr, this.__wbg_ptr, quality);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true);
var r3 = getDataViewMemory0().getInt32(retptr + 4 * 3, true);
if (r3) {
throw takeObject(r2);
}
var v1 = getArrayU8FromWasm0(r0, r1).slice();
wasm.__wbindgen_export3(r0, r1 * 1, 1);
return v1;
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
}
}
/**
* Export image as lossless WebP bytes.
* @returns {Uint8Array}
*/
get_bytes_webp() {
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
wasm.photonimage_get_bytes_webp(retptr, this.__wbg_ptr);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true);
var r3 = getDataViewMemory0().getInt32(retptr + 4 * 3, true);
if (r3) {
throw takeObject(r2);
}
var v1 = getArrayU8FromWasm0(r0, r1).slice();
wasm.__wbindgen_export3(r0, r1 * 1, 1);
return v1;
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
}
}
/**
* Get the height of the image.
* @returns {number}
*/
get_height() {
const ret = wasm.photonimage_get_height(this.__wbg_ptr);
return ret >>> 0;
}
/**
* Get the width of the image.
* @returns {number}
*/
get_width() {
const ret = wasm.photonimage_get_width(this.__wbg_ptr);
return ret >>> 0;
}
/**
* Create a new `PhotonImage` from encoded image bytes (PNG, JPEG, WebP,
* GIF).
* @param {Uint8Array} bytes
* @returns {PhotonImage}
*/
static new_from_byteslice(bytes) {
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
const ptr0 = passArray8ToWasm0(bytes, wasm.__wbindgen_export);
const len0 = WASM_VECTOR_LEN;
wasm.photonimage_new_from_byteslice(retptr, ptr0, len0);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true);
if (r2) {
throw takeObject(r1);
}
return PhotonImage.__wrap(r0);
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
}
}
}
if (Symbol.dispose) PhotonImage.prototype[Symbol.dispose] = PhotonImage.prototype.free;
/**
* Sampling filter for resize operations.
* @enum {1 | 2 | 3 | 4 | 5}
*/
export const SamplingFilter = Object.freeze({
Nearest: 1, "1": "Nearest",
Triangle: 2, "2": "Triangle",
CatmullRom: 3, "3": "CatmullRom",
Gaussian: 4, "4": "Gaussian",
Lanczos3: 5, "5": "Lanczos3",
});
/**
* Extract the before/after slices around an overlay region.
* @param {string} line
* @param {number} before_end
* @param {number} after_start
* @param {number} after_len
* @param {boolean} strict_after
* @returns {any}
*/
export function extract_segments(line, before_end, after_start, after_len, strict_after) {
const ptr0 = passStringToWasm0(line, wasm.__wbindgen_export, wasm.__wbindgen_export2);
const len0 = WASM_VECTOR_LEN;
const ret = wasm.extract_segments(ptr0, len0, before_end, after_start, after_len, strict_after);
return takeObject(ret);
}
/**
* Get list of supported languages.
* @returns {string[]}
*/
export function get_supported_languages() {
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
wasm.get_supported_languages(retptr);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var v1 = getArrayJsValueFromWasm0(r0, r1).slice();
wasm.__wbindgen_export3(r0, r1 * 4, 4);
return v1;
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
}
}
/**
* Quick check if content matches a pattern.
* @param {string} content
* @param {string} pattern
* @param {boolean} ignore_case
* @param {boolean} multiline
* @returns {boolean}
*/
export function has_match(content, pattern, ignore_case, multiline) {
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
const ptr0 = passStringToWasm0(content, wasm.__wbindgen_export, wasm.__wbindgen_export2);
const len0 = WASM_VECTOR_LEN;
const ptr1 = passStringToWasm0(pattern, wasm.__wbindgen_export, wasm.__wbindgen_export2);
const len1 = WASM_VECTOR_LEN;
wasm.has_match(retptr, ptr0, len0, ptr1, len1, ignore_case, multiline);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true);
if (r2) {
throw takeObject(r1);
}
return r0 !== 0;
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
}
}
/**
* Highlight code and return ANSI-colored lines.
*
* # Arguments
* * `code` - The source code to highlight
* * `lang` - Language identifier (e.g., "rust", "typescript", "python")
* * `colors` - Theme colors as ANSI escape sequences
*
* # Returns
* Highlighted code with ANSI color codes, or the original code if highlighting
* fails.
* @param {string} code
* @param {string | null | undefined} lang
* @param {any} colors
* @returns {string}
*/
export function highlight_code(code, lang, colors) {
let deferred3_0;
let deferred3_1;
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
const ptr0 = passStringToWasm0(code, wasm.__wbindgen_export, wasm.__wbindgen_export2);
const len0 = WASM_VECTOR_LEN;
var ptr1 = isLikeNone(lang) ? 0 : passStringToWasm0(lang, wasm.__wbindgen_export, wasm.__wbindgen_export2);
var len1 = WASM_VECTOR_LEN;
wasm.highlight_code(retptr, ptr0, len0, ptr1, len1, addHeapObject(colors));
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
deferred3_0 = r0;
deferred3_1 = r1;
return getStringFromWasm0(r0, r1);
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
wasm.__wbindgen_export3(deferred3_0, deferred3_1, 1);
}
}
/**
* Convert HTML to Markdown.
* @param {string} html
* @param {any} options
* @returns {string}
*/
export function html_to_markdown(html, options) {
let deferred3_0;
let deferred3_1;
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
const ptr0 = passStringToWasm0(html, wasm.__wbindgen_export, wasm.__wbindgen_export2);
const len0 = WASM_VECTOR_LEN;
wasm.html_to_markdown(retptr, ptr0, len0, addHeapObject(options));
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true);
var r3 = getDataViewMemory0().getInt32(retptr + 4 * 3, true);
var ptr2 = r0;
var len2 = r1;
if (r3) {
ptr2 = 0; len2 = 0;
throw takeObject(r2);
}
deferred3_0 = ptr2;
deferred3_1 = len2;
return getStringFromWasm0(ptr2, len2);
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
wasm.__wbindgen_export3(deferred3_0, deferred3_1, 1);
}
}
/**
* Resize an image to the specified dimensions.
* @param {PhotonImage} image
* @param {number} width
* @param {number} height
* @param {SamplingFilter} filter
* @returns {PhotonImage}
*/
export function resize(image, width, height, filter) {
_assertClass(image, PhotonImage);
const ret = wasm.resize(image.__wbg_ptr, width, height, filter);
return PhotonImage.__wrap(ret);
}
/**
* Search content for a pattern (one-shot, compiles pattern each time).
* For repeated searches with the same pattern, use [`CompiledPattern`].
* @param {string} content
* @param {any} options
* @returns {any}
*/
export function search(content, options) {
const ptr0 = passStringToWasm0(content, wasm.__wbindgen_export, wasm.__wbindgen_export2);
const len0 = WASM_VECTOR_LEN;
const ret = wasm.search(ptr0, len0, addHeapObject(options));
return takeObject(ret);
}
/**
* Slice a range of visible columns from a line.
* @param {string} line
* @param {number} start_col
* @param {number} length
* @param {boolean} strict
* @returns {any}
*/
export function slice_with_width(line, start_col, length, strict) {
const ptr0 = passStringToWasm0(line, wasm.__wbindgen_export, wasm.__wbindgen_export2);
const len0 = WASM_VECTOR_LEN;
const ret = wasm.slice_with_width(ptr0, len0, start_col, length, strict);
return takeObject(ret);
}
/**
* Check if a language is supported for highlighting.
* Returns true if the language has either direct support or a fallback
* mapping.
* @param {string} lang
* @returns {boolean}
*/
export function supports_language(lang) {
const ptr0 = passStringToWasm0(lang, wasm.__wbindgen_export, wasm.__wbindgen_export2);
const len0 = WASM_VECTOR_LEN;
const ret = wasm.supports_language(ptr0, len0);
return ret !== 0;
}
/**
* Truncate text to a visible width, preserving ANSI codes.
* @param {string} text
* @param {number} max_width
* @param {string} ellipsis
* @param {boolean} pad
* @returns {string}
*/
export function truncate_to_width(text, max_width, ellipsis, pad) {
let deferred3_0;
let deferred3_1;
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
const ptr0 = passStringToWasm0(text, wasm.__wbindgen_export, wasm.__wbindgen_export2);
const len0 = WASM_VECTOR_LEN;
const ptr1 = passStringToWasm0(ellipsis, wasm.__wbindgen_export, wasm.__wbindgen_export2);
const len1 = WASM_VECTOR_LEN;
wasm.truncate_to_width(retptr, ptr0, len0, max_width, ptr1, len1, pad);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
deferred3_0 = r0;
deferred3_1 = r1;
return getStringFromWasm0(r0, r1);
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
wasm.__wbindgen_export3(deferred3_0, deferred3_1, 1);
}
}
/**
* Compute the visible width of a string, ignoring ANSI codes.
* @param {string} text
* @returns {number}
*/
export function visible_width(text) {
const ptr0 = passStringToWasm0(text, wasm.__wbindgen_export, wasm.__wbindgen_export2);
const len0 = WASM_VECTOR_LEN;
const ret = wasm.visible_width(ptr0, len0);
return ret >>> 0;
}
function __wbg_get_imports() {
const import0 = {
__proto__: null,
__wbg_Error_8c4e43fe74559d73: function(arg0, arg1) {
const ret = Error(getStringFromWasm0(arg0, arg1));
return addHeapObject(ret);
},
__wbg_Number_04624de7d0e8332d: function(arg0) {
const ret = Number(getObject(arg0));
return ret;
},
__wbg_String_8f0eb39a4a4c2f66: function(arg0, arg1) {
const ret = String(getObject(arg1));
const ptr1 = passStringToWasm0(ret, wasm.__wbindgen_export, wasm.__wbindgen_export2);
const len1 = WASM_VECTOR_LEN;
getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true);
getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true);
},
__wbg___wbindgen_bigint_get_as_i64_8fcf4ce7f1ca72a2: function(arg0, arg1) {
const v = getObject(arg1);
const ret = typeof(v) === 'bigint' ? v : undefined;
getDataViewMemory0().setBigInt64(arg0 + 8 * 1, isLikeNone(ret) ? BigInt(0) : ret, true);
getDataViewMemory0().setInt32(arg0 + 4 * 0, !isLikeNone(ret), true);
},
__wbg___wbindgen_boolean_get_bbbb1c18aa2f5e25: function(arg0) {
const v = getObject(arg0);
const ret = typeof(v) === 'boolean' ? v : undefined;
return isLikeNone(ret) ? 0xFFFFFF : ret ? 1 : 0;
},
__wbg___wbindgen_debug_string_0bc8482c6e3508ae: function(arg0, arg1) {
const ret = debugString(getObject(arg1));
const ptr1 = passStringToWasm0(ret, wasm.__wbindgen_export, wasm.__wbindgen_export2);
const len1 = WASM_VECTOR_LEN;
getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true);
getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true);
},
__wbg___wbindgen_in_47fa6863be6f2f25: function(arg0, arg1) {
const ret = getObject(arg0) in getObject(arg1);
return ret;
},
__wbg___wbindgen_is_bigint_31b12575b56f32fc: function(arg0) {
const ret = typeof(getObject(arg0)) === 'bigint';
return ret;
},
__wbg___wbindgen_is_null_ac34f5003991759a: function(arg0) {
const ret = getObject(arg0) === null;
return ret;
},
__wbg___wbindgen_is_object_5ae8e5880f2c1fbd: function(arg0) {
const val = getObject(arg0);
const ret = typeof(val) === 'object' && val !== null;
return ret;
},
__wbg___wbindgen_is_string_cd444516edc5b180: function(arg0) {
const ret = typeof(getObject(arg0)) === 'string';
return ret;
},
__wbg___wbindgen_is_undefined_9e4d92534c42d778: function(arg0) {
const ret = getObject(arg0) === undefined;
return ret;
},
__wbg___wbindgen_jsval_eq_11888390b0186270: function(arg0, arg1) {
const ret = getObject(arg0) === getObject(arg1);
return ret;
},
__wbg___wbindgen_jsval_loose_eq_9dd77d8cd6671811: function(arg0, arg1) {
const ret = getObject(arg0) == getObject(arg1);
return ret;
},
__wbg___wbindgen_number_get_8ff4255516ccad3e: function(arg0, arg1) {
const obj = getObject(arg1);
const ret = typeof(obj) === 'number' ? obj : undefined;
getDataViewMemory0().setFloat64(arg0 + 8 * 1, isLikeNone(ret) ? 0 : ret, true);
getDataViewMemory0().setInt32(arg0 + 4 * 0, !isLikeNone(ret), true);
},
__wbg___wbindgen_string_get_72fb696202c56729: function(arg0, arg1) {
const obj = getObject(arg1);
const ret = typeof(obj) === 'string' ? obj : undefined;
var ptr1 = isLikeNone(ret) ? 0 : passStringToWasm0(ret, wasm.__wbindgen_export, wasm.__wbindgen_export2);
var len1 = WASM_VECTOR_LEN;
getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true);
getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true);
},
__wbg___wbindgen_throw_be289d5034ed271b: function(arg0, arg1) {
throw new Error(getStringFromWasm0(arg0, arg1));
},
__wbg_entries_58c7934c745daac7: function(arg0) {
const ret = Object.entries(getObject(arg0));
return addHeapObject(ret);
},
__wbg_error_7534b8e9a36f1ab4: function(arg0, arg1) {
let deferred0_0;
let deferred0_1;
try {
deferred0_0 = arg0;
deferred0_1 = arg1;
console.error(getStringFromWasm0(arg0, arg1));
} finally {
wasm.__wbindgen_export3(deferred0_0, deferred0_1, 1);
}
},
__wbg_get_9b94d73e6221f75c: function(arg0, arg1) {
const ret = getObject(arg0)[arg1 >>> 0];
return addHeapObject(ret);
},
__wbg_get_with_ref_key_1dc361bd10053bfe: function(arg0, arg1) {
const ret = getObject(arg0)[getObject(arg1)];
return addHeapObject(ret);
},
__wbg_instanceof_ArrayBuffer_c367199e2fa2aa04: function(arg0) {
let result;
try {
result = getObject(arg0) instanceof ArrayBuffer;
} catch (_) {
result = false;
}
const ret = result;
return ret;
},
__wbg_instanceof_Uint8Array_9b9075935c74707c: function(arg0) {
let result;
try {
result = getObject(arg0) instanceof Uint8Array;
} catch (_) {
result = false;
}
const ret = result;
return ret;
},
__wbg_isSafeInteger_bfbc7332a9768d2a: function(arg0) {
const ret = Number.isSafeInteger(getObject(arg0));
return ret;
},
__wbg_length_32ed9a279acd054c: function(arg0) {
const ret = getObject(arg0).length;
return ret;
},
__wbg_length_35a7bace40f36eac: function(arg0) {
const ret = getObject(arg0).length;
return ret;
},
__wbg_new_361308b2356cecd0: function() {
const ret = new Object();
return addHeapObject(ret);
},
__wbg_new_3eb36ae241fe6f44: function() {
const ret = new Array();
return addHeapObject(ret);
},
__wbg_new_8a6f238a6ece86ea: function() {
const ret = new Error();
return addHeapObject(ret);
},
__wbg_new_dd2b680c8bf6ae29: function(arg0) {
const ret = new Uint8Array(getObject(arg0));
return addHeapObject(ret);
},
__wbg_prototypesetcall_bdcdcc5842e4d77d: function(arg0, arg1, arg2) {
Uint8Array.prototype.set.call(getArrayU8FromWasm0(arg0, arg1), getObject(arg2));
},
__wbg_set_3f1d0b984ed272ed: function(arg0, arg1, arg2) {
getObject(arg0)[takeObject(arg1)] = takeObject(arg2);
},
__wbg_set_f43e577aea94465b: function(arg0, arg1, arg2) {
getObject(arg0)[arg1 >>> 0] = takeObject(arg2);
},
__wbg_stack_0ed75d68575b0f3c: function(arg0, arg1) {
const ret = getObject(arg1).stack;
const ptr1 = passStringToWasm0(ret, wasm.__wbindgen_export, wasm.__wbindgen_export2);
const len1 = WASM_VECTOR_LEN;
getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true);
getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true);
},
__wbindgen_cast_0000000000000001: function(arg0) {
// Cast intrinsic for `F64 -> Externref`.
const ret = arg0;
return addHeapObject(ret);
},
__wbindgen_cast_0000000000000002: function(arg0, arg1) {
// Cast intrinsic for `Ref(String) -> Externref`.
const ret = getStringFromWasm0(arg0, arg1);
return addHeapObject(ret);
},
__wbindgen_cast_0000000000000003: function(arg0) {
// Cast intrinsic for `U64 -> Externref`.
const ret = BigInt.asUintN(64, arg0);
return addHeapObject(ret);
},
__wbindgen_object_clone_ref: function(arg0) {
const ret = getObject(arg0);
return addHeapObject(ret);
},
__wbindgen_object_drop_ref: function(arg0) {
takeObject(arg0);
},
};
return {
__proto__: null,
"./pi_natives_bg.js": import0,
};
}
const CompiledPatternFinalization = (typeof FinalizationRegistry === 'undefined')
? { register: () => {}, unregister: () => {} }
: new FinalizationRegistry(ptr => wasm.__wbg_compiledpattern_free(ptr >>> 0, 1));
const PhotonImageFinalization = (typeof FinalizationRegistry === 'undefined')
? { register: () => {}, unregister: () => {} }
: new FinalizationRegistry(ptr => wasm.__wbg_photonimage_free(ptr >>> 0, 1));
function addHeapObject(obj) {
if (heap_next === heap.length) heap.push(heap.length + 1);
const idx = heap_next;
heap_next = heap[idx];
heap[idx] = obj;
return idx;
}
function _assertClass(instance, klass) {
if (!(instance instanceof klass)) {
throw new Error(`expected instance of ${klass.name}`);
}
}
function debugString(val) {
// primitive types
const type = typeof val;
if (type == 'number' || type == 'boolean' || val == null) {
return `${val}`;
}
if (type == 'string') {
return `"${val}"`;
}
if (type == 'symbol') {
const description = val.description;
if (description == null) {
return 'Symbol';
} else {
return `Symbol(${description})`;
}
}
if (type == 'function') {
const name = val.name;
if (typeof name == 'string' && name.length > 0) {
return `Function(${name})`;
} else {
return 'Function';
}
}
// objects
if (Array.isArray(val)) {
const length = val.length;
let debug = '[';
if (length > 0) {
debug += debugString(val[0]);
}
for(let i = 1; i < length; i++) {
debug += ', ' + debugString(val[i]);
}
debug += ']';
return debug;
}
// Test for built-in
const builtInMatches = /\[object ([^\]]+)\]/.exec(toString.call(val));
let className;
if (builtInMatches && builtInMatches.length > 1) {
className = builtInMatches[1];
} else {
// Failed to match the standard '[object ClassName]'
return toString.call(val);
}
if (className == 'Object') {
// we're a user defined class or Object
// JSON.stringify avoids problems with cycles, and is generally much
// easier than looping through ownProperties of `val`.
try {
return 'Object(' + JSON.stringify(val) + ')';
} catch (_) {
return 'Object';
}
}
// errors
if (val instanceof Error) {
return `${val.name}: ${val.message}\n${val.stack}`;
}
// TODO we could test for more things here, like `Set`s and `Map`s.
return className;
}
function dropObject(idx) {
if (idx < 132) return;
heap[idx] = heap_next;
heap_next = idx;
}
function getArrayJsValueFromWasm0(ptr, len) {
ptr = ptr >>> 0;
const mem = getDataViewMemory0();
const result = [];
for (let i = ptr; i < ptr + 4 * len; i += 4) {
result.push(takeObject(mem.getUint32(i, true)));
}
return result;
}
function getArrayU8FromWasm0(ptr, len) {
ptr = ptr >>> 0;
return getUint8ArrayMemory0().subarray(ptr / 1, ptr / 1 + len);
}
let cachedDataViewMemory0 = null;
function getDataViewMemory0() {
if (cachedDataViewMemory0 === null || cachedDataViewMemory0.buffer.detached === true || (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer)) {
cachedDataViewMemory0 = new DataView(wasm.memory.buffer);
}
return cachedDataViewMemory0;
}
function getStringFromWasm0(ptr, len) {
ptr = ptr >>> 0;
return decodeText(ptr, len);
}
let cachedUint8ArrayMemory0 = null;
function getUint8ArrayMemory0() {
if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) {
cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer);
}
return cachedUint8ArrayMemory0;
}
function getObject(idx) { return heap[idx]; }
let heap = new Array(128).fill(undefined);
heap.push(undefined, null, true, false);
let heap_next = heap.length;
function isLikeNone(x) {
return x === undefined || x === null;
}
function passArray8ToWasm0(arg, malloc) {
const ptr = malloc(arg.length * 1, 1) >>> 0;
getUint8ArrayMemory0().set(arg, ptr / 1);
WASM_VECTOR_LEN = arg.length;
return ptr;
}
function passStringToWasm0(arg, malloc, realloc) {
if (realloc === undefined) {
const buf = cachedTextEncoder.encode(arg);
const ptr = malloc(buf.length, 1) >>> 0;
getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf);
WASM_VECTOR_LEN = buf.length;
return ptr;
}
let len = arg.length;
let ptr = malloc(len, 1) >>> 0;
const mem = getUint8ArrayMemory0();
let offset = 0;
for (; offset < len; offset++) {
const code = arg.charCodeAt(offset);
if (code > 0x7F) break;
mem[ptr + offset] = code;
}
if (offset !== len) {
if (offset !== 0) {
arg = arg.slice(offset);
}
ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0;
const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len);
const ret = cachedTextEncoder.encodeInto(arg, view);
offset += ret.written;
ptr = realloc(ptr, len, offset, 1) >>> 0;
}
WASM_VECTOR_LEN = offset;
return ptr;
}
function takeObject(idx) {
const ret = getObject(idx);
dropObject(idx);
return ret;
}
let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });
cachedTextDecoder.decode();
function decodeText(ptr, len) {
return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len));
}
const cachedTextEncoder = new TextEncoder();
let WASM_VECTOR_LEN = 0;
const wasmUrl = import.meta.resolve(wasmPath);
const wasmInstantiated = await WebAssembly.instantiateStreaming(fetch(wasmUrl), __wbg_get_imports());
const wasm = wasmInstantiated.instance.exports;
packages/natives/wasm/pi_natives_bg.wasm
Vendored
-3
View File
@@ -1,3 +0,0 @@
version https://git-lfs.github.com/spec/v1
oid sha256:6b5e30b4f15ac967b02911abb51654e05b7c7033b557ee7c6f0c7fee662a8d6c
size 3720495
packages/natives/wasm/pi_natives_bg.wasm.d.ts
Vendored
-32
View File
@@ -1,32 +0,0 @@
/* tslint:disable */
/* eslint-disable */
export const memory: WebAssembly.Memory;
export const __wbg_compiledpattern_free: (a: number, b: number) => void;
export const __wbg_photonimage_free: (a: number, b: number) => void;
export const compiledpattern_has_match: (a: number, b: number, c: number) => number;
export const compiledpattern_new: (a: number, b: number) => void;
export const compiledpattern_search: (a: number, b: number, c: number, d: number, e: number) => number;
export const extract_segments: (a: number, b: number, c: number, d: number, e: number, f: number) => number;
export const get_supported_languages: (a: number) => void;
export const has_match: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => void;
export const highlight_code: (a: number, b: number, c: number, d: number, e: number, f: number) => void;
export const html_to_markdown: (a: number, b: number, c: number, d: number) => void;
export const photonimage_get_bytes: (a: number, b: number) => void;
export const photonimage_get_bytes_gif: (a: number, b: number) => void;
export const photonimage_get_bytes_jpeg: (a: number, b: number, c: number) => void;
export const photonimage_get_bytes_webp: (a: number, b: number) => void;
export const photonimage_get_height: (a: number) => number;
export const photonimage_get_width: (a: number) => number;
export const photonimage_new_from_byteslice: (a: number, b: number, c: number) => void;
export const resize: (a: number, b: number, c: number, d: number) => number;
export const search: (a: number, b: number, c: number) => number;
export const slice_with_width: (a: number, b: number, c: number, d: number, e: number) => number;
export const supports_language: (a: number, b: number) => number;
export const truncate_to_width: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => void;
export const visible_width: (a: number, b: number) => number;
export const compiledpattern_has_match_bytes: (a: number, b: number, c: number) => number;
export const compiledpattern_search_bytes: (a: number, b: number, c: number, d: number, e: number) => number;
export const __wbindgen_export: (a: number, b: number) => number;
export const __wbindgen_export2: (a: number, b: number, c: number, d: number) => number;
export const __wbindgen_export3: (a: number, b: number, c: number) => void;
export const __wbindgen_add_to_stack_pointer: (a: number) => number;