feat(builtins): sweep of GNU/BSD compat fixes + str performance opts

Addresses a broad audit of built-in shell utilities against their real counterparts: timeout gains signal delivery, -s/-k/--preserve-status/--foreground/-v, and GNU exit codes; diff defaults to normal format and gains -w/-b/-B/-i/-c/-x/-L/-s/--strip-trailing-cr and proper -r gating; find fixes -newerXY timestamp comparison direction, anchors -regex to whole paths, and gains BSD -perm +mode, -type lists, -size T/P suffixes, -E/-x/-s flags; date gains BSD -r epoch, -v adjustments, -j -f strptime, and non-greedy -I; tail/head accept obsolete -N/+N at any position with any file count; rg resolves case flags by last occurrence and gains --path-separator and clean -0 output; stat prints integer epochs for %X/%Y/%Z and gains BSD -s/-x/-t; cksum is registered as a builtin; truncate implements -o/--io-blocks and b/= size suffixes; sleep/timeout accept infinity; yes/errno/kill accept hyphen-prefixed operands; nohup -- cmd no longer runs --; which gains BSD -s.
This commit is contained in:
can1357
2026-08-20 01:45:28 +02:00
parent 219aae1303
commit edc0caeb0f
51 changed files with 5909 additions and 980 deletions
+1 -7
View File
@@ -28,6 +28,7 @@ ast-grep-core.workspace = true
base64.workspace = true
clap.workspace = true
globset.workspace = true
heapless.workspace = true
fontdue.workspace = true
grep-matcher.workspace = true
futures.workspace = true
@@ -63,13 +64,8 @@ syntect.workspace = true
tokio.workspace = true
tokio-util.workspace = true
toml.workspace = true
unicode-segmentation.workspace = true
unicode-normalization.workspace = true
unicode-properties.workspace = true
unicode-script.workspace = true
xutf.workspace = true
zstd.workspace = true
unicode-width.workspace = true
xxhash-rust.workspace = true
[target.'cfg(target_os = "linux")'.dependencies]
@@ -78,8 +74,6 @@ atspi = { version = "=0.30.0", features = ["tokio", "zbus"] }
pipewire = { version = "=0.9.2", optional = true }
reis = { version = "=0.5.0", features = ["tokio"] }
x11rb = { version = "=0.13.2", features = ["randr", "xinput", "xtest"] }
fancy-regex.workspace = true # utok scanner differential oracle
tiktoken-rs.workspace = true # utok openai differential oracle
xkeysym = "=0.2.1"
[target.'cfg(any(target_os = "macos", target_os = "windows"))'.dependencies]
+6 -6
View File
@@ -8,10 +8,10 @@ use std::io::Cursor;
use arboard::{Clipboard, Error as ClipboardError, ImageData};
use image::{DynamicImage, ImageFormat, RgbaImage};
use napi::bindgen_prelude::*;
use napi::{JsString, bindgen_prelude::*};
use napi_derive::napi;
use crate::task;
use crate::{js, task};
/// Clipboard image payload encoded as PNG bytes.
#[napi(object)]
@@ -135,8 +135,8 @@ fn read_raw_cf_dib() -> Option<Vec<u8>> {
/// # Errors
/// Returns an error if clipboard access fails.
#[napi]
pub fn copy_to_clipboard(text: String) -> Result<()> {
set_clipboard_text(text)
pub fn copy_to_clipboard(text: JsString) -> Result<()> {
set_clipboard_text(&js::utf8(text)?)
}
/// Linux: keep a single `arboard::Clipboard` alive for the whole process.
@@ -153,7 +153,7 @@ pub fn copy_to_clipboard(text: String) -> Result<()> {
/// (`wl-clipboard-rs` forks its own serving process) but sharing the instance
/// is harmless there.
#[cfg(target_os = "linux")]
fn set_clipboard_text(text: String) -> Result<()> {
fn set_clipboard_text(text: &str) -> Result<()> {
use std::sync::OnceLock;
use parking_lot::Mutex;
@@ -180,7 +180,7 @@ fn set_clipboard_text(text: String) -> Result<()> {
/// calling thread also avoids worker-thread `AppKit` pasteboard warnings on
/// macOS.
#[cfg(not(target_os = "linux"))]
fn set_clipboard_text(text: String) -> Result<()> {
fn set_clipboard_text(text: &str) -> Result<()> {
let mut clipboard = Clipboard::new()
.map_err(|err| Error::from_reason(format!("Failed to access clipboard: {err}")))?;
clipboard
+21 -11
View File
@@ -24,9 +24,11 @@
use std::{collections::HashMap, rc::Rc};
use napi::bindgen_prelude::*;
use napi::{JsString, bindgen_prelude::*};
use napi_derive::napi;
use crate::js;
/// UTF-16 code unit for `\n`.
const LF: u16 = 0x000a;
@@ -333,8 +335,10 @@ fn concat_tokens(tokens: &[&[u16]]) -> Vec<u16> {
/// options). Change values keep line terminators, and common runs are joined
/// from the new text.
#[napi]
pub fn diff_lines(old_text: Utf16String, new_text: Utf16String) -> Vec<DiffChange> {
diff_lines_impl(&old_text, &new_text)
pub fn diff_lines(old_text: JsString, new_text: JsString) -> Result<Vec<DiffChange>> {
let old_text = js::utf16(old_text)?;
let new_text = js::utf16(new_text)?;
Ok(diff_lines_impl(&old_text, &new_text))
}
fn diff_lines_impl(old_text: &[u16], new_text: &[u16]) -> Vec<DiffChange> {
@@ -351,8 +355,10 @@ fn diff_lines_impl(old_text: &[u16], new_text: &[u16]) -> Vec<DiffChange> {
/// Callers that map line numbers — like hashline recovery — need the counts,
/// not another copy of the text.
#[napi]
pub fn diff_line_runs(old_text: Utf16String, new_text: Utf16String) -> Vec<DiffRun> {
diff_line_runs_impl(&old_text, &new_text)
pub fn diff_line_runs(old_text: JsString, new_text: JsString) -> Result<Vec<DiffRun>> {
let old_text = js::utf16(old_text)?;
let new_text = js::utf16(new_text)?;
Ok(diff_line_runs_impl(&old_text, &new_text))
}
fn diff_line_runs_impl(old_text: &[u16], new_text: &[u16]) -> Vec<DiffRun> {
@@ -387,11 +393,13 @@ fn no_newline_marker() -> Vec<u16> {
/// semantics. `context` defaults to 4 like jsdiff.
#[napi]
pub fn structured_patch_hunks(
old_text: Utf16String,
new_text: Utf16String,
old_text: JsString,
new_text: JsString,
context: Option<u32>,
) -> Vec<PatchHunk> {
structured_patch_hunks_impl(&old_text, &new_text, context)
) -> Result<Vec<PatchHunk>> {
let old_text = js::utf16(old_text)?;
let new_text = js::utf16(new_text)?;
Ok(structured_patch_hunks_impl(&old_text, &new_text, context))
}
fn structured_patch_hunks_impl(
@@ -894,8 +902,10 @@ fn word_post_process(changes: &mut [DiffChange]) {
/// Tokens carry surrounding whitespace, equality ignores it, and the
/// post-pass dedupes whitespace across change boundaries.
#[napi]
pub fn diff_words(old_text: Utf16String, new_text: Utf16String) -> Vec<DiffChange> {
diff_words_impl(&old_text, &new_text)
pub fn diff_words(old_text: JsString, new_text: JsString) -> Result<Vec<DiffChange>> {
let old_text = js::utf16(old_text)?;
let new_text = js::utf16(new_text)?;
Ok(diff_words_impl(&old_text, &new_text))
}
fn diff_words_impl(old_text: &[u16], new_text: &[u16]) -> Vec<DiffChange> {
+9 -2
View File
@@ -5,8 +5,11 @@
//! on a persistent sidecar because they lack a process-owned in-memory name
//! registry with automatic crash recovery.
use napi::JsString;
use napi_derive::napi;
use crate::js;
#[cfg(target_os = "linux")]
mod linux;
#[cfg(all(unix, not(target_os = "linux")))]
@@ -48,9 +51,13 @@ pub struct FileLock {
impl FileLock {
/// Try to acquire `path` without blocking.
#[napi(factory)]
pub fn try_acquire(path: String) -> napi::Result<Self> {
pub fn try_acquire(path: JsString) -> napi::Result<Self> {
let path = js::utf8(path)?;
let inner = platform::try_acquire(&path).map_err(|error| {
napi::Error::from_reason(format!("Failed to acquire native file lock for {path}: {error}"))
napi::Error::from_reason(format!(
"Failed to acquire native file lock for {}: {error}",
&*path
))
})?;
Ok(Self { inner })
}
+84 -50
View File
@@ -7,11 +7,22 @@
use std::{cell::RefCell, collections::HashMap, sync::OnceLock};
use napi::{JsString, Result};
use napi_derive::napi;
use syntect::parsing::{
ParseState, Scope, ScopeStack, ScopeStackOp, SyntaxDefinition, SyntaxReference, SyntaxSet,
};
use crate::js::{self, InlineStr};
/// One theme colour: an ANSI escape sequence such as `\x1b[38;2;255;0;0m`.
///
/// Decoded inline, so a whole palette crosses the boundary without touching
/// the heap. The longest sequence a theme can produce sets attributes plus
/// truecolor foreground and background — `\x1b[1;3;4;38;2;255;255;255;48;2;
/// 255;255;255m`, 42 bytes — which the 47 usable bytes cover.
pub type Color = InlineStr<48>;
static SYNTAX_SET: OnceLock<SyntaxSet> = OnceLock::new();
static SCOPE_MATCHERS: OnceLock<ScopeMatchers> = OnceLock::new();
@@ -150,27 +161,38 @@ fn get_scope_matchers() -> &'static ScopeMatchers {
#[napi(object)]
pub struct HighlightColors {
/// ANSI color for comments.
pub comment: String,
#[napi(ts_type = "string")]
pub comment: Color,
/// ANSI color for keywords.
pub keyword: String,
#[napi(ts_type = "string")]
pub keyword: Color,
/// ANSI color for function names.
pub function: String,
#[napi(ts_type = "string")]
pub function: Color,
/// ANSI color for variables and identifiers.
pub variable: String,
#[napi(ts_type = "string")]
pub variable: Color,
/// ANSI color for string literals.
pub string: String,
#[napi(ts_type = "string")]
pub string: Color,
/// ANSI color for numeric literals.
pub number: String,
#[napi(ts_type = "string")]
pub number: Color,
/// ANSI color for type identifiers.
pub r#type: String,
#[napi(ts_type = "string")]
pub r#type: Color,
/// ANSI color for operators.
pub operator: String,
#[napi(ts_type = "string")]
pub operator: Color,
/// ANSI color for punctuation tokens.
pub punctuation: String,
#[napi(ts_type = "string")]
pub punctuation: Color,
/// ANSI color for diff inserted lines.
pub inserted: Option<String>,
#[napi(ts_type = "string")]
pub inserted: Option<Color>,
/// ANSI color for diff deleted lines.
pub deleted: Option<String>,
#[napi(ts_type = "string")]
pub deleted: Option<Color>,
}
/// Language alias mappings: (aliases, target syntax name).
@@ -383,29 +405,39 @@ fn find_syntax<'a>(ss: &'a SyntaxSet, lang: &str) -> Option<&'a SyntaxReference>
/// Highlighted code with ANSI color codes, or the original code if highlighting
/// fails.
#[napi]
pub fn highlight_code(code: String, lang: Option<String>, colors: HighlightColors) -> String {
pub fn highlight_code(
code: JsString,
lang: Option<JsString>,
colors: HighlightColors,
) -> Result<String> {
let code = js::utf8(code)?;
let lang = lang.map(js::utf8).transpose()?;
Ok(highlight_code_impl(&code, lang.as_deref(), &colors))
}
fn highlight_code_impl(code: &str, lang: Option<&str>, colors: &HighlightColors) -> String {
let inserted = colors.inserted.as_deref().unwrap_or("");
let deleted = colors.deleted.as_deref().unwrap_or("");
// Color palette as array for quick indexing
let palette = [
colors.comment.as_str(), // 0
colors.keyword.as_str(), // 1
colors.function.as_str(), // 2
colors.variable.as_str(), // 3
colors.string.as_str(), // 4
colors.number.as_str(), // 5
colors.r#type.as_str(), // 6
colors.operator.as_str(), // 7
colors.punctuation.as_str(), // 8
inserted, // 9
deleted, // 10
&*colors.comment, // 0
&*colors.keyword, // 1
&*colors.function, // 2
&*colors.variable, // 3
&*colors.string, // 4
&*colors.number, // 5
&*colors.r#type, // 6
&*colors.operator, // 7
&*colors.punctuation, // 8
inserted, // 9
deleted, // 10
];
let ss = get_syntax_set();
// Find syntax for the language
let syntax = match &lang {
let syntax = match lang {
Some(l) => find_syntax(ss, l),
None => None,
}
@@ -415,7 +447,7 @@ pub fn highlight_code(code: String, lang: Option<String>, colors: HighlightColor
let mut scope_stack = ScopeStack::new();
let mut result = String::with_capacity(code.len() * 2);
for line in syntect::util::LinesWithEndings::from(code.as_str()) {
for line in syntect::util::LinesWithEndings::from(code) {
let Ok(ops) = parse_state.parse_line(line, ss) else {
// Parse error - append unhighlighted line and continue
result.push_str(line);
@@ -477,16 +509,19 @@ pub fn highlight_code(code: String, lang: Option<String>, colors: HighlightColor
/// Returns true if the language has either direct support or a fallback
/// mapping.
#[napi]
pub fn supports_language(lang: String) -> bool {
if is_known_alias(&lang) {
pub fn supports_language(lang: JsString) -> Result<bool> {
Ok(supports_language_impl(&js::utf8(lang)?))
}
fn supports_language_impl(lang: &str) -> bool {
if is_known_alias(lang) {
return true;
}
// Fall back to direct syntax lookup
let ss = get_syntax_set();
find_syntax(ss, &lang).is_some()
find_syntax(ss, lang).is_some()
}
/// Get list of supported languages.
#[napi]
pub fn get_supported_languages() -> Vec<String> {
@@ -500,15 +535,15 @@ mod tests {
fn test_colors() -> HighlightColors {
HighlightColors {
comment: "<c>".to_string(),
keyword: "<k>".to_string(),
function: "<f>".to_string(),
variable: "<v>".to_string(),
string: "<s>".to_string(),
number: "<n>".to_string(),
r#type: "<t>".to_string(),
operator: "<o>".to_string(),
punctuation: "<p>".to_string(),
comment: Color::new("<c>").unwrap(),
keyword: Color::new("<k>").unwrap(),
function: Color::new("<f>").unwrap(),
variable: Color::new("<v>").unwrap(),
string: Color::new("<s>").unwrap(),
number: Color::new("<n>").unwrap(),
r#type: Color::new("<t>").unwrap(),
operator: Color::new("<o>").unwrap(),
punctuation: Color::new("<p>").unwrap(),
inserted: None,
deleted: None,
}
@@ -517,14 +552,13 @@ mod tests {
#[test]
fn highlights_nix_vendored_syntax() {
assert!(get_supported_languages().contains(&"Nix".to_string()));
assert!(supports_language("nix".to_string()));
assert!(supports_language_impl("nix"));
let out = highlight_code(
let out = highlight_code_impl(
"{ pkgs ? import <nixpkgs> {} }:\nlet message = \"hello\"; in pkgs.writeText \"msg\" \
message # greeting\n"
.to_string(),
Some("nix".to_string()),
test_colors(),
message # greeting\n",
Some("nix"),
&test_colors(),
);
assert!(out.contains("<k>let"));
assert!(out.contains("<s>hello"));
@@ -534,13 +568,13 @@ mod tests {
#[test]
fn highlights_mermaid_vendored_syntax() {
assert!(get_supported_languages().contains(&"Mermaid".to_string()));
assert!(supports_language("mermaid".to_string()));
assert!(supports_language("mmd".to_string()));
assert!(supports_language_impl("mermaid"));
assert!(supports_language_impl("mmd"));
let out = highlight_code(
"graph TD\n A[\"Start\"] --> B\n %% note\n".to_string(),
Some("mermaid".to_string()),
test_colors(),
let out = highlight_code_impl(
"graph TD\n A[\"Start\"] --> B\n %% note\n",
Some("mermaid"),
&test_colors(),
);
assert!(out.contains("<k>graph"));
assert!(out.contains("<s>Start"));
+7 -6
View File
@@ -3,10 +3,10 @@
use html_to_markdown_rs::{
ConversionOptions, PreprocessingOptions, PreprocessingPreset, WarningKind, convert,
};
use napi::bindgen_prelude::*;
use napi::{JsString, bindgen_prelude::*};
use napi_derive::napi;
use crate::task;
use crate::{js::into_string, task};
/// Options for HTML to Markdown conversion.
#[napi(object)]
@@ -24,14 +24,15 @@ pub struct HtmlToMarkdownOptions {
/// Returns an error if the conversion fails or the worker task aborts.
#[napi]
pub fn html_to_markdown(
html: String,
html: JsString,
options: Option<HtmlToMarkdownOptions>,
) -> task::Promise<String> {
) -> Result<task::Promise<String>> {
let html = into_string(html)?;
let options = options.unwrap_or_default();
let clean_content = options.clean_content.unwrap_or(false);
let skip_images = options.skip_images.unwrap_or(false);
task::blocking("html_to_markdown", (), move |_| {
Ok(task::blocking("html_to_markdown", (), move |_| {
let conversion_opts = ConversionOptions {
skip_images,
preprocessing: PreprocessingOptions {
@@ -54,5 +55,5 @@ pub fn html_to_markdown(
return Err(Error::from_reason(format!("Conversion error: {}", warning.message)));
}
Ok(result.content.unwrap_or_default())
})
}))
}
+6 -3
View File
@@ -4,9 +4,11 @@
//! JavaScript-facing shapes plus conversions between walker entries and N-API
//! payloads.
use napi::bindgen_prelude::*;
use napi::{JsString, bindgen_prelude::*};
use napi_derive::napi;
use crate::js;
/// Resolved filesystem entry kind for glob filters and match metadata.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[napi]
@@ -75,9 +77,10 @@ pub(crate) fn map_walker_error<E: std::fmt::Display>(err: pi_walker::WalkError<E
/// Intended to be called after agent file mutations: write, edit, rename, or
/// delete.
#[napi]
pub fn invalidate_fs_scan_cache(path: Option<String>) {
pub fn invalidate_fs_scan_cache(path: Option<JsString>) -> Result<()> {
match path {
Some(path) => pi_walker::invalidate_path_string(&path),
Some(path) => pi_walker::invalidate_path_string(&js::utf8(path)?),
None => pi_walker::invalidate_all(),
}
Ok(())
}
+7 -3
View File
@@ -22,7 +22,10 @@ use napi::bindgen_prelude::*;
use napi_derive::napi;
use pi_iso::{BackendKind, ChangeKind, Diff, FileChange, IsoError, IsolationBackend};
use crate::js;
const ISO_UNAVAILABLE_PREFIX: &str = "ISO_UNAVAILABLE:";
const ISO_UNAVAILABLE_WITH_LEADING_SPACE: &str = " ISO_UNAVAILABLE:";
/// Isolation backend identifier. Numeric so the JS side can `switch` on
/// the enum without string comparisons.
@@ -174,9 +177,10 @@ pub async fn iso_diff(lower: String, merged: String) -> Result<IsoDiff> {
/// Use this to distinguish "this backend isn't installed" from a hard
/// failure when handling caught errors on the JS side.
#[napi]
pub fn iso_is_unavailable_error(message: String) -> bool {
message.starts_with(ISO_UNAVAILABLE_PREFIX)
|| message.contains(&format!(" {ISO_UNAVAILABLE_PREFIX}"))
pub fn iso_is_unavailable_error(message: napi::JsString) -> Result<bool> {
let message = js::utf8(message)?;
Ok(message.starts_with(ISO_UNAVAILABLE_PREFIX)
|| message.contains(ISO_UNAVAILABLE_WITH_LEADING_SPACE))
}
const fn to_napi_kind(kind: BackendKind) -> IsoBackendKind {
+475
View File
@@ -0,0 +1,475 @@
//! Borrowing JavaScript strings at the N-API boundary.
//!
//! Node-API never hands out a pointer into a JS string's backing store: every
//! accessor writes code units into a caller-owned buffer. The copy is
//! unavoidable, the *allocation* is not — [`utf16`] and [`utf8`] read into a
//! fixed per-thread scratch arena and hand back a guard that derefs to the
//! borrowed text, releasing its range on drop. Text that fits the arena costs
//! zero allocations; longer text spills to one owned `Vec`. Several guards can
//! be live at once (diff pairs, colour palettes) — each owns a disjoint range.
//!
//! [`utf16`] is the default: it is the JS string's own encoding, so it is the
//! only accessor that never transcodes. [`utf8`] exists for algorithms that are
//! byte- or `str`-shaped (terminal escape parsing, syntect, paths). Reach for
//! [`into_string`] only when the text must outlive the N-API callback — a value
//! moved into a worker task, a channel, or an async body.
//! Short, bounded strings — ANSI colours, font names, language ids — skip the
//! arena entirely: [`InlineStr`] decodes them into a fixed-size array that
//! lives in the struct itself, so a whole options object crosses with no
//! allocation.
use std::{
cell::{Cell, UnsafeCell},
fmt,
ops::{Deref, Range},
ptr::{self, NonNull},
slice, str,
};
use napi::{
Error, JsString, JsValue, Result, Status,
bindgen_prelude::{FromNapiValue, ToNapiValue, TypeName, ValidateNapiValue},
sys,
};
/// Scratch bytes per thread. Only the JS thread reaches this module (N-API
/// handles are not `Send`), so the footprint is effectively process-global.
const SCRATCH_LEN: usize = 64 * 1024;
/// Fixed-size bump arena backing [`Utf16`]/[`Utf8`] guards.
///
/// The base address is stable for the thread's lifetime (the array never
/// grows), so guards may hold raw pointers into it. Soundness rests on range
/// discipline, not a borrow flag: every committed range is disjoint, new reads
/// only touch bytes past `offset`, and no reference to the whole array is ever
/// formed — all access goes through raw pointers into a caller's own range.
struct Arena {
/// Stored as `u16` units purely for the 2-alignment UTF-16 fills need;
/// UTF-8 fills reinterpret the same bytes at alignment 1.
buf: UnsafeCell<[u16; SCRATCH_LEN / 2]>,
/// Bytes handed out. Fills bump it; drops roll it back (see [`Self::release`]).
offset: Cell<usize>,
/// Live scratch-backed guards. Hitting zero resets `offset`, so a non-LIFO
/// drop order leaks at most until the last guard goes away.
live: Cell<usize>,
}
thread_local! {
static ARENA: Arena = const {
Arena {
buf: UnsafeCell::new([0; SCRATCH_LEN / 2]),
offset: Cell::new(0),
live: Cell::new(0),
}
};
}
impl Arena {
const fn base(&self) -> *mut u8 {
self.buf.get().cast()
}
/// Free tail aligned to `align` (a power of two): byte offset and length.
const fn tail(&self, align: usize) -> (usize, usize) {
let start = (self.offset.get() + align - 1) & !(align - 1);
(start, SCRATCH_LEN.saturating_sub(start))
}
/// Record `start..start + len` as owned by a new guard.
fn commit(&self, start: usize, len: usize) {
self.offset.set(start + len);
self.live.set(self.live.get() + 1);
}
/// Return `start..end`. The topmost range rolls the bump pointer back
/// (LIFO drops recycle immediately); otherwise the bytes are stranded
/// until `live` reaches zero and the whole arena resets.
fn release(&self, start: usize, end: usize) {
let live = self.live.get() - 1;
self.live.set(live);
if live == 0 {
self.offset.set(0);
} else if self.offset.get() == end {
self.offset.set(start);
}
}
}
/// Text read from a JS string: borrowed from the thread's scratch arena when
/// it fits, spilled to one owned `Vec` when it does not.
enum TextRepr<T> {
/// Range inside [`ARENA`]; `Drop` releases it. `NonNull` keeps this
/// variant `!Send`, so the pointer can never outlive its thread's TLS.
Scratch { ptr: NonNull<T>, len: usize },
/// Heap spill for text longer than the arena's free tail.
Owned(Vec<T>),
}
impl<T> TextRepr<T> {
#[inline]
fn as_slice(&self) -> &[T] {
match self {
// SAFETY: the constructor committed `ptr..ptr + len` to this guard;
// the arena never moves and no other guard overlaps the range.
Self::Scratch { ptr, len } => unsafe { slice::from_raw_parts(ptr.as_ptr(), *len) },
Self::Owned(vec) => vec,
}
}
}
impl<T> Drop for TextRepr<T> {
fn drop(&mut self) {
if let Self::Scratch { ptr, len } = *self {
ARENA.with(|arena| {
let start = ptr.as_ptr().addr() - arena.base().addr();
arena.release(start, start + len * size_of::<T>());
});
}
}
}
/// Borrowed UTF-16 code units of a JS string, backed by the scratch arena.
///
/// Unlike `JsString::into_utf16`, the view excludes the NUL terminator
/// Node-API appends, so `&*guard` is exactly the string's code units.
pub struct Utf16(TextRepr<u16>);
impl Deref for Utf16 {
type Target = [u16];
#[inline]
fn deref(&self) -> &[u16] {
self.0.as_slice()
}
}
/// Borrowed UTF-8 bytes of a JS string, backed by the scratch arena.
pub struct Utf8(TextRepr<u8>);
impl Deref for Utf8 {
type Target = str;
#[inline]
fn deref(&self) -> &str {
// SAFETY: utf8 validates the bytes before constructing Utf8.
unsafe { str::from_utf8_unchecked(self.0.as_slice()) }
}
}
/// Borrow `value` as UTF-16 code units using the thread's scratch arena.
///
/// The happy path is a single N-API call into the arena's free tail; text
/// that does not fit is measured and read into an owned spill buffer.
#[inline]
pub fn utf16(value: JsString<'_>) -> Result<Utf16> {
let raw = value.value();
ARENA.with(|arena| {
let (start, avail_bytes) = arena.tail(2);
let avail = avail_bytes / 2;
if avail >= 2 {
// SAFETY: `start..start + avail_bytes` is past every committed range,
// and the base is 2-aligned with `start` aligned up.
let ptr = unsafe { arena.base().add(start) }.cast::<u16>();
let mut written = 0;
// SAFETY: `raw` is a JS string owned by the live callback; Node-API
// writes at most `avail - 1` units plus a NUL into the free tail.
let status = unsafe {
sys::napi_get_value_string_utf16(raw.env, raw.value, ptr, avail, &mut written)
};
napi::check_status!(status, "Failed to read JavaScript string")?;
if written < avail - 1 {
arena.commit(start, written * 2);
return Ok(Utf16(TextRepr::Scratch {
ptr: NonNull::new(ptr).unwrap(),
len: written,
}));
}
}
let mut len = 0;
// SAFETY: a null buffer asks Node-API for the code-unit length only.
let status = unsafe {
sys::napi_get_value_string_utf16(raw.env, raw.value, ptr::null_mut(), 0, &mut len)
};
napi::check_status!(status, "Failed to measure JavaScript string")?;
let mut buf: Vec<u16> = Vec::with_capacity(len + 1);
let mut written = 0;
// SAFETY: `buf` holds the measured length plus the NUL slot.
let status = unsafe {
sys::napi_get_value_string_utf16(
raw.env,
raw.value,
buf.as_mut_ptr(),
len + 1,
&mut written,
)
};
napi::check_status!(status, "Failed to read JavaScript string")?;
// SAFETY: Node-API initialised `written` units.
unsafe { buf.set_len(written) };
Ok(Utf16(TextRepr::Owned(buf)))
})
}
/// Borrow `value` as UTF-8 using the thread's scratch arena.
///
/// Same shape as [`utf16`], plus UTF-8 validation before the guard exists.
#[inline]
pub fn utf8(value: JsString<'_>) -> Result<Utf8> {
let raw = value.value();
ARENA.with(|arena| {
let (start, avail) = arena.tail(1);
if avail >= 2 {
// SAFETY: `start..start + avail` is past every committed range.
let ptr = unsafe { arena.base().add(start) };
let mut written = 0;
// SAFETY: `raw` is a JS string owned by the live callback; Node-API
// writes at most `avail - 1` bytes plus a NUL into the free tail.
let status = unsafe {
sys::napi_get_value_string_utf8(raw.env, raw.value, ptr.cast(), avail, &mut written)
};
napi::check_status!(status, "Failed to read JavaScript string")?;
if written < avail - 1 {
// SAFETY: Node-API initialised `written` bytes at `ptr`.
let bytes = unsafe { slice::from_raw_parts(ptr, written) };
if let Err(error) = str::from_utf8(bytes) {
return Err(Error::new(Status::InvalidArg, error.to_string()));
}
arena.commit(start, written);
return Ok(Utf8(TextRepr::Scratch {
ptr: NonNull::new(ptr).unwrap(),
len: written,
}));
}
}
let mut len = 0;
// SAFETY: a null buffer asks Node-API for the byte length only.
let status = unsafe {
sys::napi_get_value_string_utf8(raw.env, raw.value, ptr::null_mut(), 0, &mut len)
};
napi::check_status!(status, "Failed to measure JavaScript string")?;
let mut buf: Vec<u8> = Vec::with_capacity(len + 1);
let mut written = 0;
// SAFETY: `buf` holds the measured length plus the NUL slot.
let status = unsafe {
sys::napi_get_value_string_utf8(
raw.env,
raw.value,
buf.as_mut_ptr().cast(),
len + 1,
&mut written,
)
};
napi::check_status!(status, "Failed to read JavaScript string")?;
// SAFETY: Node-API initialised `written` bytes.
unsafe { buf.set_len(written) };
if let Err(error) = str::from_utf8(&buf) {
return Err(Error::new(Status::InvalidArg, error.to_string()));
}
Ok(Utf8(TextRepr::Owned(buf)))
})
}
/// Append `value`'s UTF-16 code units to `out` and return their span.
///
/// For batches: one growing buffer holds every element, so an array costs a
/// single allocation instead of one per string, and the spans can then be
/// counted in parallel while the reads themselves stay on the JS thread —
/// Node-API handles are not `Send`.
pub fn utf16_append(value: JsString<'_>, out: &mut Vec<u16>) -> Result<Range<usize>> {
let raw = value.value();
let start = out.len();
let mut len = 0;
// SAFETY: `raw` is a JS string owned by the live callback; a null buffer asks
// Node-API for the code-unit length only.
let status =
unsafe { sys::napi_get_value_string_utf16(raw.env, raw.value, ptr::null_mut(), 0, &mut len) };
napi::check_status!(status, "Failed to measure JavaScript string")?;
out.resize(start + len + 1, 0);
let mut written = 0;
// SAFETY: same string, and the tail from `start` holds the measured length
// plus the NUL slot Node-API writes.
let status = unsafe {
sys::napi_get_value_string_utf16(
raw.env,
raw.value,
out[start..].as_mut_ptr(),
len + 1,
&mut written,
)
};
napi::check_status!(status, "Failed to read JavaScript string")?;
out.truncate(start + written);
Ok(start..out.len())
}
/// Copy `value` into an owned `String`.
///
/// Only for text that outlives the N-API callback: a worker task, a channel
/// message, or an async body. Synchronous consumers must borrow instead.
pub fn into_string(value: JsString<'_>) -> Result<String> {
let raw = value.value();
// SAFETY: `raw` is a validated JS string from the current callback.
unsafe { String::from_napi_value(raw.env, raw.value) }
}
/// A JS string decoded into a fixed-capacity inline buffer, no heap involved.
///
/// Holds `N` bytes with a `u8` length, so the whole value is `N + 1` bytes and
/// an options struct full of them costs nothing to build. Node-API needs one
/// byte for its NUL terminator, leaving [`Self::CAPACITY`] usable; longer input
/// is a caller error rather than a silent truncation, so an escape sequence can
/// never arrive half-copied.
///
/// Storage is UTF-8: every consumer of these values wants `&str`, and the
/// inputs are ASCII, so this is the encoding that avoids a transcode at the
/// point of use. Use [`utf16`] for text whose consumer works in code units.
#[derive(Clone)]
pub struct InlineStr<const N: usize>(heapless::Vec<u8, N, u8>);
impl<const N: usize> InlineStr<N> {
/// Usable bytes, excluding the NUL slot Node-API requires.
pub const CAPACITY: usize = N - 1;
/// Build from Rust text, for tests and native-side defaults.
pub fn new(text: &str) -> Result<Self> {
if text.len() > Self::CAPACITY {
return Err(too_long(text.len(), Self::CAPACITY));
}
heapless::Vec::from_slice(text.as_bytes())
.map(Self)
.map_err(|_| too_long(text.len(), Self::CAPACITY))
}
}
fn too_long(len: usize, capacity: usize) -> Error {
Error::new(Status::InvalidArg, format!("string is {len} bytes, expected at most {capacity}"))
}
impl<const N: usize> Deref for InlineStr<N> {
type Target = str;
fn deref(&self) -> &str {
// SAFETY: both constructors validate the bytes as UTF-8 before storing
// them, and the buffer is immutable afterwards.
unsafe { str::from_utf8_unchecked(&self.0) }
}
}
impl<const N: usize> fmt::Debug for InlineStr<N> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Debug::fmt(&**self, f)
}
}
impl<const N: usize> TypeName for InlineStr<N> {
fn type_name() -> &'static str {
"String"
}
fn value_type() -> napi::ValueType {
napi::ValueType::String
}
}
impl<const N: usize> ValidateNapiValue for InlineStr<N> {}
impl<const N: usize> FromNapiValue for InlineStr<N> {
unsafe fn from_napi_value(env: sys::napi_env, napi_val: sys::napi_value) -> Result<Self> {
let mut len = 0;
// SAFETY: `napi_val` is a JS string owned by the live callback; a null
// buffer asks Node-API for the byte length only.
let status =
unsafe { sys::napi_get_value_string_utf8(env, napi_val, ptr::null_mut(), 0, &mut len) };
napi::check_status!(status, "Failed to measure JavaScript string")?;
if len > Self::CAPACITY {
return Err(too_long(len, Self::CAPACITY));
}
let mut buf: heapless::Vec<u8, N, u8> = heapless::Vec::new();
buf.resize_default(N)
.map_err(|_| too_long(len, Self::CAPACITY))?;
let mut written = 0;
// SAFETY: same string, and `buf` is filled to `N`, which holds the measured
// length plus the NUL terminator Node-API writes.
let status = unsafe {
sys::napi_get_value_string_utf8(env, napi_val, buf.as_mut_ptr().cast(), N, &mut written)
};
napi::check_status!(status, "Failed to read JavaScript string")?;
buf.truncate(written);
if let Err(error) = str::from_utf8(&buf) {
return Err(Error::new(Status::InvalidArg, error.to_string()));
}
Ok(Self(buf))
}
}
impl<const N: usize> ToNapiValue for InlineStr<N> {
unsafe fn to_napi_value(env: sys::napi_env, val: Self) -> Result<sys::napi_value> {
// SAFETY: `env` is the live callback environment.
unsafe { ToNapiValue::to_napi_value(env, &*val) }
}
}
#[cfg(test)]
mod tests {
use super::*;
fn arena() -> Arena {
Arena {
buf: UnsafeCell::new([0; SCRATCH_LEN / 2]),
offset: Cell::new(0),
live: Cell::new(0),
}
}
/// Live guards own disjoint ranges; overlap would alias the derefs (UB).
#[test]
fn commits_never_overlap_live_ranges() {
let a = arena();
let (s1, _) = a.tail(1);
a.commit(s1, 100);
let (s2, _) = a.tail(2);
assert!(s2 >= s1 + 100);
a.commit(s2, 50);
let (s3, _) = a.tail(1);
assert!(s3 >= s2 + 50);
}
/// LIFO drops recycle immediately; the next fill reuses the range.
#[test]
fn lifo_release_rolls_back() {
let a = arena();
a.commit(0, 100);
a.commit(100, 50);
a.release(100, 150);
assert_eq!(a.tail(1).0, 100);
a.release(0, 100);
assert_eq!(a.tail(1).0, 0);
}
/// Non-LIFO drops strand bytes only until the last guard goes away.
#[test]
fn arena_resets_when_last_guard_drops() {
let a = arena();
a.commit(0, 100);
a.commit(100, 50);
a.release(0, 100);
assert_eq!(a.tail(1).0, 150, "inner range stays stranded while a guard is live");
a.release(100, 150);
assert_eq!(a.tail(1).0, 0);
}
/// A utf16 fill after an odd utf8 commit must get a 2-aligned range.
#[test]
fn utf16_tail_is_aligned() {
let a = arena();
a.commit(0, 7);
let (start, len) = a.tail(2);
assert_eq!(start, 8);
assert_eq!(len, SCRATCH_LEN - 8);
}
}
+37 -27
View File
@@ -12,9 +12,12 @@
use std::borrow::Cow;
use napi::{JsString, Result};
use napi_derive::napi;
use phf::phf_map;
use crate::js;
const LOCK_MASK: u32 = 64 + 128;
// Internal sentinel codes for CSI 1;mod <letter> forms:
@@ -298,11 +301,20 @@ static LETTERS: [&str; 26] = [
/// base layout key) and modifier bits.
#[napi]
pub fn matches_kitty_sequence(
data: String,
data: JsString,
expected_codepoint: i32,
expected_modifier: u32,
) -> Result<bool> {
let data = js::utf8(data)?;
Ok(matches_kitty_sequence_inner(data.as_bytes(), expected_codepoint, expected_modifier))
}
fn matches_kitty_sequence_inner(
data: &[u8],
expected_codepoint: i32,
expected_modifier: u32,
) -> bool {
let Some(parsed) = parse_kitty_sequence_bytes(data.as_bytes()) else {
let Some(parsed) = parse_kitty_sequence_bytes(data) else {
return false;
};
@@ -378,40 +390,46 @@ const fn is_symbol_key(cp: i32) -> bool {
///
/// Returns a key id like "escape" or "ctrl+c", or None if unrecognized.
#[napi]
pub fn parse_key(data: String, kitty_protocol_active: bool) -> Option<String> {
parse_key_inner(data.as_bytes(), kitty_protocol_active).map(|s| s.into_owned())
pub fn parse_key(data: JsString, kitty_protocol_active: bool) -> Result<Option<String>> {
let data = js::utf8(data)?;
Ok(parse_key_inner(data.as_bytes(), kitty_protocol_active).map(|key| key.into_owned()))
}
/// Check if input matches a legacy escape sequence for the given key name.
///
/// Returns true only when the byte sequence maps to the exact key identifier.
#[napi]
pub fn matches_legacy_sequence(data: String, key_name: String) -> bool {
LEGACY_SEQUENCES
pub fn matches_legacy_sequence(data: JsString, key_name: JsString) -> Result<bool> {
let data = js::utf8(data)?;
let key_name = js::utf8(key_name)?;
Ok(LEGACY_SEQUENCES
.get(data.as_bytes())
.is_some_and(|&id| id == key_name)
.is_some_and(|&id| id == &*key_name))
}
/// Match input data against a key identifier string.
///
/// Returns true when the bytes represent the specified key with modifiers.
#[napi]
pub fn matches_key(data: String, key_id: String, kitty_protocol_active: bool) -> bool {
matches_key_inner(data.as_bytes(), &key_id, kitty_protocol_active)
pub fn matches_key(data: JsString, key_id: JsString, kitty_protocol_active: bool) -> Result<bool> {
let data = js::utf8(data)?;
let key_id = js::utf8(key_id)?;
Ok(matches_key_inner(data.as_bytes(), &key_id, kitty_protocol_active))
}
/// Parse a Kitty keyboard protocol sequence.
///
/// Returns a structured parse result when the input is a valid Kitty sequence.
#[napi]
pub fn parse_kitty_sequence(data: String) -> Option<ParsedKittyResult> {
parse_kitty_sequence_bytes(data.as_bytes()).map(|p| ParsedKittyResult {
codepoint: p.codepoint,
shifted_key: p.shifted_key,
base_layout_key: p.base_layout_key,
modifier: p.modifier,
event_type: optional_kitty_event_type(p.event_type),
})
pub fn parse_kitty_sequence(data: JsString) -> Result<Option<ParsedKittyResult>> {
let data = js::utf8(data)?;
Ok(parse_kitty_sequence_bytes(data.as_bytes()).map(|parsed| ParsedKittyResult {
codepoint: parsed.codepoint,
shifted_key: parsed.shifted_key,
base_layout_key: parsed.base_layout_key,
modifier: parsed.modifier,
event_type: optional_kitty_event_type(parsed.event_type),
}))
}
// =============================================================================
@@ -1590,20 +1608,12 @@ mod tests {
let plain_cyrillic_c = b"\x1b[1089::99u";
assert!(!matches_key_inner(plain_cyrillic_c, "c", true));
assert_eq!(parse_key_inner(plain_cyrillic_c, true).as_deref(), None);
assert!(!matches_kitty_sequence(
String::from_utf8_lossy(plain_cyrillic_c).into_owned(),
i32::from(b'c'),
0,
));
assert!(!matches_kitty_sequence_inner(plain_cyrillic_c, i32::from(b'c'), 0));
let ctrl_cyrillic_c = b"\x1b[1089::99;5u";
assert!(matches_key_inner(ctrl_cyrillic_c, "ctrl+c", true));
assert_eq!(parse_key_inner(ctrl_cyrillic_c, true).as_deref(), Some("ctrl+c"));
assert!(matches_kitty_sequence(
String::from_utf8_lossy(ctrl_cyrillic_c).into_owned(),
i32::from(b'c'),
MOD_CTRL,
));
assert!(matches_kitty_sequence_inner(ctrl_cyrillic_c, i32::from(b'c'), MOD_CTRL));
}
#[test]
+1
View File
@@ -39,6 +39,7 @@ pub mod grep;
pub mod highlight;
pub mod html;
pub mod iofs;
pub mod js;
pub mod keys;
pub mod live;
/// PDF inspection and Markdown conversion.
+5 -5
View File
@@ -8,14 +8,14 @@
use std::time::Duration;
use napi::{
Env, Result,
Env, JsString, Result,
bindgen_prelude::{PromiseRaw, Unknown},
};
use napi_derive::napi;
use pi_shell::process::{self as core_process, ProcessStatus as CoreProcessStatus};
pub use pi_shell::process::{KILL_SIGNAL, TERM_SIGNAL, TerminationTargets, kill_process_group};
use crate::task;
use crate::{js::into_string, task};
#[derive(Default)]
#[napi(object)]
@@ -81,11 +81,11 @@ impl Process {
/// Open stable process references whose executable path matches exactly.
#[napi]
pub fn from_path(path: String) -> Vec<Process> {
core_process::Process::from_path(path)
pub fn from_path(path: JsString) -> Result<Vec<Process>> {
Ok(core_process::Process::from_path(into_string(path)?)
.into_iter()
.map(Self::from_inner)
.collect()
.collect())
}
/// Operating-system process identifier for this process reference.
+4 -3
View File
@@ -13,6 +13,7 @@ use std::{
};
use napi::{
JsString,
bindgen_prelude::*,
threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode},
};
@@ -20,7 +21,7 @@ use napi_derive::napi;
use parking_lot::Mutex;
use portable_pty::{Child, CommandBuilder, PtySize, native_pty_system};
use crate::{ps, task};
use crate::{js::into_string, ps, task};
/// Options for running a command in a PTY session.
#[napi(object)]
@@ -178,8 +179,8 @@ impl PtySession {
/// Write raw input bytes to PTY stdin.
#[napi]
pub fn write(&self, data: String) -> Result<()> {
self.send_control(ControlMessage::Input(data))
pub fn write(&self, data: JsString) -> Result<()> {
self.send_control(ControlMessage::Input(into_string(data)?))
}
/// Resize the active PTY.
+10 -7
View File
@@ -48,10 +48,10 @@ use std::{
use base64::{Engine as _, engine::general_purpose::STANDARD};
use fontdue::{Font as TtfFace, FontSettings, Metrics};
use napi::bindgen_prelude::*;
use napi::{JsString, bindgen_prelude::*};
use napi_derive::napi;
use crate::task;
use crate::{js, task};
/// Upper bound on the frame edge: a hard stop against absurd allocations
/// (`size * size` pixel buffer), far above the 2576px production frame.
@@ -1162,14 +1162,17 @@ pub struct SnapcompactRenderOptions {
/// the selected native font has a glyph for it; renderer control codes are
/// considered renderable because they are interpreted outside font lookup.
#[napi]
pub fn snapcompact_supported_chars(font: String, chars: String) -> Result<String> {
let font = resolve_font(&font).ok_or_else(|| {
pub fn snapcompact_supported_chars(font: JsString, chars: JsString) -> Result<String> {
let font_name = js::utf8(font)?;
let font = resolve_font(&font_name).ok_or_else(|| {
Error::from_reason(format!(
"Unknown snapcompact font {font:?}: expected \"5x8\", \"8x8\", \"6x12\", \"8x13\", or \
\"silver\""
"Unknown snapcompact font {:?}: expected \"5x8\", \"8x8\", \"6x12\", \"8x13\", or \
\"silver\"",
&*font_name
))
})?;
let mut supported = String::new();
let chars = js::utf8(chars)?;
let mut supported = String::with_capacity(chars.len());
for ch in chars.chars() {
if matches!(ch as u32, DIM_ON | DIM_OFF | FULL_BLOCK | 0x0a) || font.supports(ch as u32) {
supported.push(ch);
+45 -43
View File
@@ -16,8 +16,8 @@ use std::{
use napi::{JsString, bindgen_prelude::*};
use napi_derive::napi;
use smallvec::{SmallVec, smallvec};
use unicode_segmentation::UnicodeSegmentation;
use unicode_width::{UnicodeWidthChar, UnicodeWidthStr};
use crate::js;
const MIN_TAB_WIDTH: u32 = 1;
const MAX_TAB_WIDTH: u32 = 16;
@@ -45,8 +45,7 @@ fn build_utf16_string(mut data: Vec<u16>) -> Utf16String {
while data.last() == Some(&0) {
data.pop();
}
// SAFETY: we know Utf16String == struct(Vec<u16>)
unsafe { std::mem::transmute(data) }
Utf16String::from(data)
}
// ============================================================================
@@ -642,7 +641,7 @@ fn apply_hangul_compat_jamo_delta(width: usize, c: char) -> usize {
let Some(target) = hangul_compat_jamo_target_width() else {
return width;
};
let unicode_width = UnicodeWidthChar::width(c).unwrap_or(0);
let unicode_width = xutf::width_char(c);
// The zero-width filler (U+3164 HANGUL FILLER) is an invisible placeholder.
// The target is set for *visible* jamo, so only the narrow correction
// (target 1) applies to the filler; a wide terminal renders it at its
@@ -659,7 +658,7 @@ fn apply_hangul_compat_jamo_delta(width: usize, c: char) -> usize {
}
#[inline]
fn char_width_corrected(c: char) -> Option<usize> {
fn char_width_corrected(c: char) -> usize {
// Hangul Compatibility Jamo U+3131..=U+318E render as 1 cell on some
// terminals (Terminal.app, iTerm2) but follow UAX#11 at 2 cells on others
// (Ghostty, most Linux terminals). The width is resolved at runtime from the
@@ -671,13 +670,13 @@ fn char_width_corrected(c: char) -> Option<usize> {
// Zero-width filler (U+3164): only the narrow correction applies — a
// wide terminal renders it at its Unicode width (0), not the effective
// wide target set for visible jamo. See apply_hangul_compat_jamo_delta.
let unicode_width = UnicodeWidthChar::width(c).unwrap_or(0);
let unicode_width = xutf::width_char(c);
if unicode_width == 0 && target > 1 {
return Some(unicode_width);
return unicode_width;
}
return Some(target);
return target;
}
UnicodeWidthChar::width(c)
xutf::width_char(c)
}
#[inline]
@@ -690,14 +689,14 @@ fn grapheme_width_str(g: &str, tab_width: usize) -> usize {
return 0;
};
if it.next().is_none() {
return char_width_corrected(c0).unwrap_or(0);
return char_width_corrected(c0);
}
// Multi-char grapheme: keep UnicodeWidthStr as the source of truth for
// sequence-level width rules (VS16 emoji presentation, keycaps, ZWJ emoji,
// CRLF, script ligatures). A per-char sum is not equivalent. Apply only the
// same local Compatibility Jamo delta that char_width_corrected applies to
// standalone code points; the delta is a no-op when no correction is active.
let mut width = UnicodeWidthStr::width(g);
let mut width = xutf::width_str(g);
for c in g.chars() {
width = apply_hangul_compat_jamo_delta(width, c);
}
@@ -729,7 +728,7 @@ where
}
let mut utf16_pos = 0usize;
for g in scratch.graphemes(true) {
for g in xutf::graphemes_str(scratch) {
let w = grapheme_width_str(g, tab_width);
let g_u16_len: usize = g.chars().map(|c| c.len_utf16()).sum();
@@ -1255,10 +1254,12 @@ fn wrap_text_with_ansi_impl(
/// Returns UTF-16 lines with active SGR codes carried across line boundaries.
#[napi]
pub fn wrap_text_with_ansi(text: JsString, width: u32, tab_width: u32) -> Result<Vec<Utf16String>> {
let text_u16 = text.into_utf16()?;
let text = js::utf16(text)?;
let tab_width = clamp_tab_width_for_ops(tab_width);
let lines = wrap_text_with_ansi_impl(text_u16.as_slice(), width as usize, tab_width);
Ok(lines.into_iter().map(build_utf16_string).collect())
Ok(wrap_text_with_ansi_impl(&text, width as usize, tab_width)
.into_iter()
.map(build_utf16_string)
.collect())
}
// ============================================================================
@@ -1280,30 +1281,36 @@ pub fn truncate_to_width(
let ellipsis_kind = ellipsis_kind.unwrap_or(Ellipsis::Unicode);
let pad = pad.unwrap_or(false);
let tab_width = clamp_tab_width_for_ops(tab_width);
// Keep original handle so we can return it without allocating.
let original = text;
let text = js::utf16(text)?;
Ok(truncate_to_width_impl(original, &text, max_width, ellipsis_kind, pad, tab_width))
}
let text_u16 = text.into_utf16()?;
let text = text_u16.as_slice();
fn truncate_to_width_impl<'env>(
original: JsString<'env>,
text: &[u16],
max_width: usize,
ellipsis_kind: Ellipsis,
pad: bool,
tab_width: usize,
) -> Either<JsString<'env>, Utf16String> {
// Fast path: early-exit width check
let (text_w, exceeded) = visible_width_u16_up_to(text, max_width, tab_width);
if !exceeded {
if !pad {
// Return original JsString handle: zero output allocation.
return Ok(Either::A(original));
return Either::A(original);
}
if text_w < max_width {
let mut out = Vec::with_capacity(text.len() + (max_width - text_w));
out.extend_from_slice(text);
out.resize(out.len() + (max_width - text_w), b' ' as u16);
return Ok(Either::B(build_utf16_string(out)));
return Either::B(build_utf16_string(out));
}
// Exactly fits and padding requested: return original is still fine.
return Ok(Either::A(original));
return Either::A(original);
}
// Map ellipsis kind to UTF-16 data and width
@@ -1335,7 +1342,7 @@ pub fn truncate_to_width(
if pad && w < max_width {
out.resize(out.len() + (max_width - w), b' ' as u16);
}
return Ok(Either::B(build_utf16_string(out)));
return Either::B(build_utf16_string(out));
}
// Main truncation
@@ -1439,7 +1446,7 @@ pub fn truncate_to_width(
}
}
Ok(Either::B(build_utf16_string(out)))
Either::B(build_utf16_string(out))
}
// ============================================================================
@@ -1590,19 +1597,16 @@ pub fn slice_with_width(
strict: Option<bool>,
tab_width: u32,
) -> Result<SliceResult> {
let line_u16 = line.into_utf16()?;
let line = line_u16.as_slice();
let strict = strict.unwrap_or(false);
if length == 0 {
return Ok(SliceResult { text: build_utf16_string(vec![]), width: 0 });
}
let line = js::utf16(line)?;
let strict = strict.unwrap_or(false);
let tab_width = clamp_tab_width_for_ops(tab_width);
let (out, w) =
slice_with_width_impl(line, start_col as usize, length as usize, strict, tab_width);
Ok(SliceResult { text: build_utf16_string(out), width: crate::utils::clamp_u32(w as u64) })
let (out, width) =
slice_with_width_impl(&line, start_col as usize, length as usize, strict, tab_width);
Ok(SliceResult { text: build_utf16_string(out), width: crate::utils::clamp_u32(width as u64) })
}
// ============================================================================
@@ -1812,12 +1816,10 @@ pub fn extract_segments(
strict_after: bool,
tab_width: u32,
) -> Result<ExtractSegmentsResult> {
let line_u16 = line.into_utf16()?;
let line = line_u16.as_slice();
let line = js::utf16(line)?;
let tab_width = clamp_tab_width_for_ops(tab_width);
let (before, bw, after, aw) = extract_segments_impl(
line,
let (before, before_width, after, after_width) = extract_segments_impl(
&line,
before_end as usize,
after_start as usize,
after_len as usize,
@@ -1827,9 +1829,9 @@ pub fn extract_segments(
Ok(ExtractSegmentsResult {
before: build_utf16_string(before),
before_width: crate::utils::clamp_u32(bw as u64),
before_width: crate::utils::clamp_u32(before_width as u64),
after: build_utf16_string(after),
after_width: crate::utils::clamp_u32(aw as u64),
after_width: crate::utils::clamp_u32(after_width as u64),
})
}
@@ -1842,9 +1844,9 @@ pub fn extract_segments(
/// Tabs count as a fixed-width cell.
#[napi]
pub fn visible_width(text: JsString, tab_width: u32) -> Result<u32> {
let text_u16 = text.into_utf16()?;
let text = js::utf16(text)?;
let tab_width = clamp_tab_width_for_ops(tab_width);
Ok(crate::utils::clamp_u32(visible_width_u16(text_u16.as_slice(), tab_width) as u64))
Ok(crate::utils::clamp_u32(visible_width_u16(&text, tab_width) as u64))
}
#[cfg(test)]
+29 -15
View File
@@ -22,8 +22,10 @@ use napi::{
bindgen_prelude::{Array, Either},
};
use napi_derive::napi;
use pi_shell::rayon_global_pool_available;
use rayon::prelude::*;
use crate::utok;
use crate::{js, utok};
/// Tokenizer encoding to use.
#[napi(string_enum)]
@@ -70,9 +72,9 @@ impl Encoding {
/// Count tokens in `input`.
///
/// `input` may be a single string or an array of strings; an array returns
/// the sum across all elements. Always returns a single token total — use
/// this for any aggregate budget question without paying a per-element napi
/// crossing.
/// the sum across all elements (counted in parallel when the global rayon pool
/// is available). Always returns a single token total — use this for any
/// aggregate budget question without paying a per-element napi crossing.
///
/// Measures user/model content, not wire-protocol tokens: BPE encodings
/// use ordinary encoding (no special-token handling) and the Claude
@@ -86,22 +88,34 @@ pub fn count_tokens(
) -> napi::Result<u32> {
let enc = Encoding::utok(encoding);
match input {
Either::A(js_str) => {
let text = js_str.into_utf16()?;
let (_, units) = text.as_slice().split_last().expect("napi UTF-16 buffer has a terminator");
Ok(enc.count(units))
},
Either::A(text) => Ok(enc.count(&*js::utf16(text)?)),
Either::B(array) => {
let mut total = 0u32;
// Node-API handles are thread-affine, so every element is read here on
// the JS thread — into one buffer, so the batch costs one allocation
// rather than one per string. Only the counting fans out.
let mut units = Vec::new();
let mut spans = Vec::with_capacity(array.len() as usize);
for index in 0..array.len() {
let text = array
.get::<JsString>(index)?
.ok_or_else(|| napi::Error::from_reason("array changed during token counting"))?
.into_utf16()?;
let (_, units) = text.as_slice().split_last().expect("napi UTF-16 buffer has a terminator");
total += enc.count(units);
.ok_or_else(|| napi::Error::from_reason("array changed during token counting"))?;
spans.push(js::utf16_append(text, &mut units)?);
}
Ok(total)
// Scheduling a Rayon job costs more than tokenizing a small prompt
// batch. Keep those batches on the N-API thread; large batches still
// amortize the pool handoff across enough independent strings.
const PARALLEL_BATCH_MIN: usize = 16;
Ok(if spans.len() >= PARALLEL_BATCH_MIN && rayon_global_pool_available() {
spans
.par_iter()
.map(|span| enc.count(&units[span.clone()]))
.sum()
} else {
spans
.iter()
.map(|span| enc.count(&units[span.clone()]))
.sum()
})
},
}
}
+14 -6
View File
@@ -8,11 +8,13 @@
//! bit-identical to the TS versions and integer results are exactly equal.
use napi::{
Error, Result, Status,
bindgen_prelude::{Float32Array, Float64Array, Uint32Array},
Error, JsString, Result, Status,
bindgen_prelude::{Array, Float32Array, Float64Array, Uint32Array},
};
use napi_derive::napi;
use crate::js;
fn invalid<T>(message: &str) -> Result<T> {
Err(Error::new(Status::InvalidArg, message))
}
@@ -261,20 +263,26 @@ fn jaccard_sorted(a: &[Box<str>], b: &[Box<str>]) -> f64 {
reason = "mul_add rounds differently; bit-exact with the TS loops is the contract"
)]
pub fn mmr_rerank_indices(
contents: Vec<String>,
#[napi(ts_arg_type = "Array<string>")] contents: Array,
scores: Float64Array,
lambda_param: f64,
top_k: u32,
) -> Result<Uint32Array> {
if scores.len() != contents.len() {
if scores.len() != contents.len() as usize {
return invalid("scores length must equal contents length");
}
let limit = top_k as usize;
let count = contents.len();
let count = contents.len() as usize;
if limit == 0 || count == 0 {
return Ok(Uint32Array::new(Vec::new()));
}
let sets: Vec<Vec<Box<str>>> = contents.iter().map(|text| word_set(text)).collect();
let mut sets = Vec::with_capacity(count);
for index in 0..contents.len() {
let content = contents
.get::<JsString>(index)?
.ok_or_else(|| Error::new(Status::InvalidArg, "contents changed during reranking"))?;
sets.push(word_set(&js::utf8(content)?));
}
let mut selected: Vec<u32> = Vec::with_capacity(limit.min(count));
selected.push(0);
let mut remaining: Vec<u32> = (1..count as u32).collect();