feat(pi-shell/moreutils): implemented six moreutils-inspired shell builtins

- Added six builtin commands: ts, sponge, ifne, isutf8, combine, and errno to pi-shell.
- Created moreutils module with independent implementations for all tools.
- Added jiff dependency for timestamp and timezone functionality.
- Integrated builtins into shell execution pipeline with in-process execution.
- Added integration tests verifying pipe chains like ts | sponge with isutf8.
This commit is contained in:
can1357
2026-07-24 14:52:16 +02:00
parent 9f8aa87dbf
commit 5d4f1cf6b5
13 changed files with 2329 additions and 0 deletions
Generated
+1
View File
@@ -4872,6 +4872,7 @@ dependencies = [
"globset",
"ignore",
"jaq",
"jiff",
"libc",
"os_pipe",
"parking_lot",
+1
View File
@@ -79,6 +79,7 @@ uu_comm = { path = "../vendor/uu-comm" }
uu_sed = { path = "../vendor/uu-sed" }
uu_xargs = { path = "../vendor/uu-xargs" }
jaq = { path = "../vendor/jaq" }
jiff = { version = "0.2", features = ["tzdb-bundle-platform"] }
[target.'cfg(unix)'.dependencies]
libc.workspace = true
+7
View File
@@ -279,6 +279,13 @@ uutil_builtin!(pub fn comm_builtin => uu_comm::run);
uutil_builtin!(pub fn sed_builtin => uu_sed::run);
uutil_builtin!(pub fn xargs_builtin => uu_xargs::run);
uutil_builtin!(pub fn jq_builtin => jaq::run);
uutil_builtin!(pub fn ts_builtin => crate::moreutils::ts::run);
uutil_builtin!(pub fn sponge_builtin => crate::moreutils::sponge::run);
uutil_builtin!(pub fn ifne_builtin => crate::moreutils::ifne::run);
uutil_builtin!(pub fn isutf8_builtin => crate::moreutils::isutf8::run);
uutil_builtin!(pub fn combine_builtin => crate::moreutils::combine::run);
#[cfg(unix)]
uutil_builtin!(pub fn errno_builtin => crate::moreutils::errno::run);
#[cfg(test)]
mod tests {
+1
View File
@@ -3,6 +3,7 @@ mod cmp;
mod coreutils;
mod fd;
pub mod minimizer;
mod moreutils;
pub mod process;
pub mod shell;
mod which;
+15
View File
@@ -0,0 +1,15 @@
//! In-process implementations of selected moreutils tools.
//!
//! Each module follows the same contract as [`crate::cmp`]: a
//! `pub fn run(argv: Vec<OsString>) -> i32` entry point that performs all I/O
//! through [`pi_uutils_ctx`] (scoped stdin/stdout/stderr, shell-relative path
//! resolution, cancellation), registered as a shell builtin via the
//! `uutil_builtin!` macro in [`crate::coreutils`].
pub mod combine;
#[cfg(unix)]
pub mod errno;
pub mod ifne;
pub mod isutf8;
pub mod sponge;
pub mod ts;
+371
View File
@@ -0,0 +1,371 @@
//! moreutils `combine` builtin: boolean operations on the lines of two files.
//!
//! `combine FILE1 OP FILE2` where OP (case-insensitive) is `and`, `not`, `or`,
//! or `xor`. `-` names scoped stdin; only one side may be `-`. Lines are raw
//! byte strings; membership comparison strips a trailing `\n`, so a final line
//! without a newline still matches, and is emitted as-is (no newline added).
//! Usage errors (wrong arg count, unknown OP, both sides stdin) exit 1, unlike
//! moreutils' die() exit 255.
use std::{
collections::HashSet,
ffi::{OsStr, OsString},
fs::File,
io::{BufRead, BufReader, Write},
};
use clap::{Arg, ArgAction, Command, value_parser};
use pi_uutils_ctx::format_usage;
const ARG_FILE1: &str = "file1";
const ARG_OP: &str = "op";
const ARG_FILE2: &str = "file2";
#[derive(Clone, Copy, PartialEq)]
enum Op {
And,
Not,
Or,
Xor,
}
enum Error {
Cancelled,
Msg(String),
}
impl From<String> for Error {
fn from(msg: String) -> Self {
Error::Msg(msg)
}
}
/// Runs `combine` against invocation-scoped stdin/stdout/stderr and
/// shell-relative paths.
pub fn run(argv: Vec<OsString>) -> i32 {
let matches = match command().try_get_matches_from(argv) {
Ok(matches) => matches,
Err(err) => {
let rendered = err.to_string();
if err.use_stderr() {
let _ = write!(pi_uutils_ctx::stderr(), "{rendered}");
return 1;
}
let _ = write!(pi_uutils_ctx::stdout(), "{rendered}");
return err.exit_code();
},
};
let file1 = matches.get_one::<OsString>(ARG_FILE1).expect("required").clone();
let op = matches.get_one::<String>(ARG_OP).expect("required");
let file2 = matches.get_one::<OsString>(ARG_FILE2).expect("required").clone();
match execute(&file1, op, &file2) {
Ok(()) => 0,
Err(Error::Cancelled) => 130,
Err(Error::Msg(message)) => {
let _ = writeln!(pi_uutils_ctx::stderr(), "combine: {message}");
1
},
}
}
fn command() -> Command {
Command::new("combine")
.version(concat!("combine (pi-shell) ", env!("CARGO_PKG_VERSION")))
.about("Combine the lines of two files using boolean operations.")
.override_usage(format_usage("combine FILE1 and|not|or|xor FILE2"))
.disable_help_flag(true)
.disable_version_flag(true)
.arg(Arg::new("help").long("help").action(ArgAction::Help))
.arg(
Arg::new(ARG_FILE1)
.value_name("FILE1")
.required(true)
.value_parser(value_parser!(OsString)),
)
.arg(Arg::new(ARG_OP).value_name("OP").required(true))
.arg(
Arg::new(ARG_FILE2)
.value_name("FILE2")
.required(true)
.value_parser(value_parser!(OsString)),
)
}
fn execute(file1: &OsStr, op: &str, file2: &OsStr) -> Result<(), Error> {
let op = match op.to_ascii_lowercase().as_str() {
"and" => Op::And,
"not" => Op::Not,
"or" => Op::Or,
"xor" => Op::Xor,
other => {
return Err(Error::Msg(format!(
"unknown operation '{other}' (expected and, not, or, xor)"
)));
},
};
let dash = OsStr::new("-");
if file1 == dash && file2 == dash {
return Err(Error::Msg("only one file can be stdin".into()));
}
// Open both up front so a missing FILE2 fails before stdin is consumed.
let mut input1 = open_input(file1)?;
let mut input2 = open_input(file2)?;
let mut out = pi_uutils_ctx::stdout();
match op {
Op::And | Op::Not => {
// Membership side must be fully loaded before streaming FILE1.
let lines2 = read_lines(&mut input2, file2)?;
let set2: HashSet<&[u8]> = lines2.iter().map(|line| key(line)).collect();
let keep_member = op == Op::And;
each_line(&mut input1, file1, |line| {
if set2.contains(key(line)) == keep_member {
write_line(&mut out, line)?;
}
Ok(())
})?;
},
Op::Or => {
each_line(&mut input1, file1, |line| write_line(&mut out, line))?;
each_line(&mut input2, file2, |line| write_line(&mut out, line))?;
},
Op::Xor => {
let lines1 = read_lines(&mut input1, file1)?;
let lines2 = read_lines(&mut input2, file2)?;
let set1: HashSet<&[u8]> = lines1.iter().map(|line| key(line)).collect();
let set2: HashSet<&[u8]> = lines2.iter().map(|line| key(line)).collect();
for line in &lines1 {
if !set2.contains(key(line)) {
write_line(&mut out, line)?;
}
}
for line in &lines2 {
if !set1.contains(key(line)) {
write_line(&mut out, line)?;
}
}
},
}
out.flush().map_err(|err| Error::Msg(err.to_string()))?;
Ok(())
}
fn open_input(name: &OsStr) -> Result<Box<dyn BufRead>, Error> {
if name == OsStr::new("-") {
return Ok(Box::new(BufReader::new(pi_uutils_ctx::stdin())));
}
let path = pi_uutils_ctx::resolve(name);
let file =
File::open(path).map_err(|err| Error::Msg(input_error(name, &err.to_string())))?;
Ok(Box::new(BufReader::new(file)))
}
/// Streams `reader` line by line (trailing `\n` retained when present),
/// polling for cancellation between lines.
fn each_line(
reader: &mut dyn BufRead,
name: &OsStr,
mut f: impl FnMut(&[u8]) -> Result<(), Error>,
) -> Result<(), Error> {
let mut line = Vec::new();
loop {
if pi_uutils_ctx::is_cancelled() {
return Err(Error::Cancelled);
}
line.clear();
let n = reader
.read_until(b'\n', &mut line)
.map_err(|err| Error::Msg(input_error(name, &err.to_string())))?;
if n == 0 {
return Ok(());
}
f(&line)?;
}
}
fn read_lines(reader: &mut dyn BufRead, name: &OsStr) -> Result<Vec<Vec<u8>>, Error> {
let mut lines = Vec::new();
each_line(reader, name, |line| {
lines.push(line.to_vec());
Ok(())
})?;
Ok(lines)
}
/// Membership key: the line with any trailing newline stripped, so `foo`
/// (no newline) matches `foo\n`.
fn key(line: &[u8]) -> &[u8] {
line.strip_suffix(b"\n").unwrap_or(line)
}
fn write_line(out: &mut impl Write, line: &[u8]) -> Result<(), Error> {
out.write_all(line).map_err(|err| Error::Msg(err.to_string()))
}
fn input_error(name: &OsStr, err: &str) -> String {
format!("{}: {}", name.to_string_lossy(), err)
}
#[cfg(test)]
mod tests {
use std::{
collections::HashMap,
ffi::OsString,
fs,
io::{Cursor, Write},
path::PathBuf,
sync::{Arc, atomic::AtomicBool},
};
use parking_lot::Mutex;
use pi_uutils_ctx::ScopeIo;
use super::run;
#[derive(Clone)]
struct SharedWriter(Arc<Mutex<Vec<u8>>>);
impl Write for SharedWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self.0.lock().write(buf)
}
fn flush(&mut self) -> std::io::Result<()> {
self.0.lock().flush()
}
}
fn run_in(cwd: PathBuf, stdin: &[u8], args: &[&str]) -> (i32, Vec<u8>, String) {
let stdout = Arc::new(Mutex::new(Vec::new()));
let stderr = Arc::new(Mutex::new(Vec::new()));
let io = ScopeIo {
stdin: Box::new(Cursor::new(stdin.to_vec())),
stdin_fd: None,
stdin_is_search_input: false,
stdout: Box::new(SharedWriter(Arc::clone(&stdout))),
stderr: Box::new(SharedWriter(Arc::clone(&stderr))),
cwd,
env: HashMap::new(),
cancel: Arc::new(AtomicBool::new(false)),
};
let argv = std::iter::once("combine")
.chain(args.iter().copied())
.map(OsString::from)
.collect();
let code = pi_uutils_ctx::scope(io, || run(argv));
let stdout = stdout.lock().clone();
let stderr = String::from_utf8(stderr.lock().clone()).unwrap();
(code, stdout, stderr)
}
fn tempdir() -> (tempfile::TempDir, PathBuf) {
let dir = tempfile::tempdir().unwrap();
let path = fs::canonicalize(dir.path()).unwrap();
(dir, path)
}
fn fixture() -> (tempfile::TempDir, PathBuf) {
let (dir, root) = tempdir();
fs::write(root.join("one"), b"a\nb\na\nc\n").unwrap();
fs::write(root.join("two"), b"a\nc\nd\n").unwrap();
(dir, root)
}
#[test]
fn and_keeps_file1_order_and_duplicates() {
let (_dir, root) = fixture();
let (code, stdout, stderr) = run_in(root, b"", &["one", "and", "two"]);
assert_eq!((code, stdout.as_slice(), stderr.as_str()), (0, b"a\na\nc\n".as_slice(), ""));
}
#[test]
fn not_removes_file2_members() {
let (_dir, root) = fixture();
let (code, stdout, stderr) = run_in(root, b"", &["one", "not", "two"]);
assert_eq!((code, stdout.as_slice(), stderr.as_str()), (0, b"b\n".as_slice(), ""));
}
#[test]
fn or_concatenates_both_files() {
let (_dir, root) = fixture();
let (code, stdout, stderr) = run_in(root, b"", &["one", "or", "two"]);
assert_eq!(
(code, stdout.as_slice(), stderr.as_str()),
(0, b"a\nb\na\nc\na\nc\nd\n".as_slice(), "")
);
}
#[test]
fn xor_emits_exclusive_lines_from_both_sides() {
let (_dir, root) = fixture();
let (code, stdout, stderr) = run_in(root, b"", &["one", "XOR", "two"]);
assert_eq!((code, stdout.as_slice(), stderr.as_str()), (0, b"b\nd\n".as_slice(), ""));
}
#[test]
fn dash_reads_file1_from_stdin() {
let (_dir, root) = fixture();
let (code, stdout, stderr) = run_in(root, b"a\nb\na\nc\n", &["-", "and", "two"]);
assert_eq!((code, stdout.as_slice(), stderr.as_str()), (0, b"a\na\nc\n".as_slice(), ""));
}
#[test]
fn both_sides_stdin_is_rejected() {
let (_dir, root) = tempdir();
let (code, stdout, stderr) = run_in(root, b"", &["-", "and", "-"]);
assert_eq!(code, 1);
assert!(stdout.is_empty());
assert_eq!(stderr, "combine: only one file can be stdin\n");
}
#[test]
fn non_utf8_lines_survive_byte_exact() {
let (_dir, root) = tempdir();
fs::write(root.join("one"), b"\xff\xfe\n\x80ok\n").unwrap();
fs::write(root.join("two"), b"\xff\xfe\n").unwrap();
let (code, stdout, stderr) = run_in(root, b"", &["one", "and", "two"]);
assert_eq!((code, stdout.as_slice(), stderr.as_str()), (0, b"\xff\xfe\n".as_slice(), ""));
}
#[test]
fn missing_file_reports_error_exit_1() {
let (_dir, root) = tempdir();
fs::write(root.join("one"), b"a\n").unwrap();
let (code, stdout, stderr) = run_in(root, b"", &["one", "and", "nope"]);
assert_eq!(code, 1);
assert!(stdout.is_empty());
assert!(stderr.starts_with("combine: nope: "), "stderr: {stderr}");
}
#[test]
fn unknown_op_is_usage_error_exit_1() {
let (_dir, root) = fixture();
let (code, stdout, stderr) = run_in(root, b"", &["one", "nand", "two"]);
assert_eq!(code, 1);
assert!(stdout.is_empty());
assert_eq!(stderr, "combine: unknown operation 'nand' (expected and, not, or, xor)\n");
}
#[test]
fn wrong_arg_count_is_usage_error_exit_1() {
let (_dir, root) = tempdir();
let (code, stdout, stderr) = run_in(root, b"", &["only-one"]);
assert_eq!(code, 1);
assert!(stdout.is_empty());
assert!(stderr.contains("Usage"), "stderr: {stderr}");
}
#[test]
fn last_line_without_newline_matches_and_is_emitted_as_is() {
// `b` without a trailing newline still counts as a line, matches
// `b\n` in the other file, and is emitted without adding a newline.
let (_dir, root) = tempdir();
fs::write(root.join("one"), b"a\nb").unwrap();
fs::write(root.join("two"), b"b\n").unwrap();
let (code, stdout, stderr) = run_in(root, b"", &["one", "and", "two"]);
assert_eq!((code, stdout.as_slice(), stderr.as_str()), (0, b"b".as_slice(), ""));
}
}
+371
View File
@@ -0,0 +1,371 @@
//! moreutils `errno` builtin: look up errno names, numbers, and descriptions.
//!
//! Lookup semantics: a name argument prints its `NAME NUMBER Description`
//! line; an unknown name reports `errno: unknown errno NAME` on stderr and
//! exits 1. A numeric argument reverse-maps to the first-listed (canonical)
//! name; an unknown number prints nothing and exits 1. With multiple
//! arguments each is processed and the exit code is 1 if any lookup failed.
//! `--list` prints every table entry (aliases included) sorted by number
//! then name; `--search` prints entries whose description contains all the
//! given words case-insensitively.
use std::ffi::OsString;
use std::io::Write;
use clap::{Arg, ArgAction, Command};
use pi_uutils_ctx::format_usage;
const OPT_LIST: &str = "list";
const OPT_SEARCH: &str = "search";
const ARG_QUERY: &str = "query";
/// Errno NAME -> number table. Duplicate numbers are allowed (aliases such
/// as `EWOULDBLOCK`); the first-listed name for a number is canonical for
/// reverse lookup.
const ERRNOS: &[(&str, i32)] = &[
("EPERM", libc::EPERM),
("ENOENT", libc::ENOENT),
("ESRCH", libc::ESRCH),
("EINTR", libc::EINTR),
("EIO", libc::EIO),
("ENXIO", libc::ENXIO),
("E2BIG", libc::E2BIG),
("ENOEXEC", libc::ENOEXEC),
("EBADF", libc::EBADF),
("ECHILD", libc::ECHILD),
("EAGAIN", libc::EAGAIN),
("ENOMEM", libc::ENOMEM),
("EACCES", libc::EACCES),
("EFAULT", libc::EFAULT),
("ENOTBLK", libc::ENOTBLK),
("EBUSY", libc::EBUSY),
("EEXIST", libc::EEXIST),
("EXDEV", libc::EXDEV),
("ENODEV", libc::ENODEV),
("ENOTDIR", libc::ENOTDIR),
("EISDIR", libc::EISDIR),
("EINVAL", libc::EINVAL),
("ENFILE", libc::ENFILE),
("EMFILE", libc::EMFILE),
("ENOTTY", libc::ENOTTY),
("ETXTBSY", libc::ETXTBSY),
("EFBIG", libc::EFBIG),
("ENOSPC", libc::ENOSPC),
("ESPIPE", libc::ESPIPE),
("EROFS", libc::EROFS),
("EMLINK", libc::EMLINK),
("EPIPE", libc::EPIPE),
("EDOM", libc::EDOM),
("ERANGE", libc::ERANGE),
("EDEADLK", libc::EDEADLK),
("ENAMETOOLONG", libc::ENAMETOOLONG),
("ENOLCK", libc::ENOLCK),
("ENOSYS", libc::ENOSYS),
("ENOTEMPTY", libc::ENOTEMPTY),
("ELOOP", libc::ELOOP),
("ENOMSG", libc::ENOMSG),
("EIDRM", libc::EIDRM),
("EPROTO", libc::EPROTO),
("EBADMSG", libc::EBADMSG),
("EOVERFLOW", libc::EOVERFLOW),
("EILSEQ", libc::EILSEQ),
("ENOTSOCK", libc::ENOTSOCK),
("EDESTADDRREQ", libc::EDESTADDRREQ),
("EMSGSIZE", libc::EMSGSIZE),
("EPROTOTYPE", libc::EPROTOTYPE),
("ENOPROTOOPT", libc::ENOPROTOOPT),
("EPROTONOSUPPORT", libc::EPROTONOSUPPORT),
("ESOCKTNOSUPPORT", libc::ESOCKTNOSUPPORT),
("ENOTSUP", libc::ENOTSUP),
("EOPNOTSUPP", libc::EOPNOTSUPP),
("EPFNOSUPPORT", libc::EPFNOSUPPORT),
("EAFNOSUPPORT", libc::EAFNOSUPPORT),
("EADDRINUSE", libc::EADDRINUSE),
("EADDRNOTAVAIL", libc::EADDRNOTAVAIL),
("ENETDOWN", libc::ENETDOWN),
("ENETUNREACH", libc::ENETUNREACH),
("ENETRESET", libc::ENETRESET),
("ECONNABORTED", libc::ECONNABORTED),
("ECONNRESET", libc::ECONNRESET),
("ENOBUFS", libc::ENOBUFS),
("EISCONN", libc::EISCONN),
("ENOTCONN", libc::ENOTCONN),
("ESHUTDOWN", libc::ESHUTDOWN),
("ETOOMANYREFS", libc::ETOOMANYREFS),
("ETIMEDOUT", libc::ETIMEDOUT),
("ECONNREFUSED", libc::ECONNREFUSED),
("EHOSTDOWN", libc::EHOSTDOWN),
("EHOSTUNREACH", libc::EHOSTUNREACH),
("EALREADY", libc::EALREADY),
("EINPROGRESS", libc::EINPROGRESS),
("ESTALE", libc::ESTALE),
("EDQUOT", libc::EDQUOT),
("ECANCELED", libc::ECANCELED),
("EOWNERDEAD", libc::EOWNERDEAD),
("ENOTRECOVERABLE", libc::ENOTRECOVERABLE),
("EWOULDBLOCK", libc::EWOULDBLOCK),
];
/// Runs `errno` against invocation-scoped stdout/stderr.
pub fn run(argv: Vec<OsString>) -> i32 {
let matches = match command().try_get_matches_from(argv) {
Ok(matches) => matches,
Err(err) => {
let code = err.exit_code();
let rendered = err.to_string();
if err.use_stderr() {
let _ = write!(pi_uutils_ctx::stderr(), "{rendered}");
} else {
let _ = write!(pi_uutils_ctx::stdout(), "{rendered}");
}
return code;
},
};
let args: Vec<String> = matches
.get_many::<String>(ARG_QUERY)
.map(|values| values.cloned().collect())
.unwrap_or_default();
if matches.get_flag(OPT_LIST) {
return list_all();
}
if matches.get_flag(OPT_SEARCH) {
return search(&args);
}
if args.is_empty() {
let _ = writeln!(pi_uutils_ctx::stderr(), "errno: no errno name or number given");
return 1;
}
let mut failed = false;
for arg in &args {
if !lookup(arg) {
failed = true;
}
}
if failed { 1 } else { 0 }
}
fn command() -> Command {
Command::new("errno")
.version(concat!("errno (pi-shell) ", env!("CARGO_PKG_VERSION")))
.about("Look up errno names and descriptions.")
.override_usage(format_usage("errno [-ls] [--] [name-or-number...]"))
.disable_help_flag(true)
.disable_version_flag(true)
.infer_long_args(true)
.arg(
Arg::new(OPT_LIST)
.short('l')
.long("list")
.help("List all errno values")
.action(ArgAction::SetTrue),
)
.arg(
Arg::new(OPT_SEARCH)
.short('s')
.long("search")
.help("Search errno descriptions for the given words")
.action(ArgAction::SetTrue),
)
.arg(Arg::new("help").long("help").action(ArgAction::Help))
.arg(
Arg::new(ARG_QUERY)
.value_name("NAME-OR-NUMBER")
.num_args(0..)
.action(ArgAction::Append),
)
}
/// Formats the OS description for an errno number, without std's
/// ` (os error N)` suffix.
fn description(number: i32) -> String {
let text = std::io::Error::from_raw_os_error(number).to_string();
match text.rfind(" (os error ") {
Some(index) => text[..index].to_string(),
None => text,
}
}
fn print_entry(name: &str, number: i32) {
let _ = writeln!(pi_uutils_ctx::stdout(), "{name} {number} {}", description(number));
}
/// Looks up one name or number argument; returns false on failure.
fn lookup(arg: &str) -> bool {
if let Ok(number) = arg.parse::<i32>() {
// Reverse lookup: first-listed name for the number is canonical.
match ERRNOS.iter().find(|(_, value)| *value == number) {
Some((name, value)) => {
print_entry(name, *value);
true
},
None => false,
}
} else {
match ERRNOS.iter().find(|(name, _)| name.eq_ignore_ascii_case(arg)) {
Some((name, value)) => {
print_entry(name, *value);
true
},
None => {
let _ = writeln!(pi_uutils_ctx::stderr(), "errno: unknown errno {arg}");
false
},
}
}
}
/// Prints every table entry (aliases included) sorted by number, then name.
fn list_all() -> i32 {
let mut entries: Vec<(&str, i32)> = ERRNOS.to_vec();
entries.sort_unstable_by(|a, b| a.1.cmp(&b.1).then(a.0.cmp(b.0)));
for (name, number) in entries {
print_entry(name, number);
}
0
}
/// Prints entries whose description contains all words, case-insensitively.
fn search(words: &[String]) -> i32 {
let lowered: Vec<String> = words.iter().map(|word| word.to_lowercase()).collect();
let mut entries: Vec<(&str, i32)> = ERRNOS.to_vec();
entries.sort_unstable_by(|a, b| a.1.cmp(&b.1).then(a.0.cmp(b.0)));
for (name, number) in entries {
let text = description(number).to_lowercase();
if lowered.iter().all(|word| text.contains(word.as_str())) {
print_entry(name, number);
}
}
0
}
#[cfg(test)]
mod tests {
use std::{
collections::HashMap,
ffi::OsString,
io::{Cursor, Write},
sync::{Arc, atomic::AtomicBool},
};
use parking_lot::Mutex;
use pi_uutils_ctx::ScopeIo;
use super::run;
#[derive(Clone)]
struct SharedWriter(Arc<Mutex<Vec<u8>>>);
impl Write for SharedWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self.0.lock().write(buf)
}
fn flush(&mut self) -> std::io::Result<()> {
self.0.lock().flush()
}
}
fn run_errno(args: &[&str]) -> (i32, String, String) {
let stdout = Arc::new(Mutex::new(Vec::new()));
let stderr = Arc::new(Mutex::new(Vec::new()));
let io = ScopeIo {
stdin: Box::new(Cursor::new(Vec::new())),
stdin_fd: None,
stdin_is_search_input: false,
stdout: Box::new(SharedWriter(Arc::clone(&stdout))),
stderr: Box::new(SharedWriter(Arc::clone(&stderr))),
cwd: std::env::temp_dir(),
env: HashMap::new(),
cancel: Arc::new(AtomicBool::new(false)),
};
let argv = std::iter::once("errno")
.chain(args.iter().copied())
.map(OsString::from)
.collect();
let code = pi_uutils_ctx::scope(io, || run(argv));
let stdout = String::from_utf8(stdout.lock().clone()).unwrap();
let stderr = String::from_utf8(stderr.lock().clone()).unwrap();
(code, stdout, stderr)
}
#[test]
fn looks_up_name() {
let (code, stdout, stderr) = run_errno(&["ENOENT"]);
assert_eq!(code, 0);
assert!(stdout.starts_with("ENOENT 2 "), "stdout: {stdout:?}");
assert!(stdout.trim_end().len() > "ENOENT 2 ".len(), "missing description: {stdout:?}");
assert!(stderr.is_empty());
}
#[test]
fn reverse_lookup_by_number() {
let (code, stdout, _) = run_errno(&["2"]);
assert_eq!(code, 0);
assert!(stdout.starts_with("ENOENT 2 "), "stdout: {stdout:?}");
}
#[test]
fn unknown_name_fails_with_stderr() {
let (code, stdout, stderr) = run_errno(&["ENOSUCHTHING"]);
assert_eq!(code, 1);
assert!(stdout.is_empty());
assert!(stderr.contains("unknown errno ENOSUCHTHING"), "stderr: {stderr:?}");
}
#[test]
fn unknown_number_fails_silently() {
let (code, stdout, stderr) = run_errno(&["99999"]);
assert_eq!(code, 1);
assert!(stdout.is_empty());
assert!(stderr.is_empty());
}
#[test]
fn list_is_sorted_by_number() {
let (code, stdout, _) = run_errno(&["-l"]);
assert_eq!(code, 0);
let lines: Vec<&str> = stdout.lines().collect();
let eperm = lines.iter().position(|line| line.starts_with("EPERM 1 "));
let enoent = lines.iter().position(|line| line.starts_with("ENOENT 2 "));
assert!(eperm.is_some(), "EPERM missing from list");
assert!(enoent.is_some(), "ENOENT missing from list");
assert!(eperm.unwrap() < enoent.unwrap(), "list not number-sorted");
let numbers: Vec<i32> = lines
.iter()
.map(|line| line.split(' ').nth(1).unwrap().parse().unwrap())
.collect();
let mut sorted = numbers.clone();
sorted.sort_unstable();
assert_eq!(numbers, sorted, "list not sorted by number");
}
#[test]
fn search_is_case_insensitive() {
let (code, stdout, _) = run_errno(&["-s", "No", "SUCH"]);
assert_eq!(code, 0);
assert!(
stdout.lines().any(|line| line.starts_with("ENOENT 2 ")),
"search missed ENOENT: {stdout:?}"
);
}
#[test]
fn multiple_args_aggregate_exit_code() {
let (code, stdout, stderr) = run_errno(&["ENOENT", "ENOSUCHTHING", "EPERM"]);
assert_eq!(code, 1);
assert!(stdout.lines().any(|line| line.starts_with("ENOENT 2 ")));
assert!(stdout.lines().any(|line| line.starts_with("EPERM 1 ")));
assert!(stderr.contains("unknown errno ENOSUCHTHING"));
}
#[test]
fn no_args_is_an_error() {
let (code, stdout, stderr) = run_errno(&[]);
assert_eq!(code, 1);
assert!(stdout.is_empty());
assert!(!stderr.is_empty());
}
}
+311
View File
@@ -0,0 +1,311 @@
//! moreutils `ifne` builtin: run a command iff stdin is non-empty (`-n`
//! inverts). The command is argv exec'd directly — no shell interpretation.
use std::{
ffi::OsString,
io::{ErrorKind, Read, Write},
process::{Command, Stdio},
};
const USAGE: &str = "usage: ifne [-n] command [args...]";
const CHUNK: usize = 64 * 1024;
/// Runs `ifne` against invocation-scoped stdin/stdout/stderr; spawns the
/// command with the scope's cwd and exported environment.
///
/// Argument parsing is manual rather than clap: everything after the optional
/// leading `-n` is the child argv verbatim, and clap's hyphen pass-through
/// rules would only complicate `ifne grep -q foo`.
pub fn run(argv: Vec<OsString>) -> i32 {
let mut args = argv.into_iter().skip(1).peekable();
let mut invert = false;
match args.peek().map(|a| a.to_string_lossy().into_owned()).as_deref() {
Some("-n") => {
invert = true;
args.next();
},
Some("--help") => {
let _ = writeln!(pi_uutils_ctx::stdout(), "{USAGE}");
return 0;
},
_ => {},
}
let command: Vec<OsString> = args.collect();
if command.is_empty() {
let _ = writeln!(pi_uutils_ctx::stderr(), "{USAGE}");
return 1;
}
// Probe stdin: one byte decides which mode acts.
let mut stdin = pi_uutils_ctx::stdin();
let mut first = [0u8; 1];
let got = loop {
match stdin.read(&mut first) {
Ok(n) => break n,
Err(err) if err.kind() == ErrorKind::Interrupted => {
if pi_uutils_ctx::is_cancelled() {
return 130;
}
},
Err(err) => {
let _ = writeln!(pi_uutils_ctx::stderr(), "ifne: stdin: {err}");
return 1;
},
}
};
let empty = got == 0;
if empty != invert {
if empty {
// Default mode, empty stdin: do nothing.
return 0;
}
// -n mode, non-empty stdin: pass stdin through, don't run the command.
return match copy_cancellable(&mut stdin, &mut pi_uutils_ctx::stdout(), Some(first[0])) {
Ok(()) => 0,
Err(CopyError::Cancelled) => 130,
Err(CopyError::Io(err)) => {
let _ = writeln!(pi_uutils_ctx::stderr(), "ifne: {err}");
1
},
};
}
spawn_and_pump(&command, if empty { None } else { Some(first[0]) }, &mut stdin)
}
/// Spawns the child and pumps stdin into it while draining its stdout/stderr;
/// returns the child's exit code (128+signal when signaled, 127 on spawn
/// failure, 130 on cancel).
fn spawn_and_pump(command: &[OsString], first: Option<u8>, stdin: &mut impl Read) -> i32 {
let mut child = match Command::new(&command[0])
.args(&command[1..])
.current_dir(pi_uutils_ctx::cwd())
.env_clear()
.envs(pi_uutils_ctx::env_snapshot())
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
{
Ok(child) => child,
Err(err) => {
let _ = writeln!(
pi_uutils_ctx::stderr(),
"ifne: {}: {err}",
command[0].to_string_lossy()
);
return 127;
},
};
let mut child_stdin = child.stdin.take().expect("piped stdin");
let mut child_stdout = child.stdout.take().expect("piped stdout");
let mut child_stderr = child.stderr.take().expect("piped stderr");
// The ctx streams are thread-local, so the drain threads collect into
// buffers that this (scoped) thread flushes to ctx stdout/stderr after
// the child exits.
let (out_buf, err_buf, pump) = std::thread::scope(|s| {
let out = s.spawn(move || {
let mut buf = Vec::new();
let _ = child_stdout.read_to_end(&mut buf);
buf
});
let err = s.spawn(move || {
let mut buf = Vec::new();
let _ = child_stderr.read_to_end(&mut buf);
buf
});
// Ignore BrokenPipe: the child may exit before consuming its stdin
// (e.g. `ifne head -1`).
let pump = match copy_cancellable(stdin, &mut child_stdin, first) {
Err(CopyError::Io(err)) if err.kind() != ErrorKind::BrokenPipe => {
Err(CopyError::Io(err))
},
Err(CopyError::Cancelled) => Err(CopyError::Cancelled),
_ => Ok(()),
};
drop(child_stdin); // EOF so the child terminates.
if matches!(pump, Err(CopyError::Cancelled)) {
let _ = child.kill();
}
(out.join().unwrap_or_default(), err.join().unwrap_or_default(), pump)
});
let status = child.wait();
let _ = pi_uutils_ctx::stdout().write_all(&out_buf);
let _ = pi_uutils_ctx::stderr().write_all(&err_buf);
match pump {
Err(CopyError::Cancelled) => return 130,
Err(CopyError::Io(err)) => {
let _ = writeln!(pi_uutils_ctx::stderr(), "ifne: {err}");
return 1;
},
Ok(()) => {},
}
match status {
Ok(status) => exit_code(status),
Err(err) => {
let _ = writeln!(pi_uutils_ctx::stderr(), "ifne: {err}");
1
},
}
}
enum CopyError {
Cancelled,
Io(std::io::Error),
}
/// Copies `first` (when present) then all of `src` into `dst` in chunks,
/// polling the scope's cancel flag between reads.
fn copy_cancellable(
src: &mut impl Read,
dst: &mut impl Write,
first: Option<u8>,
) -> Result<(), CopyError> {
if let Some(byte) = first {
dst.write_all(&[byte]).map_err(CopyError::Io)?;
}
let mut buf = [0u8; CHUNK];
loop {
if pi_uutils_ctx::is_cancelled() {
return Err(CopyError::Cancelled);
}
match src.read(&mut buf) {
Ok(0) => return Ok(()),
Ok(n) => dst.write_all(&buf[..n]).map_err(CopyError::Io)?,
Err(err) if err.kind() == ErrorKind::Interrupted => {},
Err(err) => return Err(CopyError::Io(err)),
}
}
}
/// Maps a child exit status to `ifne`'s exit code: the child's code, or
/// 128+signal on unix when the child was signaled.
fn exit_code(status: std::process::ExitStatus) -> i32 {
if let Some(code) = status.code() {
return code;
}
#[cfg(unix)]
{
use std::os::unix::process::ExitStatusExt;
if let Some(sig) = status.signal() {
return 128 + sig;
}
}
1
}
#[cfg(test)]
mod tests {
use std::{
collections::HashMap,
ffi::OsString,
io::{Cursor, Write},
sync::{Arc, atomic::AtomicBool},
};
use parking_lot::Mutex;
use pi_uutils_ctx::ScopeIo;
use super::run;
#[derive(Clone)]
struct SharedWriter(Arc<Mutex<Vec<u8>>>);
impl Write for SharedWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self.0.lock().write(buf)
}
fn flush(&mut self) -> std::io::Result<()> {
self.0.lock().flush()
}
}
fn run_in(stdin: &[u8], args: &[&str]) -> (i32, String, String) {
let stdout = Arc::new(Mutex::new(Vec::new()));
let stderr = Arc::new(Mutex::new(Vec::new()));
let io = ScopeIo {
stdin: Box::new(Cursor::new(stdin.to_vec())),
stdin_fd: None,
stdin_is_search_input: false,
stdout: Box::new(SharedWriter(Arc::clone(&stdout))),
stderr: Box::new(SharedWriter(Arc::clone(&stderr))),
cwd: std::env::temp_dir(),
env: HashMap::from([("PATH".to_string(), "/usr/bin:/bin".to_string())]),
cancel: Arc::new(AtomicBool::new(false)),
};
let argv = std::iter::once("ifne")
.chain(args.iter().copied())
.map(OsString::from)
.collect();
let code = pi_uutils_ctx::scope(io, || run(argv));
let stdout = String::from_utf8(stdout.lock().clone()).unwrap();
let stderr = String::from_utf8(stderr.lock().clone()).unwrap();
(code, stdout, stderr)
}
#[cfg(unix)]
#[test]
fn nonempty_stdin_runs_command_with_stdin() {
let (code, stdout, stderr) = run_in(b"hello world\n", &["cat"]);
assert_eq!((code, stdout.as_str(), stderr.as_str()), (0, "hello world\n", ""));
}
#[cfg(unix)]
#[test]
fn empty_stdin_skips_command() {
let (code, stdout, stderr) = run_in(b"", &["sh", "-c", "echo ran"]);
assert_eq!((code, stdout.as_str(), stderr.as_str()), (0, "", ""));
}
#[cfg(unix)]
#[test]
fn invert_runs_command_on_empty_stdin() {
let (code, stdout, stderr) = run_in(b"", &["-n", "sh", "-c", "echo ran"]);
assert_eq!((code, stdout.as_str(), stderr.as_str()), (0, "ran\n", ""));
}
#[cfg(unix)]
#[test]
fn invert_passes_nonempty_stdin_through() {
let (code, stdout, stderr) = run_in(b"data\n", &["-n", "sh", "-c", "echo ran"]);
assert_eq!((code, stdout.as_str(), stderr.as_str()), (0, "data\n", ""));
}
#[cfg(unix)]
#[test]
fn child_exit_code_propagates() {
let (code, stdout, stderr) = run_in(b"x", &["sh", "-c", "exit 3"]);
assert_eq!((code, stdout.as_str(), stderr.as_str()), (3, "", ""));
}
#[test]
fn unknown_command_exits_127() {
let (code, stdout, stderr) = run_in(b"x", &["definitely-not-a-command-xyz"]);
assert_eq!(code, 127);
assert_eq!(stdout, "");
assert!(stderr.starts_with("ifne: definitely-not-a-command-xyz: "), "stderr: {stderr}");
}
#[cfg(unix)]
#[test]
fn early_exiting_child_is_not_an_error() {
let big = vec![b'a'; 1 << 20];
let (code, stdout, stderr) = run_in(&big, &["head", "-c", "1"]);
assert_eq!((code, stdout.as_str(), stderr.as_str()), (0, "a", ""));
}
#[test]
fn missing_command_is_usage_error() {
let (code, stdout, stderr) = run_in(b"", &[]);
assert_eq!(code, 1);
assert_eq!(stdout, "");
assert!(stderr.contains("usage: ifne"));
}
}
+394
View File
@@ -0,0 +1,394 @@
//! moreutils `isutf8` builtin: check whether files (or stdin) are valid UTF-8.
//!
//! Diagnostic coordinates follow moreutils semantics: `line` is 1-based
//! (counting `\n`), `char` is the 1-based character position within that line,
//! and `byte` is the 0-based file offset of the first byte of the invalid
//! sequence. Input is streamed in 64 KiB chunks; a multi-byte sequence split
//! across a chunk boundary carries its incomplete tail (at most 3 bytes) into
//! the next chunk, and an incomplete tail at EOF counts as invalid.
//!
//! Stdin (no file arguments, or the argument `-`) is reported as
//! `(standard input)`. With `--invert` the exit status and `--list` output
//! treat valid files as failures; the default diagnostic is still printed for
//! invalid files. Exit codes: 0 = every file passes the (possibly inverted)
//! predicate, 1 = at least one file fails it, 2 = an I/O error opening or
//! reading a file (reported on stderr; remaining files are still checked).
use std::{
ffi::{OsStr, OsString},
fs::File,
io::{self, Read, Write},
};
use clap::{Arg, ArgAction, Command};
use pi_uutils_ctx::format_usage;
const OPT_QUIET: &str = "quiet";
const OPT_LIST: &str = "list";
const OPT_INVERT: &str = "invert";
const ARG_FILES: &str = "files";
const CHUNK_SIZE: usize = 64 * 1024;
enum Verdict {
Valid,
Invalid { line: u64, character: u64, byte: u64 },
Cancelled,
}
/// Runs `isutf8` against invocation-scoped stdin/stdout/stderr and
/// shell-relative paths.
pub fn run(argv: Vec<OsString>) -> i32 {
let matches = match command().try_get_matches_from(argv) {
Ok(matches) => matches,
Err(err) => {
let code = err.exit_code();
let rendered = err.to_string();
if err.use_stderr() {
let _ = write!(pi_uutils_ctx::stderr(), "{rendered}");
} else {
let _ = write!(pi_uutils_ctx::stdout(), "{rendered}");
}
return code;
},
};
let quiet = matches.get_flag(OPT_QUIET);
let list = matches.get_flag(OPT_LIST);
let invert = matches.get_flag(OPT_INVERT);
let files: Vec<OsString> = matches
.get_many::<OsString>(ARG_FILES)
.map(|values| values.cloned().collect())
.unwrap_or_else(|| vec![OsString::from("-")]);
let mut any_failed = false;
let mut io_error = false;
for file in &files {
let display = display_name(file);
let valid = match validate_file(file) {
Err(err) => {
let _ = writeln!(pi_uutils_ctx::stderr(), "isutf8: {display}: {err}");
io_error = true;
continue;
},
Ok(Verdict::Cancelled) => return 130,
Ok(Verdict::Valid) => true,
Ok(Verdict::Invalid { line, character, byte }) => {
if !quiet && !list {
let _ = writeln!(
pi_uutils_ctx::stdout(),
"{display}: line {line}, char {character}, byte {byte}: invalid UTF-8 code"
);
}
false
},
};
// A file fails when its validity matches the inversion flag.
if valid == invert {
any_failed = true;
if list && !quiet {
let _ = writeln!(pi_uutils_ctx::stdout(), "{display}");
}
}
}
if io_error {
2
} else if any_failed {
1
} else {
0
}
}
fn command() -> Command {
Command::new("isutf8")
.version(concat!("isutf8 (pi-shell) ", env!("CARGO_PKG_VERSION")))
.about("Check whether files are valid UTF-8.")
.override_usage(format_usage("isutf8 [-q|--quiet] [-l|--list] [-i|--invert] [FILE]..."))
.disable_help_flag(true)
.disable_version_flag(true)
.infer_long_args(true)
.arg(
Arg::new(OPT_QUIET)
.short('q')
.long(OPT_QUIET)
.help("suppress all output; report via exit status only")
.action(ArgAction::SetTrue),
)
.arg(
Arg::new(OPT_LIST)
.short('l')
.long(OPT_LIST)
.help("print only the names of files failing the check")
.action(ArgAction::SetTrue),
)
.arg(
Arg::new(OPT_INVERT)
.short('i')
.long(OPT_INVERT)
.help("invert the check: valid files fail")
.action(ArgAction::SetTrue),
)
.arg(Arg::new("help").long("help").action(ArgAction::Help))
.arg(
Arg::new(ARG_FILES)
.value_name("FILE")
.num_args(0..)
.value_parser(clap::value_parser!(OsString)),
)
}
fn validate_file(name: &OsStr) -> io::Result<Verdict> {
if name == "-" {
validate(&mut pi_uutils_ctx::stdin())
} else {
let mut file = File::open(pi_uutils_ctx::resolve(name))?;
validate(&mut file)
}
}
/// Streams `input` in [`CHUNK_SIZE`] chunks, carrying an incomplete multi-byte
/// tail (≤ 3 bytes) across chunk boundaries.
fn validate(input: &mut impl Read) -> io::Result<Verdict> {
let mut buf = vec![0u8; CHUNK_SIZE + 3];
let mut carry = 0usize; // bytes at buf[..carry] carried from the previous chunk
let mut offset = 0u64; // file offset of buf[0]
let mut line = 1u64;
let mut chars_in_line = 0u64; // complete chars decoded on the current line
loop {
if pi_uutils_ctx::is_cancelled() {
return Ok(Verdict::Cancelled);
}
let read = input.read(&mut buf[carry..carry + CHUNK_SIZE])?;
let eof = read == 0;
let data_len = carry + read;
if data_len == 0 {
return Ok(Verdict::Valid);
}
let mut pos = 0usize;
while pos < data_len {
match std::str::from_utf8(&buf[pos..data_len]) {
Ok(_) => {
advance(&buf[pos..data_len], &mut line, &mut chars_in_line);
pos = data_len;
},
Err(err) => {
advance(&buf[pos..pos + err.valid_up_to()], &mut line, &mut chars_in_line);
pos += err.valid_up_to();
if err.error_len().is_some() || eof {
// Bad sequence, or an incomplete one truncated by EOF.
return Ok(Verdict::Invalid {
line,
character: chars_in_line + 1,
byte: offset + pos as u64,
});
}
break; // incomplete tail: carry it into the next chunk
},
}
}
if eof {
return Ok(Verdict::Valid);
}
// Slide the unconsumed tail (≤ 3 bytes) to the front of the buffer.
buf.copy_within(pos..data_len, 0);
carry = data_len - pos;
offset += pos as u64;
}
}
/// Updates line/char counters over `text`, a slice already known to be valid
/// UTF-8 (chars are counted as non-continuation bytes, so no re-decode).
fn advance(text: &[u8], line: &mut u64, chars_in_line: &mut u64) {
match text.iter().rposition(|&b| b == b'\n') {
Some(last) => {
*line += text.iter().filter(|&&b| b == b'\n').count() as u64;
*chars_in_line = count_chars(&text[last + 1..]);
},
None => *chars_in_line += count_chars(text),
}
}
fn count_chars(bytes: &[u8]) -> u64 {
bytes.iter().filter(|&&b| (b & 0xC0) != 0x80).count() as u64
}
fn display_name(name: &OsStr) -> String {
if name == "-" {
"(standard input)".to_owned()
} else {
name.to_string_lossy().into_owned()
}
}
#[cfg(test)]
mod tests {
use std::{
collections::HashMap,
ffi::OsString,
fs,
io::{Cursor, Write},
path::PathBuf,
sync::{Arc, atomic::AtomicBool},
};
use parking_lot::Mutex;
use pi_uutils_ctx::ScopeIo;
use super::run;
#[derive(Clone)]
struct SharedWriter(Arc<Mutex<Vec<u8>>>);
impl Write for SharedWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self.0.lock().write(buf)
}
fn flush(&mut self) -> std::io::Result<()> {
self.0.lock().flush()
}
}
fn run_in(cwd: PathBuf, stdin: &[u8], args: &[&str]) -> (i32, String, String) {
let stdout = Arc::new(Mutex::new(Vec::new()));
let stderr = Arc::new(Mutex::new(Vec::new()));
let io = ScopeIo {
stdin: Box::new(Cursor::new(stdin.to_vec())),
stdin_fd: None,
stdin_is_search_input: false,
stdout: Box::new(SharedWriter(Arc::clone(&stdout))),
stderr: Box::new(SharedWriter(Arc::clone(&stderr))),
cwd,
env: HashMap::new(),
cancel: Arc::new(AtomicBool::new(false)),
};
let argv = std::iter::once("isutf8")
.chain(args.iter().copied())
.map(OsString::from)
.collect();
let code = pi_uutils_ctx::scope(io, || run(argv));
let stdout = String::from_utf8(stdout.lock().clone()).unwrap();
let stderr = String::from_utf8(stderr.lock().clone()).unwrap();
(code, stdout, stderr)
}
fn tempdir() -> (tempfile::TempDir, PathBuf) {
let dir = tempfile::tempdir().unwrap();
let path = fs::canonicalize(dir.path()).unwrap();
(dir, path)
}
#[test]
fn valid_ascii_and_multibyte_pass_silently() {
let (_dir, root) = tempdir();
fs::write(root.join("ok"), "hello é 🎉\nplain ascii\n").unwrap();
assert_eq!(run_in(root, b"", &["ok"]), (0, String::new(), String::new()));
}
#[test]
fn invalid_sequence_reports_line_char_and_byte() {
let (_dir, root) = tempdir();
fs::write(root.join("bad"), b"ab\xC3(\n").unwrap();
fs::write(root.join("late"), b"a\nb\n\xFF").unwrap();
let (code, stdout, stderr) = run_in(root.clone(), b"", &["bad"]);
assert_eq!(code, 1);
assert_eq!(stdout, "bad: line 1, char 3, byte 2: invalid UTF-8 code\n");
assert_eq!(stderr, "");
let (code, stdout, _) = run_in(root, b"", &["late"]);
assert_eq!(code, 1);
assert_eq!(stdout, "late: line 3, char 1, byte 4: invalid UTF-8 code\n");
}
#[test]
fn multibyte_sequence_straddling_chunk_boundary_is_valid() {
let (_dir, root) = tempdir();
let mut bytes = vec![b'a'; 65535];
bytes.extend_from_slice("é".as_bytes()); // 0xC3 at offset 65535, 0xA9 at 65536
fs::write(root.join("straddle"), &bytes).unwrap();
assert_eq!(run_in(root, b"", &["straddle"]), (0, String::new(), String::new()));
}
#[test]
fn truncated_sequence_at_chunk_boundary_is_invalid() {
let (_dir, root) = tempdir();
let mut bytes = vec![b'a'; 65535];
bytes.push(0xC3); // incomplete at the exact chunk boundary
bytes.extend_from_slice(b"zzz");
fs::write(root.join("cut"), &bytes).unwrap();
let (code, stdout, stderr) = run_in(root, b"", &["cut"]);
assert_eq!(code, 1);
assert_eq!(stdout, "cut: line 1, char 65536, byte 65535: invalid UTF-8 code\n");
assert_eq!(stderr, "");
}
#[test]
fn truncated_multibyte_at_eof_is_invalid() {
let (_dir, root) = tempdir();
fs::write(root.join("eof"), b"abc\xE2\x82").unwrap();
let (code, stdout, _) = run_in(root, b"", &["eof"]);
assert_eq!(code, 1);
assert_eq!(stdout, "eof: line 1, char 4, byte 3: invalid UTF-8 code\n");
}
#[test]
fn quiet_suppresses_output_but_keeps_status() {
let (_dir, root) = tempdir();
fs::write(root.join("bad"), b"\xFF").unwrap();
assert_eq!(run_in(root, b"", &["-q", "bad"]), (1, String::new(), String::new()));
}
#[test]
fn list_prints_failing_names_and_invert_flips_them() {
let (_dir, root) = tempdir();
fs::write(root.join("good"), "fine\n").unwrap();
fs::write(root.join("bad"), b"\xFF\n").unwrap();
let (code, stdout, stderr) = run_in(root.clone(), b"", &["-l", "good", "bad"]);
assert_eq!((code, stdout.as_str(), stderr.as_str()), (1, "bad\n", ""));
let (code, stdout, stderr) = run_in(root, b"", &["-l", "-i", "good", "bad"]);
assert_eq!((code, stdout.as_str(), stderr.as_str()), (1, "good\n", ""));
}
#[test]
fn invert_flips_exit_status_without_list() {
let (_dir, root) = tempdir();
fs::write(root.join("good"), "fine\n").unwrap();
fs::write(root.join("bad"), b"\xFF").unwrap();
assert_eq!(run_in(root.clone(), b"", &["-q", "-i", "good"]).0, 1);
assert_eq!(run_in(root, b"", &["-q", "-i", "bad"]).0, 0);
}
#[test]
fn stdin_is_validated_when_no_files_given() {
let (_dir, root) = tempdir();
assert_eq!(run_in(root.clone(), "héllo\n".as_bytes(), &[]), (0, String::new(), String::new()));
let (code, stdout, _) = run_in(root, b"h\xFFi", &[]);
assert_eq!(code, 1);
assert_eq!(stdout, "(standard input): line 1, char 2, byte 1: invalid UTF-8 code\n");
}
#[test]
fn missing_file_reports_io_error_and_continues() {
let (_dir, root) = tempdir();
fs::write(root.join("bad"), b"\xFF").unwrap();
let (code, stdout, stderr) = run_in(root, b"", &["nope", "bad"]);
assert_eq!(code, 2);
assert_eq!(stdout, "bad: line 1, char 1, byte 0: invalid UTF-8 code\n");
assert!(stderr.starts_with("isutf8: nope: "), "stderr: {stderr}");
}
}
+291
View File
@@ -0,0 +1,291 @@
//! Context-safe, in-process moreutils `sponge` implementation for the
//! embedded shell: soak up all of stdin, then write it to a file (or stdout).
//!
//! The defining contract is that the output file is not opened or truncated
//! until stdin has been consumed to EOF, so `foo file | sponge file` works.
use std::{
ffi::{OsStr, OsString},
fs::{self, File, OpenOptions},
io::{self, Read, Write},
path::{Path, PathBuf},
sync::atomic::{AtomicU64, Ordering},
time::{SystemTime, UNIX_EPOCH},
};
use clap::{Arg, ArgAction, Command};
use pi_uutils_ctx::format_usage;
const OPT_APPEND: &str = "append";
const ARG_FILE: &str = "file";
const CHUNK_SIZE: usize = 64 * 1024;
/// Runs `sponge` against invocation-scoped stdin/stdout/stderr and
/// shell-relative paths.
pub fn run(argv: Vec<OsString>) -> i32 {
let matches = match command().try_get_matches_from(argv) {
Ok(matches) => matches,
Err(err) => {
let code = err.exit_code();
let rendered = err.to_string();
if err.use_stderr() {
let _ = write!(pi_uutils_ctx::stderr(), "{rendered}");
} else {
let _ = write!(pi_uutils_ctx::stdout(), "{rendered}");
}
return code;
},
};
let buffer = match soak_stdin() {
Ok(buffer) => buffer,
Err(SoakError::Cancelled) => return 130,
Err(SoakError::Io(err)) => {
let _ = writeln!(pi_uutils_ctx::stderr(), "sponge: stdin: {err}");
return 1;
},
};
let Some(file) = matches.get_one::<OsString>(ARG_FILE) else {
let mut out = pi_uutils_ctx::stdout();
if let Err(err) = out.write_all(&buffer).and_then(|()| out.flush()) {
let _ = writeln!(pi_uutils_ctx::stderr(), "sponge: stdout: {err}");
return 1;
}
return 0;
};
let target = pi_uutils_ctx::resolve(file);
let result = if matches.get_flag(OPT_APPEND) {
append_to(&target, &buffer)
} else {
replace_atomically(&target, &buffer)
};
match result {
Ok(()) => 0,
Err(err) => {
let _ = writeln!(pi_uutils_ctx::stderr(), "sponge: {}: {err}", file.to_string_lossy());
1
},
}
}
fn command() -> Command {
Command::new("sponge")
.version(concat!("sponge (pi-shell) ", env!("CARGO_PKG_VERSION")))
.about("Soak up all standard input, then write it to a file.")
.override_usage(format_usage("sponge [-a] [FILE]"))
.disable_help_flag(true)
.disable_version_flag(true)
.arg(
Arg::new(OPT_APPEND)
.short('a')
.long(OPT_APPEND)
.help("append the soaked input to the file instead of replacing it")
.action(ArgAction::SetTrue),
)
.arg(Arg::new("help").long("help").action(ArgAction::Help))
.arg(Arg::new("version").long("version").action(ArgAction::Version))
.arg(
Arg::new(ARG_FILE)
.value_name("FILE")
.value_parser(clap::value_parser!(OsString)),
)
}
enum SoakError {
Cancelled,
Io(io::Error),
}
/// Reads context stdin to EOF into memory, polling for cancellation between
/// chunks so an aborted pipeline never touches the output file.
fn soak_stdin() -> Result<Vec<u8>, SoakError> {
let mut stdin = pi_uutils_ctx::stdin();
let mut buffer = Vec::new();
let mut chunk = [0u8; CHUNK_SIZE];
loop {
if pi_uutils_ctx::is_cancelled() {
return Err(SoakError::Cancelled);
}
match stdin.read(&mut chunk) {
Ok(0) => return Ok(buffer),
Ok(n) => buffer.extend_from_slice(&chunk[..n]),
Err(err) if err.kind() == io::ErrorKind::Interrupted => {},
Err(err) => return Err(SoakError::Io(err)),
}
}
}
fn append_to(target: &Path, buffer: &[u8]) -> io::Result<()> {
let mut file = OpenOptions::new().append(true).create(true).open(target)?;
file.write_all(buffer)?;
file.flush()
}
/// Writes `buffer` to a fresh temporary file beside `target`, copies the
/// existing target's permissions onto it, then renames it over the target so
/// readers never observe a truncated file.
fn replace_atomically(target: &Path, buffer: &[u8]) -> io::Result<()> {
let (temp_path, mut temp) = create_sibling_temp(target)?;
let result = write_and_swap(target, &temp_path, &mut temp, buffer);
if result.is_err() {
let _ = fs::remove_file(&temp_path);
}
result
}
fn write_and_swap(target: &Path, temp_path: &Path, temp: &mut File, buffer: &[u8]) -> io::Result<()> {
temp.write_all(buffer)?;
temp.flush()?;
if let Ok(metadata) = fs::metadata(target) {
fs::set_permissions(temp_path, metadata.permissions())?;
}
fs::rename(temp_path, target)
}
/// Creates a uniquely named `.<basename>.sponge.<random>` file next to
/// `target` with `create_new`, retrying on collision.
fn create_sibling_temp(target: &Path) -> io::Result<(PathBuf, File)> {
static COUNTER: AtomicU64 = AtomicU64::new(0);
let dir = target.parent().filter(|p| !p.as_os_str().is_empty()).unwrap_or(Path::new("."));
let base = target.file_name().unwrap_or(OsStr::new("sponge")).to_string_lossy();
for _ in 0..32 {
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_or(0, |d| d.subsec_nanos() as u64);
let tag = nanos
.wrapping_mul(0x9e37_79b9_7f4a_7c15)
.wrapping_add(COUNTER.fetch_add(1, Ordering::Relaxed))
.wrapping_add(std::process::id() as u64);
let path = dir.join(format!(".{base}.sponge.{tag:016x}"));
match OpenOptions::new().write(true).create_new(true).open(&path) {
Ok(file) => return Ok((path, file)),
Err(err) if err.kind() == io::ErrorKind::AlreadyExists => {},
Err(err) => return Err(err),
}
}
Err(io::Error::new(io::ErrorKind::AlreadyExists, "could not create temporary file"))
}
#[cfg(test)]
mod tests {
use std::{
collections::HashMap,
ffi::OsString,
fs,
io::{Cursor, Write},
path::PathBuf,
sync::{Arc, atomic::AtomicBool},
};
use parking_lot::Mutex;
use pi_uutils_ctx::ScopeIo;
use super::run;
#[derive(Clone)]
struct SharedWriter(Arc<Mutex<Vec<u8>>>);
impl Write for SharedWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self.0.lock().write(buf)
}
fn flush(&mut self) -> std::io::Result<()> {
self.0.lock().flush()
}
}
fn run_in(cwd: PathBuf, stdin: &[u8], args: &[&str]) -> (i32, String, String) {
let stdout = Arc::new(Mutex::new(Vec::new()));
let stderr = Arc::new(Mutex::new(Vec::new()));
let io = ScopeIo {
stdin: Box::new(Cursor::new(stdin.to_vec())),
stdin_fd: None,
stdin_is_search_input: false,
stdout: Box::new(SharedWriter(Arc::clone(&stdout))),
stderr: Box::new(SharedWriter(Arc::clone(&stderr))),
cwd,
env: HashMap::new(),
cancel: Arc::new(AtomicBool::new(false)),
};
let argv = std::iter::once("sponge")
.chain(args.iter().copied())
.map(OsString::from)
.collect();
let code = pi_uutils_ctx::scope(io, || run(argv));
let stdout = String::from_utf8(stdout.lock().clone()).unwrap();
let stderr = String::from_utf8(stderr.lock().clone()).unwrap();
(code, stdout, stderr)
}
fn tempdir() -> (tempfile::TempDir, PathBuf) {
let dir = tempfile::tempdir().unwrap();
let path = fs::canonicalize(dir.path()).unwrap();
(dir, path)
}
#[test]
fn stdin_written_to_file_exactly() {
let (_dir, root) = tempdir();
let (code, stdout, stderr) = run_in(root.clone(), b"hello\nsponge\n", &["out"]);
assert_eq!((code, stdout.as_str(), stderr.as_str()), (0, "", ""));
assert_eq!(fs::read(root.join("out")).unwrap(), b"hello\nsponge\n");
}
#[test]
fn append_flag_appends_to_existing_content() {
let (_dir, root) = tempdir();
fs::write(root.join("log"), b"first\n").unwrap();
let (code, stdout, stderr) = run_in(root.clone(), b"second\n", &["-a", "log"]);
assert_eq!((code, stdout.as_str(), stderr.as_str()), (0, "", ""));
assert_eq!(fs::read(root.join("log")).unwrap(), b"first\nsecond\n");
}
#[test]
fn no_file_writes_stdin_to_stdout() {
let (_dir, root) = tempdir();
let (code, stdout, stderr) = run_in(root, b"passthrough", &[]);
assert_eq!((code, stdout.as_str(), stderr.as_str()), (0, "passthrough", ""));
}
#[test]
fn replaces_existing_target_and_leaves_no_temp_files() {
let (_dir, root) = tempdir();
fs::write(root.join("data"), b"old contents that are longer").unwrap();
let (code, stdout, stderr) = run_in(root.clone(), b"new", &["data"]);
assert_eq!((code, stdout.as_str(), stderr.as_str()), (0, "", ""));
assert_eq!(fs::read(root.join("data")).unwrap(), b"new");
let leftovers: Vec<_> = fs::read_dir(&root)
.unwrap()
.map(|e| e.unwrap().file_name())
.filter(|n| n != "data")
.collect();
assert_eq!(leftovers, Vec::<std::ffi::OsString>::new());
}
#[cfg(unix)]
#[test]
fn permissions_preserved_on_replace() {
use std::os::unix::fs::PermissionsExt;
let (_dir, root) = tempdir();
let target = root.join("secret");
fs::write(&target, b"old").unwrap();
fs::set_permissions(&target, fs::Permissions::from_mode(0o600)).unwrap();
let (code, _, stderr) = run_in(root, b"new", &["secret"]);
assert_eq!((code, stderr.as_str()), (0, ""));
assert_eq!(fs::metadata(&target).unwrap().permissions().mode() & 0o7777, 0o600);
}
#[test]
fn missing_target_directory_reports_error() {
let (_dir, root) = tempdir();
let (code, stdout, stderr) = run_in(root, b"bytes", &["nodir/out"]);
assert_eq!(code, 1);
assert_eq!(stdout, "");
assert!(stderr.starts_with("sponge: nodir/out: "), "stderr: {stderr}");
}
}
+509
View File
@@ -0,0 +1,509 @@
//! moreutils `ts` builtin: prefix each line of standard input with a
//! timestamp.
//!
//! Modes:
//! - default: current local time, strftime format `%b %d %H:%M:%S`
//! - `-i`: time elapsed since the previous line (default format `%H:%M:%S`)
//! - `-s`: time elapsed since program start (default format `%H:%M:%S`)
//! - `-m`: use the monotonic clock for `-i`/`-s` elapsed computation
//! - `-r`: rewrite an existing leading timestamp (RFC3339/ISO8601 or syslog
//! `%b %d %H:%M:%S`) into a human-relative form
//!
//! Elapsed durations are formatted as if they were seconds since the Unix
//! epoch rendered in UTC (matching moreutils), so 90 elapsed seconds with
//! `%H:%M:%S` renders `00:01:30`. The moreutils subsecond extensions `%.S`,
//! `%.s`, and `%.T` append microseconds to the seconds field in both absolute
//! and elapsed modes.
//!
//! `-r` renders relative times on a simple largest-nonzero-unit ladder —
//! `45s ago`, `12m ago`, `3h ago`, `9d ago` (or `in 2h` for future times) —
//! rather than Perl `Time::Duration`'s two-unit concise style.
use std::{
ffi::OsString,
io::{BufRead, BufReader, Write},
time::Instant,
};
use clap::{Arg, ArgAction, ArgMatches, Command};
use jiff::{Timestamp, fmt::strtime, tz::TimeZone};
use pi_uutils_ctx::format_usage;
const OPT_RELATIVE: &str = "relative";
const OPT_INCREMENTAL: &str = "incremental";
const OPT_SINCE_START: &str = "since-start";
const OPT_MONOTONIC: &str = "monotonic";
const ARG_FORMAT: &str = "format";
const DEFAULT_ABSOLUTE_FORMAT: &str = "%b %d %H:%M:%S";
const DEFAULT_ELAPSED_FORMAT: &str = "%H:%M:%S";
/// Byte length of a syslog-style `%b %d %H:%M:%S` timestamp prefix.
const SYSLOG_LEN: usize = 15;
/// Runs `ts` against invocation-scoped stdin/stdout/stderr.
pub fn run(argv: Vec<OsString>) -> i32 {
let matches = match command().try_get_matches_from(argv) {
Ok(matches) => matches,
Err(err) => {
let code = err.exit_code();
let rendered = err.to_string();
if err.use_stderr() {
let _ = write!(pi_uutils_ctx::stderr(), "{rendered}");
} else {
let _ = write!(pi_uutils_ctx::stdout(), "{rendered}");
}
return code;
},
};
match timestamp_lines(&matches) {
Ok(code) => code,
Err(message) => {
let _ = writeln!(pi_uutils_ctx::stderr(), "ts: {message}");
1
},
}
}
fn command() -> Command {
Command::new("ts")
.version(concat!("ts (pi-shell) ", env!("CARGO_PKG_VERSION")))
.about("Timestamp each line of standard input.")
.override_usage(format_usage("ts [-r] [-i | -s] [-m] [FORMAT]"))
.disable_help_flag(true)
.disable_version_flag(true)
.arg(
Arg::new(OPT_RELATIVE)
.short('r')
.help("convert existing leading timestamps to relative times")
.conflicts_with_all([OPT_INCREMENTAL, OPT_SINCE_START])
.action(ArgAction::SetTrue),
)
.arg(
Arg::new(OPT_INCREMENTAL)
.short('i')
.help("timestamp with the time elapsed since the last line")
.conflicts_with(OPT_SINCE_START)
.action(ArgAction::SetTrue),
)
.arg(
Arg::new(OPT_SINCE_START)
.short('s')
.help("timestamp with the time elapsed since program start")
.action(ArgAction::SetTrue),
)
.arg(
Arg::new(OPT_MONOTONIC)
.short('m')
.help("use the monotonic clock for elapsed timestamps")
.action(ArgAction::SetTrue),
)
.arg(Arg::new("help").long("help").action(ArgAction::Help))
.arg(Arg::new("version").long("version").action(ArgAction::Version))
.arg(Arg::new(ARG_FORMAT).value_name("FORMAT").help("strftime format string"))
}
/// Timestamping mode selected by the flags.
#[derive(Clone, Copy, PartialEq, Eq)]
enum Mode {
Absolute,
SinceLast,
SinceStart,
Relative,
}
fn timestamp_lines(matches: &ArgMatches) -> Result<i32, String> {
let mode = if matches.get_flag(OPT_RELATIVE) {
Mode::Relative
} else if matches.get_flag(OPT_INCREMENTAL) {
Mode::SinceLast
} else if matches.get_flag(OPT_SINCE_START) {
Mode::SinceStart
} else {
Mode::Absolute
};
let monotonic = matches.get_flag(OPT_MONOTONIC);
let default_format = if mode == Mode::Absolute {
DEFAULT_ABSOLUTE_FORMAT
} else {
DEFAULT_ELAPSED_FORMAT
};
let format = expand_subseconds(
matches
.get_one::<String>(ARG_FORMAT)
.map_or(default_format, String::as_str),
);
let tz = local_timezone();
let mut reader = BufReader::new(pi_uutils_ctx::stdin());
let mut out = pi_uutils_ctx::stdout();
let start_wall = Timestamp::now();
let start_mono = Instant::now();
let mut last_wall = start_wall;
let mut last_mono = start_mono;
let mut buf = Vec::new();
loop {
if pi_uutils_ctx::is_cancelled() {
return Ok(130);
}
buf.clear();
let n = reader.read_until(b'\n', &mut buf).map_err(|err| err.to_string())?;
if n == 0 {
break;
}
let had_newline = buf.last() == Some(&b'\n');
let content = if had_newline { &buf[..n - 1] } else { &buf[..] };
match mode {
Mode::Relative => {
let now = Timestamp::now();
let year = now.to_zoned(tz.clone()).year();
if let Some((consumed, then)) = parse_leading_timestamp(content, year, &tz) {
let rel = render_relative(then, now);
out.write_all(rel.as_bytes()).map_err(|err| err.to_string())?;
out.write_all(&content[consumed..]).map_err(|err| err.to_string())?;
} else {
out.write_all(content).map_err(|err| err.to_string())?;
}
},
Mode::Absolute => {
let zoned = Timestamp::now().to_zoned(tz.clone());
let stamp = strtime::format(&format, &zoned).map_err(|err| err.to_string())?;
out.write_all(stamp.as_bytes()).map_err(|err| err.to_string())?;
out.write_all(b" ").map_err(|err| err.to_string())?;
out.write_all(content).map_err(|err| err.to_string())?;
},
Mode::SinceLast | Mode::SinceStart => {
let nanos = if monotonic {
let now = Instant::now();
let anchor = if mode == Mode::SinceLast { last_mono } else { start_mono };
last_mono = now;
i128::try_from(now.duration_since(anchor).as_nanos()).unwrap_or(i128::MAX)
} else {
let now = Timestamp::now();
let anchor = if mode == Mode::SinceLast { last_wall } else { start_wall };
last_wall = now;
now.duration_since(anchor).as_nanos().max(0)
};
let stamp = format_elapsed(nanos, &format)?;
out.write_all(stamp.as_bytes()).map_err(|err| err.to_string())?;
out.write_all(b" ").map_err(|err| err.to_string())?;
out.write_all(content).map_err(|err| err.to_string())?;
},
}
if had_newline {
out.write_all(b"\n").map_err(|err| err.to_string())?;
}
// ts is commonly used on live pipes; make each line visible promptly.
out.flush().map_err(|err| err.to_string())?;
}
Ok(0)
}
/// Formats an elapsed duration (in nanoseconds) as seconds-since-epoch
/// rendered in UTC, matching moreutils' `strftime`-with-GMT behavior.
fn format_elapsed(nanos: i128, format: &str) -> Result<String, String> {
let stamp = Timestamp::from_nanosecond(nanos).map_err(|err| err.to_string())?;
strtime::format(format, &stamp.to_zoned(TimeZone::UTC)).map_err(|err| err.to_string())
}
/// Resolves the scope's timezone: `TZ` from the shell environment when valid,
/// otherwise the system timezone, otherwise UTC.
fn local_timezone() -> TimeZone {
if let Some(tz) = pi_uutils_ctx::var("TZ")
&& let Ok(tz) = TimeZone::get(&tz)
{
return tz;
}
TimeZone::try_system().unwrap_or(TimeZone::UTC)
}
/// Rewrites the moreutils subsecond extensions `%.S`, `%.s`, and `%.T` into
/// jiff strftime equivalents with a fixed 6-digit (microsecond) fraction.
fn expand_subseconds(format: &str) -> String {
let mut out = String::with_capacity(format.len());
let bytes = format.as_bytes();
let mut i = 0;
while i < bytes.len() {
if bytes[i] == b'%' && i + 1 < bytes.len() {
match &bytes[i + 1..] {
[b'%', ..] => {
out.push_str("%%");
i += 2;
continue;
},
[b'.', b'S', ..] => {
out.push_str("%S.%6f");
i += 3;
continue;
},
[b'.', b's', ..] => {
out.push_str("%s.%6f");
i += 3;
continue;
},
[b'.', b'T', ..] => {
out.push_str("%H:%M:%S.%6f");
i += 3;
continue;
},
_ => {},
}
}
// Copy the full UTF-8 sequence starting at `i`.
let len = utf8_len(bytes[i]);
out.push_str(std::str::from_utf8(&bytes[i..i + len]).unwrap_or("\u{fffd}"));
i += len;
}
out
}
/// Length of the UTF-8 sequence introduced by `first` (1 for continuation or
/// invalid bytes, which only arise from already-valid `&str` input here).
fn utf8_len(first: u8) -> usize {
match first {
0xC0..=0xDF => 2,
0xE0..=0xEF => 3,
0xF0..=0xF7 => 4,
_ => 1,
}
}
/// Parses a timestamp at the start of `line`, returning the byte length of
/// the matched prefix and the parsed instant.
///
/// Supported formats: RFC3339/ISO8601 (optional fractional seconds; offset
/// optional, civil times resolve in `tz`) and syslog `%b %d %H:%M:%S`
/// (assumed `year`, resolved in `tz`).
fn parse_leading_timestamp(line: &[u8], year: i16, tz: &TimeZone) -> Option<(usize, Timestamp)> {
let token_len = line
.iter()
.position(|b| b.is_ascii_whitespace())
.unwrap_or(line.len());
if let Ok(token) = std::str::from_utf8(&line[..token_len]) {
if let Ok(ts) = token.parse::<Timestamp>() {
return Some((token_len, ts));
}
if let Ok(dt) = token.parse::<jiff::civil::DateTime>()
&& let Ok(zoned) = dt.to_zoned(tz.clone())
{
return Some((token_len, zoned.timestamp()));
}
}
// Syslog style: exactly 15 bytes, followed by whitespace or end of line.
if line.len() < SYSLOG_LEN
|| (line.len() > SYSLOG_LEN && !line[SYSLOG_LEN].is_ascii_whitespace())
{
return None;
}
let mut prefix: [u8; SYSLOG_LEN] = line[..SYSLOG_LEN].try_into().ok()?;
// Syslog space-pads single-digit days ("Jan 1"); zero-pad for parsing.
if prefix[4] == b' ' {
prefix[4] = b'0';
}
let text = std::str::from_utf8(&prefix).ok()?;
let tm = strtime::parse("%Y %b %d %H:%M:%S", format!("{year} {text}")).ok()?;
let zoned = tm.to_datetime().ok()?.to_zoned(tz.clone()).ok()?;
Some((SYSLOG_LEN, zoned.timestamp()))
}
/// Renders `then` relative to `now` using the largest nonzero unit on an
/// `s`/`m`/`h`/`d` ladder: `45s ago`, `12m ago`, `3h ago`, `9d ago`, or
/// `in 2h` for future times.
fn render_relative(then: Timestamp, now: Timestamp) -> String {
let secs = now.duration_since(then).as_secs();
let magnitude = secs.unsigned_abs();
let (count, unit) = if magnitude >= 86_400 {
(magnitude / 86_400, 'd')
} else if magnitude >= 3_600 {
(magnitude / 3_600, 'h')
} else if magnitude >= 60 {
(magnitude / 60, 'm')
} else {
(magnitude, 's')
};
if secs >= 0 {
format!("{count}{unit} ago")
} else {
format!("in {count}{unit}")
}
}
#[cfg(test)]
mod tests {
use std::{
collections::HashMap,
ffi::OsString,
io::{Cursor, Write},
sync::{Arc, atomic::AtomicBool},
};
use jiff::{Timestamp, tz::TimeZone};
use parking_lot::Mutex;
use pi_uutils_ctx::ScopeIo;
use super::{parse_leading_timestamp, render_relative, run};
#[derive(Clone)]
struct SharedWriter(Arc<Mutex<Vec<u8>>>);
impl Write for SharedWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self.0.lock().write(buf)
}
fn flush(&mut self) -> std::io::Result<()> {
self.0.lock().flush()
}
}
fn run_in(stdin: &[u8], args: &[&str]) -> (i32, Vec<u8>, String) {
let stdout = Arc::new(Mutex::new(Vec::new()));
let stderr = Arc::new(Mutex::new(Vec::new()));
let io = ScopeIo {
stdin: Box::new(Cursor::new(stdin.to_vec())),
stdin_fd: None,
stdin_is_search_input: false,
stdout: Box::new(SharedWriter(Arc::clone(&stdout))),
stderr: Box::new(SharedWriter(Arc::clone(&stderr))),
cwd: std::env::temp_dir(),
env: HashMap::from([("TZ".to_string(), "UTC".to_string())]),
cancel: Arc::new(AtomicBool::new(false)),
};
let argv = std::iter::once("ts")
.chain(args.iter().copied())
.map(OsString::from)
.collect();
let code = pi_uutils_ctx::scope(io, || run(argv));
let stdout = stdout.lock().clone();
let stderr = String::from_utf8(stderr.lock().clone()).unwrap();
(code, stdout, stderr)
}
#[test]
fn absolute_mode_prefixes_default_format() {
let (code, stdout, stderr) = run_in(b"hello\n", &[]);
assert_eq!((code, stderr.as_str()), (0, ""));
let text = String::from_utf8(stdout).unwrap();
let re = regex::Regex::new(r"^[A-Z][a-z]{2} \d{2} \d{2}:\d{2}:\d{2} hello\n$").unwrap();
assert!(re.is_match(&text), "unexpected output: {text:?}");
}
#[test]
fn elapsed_modes_start_at_zero() {
let (code, stdout, _) = run_in(b"line\n", &["-i"]);
assert_eq!((code, stdout.as_slice()), (0, b"00:00:00 line\n".as_slice()));
let (code, stdout, _) = run_in(b"line\n", &["-s"]);
assert_eq!((code, stdout.as_slice()), (0, b"00:00:00 line\n".as_slice()));
let (code, stdout, _) = run_in(b"line\n", &["-s", "-m"]);
assert_eq!((code, stdout.as_slice()), (0, b"00:00:00 line\n".as_slice()));
}
#[test]
fn subsecond_extensions_render_microseconds() {
let (code, stdout, stderr) = run_in(b"x\n", &["-s", "%.S"]);
assert_eq!((code, stderr.as_str()), (0, ""));
let text = String::from_utf8(stdout).unwrap();
let re = regex::Regex::new(r"^\d{2}\.\d{6} x\n$").unwrap();
assert!(re.is_match(&text), "unexpected output: {text:?}");
let (code, stdout, _) = run_in(b"x\n", &["-s", "%.T"]);
assert_eq!(code, 0);
let text = String::from_utf8(stdout).unwrap();
let re = regex::Regex::new(r"^\d{2}:\d{2}:\d{2}\.\d{6} x\n$").unwrap();
assert!(re.is_match(&text), "unexpected output: {text:?}");
}
#[test]
fn binary_lines_survive_byte_for_byte() {
let (code, stdout, _) = run_in(b"ab\xff\xfecd\n", &["-s"]);
assert_eq!(code, 0);
assert_eq!(stdout, b"00:00:00 ab\xff\xfecd\n");
}
#[test]
fn final_line_without_newline_is_timestamped_without_newline() {
let (code, stdout, _) = run_in(b"first\nlast", &["-s"]);
assert_eq!(code, 0);
assert!(stdout.starts_with(b"00:00:00 first\n"), "output: {stdout:?}");
assert!(stdout.ends_with(b" last"), "output: {stdout:?}");
assert_ne!(stdout.last(), Some(&b'\n'));
}
#[test]
fn incremental_conflicts_with_since_start() {
let (code, _, stderr) = run_in(b"", &["-i", "-s"]);
assert_eq!(code, 2);
assert!(stderr.contains("cannot be used with"), "stderr: {stderr:?}");
}
#[test]
fn relative_conflicts_with_elapsed_modes() {
let (code, _, _) = run_in(b"", &["-r", "-i"]);
assert_eq!(code, 2);
}
#[test]
fn render_relative_uses_largest_nonzero_unit() {
let now: Timestamp = "2024-06-01T12:00:00Z".parse().unwrap();
let at = |secs: i64| Timestamp::from_second(now.as_second() - secs).unwrap();
assert_eq!(render_relative(at(45), now), "45s ago");
assert_eq!(render_relative(at(12 * 60), now), "12m ago");
assert_eq!(render_relative(at(3 * 3600 + 59), now), "3h ago");
assert_eq!(render_relative(at(9 * 86_400), now), "9d ago");
assert_eq!(render_relative(at(0), now), "0s ago");
assert_eq!(render_relative(at(-2 * 3600), now), "in 2h");
}
#[test]
fn parse_leading_timestamp_accepts_supported_formats() {
let tz = TimeZone::UTC;
let (consumed, ts) =
parse_leading_timestamp(b"2024-01-01T12:00:00Z boot", 2024, &tz).unwrap();
assert_eq!(consumed, 20);
assert_eq!(ts, "2024-01-01T12:00:00Z".parse::<Timestamp>().unwrap());
let (consumed, ts) =
parse_leading_timestamp(b"2024-01-01T12:00:00.500-05:00 x", 2024, &tz).unwrap();
assert_eq!(consumed, 29);
assert_eq!(ts, "2024-01-01T17:00:00.5Z".parse::<Timestamp>().unwrap());
// Civil datetime without offset resolves in the provided timezone.
let (consumed, ts) =
parse_leading_timestamp(b"2024-01-01T12:00:00 x", 2024, &tz).unwrap();
assert_eq!(consumed, 19);
assert_eq!(ts, "2024-01-01T12:00:00Z".parse::<Timestamp>().unwrap());
// Syslog style, zero-padded and space-padded days.
let (consumed, ts) = parse_leading_timestamp(b"Jan 02 03:04:05 msg", 2024, &tz).unwrap();
assert_eq!(consumed, 15);
assert_eq!(ts, "2024-01-02T03:04:05Z".parse::<Timestamp>().unwrap());
let (consumed, ts) = parse_leading_timestamp(b"Jan 2 03:04:05 msg", 2024, &tz).unwrap();
assert_eq!(consumed, 15);
assert_eq!(ts, "2024-01-02T03:04:05Z".parse::<Timestamp>().unwrap());
assert!(parse_leading_timestamp(b"plain text line", 2024, &tz).is_none());
assert!(parse_leading_timestamp(b"Jan 02 03:04:05x", 2024, &tz).is_none());
}
#[test]
fn relative_mode_rewrites_matching_lines_and_passes_others() {
let (code, stdout, stderr) =
run_in(b"2000-01-01T00:00:00Z boot\nno timestamp here\n", &["-r"]);
assert_eq!((code, stderr.as_str()), (0, ""));
let text = String::from_utf8(stdout).unwrap();
let mut lines = text.lines();
let first = lines.next().unwrap();
assert!(first.ends_with(" boot"), "line: {first:?}");
assert!(first.contains("d ago"), "line: {first:?}");
assert!(!first.starts_with("2000"), "line: {first:?}");
assert_eq!(lines.next(), Some("no timestamp here"));
assert_eq!(lines.next(), None);
}
}
+53
View File
@@ -658,6 +658,14 @@ async fn create_session_for_run(
shell.register_builtin("sed", crate::coreutils::sed_builtin());
shell.register_builtin("xargs", crate::coreutils::xargs_builtin());
shell.register_builtin("jq", crate::coreutils::jq_builtin());
// moreutils-inspired in-process builtins (see crate::moreutils).
shell.register_builtin("ts", crate::coreutils::ts_builtin());
shell.register_builtin("sponge", crate::coreutils::sponge_builtin());
shell.register_builtin("ifne", crate::coreutils::ifne_builtin());
shell.register_builtin("isutf8", crate::coreutils::isutf8_builtin());
shell.register_builtin("combine", crate::coreutils::combine_builtin());
#[cfg(unix)]
shell.register_builtin("errno", crate::coreutils::errno_builtin());
if !uutils_env_disabled(config, "PI_DISABLE_UUTILS_DESTRUCTIVE") {
if !uutils_env_disabled(config, "PI_DISABLE_RM_BUILTIN") {
shell.register_builtin("rm", crate::coreutils::rm_builtin());
@@ -2205,6 +2213,51 @@ mod tests {
assert_eq!(exit_code(&different), 1);
}
/// The moreutils-style builtins (`ts`, `sponge`, `isutf8`, `combine`,
/// `ifne`, `errno`) dispatch in-process: the script runs with no usable
/// executable search path, and the `ts | sponge` pipeline must land its
/// output in the shell's working directory.
#[tokio::test(flavor = "multi_thread")]
async fn moreutils_builtins_are_registered_in_process() {
let dir = tempfile::tempdir().expect("temp dir");
let root = std::fs::canonicalize(dir.path()).expect("canonical temp dir");
let config = ShellConfig { session_env: None, snapshot_path: None, minimizer: None };
let mut session = create_session(&config).await.expect("create_session");
session
.shell
.set_working_dir(root.to_str().expect("utf8 temp path"))
.expect("set cwd");
let mut params = session.shell.default_exec_params();
params.set_fd(OpenFiles::STDIN_FD, null_file().expect("null stdin"));
params.set_fd(OpenFiles::STDOUT_FD, null_file().expect("null stdout"));
params.set_fd(OpenFiles::STDERR_FD, null_file().expect("null stderr"));
let source_info = SourceInfo::from("pi-natives:test");
let script = "PATH=/definitely-missing\n\
echo x | ts -s '%H:%M:%S' | sponge out || exit 10\n\
isutf8 out || exit 11\n\
echo x | combine - and out || exit 12\n\
ifne /definitely-missing/tool || exit 13";
let result =
session.shell.run_string(script, &source_info, &params).await.expect("moreutils script");
assert_eq!(exit_code(&result), 0);
// `ts -s` stamps the first line with zero elapsed time: `HH:MM:SS x`.
let out = std::fs::read_to_string(root.join("out")).expect("sponge output");
assert!(out.len() == 11 && out.ends_with(" x\n"), "unexpected ts+sponge output: {out:?}");
#[cfg(unix)]
{
let errno = session
.shell
.run_string("errno ENOENT", &source_info, &params)
.await
.expect("errno lookup");
assert_eq!(exit_code(&errno), 0);
}
}
/// The uutils-backed `mkdir` builtin must (1) create directories under the
/// shell's working directory rather than the host process cwd, (2) route
/// `-v` output through the command's (here redirected) stdout, and (3)
+4
View File
@@ -2,6 +2,10 @@
## [Unreleased]
### Added
- Added in-process moreutils-style shell builtins to the bash tool's embedded shell: `ts` (timestamp lines; `-i`/`-s` elapsed modes, `-m` monotonic clock, `-r` relative rewriting of RFC3339/syslog timestamps, `%.S`/`%.s`/`%.T` subsecond extensions), `sponge` (soak stdin fully before atomically writing the target, so `foo file | ... | sponge file` works; `-a` appends), `ifne` (run a command only when stdin is non-empty; `-n` inverts and passes non-empty stdin through), `isutf8` (streaming UTF-8 validation with line/char/byte diagnostics; `-q`, `-l`, `-i`), `combine` (boolean `and`/`not`/`or`/`xor` on the lines of two files, `-` for stdin), and `errno` (errno name/number/description lookup with `-l` list and `-s` search; unix only). Like the uutils-backed builtins, they run in-process against the command's own stdio, resolve paths against the shell working directory, honor cancellation, and are disabled by `PI_DISABLE_UUTILS_BUILTINS`.
### Changed
- Large pastes saved via the large-paste menu now insert `local://paste-N.md` references (previously `local://attachment-N`), so the saved paste carries a markdown extension and a clearer name.