From 37d5b69250cd9a1f15c88cd4b1d84b204aa79805 Mon Sep 17 00:00:00 2001 From: can1357 Date: Thu, 29 Jan 2026 11:49:46 +0100 Subject: [PATCH] feat(highlight): replaced cli-highlight with rust-based syntax highlighting - Added native syntax highlighting module powered by syntect library with ANSI color output support. - Implemented highlightCode() function that applies syntax highlighting to code with language detection and customizable color themes. - Implemented supportsLanguage() function to check if a given programming language is supported for syntax highlighting. - Implemented getSupportedLanguages() function that returns list of all supported language identifiers. - Replaced cli-highlight dependency with native syntect-based highlighting in coding-agent theme module. - Added comprehensive language alias mappings supporting 50+ programming languages including TypeScript, Python, Rust, Go, C/C++, and web languages. --- crates/pi-natives/Cargo.toml | 1 + crates/pi-natives/src/highlight.rs | 416 ++++++++++++++++++ crates/pi-natives/src/lib.rs | 1 + package.json | 1 + packages/coding-agent/package.json | 2 - packages/coding-agent/src/lsp/render.ts | 36 +- .../coding-agent/src/modes/theme/theme.ts | 72 ++- packages/natives/src/highlight/index.ts | 55 +++ packages/natives/src/index.ts | 27 ++ packages/natives/wasm/pi_natives.d.ts | 31 ++ packages/natives/wasm/pi_natives.js | 161 ++++++- packages/natives/wasm/pi_natives_bg.wasm | 4 +- packages/natives/wasm/pi_natives_bg.wasm.d.ts | 7 +- 13 files changed, 738 insertions(+), 76 deletions(-) create mode 100644 crates/pi-natives/src/highlight.rs create mode 100644 packages/natives/src/highlight/index.ts diff --git a/crates/pi-natives/Cargo.toml b/crates/pi-natives/Cargo.toml index 57730a962..1150ec233 100644 --- a/crates/pi-natives/Cargo.toml +++ b/crates/pi-natives/Cargo.toml @@ -22,6 +22,7 @@ grep-matcher = "0.1.8" image = { version = "0.25", default-features = false, features = ["png", "jpeg"] } unicode-segmentation = "1.11" unicode-width = "0.2" +syntect = { version = "5.3", default-features = false, features = ["default-syntaxes", "default-themes", "regex-fancy"] } # Better panic messages in debug builds console_error_panic_hook = { version = "0.1.7", optional = true } diff --git a/crates/pi-natives/src/highlight.rs b/crates/pi-natives/src/highlight.rs new file mode 100644 index 000000000..92b0ba6fd --- /dev/null +++ b/crates/pi-natives/src/highlight.rs @@ -0,0 +1,416 @@ +//! Syntax highlighting using syntect. +//! +//! Provides ANSI-colored output for code blocks. Takes theme colors as input +//! and maps syntect scopes to 9 semantic categories: +//! - comment, keyword, function, variable, string, number, type, operator, +//! punctuation + +use std::sync::OnceLock; + +use serde::Deserialize; +use syntect::parsing::{ParseState, Scope, ScopeStack, ScopeStackOp, SyntaxReference, SyntaxSet}; +use wasm_bindgen::prelude::*; + +static SYNTAX_SET: OnceLock = OnceLock::new(); +static SCOPE_MATCHERS: OnceLock = OnceLock::new(); + +fn get_syntax_set() -> &'static SyntaxSet { + SYNTAX_SET.get_or_init(SyntaxSet::load_defaults_newlines) +} + +/// Pre-compiled scope patterns for fast matching. +struct ScopeMatchers { + // Comment (index 0) + comment: Scope, + + // String (index 4) + string: Scope, + constant_character: Scope, + meta_string: Scope, + + // Number (index 5) + constant_numeric: Scope, + constant_integer: Scope, + constant: Scope, + + // Keyword (index 1) + keyword: Scope, + storage_type: Scope, + storage_modifier: Scope, + + // Function (index 2) + entity_name_function: Scope, + support_function: Scope, + meta_function_call: Scope, + variable_function: Scope, + + // Type (index 6) + entity_name_type: Scope, + support_type: Scope, + support_class: Scope, + entity_name_class: Scope, + entity_name_struct: Scope, + entity_name_enum: Scope, + entity_name_interface: Scope, + entity_name_trait: Scope, + + // Operator (index 7) + keyword_operator: Scope, + punctuation_accessor: Scope, + + // Punctuation (index 8) + punctuation: Scope, + + // Variable (index 3) + variable: Scope, + entity_name: Scope, + meta_path: Scope, +} + +impl ScopeMatchers { + fn new() -> Self { + Self { + comment: Scope::new("comment").unwrap(), + string: Scope::new("string").unwrap(), + constant_character: Scope::new("constant.character").unwrap(), + meta_string: Scope::new("meta.string").unwrap(), + constant_numeric: Scope::new("constant.numeric").unwrap(), + constant_integer: Scope::new("constant.integer").unwrap(), + constant: Scope::new("constant").unwrap(), + keyword: Scope::new("keyword").unwrap(), + storage_type: Scope::new("storage.type").unwrap(), + storage_modifier: Scope::new("storage.modifier").unwrap(), + entity_name_function: Scope::new("entity.name.function").unwrap(), + support_function: Scope::new("support.function").unwrap(), + meta_function_call: Scope::new("meta.function-call").unwrap(), + variable_function: Scope::new("variable.function").unwrap(), + entity_name_type: Scope::new("entity.name.type").unwrap(), + support_type: Scope::new("support.type").unwrap(), + support_class: Scope::new("support.class").unwrap(), + entity_name_class: Scope::new("entity.name.class").unwrap(), + entity_name_struct: Scope::new("entity.name.struct").unwrap(), + entity_name_enum: Scope::new("entity.name.enum").unwrap(), + entity_name_interface: Scope::new("entity.name.interface").unwrap(), + entity_name_trait: Scope::new("entity.name.trait").unwrap(), + keyword_operator: Scope::new("keyword.operator").unwrap(), + punctuation_accessor: Scope::new("punctuation.accessor").unwrap(), + punctuation: Scope::new("punctuation").unwrap(), + variable: Scope::new("variable").unwrap(), + entity_name: Scope::new("entity.name").unwrap(), + meta_path: Scope::new("meta.path").unwrap(), + } + } +} + +fn get_scope_matchers() -> &'static ScopeMatchers { + SCOPE_MATCHERS.get_or_init(ScopeMatchers::new) +} + +/// Theme colors for syntax highlighting. +/// Each color is an ANSI escape sequence (e.g., "\x1b[38;2;255;0;0m"). +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct HighlightColors { + pub comment: String, + pub keyword: String, + pub function: String, + pub variable: String, + pub string: String, + pub number: String, + pub r#type: String, + pub operator: String, + pub punctuation: String, +} + +/// Language alias mappings: (aliases, target syntax name). +/// Used for languages not in syntect's default set or with non-standard names. +const LANG_ALIASES: &[(&[&str], &str)] = &[ + (&["ts", "tsx", "typescript", "js", "jsx", "javascript", "mjs", "cjs"], "JavaScript"), + (&["py", "python"], "Python"), + (&["rb", "ruby"], "Ruby"), + (&["rs", "rust"], "Rust"), + (&["go", "golang"], "Go"), + (&["java"], "Java"), + (&["kt", "kotlin"], "Java"), + (&["swift"], "Objective-C"), + (&["c", "h"], "C"), + (&["cpp", "cc", "cxx", "c++", "hpp", "hxx", "hh"], "C++"), + (&["cs", "csharp"], "C#"), + (&["php"], "PHP"), + (&["sh", "bash", "zsh", "shell"], "Bash"), + (&["fish"], "Shell-Unix-Generic"), + (&["ps1", "powershell"], "PowerShell"), + (&["html", "htm"], "HTML"), + (&["css"], "CSS"), + (&["scss"], "SCSS"), + (&["sass"], "Sass"), + (&["less"], "LESS"), + (&["json"], "JSON"), + (&["yaml", "yml"], "YAML"), + (&["toml"], "TOML"), + (&["xml"], "XML"), + (&["md", "markdown"], "Markdown"), + (&["sql"], "SQL"), + (&["lua"], "Lua"), + (&["perl", "pl"], "Perl"), + (&["r"], "R"), + (&["scala"], "Scala"), + (&["clj", "clojure"], "Clojure"), + (&["ex", "exs", "elixir"], "Ruby"), + (&["erl", "erlang"], "Erlang"), + (&["hs", "haskell"], "Haskell"), + (&["ml", "ocaml"], "OCaml"), + (&["vim"], "VimL"), + (&["graphql", "gql"], "GraphQL"), + (&["proto", "protobuf"], "Protocol Buffers"), + (&["tf", "hcl", "terraform"], "Terraform"), + (&["dockerfile", "docker"], "Dockerfile"), + (&["makefile", "make"], "Makefile"), + (&["cmake"], "CMake"), + (&["ini", "cfg", "conf", "config", "properties"], "INI"), + (&["diff", "patch"], "Diff"), + (&["gitignore", "gitattributes", "gitmodules"], "Git Ignore"), +]; + +/// Find syntax name from alias table using case-insensitive comparison. +#[inline] +fn find_alias(lang: &str) -> Option<&'static str> { + LANG_ALIASES + .iter() + .find(|(aliases, _)| aliases.iter().any(|a| lang.eq_ignore_ascii_case(a))) + .map(|(_, target)| *target) +} + +/// Check if language is in the alias table. +#[inline] +fn is_known_alias(lang: &str) -> bool { + LANG_ALIASES + .iter() + .any(|(aliases, _)| aliases.iter().any(|a| lang.eq_ignore_ascii_case(a))) +} + +/// Determine the semantic color category from a scope stack. +/// Uses bitwise prefix matching for fast comparison. +#[inline] +fn scope_to_color_index(scope: &ScopeStack) -> usize { + let m = get_scope_matchers(); + + // Walk from innermost to outermost scope + for s in scope.as_slice().iter().rev() { + // Check for specific scopes (order matters - more specific first) + + // Comment (index 0) + if m.comment.is_prefix_of(*s) { + return 0; + } + + // String (index 4) + if m.string.is_prefix_of(*s) + || m.constant_character.is_prefix_of(*s) + || m.meta_string.is_prefix_of(*s) + { + return 4; + } + + // Number (index 5) - check specific numeric first + if m.constant_numeric.is_prefix_of(*s) || m.constant_integer.is_prefix_of(*s) { + return 5; + } + + // Keyword (index 1) + if m.keyword.is_prefix_of(*s) + || m.storage_type.is_prefix_of(*s) + || m.storage_modifier.is_prefix_of(*s) + { + return 1; + } + + // Function (index 2) + if m.entity_name_function.is_prefix_of(*s) + || m.support_function.is_prefix_of(*s) + || m.meta_function_call.is_prefix_of(*s) + || m.variable_function.is_prefix_of(*s) + { + return 2; + } + + // Type (index 6) + if m.entity_name_type.is_prefix_of(*s) + || m.support_type.is_prefix_of(*s) + || m.support_class.is_prefix_of(*s) + || m.entity_name_class.is_prefix_of(*s) + || m.entity_name_struct.is_prefix_of(*s) + || m.entity_name_enum.is_prefix_of(*s) + || m.entity_name_interface.is_prefix_of(*s) + || m.entity_name_trait.is_prefix_of(*s) + { + return 6; + } + + // Operator (index 7) + if m.keyword_operator.is_prefix_of(*s) || m.punctuation_accessor.is_prefix_of(*s) { + return 7; + } + + // Punctuation (index 8) + if m.punctuation.is_prefix_of(*s) { + return 8; + } + + // Variable (index 3) + if m.variable.is_prefix_of(*s) + || m.entity_name.is_prefix_of(*s) + || m.meta_path.is_prefix_of(*s) + { + return 3; + } + + // Generic constant -> number (index 5) + if m.constant.is_prefix_of(*s) { + return 5; + } + } + + // Default: no special color (will render as-is) + usize::MAX +} + +/// Find the appropriate syntax for a language name. +fn find_syntax<'a>(ss: &'a SyntaxSet, lang: &str) -> Option<&'a SyntaxReference> { + // Direct name/token match (syntect APIs are case-insensitive) + if let Some(syn) = ss.find_syntax_by_token(lang) { + return Some(syn); + } + + // Extension-based match + if let Some(syn) = ss.find_syntax_by_extension(lang) { + return Some(syn); + } + + // Alias lookup for languages not in syntect's default set + let alias = find_alias(lang)?; + + ss.find_syntax_by_name(alias) + .or_else(|| ss.find_syntax_by_token(alias)) +} + +/// Highlight code and return ANSI-colored lines. +/// +/// # Arguments +/// * `code` - The source code to highlight +/// * `lang` - Language identifier (e.g., "rust", "typescript", "python") +/// * `colors` - Theme colors as ANSI escape sequences +/// +/// # Returns +/// Highlighted code with ANSI color codes, or the original code if highlighting +/// fails. +#[wasm_bindgen] +pub fn highlight_code(code: &str, lang: Option, colors: JsValue) -> String { + let colors: HighlightColors = match serde_wasm_bindgen::from_value(colors) { + Ok(c) => c, + Err(_) => return code.to_string(), + }; + + // Color palette as array for quick indexing + let palette = [ + &colors.comment, // 0 + &colors.keyword, // 1 + &colors.function, // 2 + &colors.variable, // 3 + &colors.string, // 4 + &colors.number, // 5 + &colors.r#type, // 6 + &colors.operator, // 7 + &colors.punctuation, // 8 + ]; + + let ss = get_syntax_set(); + + // Find syntax for the language + let syntax = match &lang { + Some(l) => find_syntax(ss, l), + None => None, + } + .unwrap_or_else(|| ss.find_syntax_plain_text()); + + let mut parse_state = ParseState::new(syntax); + let mut scope_stack = ScopeStack::new(); + let mut result = String::with_capacity(code.len() * 2); + + for line in syntect::util::LinesWithEndings::from(code) { + let Ok(ops) = parse_state.parse_line(line, ss) else { + result.push_str(line); + continue; + }; + + let mut prev_end = 0; + for (offset, op) in ops { + let offset = offset.min(line.len()); + + // Output text BEFORE this operation using current scope + if offset > prev_end { + let text = &line[prev_end..offset]; + let color_idx = scope_to_color_index(&scope_stack); + + if color_idx < palette.len() && !palette[color_idx].is_empty() { + result.push_str(palette[color_idx]); + result.push_str(text); + result.push_str("\x1b[39m"); + } else { + result.push_str(text); + } + } + prev_end = offset; + + // Now apply scope operation for NEXT segment + match op { + ScopeStackOp::Push(scope) => { + scope_stack.push(scope); + }, + ScopeStackOp::Pop(count) => { + for _ in 0..count { + scope_stack.pop(); + } + }, + ScopeStackOp::Restore | ScopeStackOp::Clear(_) | ScopeStackOp::Noop => {}, + } + } + + // Output remaining text with current scope + if prev_end < line.len() { + let text = &line[prev_end..]; + let color_idx = scope_to_color_index(&scope_stack); + + if color_idx < palette.len() && !palette[color_idx].is_empty() { + result.push_str(palette[color_idx]); + result.push_str(text); + result.push_str("\x1b[39m"); + } else { + result.push_str(text); + } + } + } + + result +} + +/// Check if a language is supported for highlighting. +/// Returns true if the language has either direct support or a fallback +/// mapping. +#[wasm_bindgen] +pub fn supports_language(lang: &str) -> bool { + if is_known_alias(lang) { + return true; + } + + // Fall back to direct syntax lookup + let ss = get_syntax_set(); + find_syntax(ss, lang).is_some() +} + +/// Get list of supported languages. +#[wasm_bindgen] +pub fn get_supported_languages() -> Vec { + let ss = get_syntax_set(); + ss.syntaxes().iter().map(|s| s.name.clone()).collect() +} diff --git a/crates/pi-natives/src/lib.rs b/crates/pi-natives/src/lib.rs index f7f5f2a5b..eba9a7bc1 100644 --- a/crates/pi-natives/src/lib.rs +++ b/crates/pi-natives/src/lib.rs @@ -4,6 +4,7 @@ //! The JS side handles directory walking and file reading, then passes //! content here for matching. +pub mod highlight; pub mod image; pub mod text; diff --git a/package.json b/package.json index 78ea3bab9..0ad2ef8cb 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "fix": "bun run fix:ts && bun run fix:rs", "fix:ts": "biome check --write --unsafe . && bun --cwd=packages/coding-agent run format-prompts", "fix:rs": "cargo clippy --fix --allow-dirty --all-targets --no-deps --allow-staged --broken-code --allow-no-vcs && cargo fmt --all", + "build:wasm": "bun --cwd=packages/natives run build:wasm", "bench:gen-fixtures": "bun run packages/react-edit-benchmark/generate.ts --react-dir /tmp/react-source --count-per-type 3", "bench:edit": "bun --cwd=packages/react-edit-benchmark run start", "version:patch": "npm version patch -ws --no-git-tag-version && bun scripts/sync-versions.ts && rm -rf node_modules packages/*/node_modules bun.lock && bun install", diff --git a/packages/coding-agent/package.json b/packages/coding-agent/package.json index c84038bd6..a9bcdf582 100644 --- a/packages/coding-agent/package.json +++ b/packages/coding-agent/package.json @@ -89,12 +89,10 @@ "@sinclair/typebox": "^0.34.48", "ajv": "^8.17.1", "chalk": "^5.6.2", - "cli-highlight": "^2.1.11", "diff": "^8.0.3", "file-type": "^21.3.0", "glob": "^13.0.0", "handlebars": "^4.7.8", - "highlight.js": "^11.11.1", "marked": "^17.0.1", "nanoid": "^5.1.6", "node-html-parser": "^7.0.2", diff --git a/packages/coding-agent/src/lsp/render.ts b/packages/coding-agent/src/lsp/render.ts index 0100e0ca8..6d18d139b 100644 --- a/packages/coding-agent/src/lsp/render.ts +++ b/packages/coding-agent/src/lsp/render.ts @@ -8,8 +8,12 @@ * - Collapsible/expandable views */ import type { RenderResultOptions } from "@oh-my-pi/pi-agent-core"; +import { + highlightCode as nativeHighlightCode, + type HighlightColors, + supportsLanguage, +} from "@oh-my-pi/pi-natives"; import { type Component, Text } from "@oh-my-pi/pi-tui"; -import { highlight, supportsLanguage } from "cli-highlight"; import { getLanguageFromPath, type Theme } from "../modes/theme/theme"; import { formatExpandHint, @@ -291,29 +295,23 @@ function renderHover( } /** - * Syntax highlight code using highlight.ts. + * Syntax highlight code using native WASM highlighter. */ function highlightCode(codeText: string, language: string, theme: Theme): string[] { const validLang = language && supportsLanguage(language) ? language : undefined; try { - const cliTheme = { - keyword: (s: string) => theme.fg("syntaxKeyword", s), - built_in: (s: string) => theme.fg("syntaxType", s), - literal: (s: string) => theme.fg("syntaxNumber", s), - number: (s: string) => theme.fg("syntaxNumber", s), - string: (s: string) => theme.fg("syntaxString", s), - comment: (s: string) => theme.fg("syntaxComment", s), - function: (s: string) => theme.fg("syntaxFunction", s), - title: (s: string) => theme.fg("syntaxFunction", s), - class: (s: string) => theme.fg("syntaxType", s), - type: (s: string) => theme.fg("syntaxType", s), - attr: (s: string) => theme.fg("syntaxVariable", s), - variable: (s: string) => theme.fg("syntaxVariable", s), - params: (s: string) => theme.fg("syntaxVariable", s), - operator: (s: string) => theme.fg("syntaxOperator", s), - punctuation: (s: string) => theme.fg("syntaxPunctuation", s), + const colors: HighlightColors = { + comment: theme.getFgAnsi("syntaxComment"), + keyword: theme.getFgAnsi("syntaxKeyword"), + function: theme.getFgAnsi("syntaxFunction"), + variable: theme.getFgAnsi("syntaxVariable"), + string: theme.getFgAnsi("syntaxString"), + number: theme.getFgAnsi("syntaxNumber"), + type: theme.getFgAnsi("syntaxType"), + operator: theme.getFgAnsi("syntaxOperator"), + punctuation: theme.getFgAnsi("syntaxPunctuation"), }; - return highlight(codeText, { language: validLang, ignoreIllegals: true, theme: cliTheme }).split("\n"); + return nativeHighlightCode(codeText, validLang, colors).split("\n"); } catch { return codeText.split("\n"); } diff --git a/packages/coding-agent/src/modes/theme/theme.ts b/packages/coding-agent/src/modes/theme/theme.ts index 8d3e5a8eb..8b3d8b6a0 100644 --- a/packages/coding-agent/src/modes/theme/theme.ts +++ b/packages/coding-agent/src/modes/theme/theme.ts @@ -4,8 +4,12 @@ import type { EditorTheme, MarkdownTheme, SelectListTheme, SymbolTheme } from "@ import { adjustHsv, isEnoent, logger } from "@oh-my-pi/pi-utils"; import { type Static, Type } from "@sinclair/typebox"; import { TypeCompiler } from "@sinclair/typebox/compiler"; +import { + highlightCode as nativeHighlightCode, + type HighlightColors as NativeHighlightColors, + supportsLanguage as nativeSupportsLanguage, +} from "@oh-my-pi/pi-natives"; import chalk from "chalk"; -import { highlight, supportsLanguage } from "cli-highlight"; import { getCustomThemesDir } from "../../config"; // Embed theme JSON files at build time import darkThemeJson from "./dark.json" with { type: "json" }; @@ -2029,37 +2033,25 @@ export async function getThemeExportColors(themeName?: string): Promise<{ // TUI Helpers // ============================================================================ -type CliHighlightTheme = Record string>; +let cachedHighlightColorsFor: Theme | undefined; +let cachedHighlightColors: NativeHighlightColors | undefined; -let cachedHighlightThemeFor: Theme | undefined; -let cachedCliHighlightTheme: CliHighlightTheme | undefined; - -function buildCliHighlightTheme(t: Theme): CliHighlightTheme { - return { - keyword: (s: string) => t.fg("syntaxKeyword", s), - built_in: (s: string) => t.fg("syntaxType", s), - literal: (s: string) => t.fg("syntaxNumber", s), - number: (s: string) => t.fg("syntaxNumber", s), - string: (s: string) => t.fg("syntaxString", s), - comment: (s: string) => t.fg("syntaxComment", s), - function: (s: string) => t.fg("syntaxFunction", s), - title: (s: string) => t.fg("syntaxFunction", s), - class: (s: string) => t.fg("syntaxType", s), - type: (s: string) => t.fg("syntaxType", s), - attr: (s: string) => t.fg("syntaxVariable", s), - variable: (s: string) => t.fg("syntaxVariable", s), - params: (s: string) => t.fg("syntaxVariable", s), - operator: (s: string) => t.fg("syntaxOperator", s), - punctuation: (s: string) => t.fg("syntaxPunctuation", s), - }; -} - -function getCliHighlightTheme(t: Theme): CliHighlightTheme { - if (cachedHighlightThemeFor !== t || !cachedCliHighlightTheme) { - cachedHighlightThemeFor = t; - cachedCliHighlightTheme = buildCliHighlightTheme(t); +function getHighlightColors(t: Theme): NativeHighlightColors { + if (cachedHighlightColorsFor !== t || !cachedHighlightColors) { + cachedHighlightColorsFor = t; + cachedHighlightColors = { + comment: t.getFgAnsi("syntaxComment"), + keyword: t.getFgAnsi("syntaxKeyword"), + function: t.getFgAnsi("syntaxFunction"), + variable: t.getFgAnsi("syntaxVariable"), + string: t.getFgAnsi("syntaxString"), + number: t.getFgAnsi("syntaxNumber"), + type: t.getFgAnsi("syntaxType"), + operator: t.getFgAnsi("syntaxOperator"), + punctuation: t.getFgAnsi("syntaxPunctuation"), + }; } - return cachedCliHighlightTheme; + return cachedHighlightColors; } /** @@ -2067,15 +2059,9 @@ function getCliHighlightTheme(t: Theme): CliHighlightTheme { * Returns array of highlighted lines. */ export function highlightCode(code: string, lang?: string): string[] { - // Validate language before highlighting to avoid stderr spam from cli-highlight - const validLang = lang && supportsLanguage(lang) ? lang : undefined; - const opts = { - language: validLang, - ignoreIllegals: true, - theme: getCliHighlightTheme(theme), - }; + const validLang = lang && nativeSupportsLanguage(lang) ? lang : undefined; try { - return highlight(code, opts).split("\n"); + return nativeHighlightCode(code, validLang, getHighlightColors(theme)).split("\n"); } catch { return code.split("\n"); } @@ -2212,15 +2198,9 @@ export function getMarkdownTheme(): MarkdownTheme { symbols: getSymbolTheme(), getMermaidImage, highlightCode: (code: string, lang?: string): string[] => { - // Validate language before highlighting to avoid stderr spam from cli-highlight - const validLang = lang && supportsLanguage(lang) ? lang : undefined; - const opts = { - language: validLang, - ignoreIllegals: true, - theme: getCliHighlightTheme(theme), - }; + const validLang = lang && nativeSupportsLanguage(lang) ? lang : undefined; try { - return highlight(code, opts).split("\n"); + return nativeHighlightCode(code, validLang, getHighlightColors(theme)).split("\n"); } catch { return code.split("\n").map(line => theme.fg("mdCodeBlock", line)); } diff --git a/packages/natives/src/highlight/index.ts b/packages/natives/src/highlight/index.ts new file mode 100644 index 000000000..9f5a5cde5 --- /dev/null +++ b/packages/natives/src/highlight/index.ts @@ -0,0 +1,55 @@ +/** + * Syntax highlighting powered by WASM (syntect). + */ + +import * as wasm from "../../wasm/pi_natives"; + +type WasmHighlightExports = typeof wasm & { + highlight_code: (code: string, lang: string | null | undefined, colors: HighlightColors) => string; + supports_language: (lang: string) => boolean; + get_supported_languages: () => string[]; +}; + +const wasmHighlight = wasm as WasmHighlightExports; + +/** + * Theme colors for syntax highlighting. + * Each color should be an ANSI escape sequence (e.g., "\x1b[38;2;255;0;0m"). + */ +export interface HighlightColors { + comment: string; + keyword: string; + function: string; + variable: string; + string: string; + number: string; + type: string; + operator: string; + punctuation: string; +} + +/** + * Highlight code with syntax coloring. + * + * @param code - The source code to highlight + * @param lang - Optional language identifier (e.g., "rust", "typescript", "python") + * @param colors - Theme colors as ANSI escape sequences + * @returns Highlighted code as a single string with ANSI color codes + */ +export function highlightCode(code: string, lang: string | undefined, colors: HighlightColors): string { + return wasmHighlight.highlight_code(code, lang, colors); +} + +/** + * Check if a language is supported for highlighting. + */ +export function supportsLanguage(lang: string): boolean { + return wasmHighlight.supports_language(lang); +} + +/** + * Get list of all supported languages. + */ +export function getSupportedLanguages(): string[] { + return wasmHighlight.get_supported_languages(); +} diff --git a/packages/natives/src/index.ts b/packages/natives/src/index.ts index 07b3c9701..b533a668c 100644 --- a/packages/natives/src/index.ts +++ b/packages/natives/src/index.ts @@ -146,6 +146,33 @@ export { visibleWidth, } from "./text/index"; +// ============================================================================= +// Syntax highlighting +// ============================================================================= + +export { + getSupportedLanguages, + highlightCode, + type HighlightColors, + supportsLanguage, +} from "./highlight/index"; + +// ============================================================================= +// TUI layout engine +// ============================================================================= + +export { + type BoxNode, + type ColumnNode, + type LayoutNode, + type LayoutRequest, + type LayoutResult, + layoutTui, + type SpacerNode, + type TextNode, + type WrapMode, +} from "./tui/index"; + // ============================================================================= // Worker Pool (shared infrastructure) // ============================================================================= diff --git a/packages/natives/wasm/pi_natives.d.ts b/packages/natives/wasm/pi_natives.d.ts index 0f1c6f5f8..0ea6b8e5c 100644 --- a/packages/natives/wasm/pi_natives.d.ts +++ b/packages/natives/wasm/pi_natives.d.ts @@ -77,11 +77,35 @@ export enum SamplingFilter { */ export function extract_segments(line: string, before_end: number, after_start: number, after_len: number, strict_after: boolean): any; +/** + * Get list of supported languages. + */ +export function get_supported_languages(): string[]; + /** * Quick check if content matches a pattern. */ export function has_match(content: string, pattern: string, ignore_case: boolean, multiline: boolean): boolean; +/** + * Highlight code and return ANSI-colored lines. + * + * # Arguments + * * `code` - The source code to highlight + * * `lang` - Language identifier (e.g., "rust", "typescript", "python") + * * `colors` - Theme colors as ANSI escape sequences + * + * # Returns + * Highlighted code with ANSI color codes, or the original code if highlighting + * fails. + */ +export function highlight_code(code: string, lang: string | null | undefined, colors: any): string; + +/** + * Layout a component tree into terminal lines. + */ +export function layout_tui(request: any, width: number): any; + /** * Resize an image to the specified dimensions. */ @@ -98,6 +122,13 @@ export function search(content: string, options: any): any; */ export function slice_with_width(line: string, start_col: number, length: number, strict: boolean): any; +/** + * Check if a language is supported for highlighting. + * Returns true if the language has either direct support or a fallback + * mapping. + */ +export function supports_language(lang: string): boolean; + /** * Truncate text to a visible width, preserving ANSI codes. */ diff --git a/packages/natives/wasm/pi_natives.js b/packages/natives/wasm/pi_natives.js index 65632409a..ddbf6aae5 100644 --- a/packages/natives/wasm/pi_natives.js +++ b/packages/natives/wasm/pi_natives.js @@ -127,7 +127,7 @@ export class PhotonImage { throw takeObject(r2); } var v1 = getArrayU8FromWasm0(r0, r1).slice(); - wasm.__wbindgen_export3(r0, r1 * 1, 1); + wasm.__wbindgen_export4(r0, r1 * 1, 1); return v1; } finally { wasm.__wbindgen_add_to_stack_pointer(16); @@ -150,7 +150,7 @@ export class PhotonImage { throw takeObject(r2); } var v1 = getArrayU8FromWasm0(r0, r1).slice(); - wasm.__wbindgen_export3(r0, r1 * 1, 1); + wasm.__wbindgen_export4(r0, r1 * 1, 1); return v1; } finally { wasm.__wbindgen_add_to_stack_pointer(16); @@ -226,6 +226,24 @@ export function extract_segments(line, before_end, after_start, after_len, stric return takeObject(ret); } +/** + * Get list of supported languages. + * @returns {string[]} + */ +export function get_supported_languages() { + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + wasm.get_supported_languages(retptr); + var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); + var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); + var v1 = getArrayJsValueFromWasm0(r0, r1).slice(); + wasm.__wbindgen_export4(r0, r1 * 4, 4); + return v1; + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + } +} + /** * Quick check if content matches a pattern. * @param {string} content @@ -254,6 +272,54 @@ export function has_match(content, pattern, ignore_case, multiline) { } } +/** + * Highlight code and return ANSI-colored lines. + * + * # Arguments + * * `code` - The source code to highlight + * * `lang` - Language identifier (e.g., "rust", "typescript", "python") + * * `colors` - Theme colors as ANSI escape sequences + * + * # Returns + * Highlighted code with ANSI color codes, or the original code if highlighting + * fails. + * @param {string} code + * @param {string | null | undefined} lang + * @param {any} colors + * @returns {string} + */ +export function highlight_code(code, lang, colors) { + let deferred3_0; + let deferred3_1; + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_export, wasm.__wbindgen_export2); + const len0 = WASM_VECTOR_LEN; + var ptr1 = isLikeNone(lang) ? 0 : passStringToWasm0(lang, wasm.__wbindgen_export, wasm.__wbindgen_export2); + var len1 = WASM_VECTOR_LEN; + wasm.highlight_code(retptr, ptr0, len0, ptr1, len1, addHeapObject(colors)); + var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); + var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); + deferred3_0 = r0; + deferred3_1 = r1; + return getStringFromWasm0(r0, r1); + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + wasm.__wbindgen_export4(deferred3_0, deferred3_1, 1); + } +} + +/** + * Layout a component tree into terminal lines. + * @param {any} request + * @param {number} width + * @returns {any} + */ +export function layout_tui(request, width) { + const ret = wasm.layout_tui(addHeapObject(request), width); + return takeObject(ret); +} + /** * Resize an image to the specified dimensions. * @param {PhotonImage} image @@ -297,6 +363,20 @@ export function slice_with_width(line, start_col, length, strict) { return takeObject(ret); } +/** + * Check if a language is supported for highlighting. + * Returns true if the language has either direct support or a fallback + * mapping. + * @param {string} lang + * @returns {boolean} + */ +export function supports_language(lang) { + const ptr0 = passStringToWasm0(lang, wasm.__wbindgen_export, wasm.__wbindgen_export2); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.supports_language(ptr0, len0); + return ret !== 0; +} + /** * Truncate text to a visible width, preserving ANSI codes. * @param {string} text @@ -322,7 +402,7 @@ export function truncate_to_width(text, max_width, ellipsis, pad) { return getStringFromWasm0(r0, r1); } finally { wasm.__wbindgen_add_to_stack_pointer(16); - wasm.__wbindgen_export3(deferred3_0, deferred3_1, 1); + wasm.__wbindgen_export4(deferred3_0, deferred3_1, 1); } } @@ -382,6 +462,10 @@ function __wbg_get_imports() { const ret = typeof(getObject(arg0)) === 'bigint'; return ret; }, + __wbg___wbindgen_is_function_0095a73b8b156f76: function(arg0) { + const ret = typeof(getObject(arg0)) === 'function'; + return ret; + }, __wbg___wbindgen_is_object_5ae8e5880f2c1fbd: function(arg0) { const val = getObject(arg0); const ret = typeof(val) === 'object' && val !== null; @@ -420,6 +504,14 @@ function __wbg_get_imports() { __wbg___wbindgen_throw_be289d5034ed271b: function(arg0, arg1) { throw new Error(getStringFromWasm0(arg0, arg1)); }, + __wbg_call_389efe28435a9388: function() { return handleError(function (arg0, arg1) { + const ret = getObject(arg0).call(getObject(arg1)); + return addHeapObject(ret); + }, arguments); }, + __wbg_done_57b39ecd9addfe81: function(arg0) { + const ret = getObject(arg0).done; + return ret; + }, __wbg_entries_58c7934c745daac7: function(arg0) { const ret = Object.entries(getObject(arg0)); return addHeapObject(ret); @@ -432,13 +524,17 @@ function __wbg_get_imports() { deferred0_1 = arg1; console.error(getStringFromWasm0(arg0, arg1)); } finally { - wasm.__wbindgen_export3(deferred0_0, deferred0_1, 1); + wasm.__wbindgen_export4(deferred0_0, deferred0_1, 1); } }, __wbg_get_9b94d73e6221f75c: function(arg0, arg1) { const ret = getObject(arg0)[arg1 >>> 0]; return addHeapObject(ret); }, + __wbg_get_b3ed3ad4be2bc8ac: function() { return handleError(function (arg0, arg1) { + const ret = Reflect.get(getObject(arg0), getObject(arg1)); + return addHeapObject(ret); + }, arguments); }, __wbg_get_with_ref_key_1dc361bd10053bfe: function(arg0, arg1) { const ret = getObject(arg0)[getObject(arg1)]; return addHeapObject(ret); @@ -453,6 +549,16 @@ function __wbg_get_imports() { const ret = result; return ret; }, + __wbg_instanceof_Map_53af74335dec57f4: function(arg0) { + let result; + try { + result = getObject(arg0) instanceof Map; + } catch (_) { + result = false; + } + const ret = result; + return ret; + }, __wbg_instanceof_Uint8Array_9b9075935c74707c: function(arg0) { let result; try { @@ -463,10 +569,18 @@ function __wbg_get_imports() { const ret = result; return ret; }, + __wbg_isArray_d314bb98fcf08331: function(arg0) { + const ret = Array.isArray(getObject(arg0)); + return ret; + }, __wbg_isSafeInteger_bfbc7332a9768d2a: function(arg0) { const ret = Number.isSafeInteger(getObject(arg0)); return ret; }, + __wbg_iterator_6ff6560ca1568e55: function() { + const ret = Symbol.iterator; + return addHeapObject(ret); + }, __wbg_length_32ed9a279acd054c: function(arg0) { const ret = getObject(arg0).length; return ret; @@ -491,6 +605,14 @@ function __wbg_get_imports() { const ret = new Uint8Array(getObject(arg0)); return addHeapObject(ret); }, + __wbg_next_3482f54c49e8af19: function() { return handleError(function (arg0) { + const ret = getObject(arg0).next(); + return addHeapObject(ret); + }, arguments); }, + __wbg_next_418f80d8f5303233: function(arg0) { + const ret = getObject(arg0).next; + return addHeapObject(ret); + }, __wbg_prototypesetcall_bdcdcc5842e4d77d: function(arg0, arg1, arg2) { Uint8Array.prototype.set.call(getArrayU8FromWasm0(arg0, arg1), getObject(arg2)); }, @@ -507,17 +629,26 @@ function __wbg_get_imports() { getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true); getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true); }, + __wbg_value_0546255b415e96c1: function(arg0) { + const ret = getObject(arg0).value; + return addHeapObject(ret); + }, __wbindgen_cast_0000000000000001: function(arg0) { // Cast intrinsic for `F64 -> Externref`. const ret = arg0; return addHeapObject(ret); }, - __wbindgen_cast_0000000000000002: function(arg0, arg1) { + __wbindgen_cast_0000000000000002: function(arg0) { + // Cast intrinsic for `I64 -> Externref`. + const ret = arg0; + return addHeapObject(ret); + }, + __wbindgen_cast_0000000000000003: function(arg0, arg1) { // Cast intrinsic for `Ref(String) -> Externref`. const ret = getStringFromWasm0(arg0, arg1); return addHeapObject(ret); }, - __wbindgen_cast_0000000000000003: function(arg0) { + __wbindgen_cast_0000000000000004: function(arg0) { // Cast intrinsic for `U64 -> Externref`. const ret = BigInt.asUintN(64, arg0); return addHeapObject(ret); @@ -629,6 +760,16 @@ function dropObject(idx) { heap_next = idx; } +function getArrayJsValueFromWasm0(ptr, len) { + ptr = ptr >>> 0; + const mem = getDataViewMemory0(); + const result = []; + for (let i = ptr; i < ptr + 4 * len; i += 4) { + result.push(takeObject(mem.getUint32(i, true))); + } + return result; +} + function getArrayU8FromWasm0(ptr, len) { ptr = ptr >>> 0; return getUint8ArrayMemory0().subarray(ptr / 1, ptr / 1 + len); @@ -657,6 +798,14 @@ function getUint8ArrayMemory0() { function getObject(idx) { return heap[idx]; } +function handleError(f, args) { + try { + return f.apply(this, args); + } catch (e) { + wasm.__wbindgen_export3(addHeapObject(e)); + } +} + let heap = new Array(128).fill(undefined); heap.push(undefined, null, true, false); diff --git a/packages/natives/wasm/pi_natives_bg.wasm b/packages/natives/wasm/pi_natives_bg.wasm index c5e1ed345..4d576322d 100644 --- a/packages/natives/wasm/pi_natives_bg.wasm +++ b/packages/natives/wasm/pi_natives_bg.wasm @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6d40e6ace58364b5f198e84c9fcb8d28d93e756fc66c67f755324e3984c4b069 -size 1950251 +oid sha256:f834e9594d8367d45a5d870ba995e81c0603de0482f7eda8f2782c9d548e9ba4 +size 2596678 diff --git a/packages/natives/wasm/pi_natives_bg.wasm.d.ts b/packages/natives/wasm/pi_natives_bg.wasm.d.ts index 0f6333b73..5e317ba57 100644 --- a/packages/natives/wasm/pi_natives_bg.wasm.d.ts +++ b/packages/natives/wasm/pi_natives_bg.wasm.d.ts @@ -7,7 +7,10 @@ export const compiledpattern_has_match: (a: number, b: number, c: number) => num export const compiledpattern_new: (a: number, b: number) => void; export const compiledpattern_search: (a: number, b: number, c: number, d: number, e: number) => number; export const extract_segments: (a: number, b: number, c: number, d: number, e: number, f: number) => number; +export const get_supported_languages: (a: number) => void; export const has_match: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => void; +export const highlight_code: (a: number, b: number, c: number, d: number, e: number, f: number) => void; +export const layout_tui: (a: number, b: number) => number; export const photonimage_get_bytes: (a: number, b: number) => void; export const photonimage_get_bytes_jpeg: (a: number, b: number, c: number) => void; export const photonimage_get_height: (a: number) => number; @@ -16,11 +19,13 @@ export const photonimage_new_from_byteslice: (a: number, b: number, c: number) = export const resize: (a: number, b: number, c: number, d: number) => number; export const search: (a: number, b: number, c: number) => number; export const slice_with_width: (a: number, b: number, c: number, d: number, e: number) => number; +export const supports_language: (a: number, b: number) => number; export const truncate_to_width: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => void; export const visible_width: (a: number, b: number) => number; export const compiledpattern_has_match_bytes: (a: number, b: number, c: number) => number; export const compiledpattern_search_bytes: (a: number, b: number, c: number, d: number, e: number) => number; export const __wbindgen_export: (a: number, b: number) => number; export const __wbindgen_export2: (a: number, b: number, c: number, d: number) => number; -export const __wbindgen_export3: (a: number, b: number, c: number) => void; +export const __wbindgen_export3: (a: number) => void; +export const __wbindgen_export4: (a: number, b: number, c: number) => void; export const __wbindgen_add_to_stack_pointer: (a: number) => number;