From eef22705ecb911627380c8b3025983b920105524 Mon Sep 17 00:00:00 2001 From: Trevor Smith Date: Mon, 20 Apr 2026 13:00:09 -0500 Subject: [PATCH 1/2] fix: resolve tree-sitter-glimmer arm64 undefined symbol error The published tree-sitter-glimmer v0.0.1 crate has its external scanner compilation commented out in build.rs. The vendored workaround compiled scanner.c into a separate static library, which causes a linker symbol resolution failure on arm64: undefined symbol tree_sitter_glimmer_external_scanner_create. Replace the vendored workaround with a [patch.crates-io] pointing to upstream v1.6.0 (which compiles scanner.c correctly in its own build.rs) and remove the vendor directory and cc build dependency. Fixes #651, fixes #727 --- Cargo.lock | 4 +- Cargo.toml | 1 + crates/pi-natives/Cargo.toml | 1 - crates/pi-natives/build.rs | 21 -- .../vendor/tree-sitter-glimmer/parser.h | 219 ------------------ .../vendor/tree-sitter-glimmer/scanner.c | 152 ------------ 6 files changed, 2 insertions(+), 396 deletions(-) delete mode 100644 crates/pi-natives/vendor/tree-sitter-glimmer/parser.h delete mode 100644 crates/pi-natives/vendor/tree-sitter-glimmer/scanner.c diff --git a/Cargo.lock b/Cargo.lock index df1ca437f..aab6b017f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2180,7 +2180,6 @@ dependencies = [ "ast-grep-core", "brush-builtins", "brush-core", - "cc", "clap", "dashmap", "globset", @@ -3251,8 +3250,7 @@ dependencies = [ [[package]] name = "tree-sitter-glimmer" version = "0.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "295346943475457f55688342f14675e2a54ba3e8f3d224f11d10477d99e70330" +source = "git+https://github.com/ember-tooling/tree-sitter-glimmer?tag=v1.6.0-tree-sitter-glimmer#88af85568bde3b91acb5d4c352ed094d0c1f9d84" dependencies = [ "cc", "tree-sitter", diff --git a/Cargo.toml b/Cargo.toml index c67cf6b26..402eec9c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ repository = "https://github.com/can1357/oh-my-pi" [patch.crates-io] brush-core = { path = "crates/brush-core-vendored" } brush-builtins = { path = "crates/brush-builtins-vendored" } +tree-sitter-glimmer = { git = "https://github.com/ember-tooling/tree-sitter-glimmer", tag = "v1.6.0-tree-sitter-glimmer" } [profile.release] opt-level = 3 lto = "fat" diff --git a/crates/pi-natives/Cargo.toml b/crates/pi-natives/Cargo.toml index da650e457..fc6aa8033 100644 --- a/crates/pi-natives/Cargo.toml +++ b/crates/pi-natives/Cargo.toml @@ -137,6 +137,5 @@ windows-sys = { version = "0.61", features = [ ] } [build-dependencies] napi-build = "2" -cc = "1" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/crates/pi-natives/build.rs b/crates/pi-natives/build.rs index 8da7f0915..13ee63bf0 100644 --- a/crates/pi-natives/build.rs +++ b/crates/pi-natives/build.rs @@ -356,27 +356,6 @@ const GRAMMARS: &[GrammarSpec] = &[ fn main() { napi_build::setup(); generate_chunk_schema(); - - let scanner_dir = Path::new("vendor/tree-sitter-glimmer"); - let scanner_path = scanner_dir.join("scanner.c"); - let parser_header_path = scanner_dir.join("parser.h"); - - println!("cargo:rerun-if-changed={}", scanner_path.display()); - println!("cargo:rerun-if-changed={}", parser_header_path.display()); - - let mut build = cc::Build::new(); - build - .std("c11") - .include(scanner_dir) - .file(&scanner_path) - // Vendored code: suppress warnings (including the ar -D probe noise on - // macOS where Apple's ar rejects the deterministic flag). - .cargo_warnings(false); - - #[cfg(target_env = "msvc")] - build.flag("-utf-8"); - - build.compile("tree-sitter-glimmer-scanner"); } fn generate_chunk_schema() { diff --git a/crates/pi-natives/vendor/tree-sitter-glimmer/parser.h b/crates/pi-natives/vendor/tree-sitter-glimmer/parser.h deleted file mode 100644 index 82c2940e8..000000000 --- a/crates/pi-natives/vendor/tree-sitter-glimmer/parser.h +++ /dev/null @@ -1,219 +0,0 @@ -#ifndef TREE_SITTER_PARSER_H_ -#define TREE_SITTER_PARSER_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -#define ts_builtin_sym_error ((TSSymbol)-1) -#define ts_builtin_sym_end 0 -#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024 - -#ifndef TREE_SITTER_API_H_ -typedef uint16_t TSStateId; -typedef uint16_t TSSymbol; -typedef uint16_t TSFieldId; -typedef struct TSLanguage TSLanguage; -#endif - -typedef struct { - TSFieldId field_id; - uint8_t child_index; - bool inherited; -} TSFieldMapEntry; - -typedef struct { - uint16_t index; - uint16_t length; -} TSFieldMapSlice; - -typedef struct { - bool visible; - bool named; - bool supertype; -} TSSymbolMetadata; - -typedef struct TSLexer TSLexer; - -struct TSLexer { - int32_t lookahead; - TSSymbol result_symbol; - void (*advance)(TSLexer *, bool); - void (*mark_end)(TSLexer *); - uint32_t (*get_column)(TSLexer *); - bool (*is_at_included_range_start)(const TSLexer *); - bool (*eof)(const TSLexer *); -}; - -typedef enum { - TSParseActionTypeShift, - TSParseActionTypeReduce, - TSParseActionTypeAccept, - TSParseActionTypeRecover, -} TSParseActionType; - -typedef union { - struct { - uint8_t type; - TSStateId state; - bool extra; - bool repetition; - } shift; - struct { - uint8_t type; - uint8_t child_count; - TSSymbol symbol; - int16_t dynamic_precedence; - uint16_t production_id; - } reduce; - uint8_t type; -} TSParseAction; - -typedef struct { - uint16_t lex_state; - uint16_t external_lex_state; -} TSLexMode; - -typedef union { - TSParseAction action; - struct { - uint8_t count; - bool reusable; - } entry; -} TSParseActionEntry; - -typedef struct { - int32_t start; - int32_t end; -} TSCharacterRange; - -struct TSLanguage { - uint32_t version; - uint32_t symbol_count; - uint32_t alias_count; - uint32_t token_count; - uint32_t external_token_count; - uint32_t state_count; - uint32_t large_state_count; - uint32_t production_id_count; - uint32_t field_count; - uint16_t max_alias_sequence_length; - const uint16_t *parse_table; - const uint16_t *small_parse_table; - const uint32_t *small_parse_table_map; - const TSParseActionEntry *parse_actions; - const char * const *symbol_names; - const char * const *field_names; - const TSFieldMapSlice *field_map_slices; - const TSFieldMapEntry *field_map_entries; - const TSSymbolMetadata *symbol_metadata; - const TSSymbol *public_symbol_map; - const uint16_t *alias_map; - const TSSymbol *alias_sequences; - const TSLexMode *lex_modes; - bool (*lex_fn)(TSLexer *, TSStateId); - bool (*keyword_lex_fn)(TSLexer *, TSStateId); - TSSymbol keyword_capture_token; - struct { - const bool *states; - const TSSymbol *symbol_map; - void *(*create)(void); - void (*destroy)(void *); - bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist); - unsigned (*serialize)(void *, char *); - void (*deserialize)(void *, const char *, unsigned); - } external_scanner; - const TSStateId *primary_state_ids; -}; - -static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t lookahead) { - uint32_t index = 0; - uint32_t size = len - index; - while (size > 1) { - uint32_t half_size = size / 2; - uint32_t mid_index = index + half_size; - TSCharacterRange *range = &ranges[mid_index]; - if (lookahead >= range->start && lookahead <= range->end) { - return true; - } else if (lookahead > range->end) { - index = mid_index; - } - size -= half_size; - } - TSCharacterRange *range = &ranges[index]; - return (lookahead >= range->start && lookahead <= range->end); -} - -#ifdef _MSC_VER -#define UNUSED __pragma(warning(suppress : 4101)) -#else -#define UNUSED __attribute__((unused)) -#endif - -#define START_LEXER() \ - bool result = false; \ - bool skip = false; \ - UNUSED \ - bool eof = false; \ - int32_t lookahead; \ - goto start; \ - next_state: \ - lexer->advance(lexer, skip); \ - start: \ - skip = false; \ - lookahead = lexer->lookahead; - -#define ADVANCE(state_value) \ - { \ - state = state_value; \ - goto next_state; \ - } - -#define ADVANCE_MAP(...) \ - { \ - static const uint16_t map[] = { __VA_ARGS__ }; \ - for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) { \ - if (map[i] == lookahead) { \ - state = map[i + 1]; \ - goto next_state; \ - } \ - } \ - } - -#define SKIP(state_value) \ - { \ - skip = true; \ - state = state_value; \ - goto next_state; \ - } - -#define ACCEPT_TOKEN(symbol_value) \ - result = true; \ - lexer->result_symbol = symbol_value; \ - lexer->mark_end(lexer); - -#define END_STATE() return result; - -#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT) - -#define STATE(id) id - -#define ACTIONS(id) id - -#define SHIFT(state_value) \ - {{ \ - .shift = { \ - .type = TSParseActionTypeShift, \ - .state = (state_value) \ - } \ - }} - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/crates/pi-natives/vendor/tree-sitter-glimmer/scanner.c b/crates/pi-natives/vendor/tree-sitter-glimmer/scanner.c deleted file mode 100644 index 872551ac8..000000000 --- a/crates/pi-natives/vendor/tree-sitter-glimmer/scanner.c +++ /dev/null @@ -1,152 +0,0 @@ -// Vendored from tree-sitter-glimmer 0.0.1 because the published crate does -// not compile its external scanner in bindings/rust/build.rs. -#include "parser.h" -#include - -enum TokenType { - COMMENT -}; - -void *tree_sitter_glimmer_external_scanner_create() { return NULL; } -void tree_sitter_glimmer_external_scanner_destroy(void *payload) { (void)payload; } -void tree_sitter_glimmer_external_scanner_reset(void *payload) { (void)payload; } -unsigned tree_sitter_glimmer_external_scanner_serialize(void *payload, char *buffer) { - (void)payload; - (void)buffer; - return 0; -} -void tree_sitter_glimmer_external_scanner_deserialize(void *payload, const char *buffer, unsigned length) { - (void)payload; - (void)buffer; - (void)length; -} - -static void advance(TSLexer *lexer) { lexer->advance(lexer, false); } - -static bool scan_html_comment(TSLexer *lexer) { - if (lexer->lookahead != '!') return false; - advance(lexer); - if (lexer->lookahead != '-') return false; - advance(lexer); - if (lexer->lookahead != '-') return false; - advance(lexer); - - unsigned dashes = 0; - while (lexer->lookahead) { - switch (lexer->lookahead) { - case '-': - ++dashes; - break; - case '>': - if (dashes >= 2) { - lexer->result_symbol = COMMENT; - advance(lexer); - lexer->mark_end(lexer); - return true; - } - default: - dashes = 0; - } - advance(lexer); - } - - return false; -} - -static bool scan_multi_line_handlebars_comment(TSLexer *lexer) { - if (lexer->lookahead != '-') return false; - - unsigned dashes = 0; - unsigned brackets = 0; - while (lexer->lookahead) { - switch (lexer->lookahead) { - case '-': - ++dashes; - break; - case '}': - ++brackets; - if (dashes >= 2 && brackets == 2) { - lexer->result_symbol = COMMENT; - advance(lexer); - lexer->mark_end(lexer); - return true; - } else { - break; - } - default: - dashes = 0; - brackets = 0; - } - advance(lexer); - } - - return false; -} - -static bool scan_single_line_handlebars_comment(TSLexer *lexer) { - unsigned brackets = 0; - while (lexer->lookahead) { - switch (lexer->lookahead) { - case '}': - ++brackets; - if (brackets == 2) { - lexer->result_symbol = COMMENT; - advance(lexer); - lexer->mark_end(lexer); - return true; - } else { - break; - } - default: - brackets = 0; - } - advance(lexer); - } - - return false; -} - -static bool scan_handlebars_comment(TSLexer *lexer) { - if (lexer->lookahead != '{') return false; - advance(lexer); - if (lexer->lookahead != '!') return false; - advance(lexer); - - switch (lexer->lookahead) { - case '-': - advance(lexer); - return scan_multi_line_handlebars_comment(lexer); - default: - advance(lexer); - return scan_single_line_handlebars_comment(lexer); - } -} - -bool tree_sitter_glimmer_external_scanner_scan( - void *payload, - TSLexer *lexer, - const bool *valid_symbols -) { - (void)payload; - - while (iswspace(lexer->lookahead)) { - lexer->advance(lexer, true); - } - - if (valid_symbols[COMMENT]) { - switch (lexer->lookahead) { - case '<': - lexer->mark_end(lexer); - advance(lexer); - return scan_html_comment(lexer); - case '{': - lexer->mark_end(lexer); - advance(lexer); - return scan_handlebars_comment(lexer); - default: - return false; - } - } - - return false; -} From c96f15b6dac34d980b22bbe7d4803bf52ba5b716 Mon Sep 17 00:00:00 2001 From: can1357 Date: Thu, 23 Apr 2026 22:55:42 +0200 Subject: [PATCH 2/2] fix: preserve glimmer schema generation --- crates/pi-natives/build.rs | 174 ++++++++++++++++++++++++-- crates/pi-natives/src/chunk/schema.rs | 2 +- 2 files changed, 163 insertions(+), 13 deletions(-) diff --git a/crates/pi-natives/build.rs b/crates/pi-natives/build.rs index 13ee63bf0..3ed8d4aed 100644 --- a/crates/pi-natives/build.rs +++ b/crates/pi-natives/build.rs @@ -31,6 +31,11 @@ struct GrammarSpec { node_types_rel: &'static str, } +struct LockedPackage { + version: String, + source: Option, +} + #[derive(Deserialize)] struct RawTypeRef { #[serde(rename = "type")] @@ -366,15 +371,17 @@ fn generate_chunk_schema() { .expect("pi-natives should live under the workspace root"); let out_dir = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR should be set")); let output_path = out_dir.join("chunk_schema.json"); - let locked_versions = locked_package_versions(&workspace_root.join("Cargo.lock")); + let locked_packages = locked_packages(&workspace_root.join("Cargo.lock")); let registry_roots = cargo_registry_roots(); + let git_roots = cargo_git_checkout_roots(); let mut languages = BTreeMap::new(); for grammar in GRAMMARS { - let Some(version) = locked_versions.get(grammar.package) else { + let Some(locked) = locked_packages.get(grammar.package) else { continue; }; - let Some(package_dir) = find_registry_package_dir(®istry_roots, grammar.package, version) + let Some(package_dir) = + find_locked_package_dir(®istry_roots, &git_roots, grammar.package, locked) else { continue; }; @@ -632,6 +639,36 @@ fn cargo_registry_roots() -> Vec { roots } +fn cargo_git_checkout_roots() -> Vec { + let mut roots = Vec::new(); + if let Some(cargo_home) = env::var_os("CARGO_HOME") { + roots.push(PathBuf::from(cargo_home).join("git").join("checkouts")); + } + if let Some(home) = env::var_os("HOME") { + roots.push( + PathBuf::from(home) + .join(".cargo") + .join("git") + .join("checkouts"), + ); + } + roots +} + +fn find_locked_package_dir( + registry_roots: &[PathBuf], + git_roots: &[PathBuf], + package: &str, + locked: &LockedPackage, +) -> Option { + match locked.source.as_deref() { + Some(source) if source.starts_with("git+") => { + find_git_package_dir(git_roots, package, &locked.version, git_revision(source)) + }, + _ => find_registry_package_dir(registry_roots, package, &locked.version), + } +} + fn find_registry_package_dir( registry_roots: &[PathBuf], package: &str, @@ -651,32 +688,145 @@ fn find_registry_package_dir( None } -fn locked_package_versions(lock_path: &Path) -> HashMap { +fn find_git_package_dir( + git_roots: &[PathBuf], + package: &str, + version: &str, + revision: Option<&str>, +) -> Option { + for git_root in git_roots { + let Ok(checkout_dirs) = fs::read_dir(git_root) else { + continue; + }; + for checkout_dir in checkout_dirs.flatten() { + let Ok(revision_dirs) = fs::read_dir(checkout_dir.path()) else { + continue; + }; + for revision_dir in revision_dirs.flatten() { + let revision_path = revision_dir.path(); + let Some(revision_name) = revision_path.file_name().and_then(|name| name.to_str()) + else { + continue; + }; + if !revision_matches(revision_name, revision) { + continue; + } + if let Some(package_dir) = find_manifest_package_dir(&revision_path, package, version) { + return Some(package_dir); + } + } + } + } + None +} + +fn revision_matches(revision_name: &str, revision: Option<&str>) -> bool { + revision.is_none_or(|revision| { + revision.starts_with(revision_name) || revision_name.starts_with(revision) + }) +} + +fn find_manifest_package_dir(root: &Path, package: &str, version: &str) -> Option { + if manifest_matches_package(&root.join("Cargo.toml"), package, version) { + return Some(root.to_path_buf()); + } + + let Ok(entries) = fs::read_dir(root) else { + return None; + }; + for entry in entries.flatten() { + let candidate = entry.path(); + if candidate.is_dir() + && manifest_matches_package(&candidate.join("Cargo.toml"), package, version) + { + return Some(candidate); + } + } + None +} + +fn manifest_matches_package(manifest_path: &Path, package: &str, version: &str) -> bool { + let Ok(source) = fs::read_to_string(manifest_path) else { + return false; + }; + let mut in_package = false; + let mut name_matches = false; + let mut version_matches = false; + + for line in source.lines() { + let trimmed = line.trim(); + if trimmed.starts_with('[') { + in_package = trimmed == "[package]"; + continue; + } + if !in_package { + continue; + } + if let Some(value) = toml_string_value(trimmed, "name") { + name_matches = value == package; + continue; + } + if let Some(value) = toml_string_value(trimmed, "version") { + version_matches = value == version; + } + } + + name_matches && version_matches +} + +fn git_revision(source: &str) -> Option<&str> { + source.rsplit_once('#').and_then(|(_, revision)| { + if revision.is_empty() { + None + } else { + Some(revision) + } + }) +} + +fn locked_packages(lock_path: &Path) -> HashMap { let source = fs::read_to_string(lock_path).expect("Cargo.lock should be readable"); - let mut versions = HashMap::new(); + let mut packages = HashMap::new(); let mut current_name = None; let mut current_version = None; + let mut current_source = None; for line in source.lines() { let trimmed = line.trim(); if trimmed == "[[package]]" { if let (Some(name), Some(version)) = (current_name.take(), current_version.take()) { - versions.insert(name, version); + packages.insert(name, LockedPackage { version, source: current_source.take() }); } + current_source = None; continue; } - if let Some(value) = trimmed.strip_prefix("name = \"") { - current_name = value.strip_suffix('"').map(ToOwned::to_owned); + if let Some(value) = toml_string_value(trimmed, "name") { + current_name = Some(value.to_string()); continue; } - if let Some(value) = trimmed.strip_prefix("version = \"") { - current_version = value.strip_suffix('"').map(ToOwned::to_owned); + if let Some(value) = toml_string_value(trimmed, "version") { + current_version = Some(value.to_string()); + continue; + } + if let Some(value) = toml_string_value(trimmed, "source") { + current_source = Some(value.to_string()); } } if let (Some(name), Some(version)) = (current_name, current_version) { - versions.insert(name, version); + packages.insert(name, LockedPackage { version, source: current_source }); } - versions + packages +} + +fn toml_string_value<'a>(line: &'a str, key: &str) -> Option<&'a str> { + let value = line + .strip_prefix(key)? + .trim_start() + .strip_prefix('=')? + .trim_start() + .strip_prefix('"')?; + let end = value.find('"')?; + Some(&value[..end]) } diff --git a/crates/pi-natives/src/chunk/schema.rs b/crates/pi-natives/src/chunk/schema.rs index ae0126ecd..65f609ef6 100644 --- a/crates/pi-natives/src/chunk/schema.rs +++ b/crates/pi-natives/src/chunk/schema.rs @@ -123,7 +123,7 @@ mod tests { #[test] fn generated_schema_covers_expected_languages() { - for language in ["python", "nix", "toml", "typescript", "rust", "yaml"] { + for language in ["python", "nix", "toml", "typescript", "rust", "yaml", "handlebars"] { assert!(has_schema(language), "{language} should have generated schema data"); } }