feat(pi-natives/tools): implemented utok tokenizer for multiple models

- Replaced the `ctok` implementation with the `utok` universal tokenizer supporting multiple model families and UTF text encodings.
- Added tokenizer support and embedding data for Qwen3, DeepSeek V3, Kimi K2, and GLM-5 model variants.
- Added fixture generation scripts, vocabulary packers, and golden test suites for validating tokenization parity.
- Updated dependency requirements and Bazel workspace definitions for new crates and tools.
This commit is contained in:
can1357
2026-08-20 00:37:05 +02:00
parent 37fd2dbbe1
commit 0cdd37fc15
101 changed files with 746591 additions and 278 deletions
+14
View File
@@ -121,6 +121,17 @@ register_toolchains(
"@zig_sdk//libc_aware/toolchain:linux_arm64_musl",
)
# --- Host tools (crate_universe splicing / lockfile generation) ----------------
# Must be nightly: cargo's rustc target probe inherits user-level RUSTFLAGS
# (~/.cargo/config.toml commonly carries -Z flags on this repo's nightly
# toolchain), and a stable host rustc rejects them. Match rust.toolchain.
rust_host_tools = use_extension("@rules_rust//rust:extensions.bzl", "rust_host_tools")
rust_host_tools.host_tools(
name = "rust_host_tools_nightly",
version = "nightly/2026-04-29",
)
use_repo(rust_host_tools, "rust_host_tools_nightly")
# --- Third-party crates (crate_universe over the cargo workspace) --------------
crate = use_extension("@rules_rust//crate_universe:extensions.bzl", "crate")
crate.render_config(
@@ -131,6 +142,9 @@ crate.render_config(
crate.from_cargo(
name = "crates",
cargo_lockfile = "//:Cargo.lock",
# Nightly host tools (see the rust_host_tools extension above): the
# splice's rustc probe must accept user-level -Z RUSTFLAGS.
host_tools = "@rust_host_tools_nightly",
# Exactly the shipped addon triples: crate BUILD files get
# target_compatible_with selects over this set (defaults omit darwin-x64
# and musl), and features/deps resolve per-triple from Cargo.lock.