feat(pi-natives/tools): implemented utok tokenizer for multiple models
- Replaced the `ctok` implementation with the `utok` universal tokenizer supporting multiple model families and UTF text encodings. - Added tokenizer support and embedding data for Qwen3, DeepSeek V3, Kimi K2, and GLM-5 model variants. - Added fixture generation scripts, vocabulary packers, and golden test suites for validating tokenization parity. - Updated dependency requirements and Bazel workspace definitions for new crates and tools.
This commit is contained in:
@@ -121,6 +121,17 @@ register_toolchains(
|
||||
"@zig_sdk//libc_aware/toolchain:linux_arm64_musl",
|
||||
)
|
||||
|
||||
# --- Host tools (crate_universe splicing / lockfile generation) ----------------
|
||||
# Must be nightly: cargo's rustc target probe inherits user-level RUSTFLAGS
|
||||
# (~/.cargo/config.toml commonly carries -Z flags on this repo's nightly
|
||||
# toolchain), and a stable host rustc rejects them. Match rust.toolchain.
|
||||
rust_host_tools = use_extension("@rules_rust//rust:extensions.bzl", "rust_host_tools")
|
||||
rust_host_tools.host_tools(
|
||||
name = "rust_host_tools_nightly",
|
||||
version = "nightly/2026-04-29",
|
||||
)
|
||||
use_repo(rust_host_tools, "rust_host_tools_nightly")
|
||||
|
||||
# --- Third-party crates (crate_universe over the cargo workspace) --------------
|
||||
crate = use_extension("@rules_rust//crate_universe:extensions.bzl", "crate")
|
||||
crate.render_config(
|
||||
@@ -131,6 +142,9 @@ crate.render_config(
|
||||
crate.from_cargo(
|
||||
name = "crates",
|
||||
cargo_lockfile = "//:Cargo.lock",
|
||||
# Nightly host tools (see the rust_host_tools extension above): the
|
||||
# splice's rustc probe must accept user-level -Z RUSTFLAGS.
|
||||
host_tools = "@rust_host_tools_nightly",
|
||||
# Exactly the shipped addon triples: crate BUILD files get
|
||||
# target_compatible_with selects over this set (defaults omit darwin-x64
|
||||
# and musl), and features/deps resolve per-triple from Cargo.lock.
|
||||
|
||||
Reference in New Issue
Block a user