smashimo/oh-my-pi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
0cdd37fc15a9016ba8726ffc215d25856a8cd744
oh-my-pi/crates/pi-natives/fixtures/kimi_k2.json
T
can1357 0cdd37fc15 feat(pi-natives/tools): implemented utok tokenizer for multiple models 
- Replaced the `ctok` implementation with the `utok` universal tokenizer supporting multiple model families and UTF text encodings.
- Added tokenizer support and embedding data for Qwen3, DeepSeek V3, Kimi K2, and GLM-5 model variants.
- Added fixture generation scripts, vocabulary packers, and golden test suites for validating tokenization parity.
- Updated dependency requirements and Bazel workspace definitions for new crates and tools.
2026-08-20 01:45:04 +02:00

891 lines
11 KiB
JSON
Raw Blame History

{
"generator": "tiktoken 0.14.0 Encoding(kimi, tokenization_kimi.py pat_str) encode_ordinary",
"cases": [
{
"text": "",
"ids": [],
"count": 0
},
{
"text": " ",
"ids": [
220
],
"count": 1
},
{
"text": " \t\t\n\n\r\n ",
"ids": [
99000,
382,
462,
274
],
"count": 4
},
{
"text": "hello world",
"ids": [
22931,
2695
],
"count": 2
},
{
"text": "Hello, World! It's a test. We'll see; don't worry, y'all've been warned.",
"ids": [
19180,
11,
6949,
0,
8629,
261,
1812,
13,
40251,
2050,
26,
4536,
13980,
11,
364,
120581,
7962,
1479,
43495,
13
],
"count": 20
},
{
"text": "The quick brown fox jumps over the lazy dog. 1234567890 12 345 6789",
"ids": [
1008,
5072,
16331,
69275,
60062,
1312,
276,
29292,
7751,
13,
220,
6694,
12972,
16242,
15,
220,
1042,
220,
18439,
220,
22523,
24
],
"count": 22
},
{
"text": "fn main() {\n let x: Vec<u32> = (0..10).map(|i| i * 2).collect();\n println!(\"{x:?}\");\n}",
"ids": [
10964,
2777,
539,
440,
274,
2391,
1288,
25,
20431,
52794,
1202,
29,
327,
347,
15,
690,
795,
1083,
3719,
32322,
72,
91,
1032,
397,
220,
17,
1083,
28994,
1094,
274,
47647,
28547,
90,
87,
93194,
47724,
92
],
"count": 37
},
{
"text": "{\"key\": [1, 2.5, -3e8], \"nested\": {\"a\": null, \"b\": true}}",
"ids": [
8264,
1319,
1289,
793,
16,
11,
220,
17,
13,
20,
11,
635,
18,
68,
23,
2287,
414,
82046,
1289,
9392,
64,
1289,
1704,
11,
414,
65,
1289,
1653,
5375
],
"count": 29
},
{
"text": "东京は日本の首都であり、世界で最も人口の多い都市圏の一つです。深度求索发布了新一代基座模型。",
"ids": [
26268,
8831,
4546,
4663,
26476,
126596,
343,
2243,
8846,
742,
20624,
9530,
4663,
561,
9102,
16807,
321,
237,
4663,
331,
30090,
54319,
292,
13165,
124414,
27613,
32136,
1320,
2989,
15662,
292
],
"count": 31
},
{
"text": "한국어 텍스트도 테스트합니다. 안녕하세요!",
"ids": [
17079,
54078,
28763,
83194,
235,
67178,
31474,
160618,
71146,
13,
102589,
34910,
243,
13181,
102898,
0
],
"count": 16
},
{
"text": "Многоязычный текст: русский, ελληνικά, עברית, العربية.",
"ids": [
24181,
23143,
2716,
51193,
3990,
26646,
148468,
25,
139598,
32890,
11,
22121,
61500,
14287,
8711,
125776,
11,
33480,
51589,
34403,
11,
120413,
12441,
13
],
"count": 24
},
{
"text": "naïve café résumé — em-dash…ellipsis",
"ids": [
4768,
44187,
367,
70591,
23606,
7671,
1941,
4275,
1516,
150743,
4635,
151078
],
"count": 12
},
{
"text": "naïve café",
"ids": [
59911,
136,
230,
367,
51158,
37484
],
"count": 6
},
{
"text": "👍🏽 emoji test 👨‍👩‍👧‍👦 family, flags 🇹🇷🇯🇵, math 𝕏≈∑∫",
"ids": [
64390,
235,
92949,
121,
68369,
1812,
130732,
101,
67963,
64390,
102,
67963,
64390,
100,
67963,
64390,
99,
3545,
11,
10141,
67496,
117,
32166,
115,
32166,
107,
32166,
113,
11,
15601,
161507,
237,
23149,
230,
12273,
239,
12273,
104
],
"count": 38
},
{
"text": "https://example.com/path?query=value&other=%20escaped#fragment",
"ids": [
1927,
1316,
17480,
1304,
73505,
30,
4964,
100520,
5,
2348,
9458,
577,
91967,
2,
46495
],
"count": 15
},
{
"text": "supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious",
"ids": [
18785,
12440,
361,
24772,
334,
6340,
8549,
682,
338,
162249,
3945,
12440,
361,
24772,
334,
6340,
8549,
682,
338,
162249,
3945,
12440,
361,
24772,
334,
6340,
8549,
682,
338,
162249,
3945,
12440,
361,
24772,
334,
6340,
8549,
682,
338,
162249,
3945,
12440,
361,
24772,
334,
6340,
8549,
682,
338,
162249,
3945,
12440,
361,
24772,
334,
6340,
8549,
682,
338,
162249,
3945,
12440,
361,
24772,
334,
6340,
8549,
682,
338,
162249,
3945,
12440,
361,
24772,
334,
6340,
8549,
682,
338,
162249,
3945,
12440,
361,
24772,
334,
6340,
8549,
682,
338,
162249,
3945,
12440,
361,
24772,
334,
6340,
8549,
682,
338,
162249,
3945,
12440,
361,
24772,
334,
6340,
8549,
682,
338,
162249,
3945,
12440,
361,
24772,
334,
6340,
8549,
682,
338,
162249,
3945,
12440,
361,
24772,
334,
6340,
8549,
682,
338,
162249,
3945,
12440,
361,
24772,
334,
6340,
8549,
682,
338,
162249,
3945,
12440,
361,
24772,
334,
6340,
8549,
682,
338,
162249,
3945,
12440,
361,
24772,
334,
6340,
8549,
682,
338,
162249,
3945,
12440,
361,
24772,
334,
6340,
8549,
682,
338,
162249,
3945,
12440,
361,
24772,
334,
6340,
8549,
682,
338,
162249,
3945,
12440,
361,
24772,
334,
6340,
8549,
682,
338,
162249,
3945,
12440,
361,
24772,
334,
6340,
8549,
682,
338,
162249
],
"count": 200
},
{
"text": " indented\n\tmixed whitespace runs\n\n\n",
"ids": [
274,
155238,
198,
3575,
4970,
256,
60368,
257,
11082,
3898
],
"count": 10
},
{
"text": "CamelCaseIdentifier snake_case_name SCREAMING_SNAKE kebab-case-name",
"ids": [
111821,
8081,
11633,
58108,
39188,
2928,
368,
8034,
154066,
1343,
10435,
4010,
2591,
58689,
56344,
16492
],
"count": 16
},
{
"text": "中文分词是自然语言处理的基础任务之一。月之暗面发布了千亿参数模型。",
"ids": [
16717,
671,
3961,
153780,
6826,
3934,
10301,
6282,
4584,
292,
892,
685,
5519,
696,
27613,
64960,
10517,
15662,
292
],
"count": 19
},
{
"text": "汉字漢字汉字漢字",
"ids": [
38325,
1435,
95,
1855,
38325,
1435,
95,
1855
],
"count": 8
},
{
"text": "中文English中文",
"ids": [
16717,
44372,
16717
],
"count": 3
},
{
"text": "中文english中文ENGLISH中文",
"ids": [
16717,
89901,
16717,
1254,
159820,
16717
],
"count": 6
},
{
"text": "GPT4发布于2023年3月14日,共有1750亿个参数。",
"ids": [
111021,
19,
5073,
624,
2975,
18,
532,
18,
892,
1626,
778,
11,
15298,
16484,
15,
2936,
439,
10517,
292
],
"count": 19
},
{
"text": "深度学习deep learning模型model需要大量GPU资源,如A100或H100。",
"ids": [
104448,
54283,
6857,
15662,
7940,
1777,
7825,
36399,
4127,
11,
697,
32,
1570,
1081,
39,
1570,
292
],
"count": 17
},
{
"text": "价格是99.99元,折扣为8.5折。",
"ids": [
96932,
3004,
13,
3004,
1218,
11,
28799,
441,
23,
13,
20,
4692,
292
],
"count": 13
},
{
"text": "他说:'It's fine'然后离开了。",
"ids": [
9085,
16229,
15881,
8201,
6,
3250,
24750,
292
],
"count": 8
},
{
"text": "中文Word中文WORD中文word",
"ids": [
16717,
16390,
16717,
11744,
16717,
2906
],
"count": 6
},
{
"text": "日本語テスト中文한국어",
"ids": [
4546,
63011,
25875,
33360,
16717,
17079,
54078,
28763
],
"count": 8
},
{
"text": "第一行\n第二行\r\n 第三行\t结束",
"ids": [
2122,
616,
198,
2944,
616,
462,
220,
220,
4028,
616,
197,
6123
],
"count": 12
},
{
"text": "中文 English 中文 English 中文",
"ids": [
16717,
9159,
220,
16717,
220,
9159,
256,
220,
16717
],
"count": 9
},
{
"text": "12345678901234567890",
"ids": [
6694,
12972,
16242,
16349,
18439,
22523,
2788
],
"count": 7
},
{
"text": "١٢٣٤٥٦٧٨٩٠ ๑๒๓ 一二三",
"ids": [
111991,
161592,
149,
96,
149,
97,
149,
98,
149,
99,
149,
100,
149,
101,
148209,
122674,
220,
7518,
239,
7518,
240,
7518,
241,
220,
160386
],
"count": 25
},
{
"text": "éé中文éé é́中文",
"ids": [
1941,
1941,
16717,
1941,
1941,
350,
37484,
37484,
16717
],
"count": 9
},
{
"text": "foo!!!\n\nbar???\r\n",
"ids": [
10570,
20118,
382,
4566,
46518,
462
],
"count": 6
},
{
"text": " trailing spaces ",
"ids": [
256,
42711,
14803,
274
],
"count": 4
},
{
"text": " 中文 a 中文 A1中文",
"ids": [
220,
16717,
261,
220,
16717,
401,
16,
16717
],
"count": 8
}
]
}
Reference in New Issue View Git Blame Copy Permalink