smashimo/oh-my-pi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
0cdd37fc15a9016ba8726ffc215d25856a8cd744
oh-my-pi/crates/pi-natives/fixtures/qwen3.json
T
can1357 0cdd37fc15 feat(pi-natives/tools): implemented utok tokenizer for multiple models 
- Replaced the `ctok` implementation with the `utok` universal tokenizer supporting multiple model families and UTF text encodings.
- Added tokenizer support and embedding data for Qwen3, DeepSeek V3, Kimi K2, and GLM-5 model variants.
- Added fixture generation scripts, vocabulary packers, and golden test suites for validating tokenization parity.
- Updated dependency requirements and Bazel workspace definitions for new crates and tools.
2026-08-20 01:45:04 +02:00

1316 lines
16 KiB
JSON
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
{
"generator": "tools/gen-qwen-fixtures.py: HF tokenizers Tokenizer.from_file(tools/cache/qwen3.8.tokenizer.json).encode(text, add_special_tokens=False)",
"cases": [
{
"text": "",
"ids": [],
"count": 0
},
{
"text": " ",
"ids": [
220
],
"count": 1
},
{
"text": " \t\t\n\n\r\n ",
"ids": [
256,
33019,
317,
262
],
"count": 4
},
{
"text": "hello world",
"ids": [
14556,
1814
],
"count": 2
},
{
"text": "Hello, World! It's a test. We'll see; don't worry, y'all've been warned.",
"ids": [
9419,
11,
4196,
0,
1049,
579,
264,
1228,
13,
1165,
3172,
1436,
26,
1459,
914,
10635,
11,
374,
62654,
2908,
978,
18120,
13
],
"count": 23
},
{
"text": "The quick brown fox jumps over the lazy dog. 1234567890 12 345 6789",
"ids": [
760,
3841,
13477,
37550,
33075,
888,
279,
15217,
5388,
13,
220,
16,
17,
18,
19,
20,
21,
22,
23,
24,
15,
220,
16,
17,
220,
18,
19,
20,
220,
21,
22,
23,
24
],
"count": 33
},
{
"text": "fn main() {\n let x: Vec<u32> = (0..10).map(|i| i * 2).collect();\n println!(\"{x:?}\");\n}",
"ids": [
8556,
1822,
363,
313,
198,
262,
1042,
830,
25,
10985,
33682,
18,
17,
29,
283,
318,
15,
486,
16,
15,
553,
2117,
21729,
72,
91,
585,
348,
220,
17,
553,
16873,
2061,
198,
262,
13356,
85871,
87,
73858,
92,
4876,
198,
92
],
"count": 42
},
{
"text": "{\"key\": [1, 2.5, -3e8], \"nested\": {\"a\": null, \"b\": true}}",
"ids": [
4754,
767,
763,
498,
16,
11,
220,
17,
13,
20,
11,
471,
18,
68,
23,
1089,
328,
57267,
763,
5046,
64,
763,
819,
11,
328,
65,
763,
804,
3307
],
"count": 29
},
{
"text": "东京は日本の首都であり、世界で最も人口の多い都市圏の一つです。深度求索发布了新一代基座模型。",
"ids": [
110799,
14876,
161607,
110161,
174098,
5205,
96748,
15685,
180367,
99202,
15303,
151076,
99605,
246955,
190997,
36298,
1710,
99581,
96181,
96914,
111170,
105448,
96096,
96730,
103725,
1710
],
"count": 26
},
{
"text": "한국어 텍스트도 테스트합니다. 안녕하세요!",
"ids": [
174675,
30061,
225287,
46384,
187792,
149635,
13,
191972,
0
],
"count": 9
},
{
"text": "Многоязычный текст: русский, ελληνικά, עברית, العربية.",
"ids": [
36736,
13292,
14942,
58255,
148707,
170463,
185386,
25,
211075,
11,
210061,
159778,
11,
150581,
151188,
151552,
11,
171405,
13
],
"count": 19
},
{
"text": "naïve café résumé — em-dash…ellipsis",
"ids": [
3267,
36328,
571,
50203,
238976,
1892,
945,
1676,
956,
1873,
69480
],
"count": 11
},
{
"text": "naïve café",
"ids": [
3267,
36328,
571,
50203
],
"count": 4
},
{
"text": "👍🏽 emoji test 👨‍👩‍👧‍👦 family, flags 🇹🇷🇯🇵, math 𝕏≈∑∫",
"ids": [
9008,
239,
235,
9008,
237,
121,
40958,
1228,
59720,
101,
373,
235,
9008,
239,
102,
373,
235,
9008,
239,
100,
373,
235,
9008,
239,
99,
2902,
11,
7789,
10838,
229,
117,
9008,
229,
115,
9008,
229,
107,
9008,
229,
113,
11,
6673,
78449,
243,
237,
169103,
21033,
239,
21033,
104
],
"count": 50
},
{
"text": "https://example.com/path?query=value&other=%20escaped#fragment",
"ids": [
2349,
1074,
8422,
877,
49259,
30,
1574,
44984,
5,
1520,
7598,
17,
15,
63605,
2,
40802
],
"count": 16
},
{
"text": "supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious",
"ids": [
12410,
2867,
278,
331,
3965,
319,
4384,
4431,
518,
306,
74546,
2184,
5248,
331,
3965,
319,
4384,
4431,
518,
306,
74546,
2184,
5248,
331,
3965,
319,
4384,
4431,
518,
306,
74546,
2184,
5248,
331,
3965,
319,
4384,
4431,
518,
306,
74546,
2184,
5248,
331,
3965,
319,
4384,
4431,
518,
306,
74546,
2184,
5248,
331,
3965,
319,
4384,
4431,
518,
306,
74546,
2184,
5248,
331,
3965,
319,
4384,
4431,
518,
306,
74546,
2184,
5248,
331,
3965,
319,
4384,
4431,
518,
306,
74546,
2184,
5248,
331,
3965,
319,
4384,
4431,
518,
306,
74546,
2184,
5248,
331,
3965,
319,
4384,
4431,
518,
306,
74546,
2184,
5248,
331,
3965,
319,
4384,
4431,
518,
306,
74546,
2184,
5248,
331,
3965,
319,
4384,
4431,
518,
306,
74546,
2184,
5248,
331,
3965,
319,
4384,
4431,
518,
306,
74546,
2184,
5248,
331,
3965,
319,
4384,
4431,
518,
306,
74546,
2184,
5248,
331,
3965,
319,
4384,
4431,
518,
306,
74546,
2184,
5248,
331,
3965,
319,
4384,
4431,
518,
306,
74546,
2184,
5248,
331,
3965,
319,
4384,
4431,
518,
306,
74546,
2184,
5248,
331,
3965,
319,
4384,
4431,
518,
306,
74546,
2184,
5248,
331,
3965,
319,
4384,
4431,
518,
306,
74546,
2184,
5248,
331,
3965,
319,
4384,
4431,
518,
306,
74546
],
"count": 201
},
{
"text": " indented\n\tmixed whitespace runs\n\n\n",
"ids": [
262,
1214,
15397,
198,
2041,
3179,
256,
35169,
257,
8213,
1358
],
"count": 11
},
{
"text": "CamelCaseIdentifier snake_case_name SCREAMING_SNAKE kebab-case-name",
"ids": [
24601,
300,
4068,
8449,
24463,
18521,
1224,
7295,
15446,
1658,
1063,
7096,
3281,
1895,
46133,
37208,
11163
],
"count": 17
},
{
"text": "毛泽东",
"ids": [
97008,
98340,
96265
],
"count": 3
},
{
"text": "俱乐部",
"ids": [
98112,
96404,
95938
],
"count": 3
},
{
"text": "全心全意为人民",
"ids": [
100204,
95901,
102656,
96504
],
"count": 4
},
{
"text": "承担一切因您的行为而直接或间接",
"ids": [
102870,
97157,
96051,
106891
],
"count": 4
},
{
"text": "足球俱乐部",
"ids": [
100790,
98112,
96404,
95938
],
"count": 4
},
{
"text": "材料",
"ids": [
97320
],
"count": 1
},
{
"text": "他研究毛泽东思想，加入足球俱乐部，去过新加坡和加拿大。",
"ids": [
95948,
96605,
97008,
98340,
96265,
97331,
3709,
97992,
100790,
98112,
96404,
95938,
3709,
119885,
95882,
95984,
98091,
95793,
95984,
97048,
95779,
1710
],
"count": 22
},
{
"text": "众所周知，勤勤恳恳、兢兢业业，跃跃欲试。",
"ids": [
96546,
102662,
96070,
3709,
97270,
102861,
99435,
5205,
102792,
95829,
95829,
3709,
102890,
97866,
96316,
1710
],
"count": 16
},
{
"text": "матри материал експерт",
"ids": [
6240,
7721,
29119,
153982,
194218,
31608,
153922
],
"count": 7
},
{
"text": " експерт",
"ids": [
194218,
31608,
153922
],
"count": 3
},
{
"text": "สังหาริมทรัพย์ มิถุนายน",
"ids": [
150402,
148819,
158714,
160722,
92175,
149088,
150176
],
"count": 7
},
{
"text": "بسبب الأسبوع سبب",
"ids": [
219713,
20704,
20704,
149571,
20704,
149007,
57823,
20704,
20704
],
"count": 9
},
{
"text": "Selanjutnya masyarakat terdapat",
"ids": [
37748,
50341,
330,
22178,
170428,
1915,
67,
24940
],
"count": 8
},
{
"text": "深度学习模型的训练需要大量的计算资源和高质量的数据集。近年来，随着硬件技术的飞速发展，大规模预训练语言模型在自然语言处理领域取得了突破性进展。",
"ids": [
124089,
122826,
99476,
96442,
103933,
97792,
116494,
100740,
104027,
96275,
1710,
108641,
3709,
98265,
101874,
108798,
135588,
3709,
105019,
96468,
99476,
98277,
103725,
95772,
96963,
98277,
96961,
98260,
108939,
144064,
104032,
1710
],
"count": 32
},
{
"text": "白日依山尽，黄河入海流。欲穷千里目，更上一层楼。",
"ids": [
136272,
136197,
96586,
3709,
109876,
96032,
96143,
96114,
1710,
97866,
98792,
110028,
96060,
3709,
137959,
1710
],
"count": 16
},
{
"text": "中华人民共和国全国人民代表大会常务委员会",
"ids": [
99584,
97701,
140133
],
"count": 3
},
{
"text": "你好，世界！这是一个测试。２０２４年（全角数字）",
"ids": [
109266,
3709,
96748,
6115,
109455,
99449,
1710,
24128,
25191,
24128,
44085,
95859,
9616,
95901,
96518,
99129,
7313
],
"count": 17
},
{
"text": "1234567890",
"ids": [
16,
17,
18,
19,
20,
21,
22,
23,
24,
15
],
"count": 10
},
{
"text": "3.14159265358979",
"ids": [
18,
13,
16,
19,
16,
20,
24,
17,
21,
20,
18,
20,
23,
24,
22,
24
],
"count": 16
},
{
"text": "電話番号は0123456789です",
"ids": [
247887,
14876,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
36298
],
"count": 13
},
{
"text": "١٢٣٤٥ ௧௨௩ ৪৫৬",
"ids": [
149,
94,
149,
95,
149,
96,
149,
97,
149,
98,
220,
30464,
100,
30464,
101,
30464,
102,
220,
26064,
103,
26064,
104,
26064,
105
],
"count": 24
},
{
"text": "Ⅻ Ⅷ ½ ⅓ ①②③",
"ids": [
68086,
104,
220,
68086,
100,
220,
25229,
220,
68086,
241,
220,
46701,
254,
46701,
94,
46701,
95
],
"count": 17
},
{
"text": "42nd 100th x1 x22 x333",
"ids": [
19,
17,
302,
220,
16,
15,
15,
337,
830,
16,
830,
17,
17,
830,
18,
18,
18
],
"count": 17
},
{
"text": "naïve café résumé",
"ids": [
3267,
36328,
571,
50203,
238976
],
"count": 5
},
{
"text": "한국어 텍스트",
"ids": [
174675,
30061,
225287
],
"count": 3
},
{
"text": "Ångström ế ộ",
"ids": [
169111,
938,
485,
82628,
220,
25235,
220,
36430
],
"count": 8
},
{
"text": "é́clair",
"ids": [
933,
52033,
46565
],
"count": 3
},
{
"text": "ḍ̇ ḍ̇",
"ids": [
157,
116,
235,
136,
229,
83117,
116,
235,
136,
229
],
"count": 10
},
{
"text": "DON'T I'LL HE'S WE'RE THEY'VE I'M YOU'D",
"ids": [
81720,
16813,
353,
6,
3950,
11349,
12887,
19076,
90902,
60388,
6,
4343,
353,
26708,
14551,
26807
],
"count": 16
},
{
"text": "don't i'll he's we're they've i'm you'd",
"ids": [
14572,
914,
585,
3172,
551,
579,
567,
2224,
781,
2908,
585,
2688,
488,
4035
],
"count": 14
},
{
"text": "Mixed'S cAsE'Ll",
"ids": [
83445,
12887,
272,
2053,
36,
88279,
75
],
"count": 7
},
{
"text": "it'ſ IT'S don'T x'Ll they'RE we'VE i'M you'D",
"ids": [
275,
6,
129,
123,
8435,
12887,
1459,
16813,
830,
88279,
75,
781,
90902,
567,
6,
4343,
585,
26708,
488,
26807
],
"count": 20
},
{
"text": "can't've y'all'll've",
"ids": [
4660,
914,
2908,
374,
62654,
3172,
2908
],
"count": 7
},
{
"text": "a b c d",
"ids": [
64,
220,
292,
256,
272,
262,
293
],
"count": 7
},
{
"text": "end ",
"ids": [
400,
262
],
"count": 2
},
{
"text": "tabs\t\t\tthen spaces \n newline",
"ids": [
29975,
297,
56651,
220,
12258,
695,
220,
37734
],
"count": 8
},
{
"text": "\n\n\n",
"ids": [
1358
],
"count": 1
},
{
"text": " ",
"ids": [
262
],
"count": 1
},
{
"text": "$var _under #tag @user",
"ids": [
3,
917,
696,
7744,
653,
4429,
554,
846
],
"count": 8
},
{
"text": "«guillemets» “curly” ‘quotes’",
"ids": [
22961,
8452,
319,
3323,
1367,
12620,
1001,
2276,
391,
828,
3260,
51492,
515
],
"count": 13
},
{
"text": "ab́c ́x combining",
"ids": [
365,
52033,
66,
220,
52033,
87,
33041
],
"count": 7
},
{
"text": "。汉字，测试！Ｑｗｅｎ全角fifl",
"ids": [
1710,
112254,
3709,
99449,
6115,
1486,
109,
154981,
169665,
14945,
236,
95901,
96518,
169752,
171,
105,
224
],
"count": 17
},
{
"text": "x \r\n \n y",
"ids": [
87,
2449,
695,
374
],
"count": 4
},
{
"text": "a\r\nb\rc\nd",
"ids": [
64,
317,
65,
201,
66,
198,
67
],
"count": 7
},
{
"text": "para.\n\n Indented after blank.\r\n\r\nEnd",
"ids": [
14365,
13,
271,
220,
2190,
15397,
1238,
9813,
13,
845,
3601
],
"count": 11
}
]
}
Reference in New Issue View Git Blame Copy Permalink