can1357
0cdd37fc15
- Replaced the `ctok` implementation with the `utok` universal tokenizer supporting multiple model families and UTF text encodings. - Added tokenizer support and embedding data for Qwen3, DeepSeek V3, Kimi K2, and GLM-5 model variants. - Added fixture generation scripts, vocabulary packers, and golden test suites for validating tokenization parity. - Updated dependency requirements and Bazel workspace definitions for new crates and tools.
841 lines
10 KiB
JSON
841 lines
10 KiB
JSON
{
|
|
"generator": "uv run --with tokenizers tools/gen-glm-fixtures.py (tokenizers reference, add_special_tokens=False)",
|
|
"cases": [
|
|
{
|
|
"text": "",
|
|
"ids": [],
|
|
"count": 0
|
|
},
|
|
{
|
|
"text": " ",
|
|
"ids": [
|
|
220
|
|
],
|
|
"count": 1
|
|
},
|
|
{
|
|
"text": " \t\t\n\n\r\n ",
|
|
"ids": [
|
|
256,
|
|
33984,
|
|
319,
|
|
262
|
|
],
|
|
"count": 4
|
|
},
|
|
{
|
|
"text": "hello world",
|
|
"ids": [
|
|
14978,
|
|
1879
|
|
],
|
|
"count": 2
|
|
},
|
|
{
|
|
"text": "Hello, World! It's a test. We'll see; don't worry, y'all've been warned.",
|
|
"ids": [
|
|
9703,
|
|
11,
|
|
4337,
|
|
0,
|
|
1084,
|
|
594,
|
|
264,
|
|
1273,
|
|
13,
|
|
1205,
|
|
3278,
|
|
1490,
|
|
26,
|
|
1513,
|
|
944,
|
|
10950,
|
|
11,
|
|
379,
|
|
64374,
|
|
3003,
|
|
1012,
|
|
18650,
|
|
13
|
|
],
|
|
"count": 23
|
|
},
|
|
{
|
|
"text": "The quick brown fox jumps over the lazy dog. 1234567890 12 345 6789",
|
|
"ids": [
|
|
785,
|
|
3974,
|
|
13867,
|
|
38627,
|
|
34041,
|
|
916,
|
|
279,
|
|
15666,
|
|
5562,
|
|
13,
|
|
220,
|
|
108714,
|
|
100461,
|
|
21,
|
|
100928,
|
|
24,
|
|
15,
|
|
220,
|
|
98886,
|
|
220,
|
|
18,
|
|
100461,
|
|
220,
|
|
21,
|
|
100928,
|
|
24
|
|
],
|
|
"count": 26
|
|
},
|
|
{
|
|
"text": "fn main() {\n let x: Vec<u32> = (0..10).map(|i| i * 2).collect();\n println!(\"{x:?}\");\n}",
|
|
"ids": [
|
|
8821,
|
|
1887,
|
|
368,
|
|
341,
|
|
262,
|
|
1077,
|
|
856,
|
|
25,
|
|
11307,
|
|
34664,
|
|
101175,
|
|
29,
|
|
284,
|
|
320,
|
|
15,
|
|
496,
|
|
98668,
|
|
568,
|
|
2186,
|
|
22369,
|
|
72,
|
|
91,
|
|
600,
|
|
353,
|
|
220,
|
|
17,
|
|
568,
|
|
17362,
|
|
543,
|
|
262,
|
|
13742,
|
|
88193,
|
|
87,
|
|
75867,
|
|
20264,
|
|
92
|
|
],
|
|
"count": 36
|
|
},
|
|
{
|
|
"text": "{\"key\": [1, 2.5, -3e8], \"nested\": {\"a\": null, \"b\": true}}",
|
|
"ids": [
|
|
4913,
|
|
792,
|
|
788,
|
|
508,
|
|
16,
|
|
11,
|
|
220,
|
|
17,
|
|
13,
|
|
20,
|
|
11,
|
|
481,
|
|
18,
|
|
68,
|
|
23,
|
|
1125,
|
|
330,
|
|
58860,
|
|
788,
|
|
5212,
|
|
64,
|
|
788,
|
|
845,
|
|
11,
|
|
330,
|
|
65,
|
|
788,
|
|
830,
|
|
3417
|
|
],
|
|
"count": 29
|
|
},
|
|
{
|
|
"text": "东京は日本の首都であり、世界で最も人口の多い都市圏の一つです。深度求索发布了新一代基座模型。",
|
|
"ids": [
|
|
105935,
|
|
15310,
|
|
99799,
|
|
15755,
|
|
106552,
|
|
145713,
|
|
5373,
|
|
99011,
|
|
16147,
|
|
98430,
|
|
31742,
|
|
100742,
|
|
15755,
|
|
134584,
|
|
103266,
|
|
12268,
|
|
237,
|
|
138683,
|
|
58236,
|
|
37346,
|
|
1773,
|
|
102148,
|
|
98624,
|
|
99459,
|
|
108284,
|
|
110330,
|
|
98526,
|
|
99218,
|
|
100484,
|
|
1773
|
|
],
|
|
"count": 30
|
|
},
|
|
{
|
|
"text": "한국어 텍스트도 테스트합니다. 안녕하세요!",
|
|
"ids": [
|
|
23508,
|
|
130419,
|
|
30953,
|
|
10759,
|
|
43849,
|
|
52836,
|
|
47683,
|
|
10759,
|
|
71953,
|
|
52836,
|
|
60412,
|
|
13,
|
|
94372,
|
|
73588,
|
|
243,
|
|
90385,
|
|
0
|
|
],
|
|
"count": 17
|
|
},
|
|
{
|
|
"text": "Многоязычный текст: русский, ελληνικά, עברית, العربية.",
|
|
"ids": [
|
|
37793,
|
|
38592,
|
|
62539,
|
|
4552,
|
|
133834,
|
|
70363,
|
|
25,
|
|
145232,
|
|
11,
|
|
58738,
|
|
137790,
|
|
130624,
|
|
11,
|
|
17263,
|
|
95,
|
|
74926,
|
|
49904,
|
|
41995,
|
|
103,
|
|
11,
|
|
137930,
|
|
13
|
|
],
|
|
"count": 22
|
|
},
|
|
{
|
|
"text": "naïve café résumé — em-dash…ellipsis",
|
|
"ids": [
|
|
3376,
|
|
37377,
|
|
586,
|
|
51609,
|
|
9330,
|
|
1242,
|
|
963,
|
|
1959,
|
|
976,
|
|
1737,
|
|
988,
|
|
1940,
|
|
71373
|
|
],
|
|
"count": 13
|
|
},
|
|
{
|
|
"text": "naïve café",
|
|
"ids": [
|
|
77,
|
|
2143,
|
|
151464,
|
|
586,
|
|
40702,
|
|
53481
|
|
],
|
|
"count": 6
|
|
},
|
|
{
|
|
"text": "👍🏽 emoji test 👨👩👧👦 family, flags 🇹🇷🇯🇵, math 𝕏≈∑∫",
|
|
"ids": [
|
|
151901,
|
|
151821,
|
|
42123,
|
|
1273,
|
|
61370,
|
|
101,
|
|
124564,
|
|
151929,
|
|
124564,
|
|
151927,
|
|
124564,
|
|
151926,
|
|
2997,
|
|
11,
|
|
8041,
|
|
11157,
|
|
229,
|
|
117,
|
|
9281,
|
|
229,
|
|
115,
|
|
9281,
|
|
229,
|
|
107,
|
|
9281,
|
|
229,
|
|
113,
|
|
11,
|
|
6888,
|
|
80590,
|
|
243,
|
|
237,
|
|
153445,
|
|
120775,
|
|
153417
|
|
],
|
|
"count": 35
|
|
},
|
|
{
|
|
"text": "https://example.com/path?query=value&other=%20escaped#fragment",
|
|
"ids": [
|
|
2428,
|
|
1110,
|
|
8686,
|
|
905,
|
|
50642,
|
|
30,
|
|
1631,
|
|
46252,
|
|
5,
|
|
1575,
|
|
7846,
|
|
98360,
|
|
65346,
|
|
2,
|
|
41962
|
|
],
|
|
"count": 15
|
|
},
|
|
{
|
|
"text": "supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious",
|
|
"ids": [
|
|
12770,
|
|
2962,
|
|
278,
|
|
333,
|
|
4101,
|
|
321,
|
|
4532,
|
|
4580,
|
|
530,
|
|
307,
|
|
76570,
|
|
2256,
|
|
5416,
|
|
333,
|
|
4101,
|
|
321,
|
|
4532,
|
|
4580,
|
|
530,
|
|
307,
|
|
76570,
|
|
2256,
|
|
5416,
|
|
333,
|
|
4101,
|
|
321,
|
|
4532,
|
|
4580,
|
|
530,
|
|
307,
|
|
76570,
|
|
2256,
|
|
5416,
|
|
333,
|
|
4101,
|
|
321,
|
|
4532,
|
|
4580,
|
|
530,
|
|
307,
|
|
76570,
|
|
2256,
|
|
5416,
|
|
333,
|
|
4101,
|
|
321,
|
|
4532,
|
|
4580,
|
|
530,
|
|
307,
|
|
76570,
|
|
2256,
|
|
5416,
|
|
333,
|
|
4101,
|
|
321,
|
|
4532,
|
|
4580,
|
|
530,
|
|
307,
|
|
76570,
|
|
2256,
|
|
5416,
|
|
333,
|
|
4101,
|
|
321,
|
|
4532,
|
|
4580,
|
|
530,
|
|
307,
|
|
76570,
|
|
2256,
|
|
5416,
|
|
333,
|
|
4101,
|
|
321,
|
|
4532,
|
|
4580,
|
|
530,
|
|
307,
|
|
76570,
|
|
2256,
|
|
5416,
|
|
333,
|
|
4101,
|
|
321,
|
|
4532,
|
|
4580,
|
|
530,
|
|
307,
|
|
76570,
|
|
2256,
|
|
5416,
|
|
333,
|
|
4101,
|
|
321,
|
|
4532,
|
|
4580,
|
|
530,
|
|
307,
|
|
76570,
|
|
2256,
|
|
5416,
|
|
333,
|
|
4101,
|
|
321,
|
|
4532,
|
|
4580,
|
|
530,
|
|
307,
|
|
76570,
|
|
2256,
|
|
5416,
|
|
333,
|
|
4101,
|
|
321,
|
|
4532,
|
|
4580,
|
|
530,
|
|
307,
|
|
76570,
|
|
2256,
|
|
5416,
|
|
333,
|
|
4101,
|
|
321,
|
|
4532,
|
|
4580,
|
|
530,
|
|
307,
|
|
76570,
|
|
2256,
|
|
5416,
|
|
333,
|
|
4101,
|
|
321,
|
|
4532,
|
|
4580,
|
|
530,
|
|
307,
|
|
76570,
|
|
2256,
|
|
5416,
|
|
333,
|
|
4101,
|
|
321,
|
|
4532,
|
|
4580,
|
|
530,
|
|
307,
|
|
76570,
|
|
2256,
|
|
5416,
|
|
333,
|
|
4101,
|
|
321,
|
|
4532,
|
|
4580,
|
|
530,
|
|
307,
|
|
76570,
|
|
2256,
|
|
5416,
|
|
333,
|
|
4101,
|
|
321,
|
|
4532,
|
|
4580,
|
|
530,
|
|
307,
|
|
76570,
|
|
2256,
|
|
5416,
|
|
333,
|
|
4101,
|
|
321,
|
|
4532,
|
|
4580,
|
|
530,
|
|
307,
|
|
76570,
|
|
2256,
|
|
5416,
|
|
333,
|
|
4101,
|
|
321,
|
|
4532,
|
|
4580,
|
|
530,
|
|
307,
|
|
76570,
|
|
2256,
|
|
5416,
|
|
333,
|
|
4101,
|
|
321,
|
|
4532,
|
|
4580,
|
|
530,
|
|
307,
|
|
76570
|
|
],
|
|
"count": 201
|
|
},
|
|
{
|
|
"text": " indented\n\tmixed whitespace runs\n\n\n",
|
|
"ids": [
|
|
262,
|
|
1257,
|
|
15852,
|
|
198,
|
|
2109,
|
|
3286,
|
|
256,
|
|
36188,
|
|
257,
|
|
8472,
|
|
1406
|
|
],
|
|
"count": 11
|
|
},
|
|
{
|
|
"text": "CamelCaseIdentifier snake_case_name SCREAMING_SNAKE kebab-case-name",
|
|
"ids": [
|
|
25328,
|
|
301,
|
|
4207,
|
|
8713,
|
|
25187,
|
|
19061,
|
|
1269,
|
|
7531,
|
|
15903,
|
|
1718,
|
|
1098,
|
|
7326,
|
|
3390,
|
|
1962,
|
|
47427,
|
|
38278,
|
|
11489
|
|
],
|
|
"count": 17
|
|
},
|
|
{
|
|
"text": "智谱清言是由北京智谱华章科技有限公司开发的大语言模型。",
|
|
"ids": [
|
|
99126,
|
|
100789,
|
|
98691,
|
|
98856,
|
|
102047,
|
|
99334,
|
|
99126,
|
|
100789,
|
|
98762,
|
|
98648,
|
|
102595,
|
|
99495,
|
|
99707,
|
|
100132,
|
|
100484,
|
|
1773
|
|
],
|
|
"count": 16
|
|
},
|
|
{
|
|
"text": "你好，世界！这是一个测试。",
|
|
"ids": [
|
|
109377,
|
|
3837,
|
|
99011,
|
|
6313,
|
|
103974,
|
|
100838,
|
|
1773
|
|
],
|
|
"count": 7
|
|
},
|
|
{
|
|
"text": "人工智能正在改变世界，深度学习模型的参数规模不断增长。",
|
|
"ids": [
|
|
104668,
|
|
100296,
|
|
100070,
|
|
99011,
|
|
3837,
|
|
102148,
|
|
98935,
|
|
111595,
|
|
100955,
|
|
100200,
|
|
99421,
|
|
99816,
|
|
1773
|
|
],
|
|
"count": 13
|
|
},
|
|
{
|
|
"text": "中英文混排 mixed CJK and English 123 数字。",
|
|
"ids": [
|
|
98322,
|
|
103577,
|
|
99817,
|
|
98930,
|
|
9515,
|
|
356,
|
|
33905,
|
|
323,
|
|
6364,
|
|
220,
|
|
108714,
|
|
124543,
|
|
1773
|
|
],
|
|
"count": 13
|
|
},
|
|
{
|
|
"text": " 全角空格和标点符号：《引号》、【括号】——破折号……省略号",
|
|
"ids": [
|
|
22382,
|
|
98402,
|
|
99025,
|
|
98745,
|
|
98639,
|
|
98327,
|
|
98593,
|
|
98428,
|
|
105423,
|
|
102462,
|
|
98662,
|
|
98760,
|
|
106843,
|
|
10899,
|
|
99231,
|
|
98760,
|
|
10953,
|
|
8544,
|
|
99222,
|
|
100048,
|
|
98760,
|
|
14044,
|
|
98737,
|
|
99260,
|
|
98760
|
|
],
|
|
"count": 25
|
|
},
|
|
{
|
|
"text": " 参考",
|
|
"ids": [
|
|
99855
|
|
],
|
|
"count": 1
|
|
},
|
|
{
|
|
"text": " 参考资料",
|
|
"ids": [
|
|
99924
|
|
],
|
|
"count": 1
|
|
},
|
|
{
|
|
"text": " 而",
|
|
"ids": [
|
|
101502
|
|
],
|
|
"count": 1
|
|
},
|
|
{
|
|
"text": " 者",
|
|
"ids": [
|
|
102222
|
|
],
|
|
"count": 1
|
|
},
|
|
{
|
|
"text": " 王",
|
|
"ids": [
|
|
102322
|
|
],
|
|
"count": 1
|
|
},
|
|
{
|
|
"text": " 参考龘",
|
|
"ids": [
|
|
26767,
|
|
224,
|
|
98580,
|
|
82225,
|
|
246
|
|
],
|
|
"count": 5
|
|
},
|
|
{
|
|
"text": " 参考资料龘",
|
|
"ids": [
|
|
26767,
|
|
224,
|
|
98580,
|
|
99304,
|
|
82225,
|
|
246
|
|
],
|
|
"count": 6
|
|
},
|
|
{
|
|
"text": " 而龘",
|
|
"ids": [
|
|
8905,
|
|
222,
|
|
234,
|
|
82225,
|
|
246
|
|
],
|
|
"count": 5
|
|
},
|
|
{
|
|
"text": " 者龘",
|
|
"ids": [
|
|
8905,
|
|
222,
|
|
227,
|
|
82225,
|
|
246
|
|
],
|
|
"count": 5
|
|
},
|
|
{
|
|
"text": " 王龘",
|
|
"ids": [
|
|
10231,
|
|
236,
|
|
233,
|
|
82225,
|
|
246
|
|
],
|
|
"count": 5
|
|
},
|
|
{
|
|
"text": "龘 参考",
|
|
"ids": [
|
|
82225,
|
|
246,
|
|
99855
|
|
],
|
|
"count": 3
|
|
},
|
|
{
|
|
"text": "龘 参考资料",
|
|
"ids": [
|
|
82225,
|
|
246,
|
|
99924
|
|
],
|
|
"count": 3
|
|
},
|
|
{
|
|
"text": "龘 而",
|
|
"ids": [
|
|
82225,
|
|
246,
|
|
101502
|
|
],
|
|
"count": 3
|
|
},
|
|
{
|
|
"text": "龘 者",
|
|
"ids": [
|
|
82225,
|
|
246,
|
|
102222
|
|
],
|
|
"count": 3
|
|
},
|
|
{
|
|
"text": "龘 王",
|
|
"ids": [
|
|
82225,
|
|
246,
|
|
102322
|
|
],
|
|
"count": 3
|
|
}
|
|
]
|
|
}
|