smashimo/oh-my-pi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
0cdd37fc15a9016ba8726ffc215d25856a8cd744
oh-my-pi/crates/pi-natives/fixtures/deepseek3.json
T
can1357 0cdd37fc15 feat(pi-natives/tools): implemented utok tokenizer for multiple models 
- Replaced the `ctok` implementation with the `utok` universal tokenizer supporting multiple model families and UTF text encodings.
- Added tokenizer support and embedding data for Qwen3, DeepSeek V3, Kimi K2, and GLM-5 model variants.
- Added fixture generation scripts, vocabulary packers, and golden test suites for validating tokenization parity.
- Updated dependency requirements and Bazel workspace definitions for new crates and tools.
2026-08-20 01:45:04 +02:00

1035 lines
12 KiB
JSON
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
{
"generator": "uv run --with tokenizers tools/gen-deepseek-fixtures.py (tokenizers, cache/deepseek-v4.tokenizer.json, add_special_tokens=False, encode_special_tokens=True)",
"cases": [
{
"text": "",
"ids": [],
"count": 0
},
{
"text": " ",
"ids": [
223
],
"count": 1
},
{
"text": " \t\t\n\n\r\n ",
"ids": [
262,
998,
271,
204,
201,
361
],
"count": 6
},
{
"text": "hello world",
"ids": [
33310,
2058
],
"count": 2
},
{
"text": "Hello, World! It's a test. We'll see; don't worry, y'all've been warned.",
"ids": [
19923,
14,
4495,
3,
983,
734,
260,
1950,
16,
1350,
5922,
1762,
29,
2090,
1664,
14129,
14,
383,
98513,
5270,
1047,
29363,
16
],
"count": 23
},
{
"text": "The quick brown fox jumps over the lazy dog. 1234567890 12 345 6789",
"ids": [
671,
4787,
13769,
46012,
54994,
1060,
270,
41638,
6397,
16,
223,
6895,
18009,
25744,
18,
223,
736,
223,
18014,
223,
27183,
27
],
"count": 22
},
{
"text": "fn main() {\n let x: Vec<u32> = (0..10).map(|i| i * 2).collect();\n println!(\"{x:?}\");\n}",
"ids": [
28952,
1840,
1393,
875,
361,
2513,
1527,
28,
50526,
30,
87,
2111,
32,
438,
343,
18,
426,
553,
797,
10865,
74873,
75,
94,
1008,
982,
223,
20,
797,
45684,
2745,
361,
42432,
64325,
93,
90,
28,
33,
88759,
95
],
"count": 39
},
{
"text": "{\"key\": [1, 2.5, -3e8], \"nested\": {\"a\": null, \"b\": true}}",
"ids": [
24313,
4989,
3362,
764,
19,
14,
223,
20,
16,
23,
14,
565,
21,
71,
26,
3031,
582,
80,
12904,
3362,
28612,
67,
3362,
5100,
14,
582,
68,
3362,
3292,
3831
],
"count": 30
},
{
"text": "东京は日本の首都であり、世界で最も人口の多い都市圏の一つです。深度求索发布了新一代基座模型。",
"ids": [
39813,
2841,
82619,
41175,
67287,
410,
3427,
2539,
103948,
11733,
1576,
53837,
26922,
365,
240,
1576,
94637,
8262,
320,
17180,
1645,
4568,
53961,
67547,
1475,
4683,
8842,
320
],
"count": 28
},
{
"text": "한국어 텍스트도 테스트합니다. 안녕하세요!",
"ids": [
5634,
17564,
9227,
61291,
238,
45652,
9260,
78758,
45652,
32900,
16,
38880,
11939,
246,
4567,
73527,
3
],
"count": 17
},
{
"text": "Многоязычный текст: русский, ελληνικά, עברית, العربية.",
"ids": [
10331,
4818,
738,
9672,
45575,
121212,
28,
39641,
16980,
14,
57688,
14,
6166,
15773,
8170,
14,
55595,
16
],
"count": 18
},
{
"text": "naïve café résumé — em-dash…ellipsis",
"ids": [
2720,
91223,
57664,
21538,
90902,
2136,
980,
2417,
1844,
1248,
574,
4198,
278
],
"count": 13
},
{
"text": "naïve café",
"ids": [
58009,
66383,
427,
69292,
17793
],
"count": 5
},
{
"text": "👍🏽 emoji test 👨‍👩‍👧‍👦 family, flags 🇹🇷🇯🇵, math 𝕏≈∑∫",
"ids": [
92526,
40965,
124,
980,
82644,
1950,
52780,
104,
46088,
25730,
105,
46088,
25730,
103,
46088,
25730,
102,
3096,
14,
30698,
77579,
120,
43754,
118,
43754,
110,
43754,
116,
14,
7704,
83689,
246,
240,
45181,
25951,
36705
],
"count": 36
},
{
"text": "https://example.com/path?query=value&other=%20escaped#fragment",
"ids": [
5395,
2272,
30357,
2193,
9391,
901,
33,
12066,
31,
5197,
8,
1672,
61732,
397,
25441,
14807,
5,
72,
29308
],
"count": 19
},
{
"text": "supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious supercalifragilisticexpialidocious",
"ids": [
27196,
5096,
394,
3174,
321,
435,
722,
66562,
536,
329,
95790,
3777,
5096,
394,
3174,
321,
435,
722,
66562,
536,
329,
95790,
3777,
5096,
394,
3174,
321,
435,
722,
66562,
536,
329,
95790,
3777,
5096,
394,
3174,
321,
435,
722,
66562,
536,
329,
95790,
3777,
5096,
394,
3174,
321,
435,
722,
66562,
536,
329,
95790,
3777,
5096,
394,
3174,
321,
435,
722,
66562,
536,
329,
95790,
3777,
5096,
394,
3174,
321,
435,
722,
66562,
536,
329,
95790,
3777,
5096,
394,
3174,
321,
435,
722,
66562,
536,
329,
95790,
3777,
5096,
394,
3174,
321,
435,
722,
66562,
536,
329,
95790,
3777,
5096,
394,
3174,
321,
435,
722,
66562,
536,
329,
95790,
3777,
5096,
394,
3174,
321,
435,
722,
66562,
536,
329,
95790,
3777,
5096,
394,
3174,
321,
435,
722,
66562,
536,
329,
95790,
3777,
5096,
394,
3174,
321,
435,
722,
66562,
536,
329,
95790,
3777,
5096,
394,
3174,
321,
435,
722,
66562,
536,
329,
95790,
3777,
5096,
394,
3174,
321,
435,
722,
66562,
536,
329,
95790,
3777,
5096,
394,
3174,
321,
435,
722,
66562,
536,
329,
95790,
3777,
5096,
394,
3174,
321,
435,
722,
66562,
536,
329,
95790,
3777,
5096,
394,
3174,
321,
435,
722,
66562,
536,
329,
95790,
3777,
5096,
394,
3174,
321,
435,
722,
66562,
536,
329,
95790,
3777,
5096,
394,
3174,
321,
435,
722,
66562,
536,
329,
95790
],
"count": 220
},
{
"text": " indented\n\tmixed whitespace runs\n\n\n",
"ids": [
361,
1126,
19686,
201,
21840,
5053,
262,
37899,
5790,
290,
12122,
6328
],
"count": 12
},
{
"text": "CamelCaseIdentifier snake_case_name SCREAMING_SNAKE kebab-case-name",
"ids": [
37,
31105,
15434,
45282,
34951,
78392,
7907,
17283,
89879,
4951,
10993,
3794,
13969,
473,
39637,
54690,
42391
],
"count": 17
},
{
"text": "abc123def一二三ghi",
"ids": [
32372,
6895,
3465,
49548,
1131,
46382
],
"count": 6
},
{
"text": "1234",
"ids": [
6895,
22
],
"count": 2
},
{
"text": "12345 678901 3.14159265358979",
"ids": [
6895,
1883,
223,
27183,
25622,
223,
21,
16,
9926,
25403,
26681,
27471,
3981
],
"count": 13
},
{
"text": "2024-08-19T12:34:56.789Z",
"ids": [
939,
22,
15,
3019,
15,
511,
54,
736,
28,
2012,
28,
3661,
16,
25744,
60
],
"count": 15
},
{
"text": "10000000 tokens cost $0.00042",
"ids": [
1457,
1320,
504,
35977,
2753,
957,
18,
16,
1320,
3180
],
"count": 10
},
{
"text": "一2三45六789零 第123章 第1234章",
"ids": [
378,
20,
1131,
1883,
3377,
25744,
6143,
223,
1056,
6895,
2487,
223,
1056,
6895,
22,
2487
],
"count": 16
},
{
"text": "深度求索发布了新一代基座模型，性能大幅提升。",
"ids": [
17180,
1645,
4568,
53961,
67547,
1475,
4683,
8842,
303,
10861,
22997,
6161,
320
],
"count": 13
},
{
"text": "こんにちは世界！カタカナ・テストです。",
"ids": [
4549,
7245,
2298,
12457,
2841,
3427,
1175,
15961,
11767,
15961,
27071,
4825,
109288,
8262,
320
],
"count": 15
},
{
"text": "ゟ゠ヿ",
"ids": [
665,
256,
665,
257,
912,
126
],
"count": 6
},
{
"text": "中文English日本語한국어mixed",
"ids": [
21134,
17530,
88768,
5634,
17564,
9227,
122545
],
"count": 7
},
{
"text": ".NET",
"ids": [
48635
],
"count": 1
},
{
"text": ".NET Framework 4.8",
"ids": [
48635,
29206,
223,
22,
16,
26
],
"count": 6
},
{
"text": "(foo)",
"ids": [
5123,
6379,
11
],
"count": 3
},
{
"text": "(int)x + (float)y",
"ids": [
5047,
50964,
940,
343,
15891,
99892
],
"count": 6
},
{
"text": "#include <stdio.h>",
"ids": [
6201,
818,
21968,
5196,
32
],
"count": 5
},
{
"text": "#pragma once",
"ids": [
57344,
4387
],
"count": 2
},
{
"text": "[foo]bar {baz}qux",
"ids": [
61,
40897,
63,
6515,
680,
68,
1521,
95,
403,
90
],
"count": 10
},
{
"text": "@user mentioned ~home and $PATH",
"ids": [
34,
5265,
9475,
6883,
21983,
305,
957,
89849
],
"count": 8
},
{
"text": "a.b.c e.g. i.e. etc.",
"ids": [
67,
6229,
4001,
312,
3588,
16,
1008,
4987,
16,
5655,
16
],
"count": 11
},
{
"text": "'quoted' \"double\" `backtick`",
"ids": [
9,
403,
12284,
9,
582,
20563,
4,
3608,
3555,
86,
1193,
66
],
"count": 12
},
{
"text": "C++ -O2 --flag=value",
"ids": [
37,
3312,
565,
49,
20,
3820,
25002,
31,
5197
],
"count": 9
},
{
"text": "_underscore __dunder__",
"ids": [
65,
46492,
23982,
7854,
664,
4373,
848
],
"count": 7
},
{
"text": "word \nnext ",
"ids": [
3899,
14320,
6695,
262
],
"count": 4
},
{
"text": " leading and trailing ",
"ids": [
223,
6646,
305,
52964,
262
],
"count": 5
},
{
"text": "<｜begin▁of▁sentence｜>",
"ids": [
30,
28217,
8277,
5487,
226,
2154,
5487,
226,
85,
51015,
28217,
32
],
"count": 12
},
{
"text": "<｜end▁of▁sentence｜>hello<｜▁pad▁｜>",
"ids": [
30,
28217,
523,
5487,
226,
2154,
5487,
226,
85,
51015,
28217,
32,
33310,
30,
28217,
5487,
226,
24591,
5487,
226,
28217,
32
],
"count": 22
}
],
"v3_parity": {
"text": "DeepSeek V3参数量6710亿, released 2024-12-26. (fn)main一二三",
"ids": [
53091,
4374,
1465,
721,
21,
2294,
9853,
26612,
18,
5528,
14,
9400,
223,
939,
22,
15,
736,
15,
1450,
16,
343,
28952,
11,
12993,
49548,
1131
],
"count": 26
}
}
Reference in New Issue View Git Blame Copy Permalink