[
{
"text": "",
"tokens": [],
"count": 0,
"decoded": ""
},
{
"text": " ",
"tokens": [
220
],
"count": 1,
"decoded": " "
},
{
"text": " ",
"tokens": [
256
],
"count": 1,
"decoded": " "
},
{
"text": "\n",
"tokens": [
198
],
"count": 1,
"decoded": "\n"
},
{
"text": "\t",
"tokens": [
197
],
"count": 1,
"decoded": "\t"
},
{
"text": "hello",
"tokens": [
15339
],
"count": 1,
"decoded": "hello"
},
{
"text": "Hello, world!",
"tokens": [
9906,
11,
1917,
0
],
"count": 4,
"decoded": "Hello, world!"
},
{
"text": "I'm don't they're we've she'll it'd",
"tokens": [
40,
2846,
1541,
956,
814,
2351,
584,
3077,
1364,
3358,
433,
4265
],
"count": 12,
"decoded": "I'm don't they're we've she'll it'd"
},
{
"text": "The quick brown fox jumps over the lazy dog.",
"tokens": [
791,
4062,
14198,
39935,
35308,
927,
279,
16053,
5679,
13
],
"count": 10,
"decoded": "The quick brown fox jumps over the lazy dog."
},
{
"text": " hello world ",
"tokens": [
256,
24748,
256,
1917,
262
],
"count": 5,
"decoded": " hello world "
},
{
"text": "line1\nline2\nline3",
"tokens": [
1074,
16,
198,
1074,
17,
198,
1074,
18
],
"count": 8,
"decoded": "line1\nline2\nline3"
},
{
"text": "\n\n\n",
"tokens": [
1432
],
"count": 1,
"decoded": "\n\n\n"
},
{
"text": " \t\n \r\n ",
"tokens": [
99351,
10636,
256
],
"count": 3,
"decoded": " \t\n \r\n "
},
{
"text": "123",
"tokens": [
4513
],
"count": 1,
"decoded": "123"
},
{
"text": "100% of $1,000",
"tokens": [
1041,
4,
315,
400,
16,
11,
931
],
"count": 7,
"decoded": "100% of $1,000"
},
{
"text": "3.14159265358979",
"tokens": [
18,
13,
9335,
20128,
21598,
22905,
4643
],
"count": 7,
"decoded": "3.14159265358979"
},
{
"text": "1234567890",
"tokens": [
4513,
10961,
16474,
15
],
"count": 4,
"decoded": "1234567890"
},
{
"text": "def fibonacci(n):\n if n <= 1:\n return n\n return fibonacci(n-1) + fibonacci(n-2)\n",
"tokens": [
755,
76798,
1471,
997,
262,
422,
308,
2717,
220,
16,
512,
286,
471,
308,
198,
262,
471,
76798,
1471,
12,
16,
8,
489,
76798,
1471,
12,
17,
340
],
"count": 28,
"decoded": "def fibonacci(n):\n if n <= 1:\n return n\n return fibonacci(n-1) + fibonacci(n-2)\n"
},
{
"text": "fn main() { println!(\"Hello\"); }",
"tokens": [
8998,
1925,
368,
314,
14069,
17667,
9906,
5146,
335
],
"count": 9,
"decoded": "fn main() { println!(\"Hello\"); }"
},
{
"text": "const x = () => { return 42; };",
"tokens": [
1040,
865,
284,
1754,
591,
314,
471,
220,
2983,
26,
20667
],
"count": 11,
"decoded": "const x = () => { return 42; };"
},
{
"text": "你好世界",
"tokens": [
57668,
53901,
102616
],
"count": 3,
"decoded": "你好世界"
},
{
"text": "こんにちは世界",
"tokens": [
90115,
102616
],
"count": 2,
"decoded": "こんにちは世界"
},
{
"text": "안녕하세요 세계",
"tokens": [
101193,
124409,
110572
],
"count": 3,
"decoded": "안녕하세요 세계"
},
{
"text": "你好 hello 世界 world",
"tokens": [
57668,
53901,
24748,
127365,
1917
],
"count": 5,
"decoded": "你好 hello 世界 world"
},
{
"text": "🎉🚀💡",
"tokens": [
9468,
236,
231,
9468,
248,
222,
93273,
94
],
"count": 8,
"decoded": "🎉🚀💡"
},
{
"text": "Hello 🌍 World",
"tokens": [
9906,
11410,
234,
235,
4435
],
"count": 5,
"decoded": "Hello 🌍 World"
},
{
"text": "👨👩👧👦",
"tokens": [
9468,
239,
101,
102470,
9468,
239,
102,
102470,
9468,
239,
100,
102470,
9468,
239,
99
],
"count": 15,
"decoded": "👨👩👧👦"
},
{
"text": "café résumé naïve",
"tokens": [
936,
59958,
9517,
1264,
978,
95980,
588
],
"count": 7,
"decoded": "café résumé naïve"
},
{
"text": "hello world! 你好 🚀 test 123",
"tokens": [
15339,
1917,
0,
118195,
53901,
11410,
248,
222,
1296,
220,
4513
],
"count": 11,
"decoded": "hello world! 你好 🚀 test 123"
},
{
"text": "a@b.com https://example.com/path?q=1&r=2",
"tokens": [
64,
31,
65,
916,
3788,
1129,
8858,
916,
52076,
44882,
28,
16,
61717,
28,
17
],
"count": 15,
"decoded": "a@b.com https://example.com/path?q=1&r=2"
},
{
"text": "a",
"tokens": [
64
],
"count": 1,
"decoded": "a"
},
{
"text": "ab",
"tokens": [
370
],
"count": 1,
"decoded": "ab"
},
{
"text": "abc",
"tokens": [
13997
],
"count": 1,
"decoded": "abc"
},
{
"text": "The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. ",
"tokens": [
791,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
578,
4062,
14198,
39935,
13,
220
],
"count": 251,
"decoded": "The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. The quick brown fox. "
},
{
"text": "hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello ",
"tokens": [
15339,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
24748,
220
],
"count": 101,
"decoded": "hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello "
},
{
"text": "\"quoted\" and 'single'",
"tokens": [
1,
64825,
1,
323,
364,
15698,
6
],
"count": 7,
"decoded": "\"quoted\" and 'single'"
},
{
"text": "tabs\there\tand\tthere",
"tokens": [
32093,
197,
6881,
53577,
197,
19041
],
"count": 6,
"decoded": "tabs\there\tand\tthere"
},
{
"text": "back\\slash and for/ward",
"tokens": [
1445,
59,
51356,
323,
369,
14,
1637
],
"count": 7,
"decoded": "back\\slash and for/ward"
},
{
"text": " ",
"tokens": [
4194
],
"count": 1,
"decoded": " "
},
{
"text": " ",
"tokens": [
23249
],
"count": 1,
"decoded": " "
},
{
"text": " hello ",
"tokens": [
4194,
15339,
4194
],
"count": 3,
"decoded": " hello "
}
]