rs-chunks 0.6.1

Fast, high-fidelity document chunking for RAG — a pure-Rust engine covering 36 file formats (Office, OpenDocument, PDF, email, ebooks, notebooks, and more).
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281

// GENERATED DATA — do not edit by hand.
//
// Derived from the `lopdf` crate (MIT), which in turn encodes the Adobe Glyph
// List and the PDF specification's Appendix D encoding tables. Regenerating is
// a one-off: these tables are fixed by the PDF spec and do not track lopdf.

//! The PDF specification's Appendix D simple-font encodings, as direct
//! code → Unicode tables. `None` is an unused code point in that encoding.
pub(crate) type CodeToUnicode = [Option<u16>; 256];

/// StandardEncoding — the built-in encoding of the base-14 text fonts.
pub(crate) static STANDARD_ENCODING: CodeToUnicode = [
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    Some(0x0020), Some(0x0021), Some(0x0022), Some(0x0023),
    Some(0x0024), Some(0x0025), Some(0x0026), Some(0x2019),
    Some(0x0028), Some(0x0029), Some(0x002a), Some(0x002b),
    Some(0x002c), Some(0x002d), Some(0x002e), Some(0x002f),
    Some(0x0030), Some(0x0031), Some(0x0032), Some(0x0033),
    Some(0x0034), Some(0x0035), Some(0x0036), Some(0x0037),
    Some(0x0038), Some(0x0039), Some(0x003a), Some(0x003b),
    Some(0x003c), Some(0x003d), Some(0x003e), Some(0x003f),
    Some(0x0040), Some(0x0041), Some(0x0042), Some(0x0043),
    Some(0x0044), Some(0x0045), Some(0x0046), Some(0x0047),
    Some(0x0048), Some(0x0049), Some(0x004a), Some(0x004b),
    Some(0x004c), Some(0x004d), Some(0x004e), Some(0x004f),
    Some(0x0050), Some(0x0051), Some(0x0052), Some(0x0053),
    Some(0x0054), Some(0x0055), Some(0x0056), Some(0x0057),
    Some(0x0058), Some(0x0059), Some(0x005a), Some(0x005b),
    Some(0x005c), Some(0x005d), Some(0x005e), Some(0x005f),
    Some(0x2018), Some(0x0061), Some(0x0062), Some(0x0063),
    Some(0x0064), Some(0x0065), Some(0x0066), Some(0x0067),
    Some(0x0068), Some(0x0069), Some(0x006a), Some(0x006b),
    Some(0x006c), Some(0x006d), Some(0x006e), Some(0x006f),
    Some(0x0070), Some(0x0071), Some(0x0072), Some(0x0073),
    Some(0x0074), Some(0x0075), Some(0x0076), Some(0x0077),
    Some(0x0078), Some(0x0079), Some(0x007a), Some(0x007b),
    Some(0x007c), Some(0x007d), Some(0x007e), None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, Some(0x00a1), Some(0x00a2), Some(0x00a3),
    Some(0x2044), Some(0x00a5), Some(0x0192), Some(0x00a7),
    Some(0x00a4), Some(0x0027), Some(0x201c), Some(0x00ab),
    Some(0x2039), Some(0x203a), Some(0xfb01), Some(0xfb02),
    None, Some(0x2013), Some(0x2020), Some(0x2021),
    Some(0x00b7), None, Some(0x00b6), Some(0x2022),
    Some(0x201a), Some(0x201e), Some(0x201d), Some(0x00bb),
    Some(0x2026), Some(0x2030), None, Some(0x00bf),
    None, Some(0x0060), Some(0x00b4), Some(0x02c6),
    Some(0x02dc), Some(0x00af), Some(0x02d8), Some(0x02d9),
    Some(0x00a8), None, Some(0x02da), Some(0x00b8),
    None, Some(0x02dd), Some(0x02db), Some(0x02c7),
    Some(0x2014), None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, Some(0x00c6), None, Some(0x00aa),
    None, None, None, None,
    Some(0x0141), Some(0x00d8), Some(0x0152), Some(0x00ba),
    None, None, None, None,
    None, Some(0x00e6), None, None,
    None, Some(0x0131), None, None,
    Some(0x0142), Some(0x00f8), Some(0x0153), Some(0x00df),
    None, None, None, None,
];

/// WinAnsiEncoding — Windows code page 1252.
pub(crate) static WIN_ANSI_ENCODING: CodeToUnicode = [
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    Some(0x0020), Some(0x0021), Some(0x0022), Some(0x0023),
    Some(0x0024), Some(0x0025), Some(0x0026), Some(0x0027),
    Some(0x0028), Some(0x0029), Some(0x002a), Some(0x002b),
    Some(0x002c), Some(0x002d), Some(0x002e), Some(0x002f),
    Some(0x0030), Some(0x0031), Some(0x0032), Some(0x0033),
    Some(0x0034), Some(0x0035), Some(0x0036), Some(0x0037),
    Some(0x0038), Some(0x0039), Some(0x003a), Some(0x003b),
    Some(0x003c), Some(0x003d), Some(0x003e), Some(0x003f),
    Some(0x0040), Some(0x0041), Some(0x0042), Some(0x0043),
    Some(0x0044), Some(0x0045), Some(0x0046), Some(0x0047),
    Some(0x0048), Some(0x0049), Some(0x004a), Some(0x004b),
    Some(0x004c), Some(0x004d), Some(0x004e), Some(0x004f),
    Some(0x0050), Some(0x0051), Some(0x0052), Some(0x0053),
    Some(0x0054), Some(0x0055), Some(0x0056), Some(0x0057),
    Some(0x0058), Some(0x0059), Some(0x005a), Some(0x005b),
    Some(0x005c), Some(0x005d), Some(0x005e), Some(0x005f),
    Some(0x0060), Some(0x0061), Some(0x0062), Some(0x0063),
    Some(0x0064), Some(0x0065), Some(0x0066), Some(0x0067),
    Some(0x0068), Some(0x0069), Some(0x006a), Some(0x006b),
    Some(0x006c), Some(0x006d), Some(0x006e), Some(0x006f),
    Some(0x0070), Some(0x0071), Some(0x0072), Some(0x0073),
    Some(0x0074), Some(0x0075), Some(0x0076), Some(0x0077),
    Some(0x0078), Some(0x0079), Some(0x007a), Some(0x007b),
    Some(0x007c), Some(0x007d), Some(0x007e), Some(0x2022),
    Some(0x20ac), Some(0x2022), Some(0x201a), Some(0x0192),
    Some(0x201e), Some(0x2026), Some(0x2020), Some(0x2021),
    Some(0x02c6), Some(0x2030), Some(0x0160), Some(0x2039),
    Some(0x0152), Some(0x2022), Some(0x017d), Some(0x2022),
    Some(0x2022), Some(0x2018), Some(0x2019), Some(0x201c),
    Some(0x201d), Some(0x2022), Some(0x2013), Some(0x2014),
    Some(0x02dc), Some(0x2122), Some(0x0161), Some(0x203a),
    Some(0x0153), Some(0x2022), Some(0x017e), Some(0x0178),
    Some(0x0020), Some(0x00a1), Some(0x00a2), Some(0x00a3),
    Some(0x00a4), Some(0x00a5), Some(0x00a6), Some(0x00a7),
    Some(0x00a8), Some(0x00a9), Some(0x00aa), Some(0x00ab),
    Some(0x00ac), Some(0x002d), Some(0x00ae), Some(0x00af),
    Some(0x00b0), Some(0x00b1), Some(0x00b2), Some(0x00b3),
    Some(0x00b4), Some(0x00b5), Some(0x00b6), Some(0x00b7),
    Some(0x00b8), Some(0x00b9), Some(0x00ba), Some(0x00bb),
    Some(0x00bc), Some(0x00bd), Some(0x00be), Some(0x00bf),
    Some(0x00c0), Some(0x00c1), Some(0x00c2), Some(0x00c3),
    Some(0x00c4), Some(0x00c5), Some(0x00c6), Some(0x00c7),
    Some(0x00c8), Some(0x00c9), Some(0x00ca), Some(0x00cb),
    Some(0x00cc), Some(0x00cd), Some(0x00ce), Some(0x00cf),
    Some(0x00d0), Some(0x00d1), Some(0x00d2), Some(0x00d3),
    Some(0x00d4), Some(0x00d5), Some(0x00d6), Some(0x00d7),
    Some(0x00d8), Some(0x00d9), Some(0x00da), Some(0x00db),
    Some(0x00dc), Some(0x00dd), Some(0x00de), Some(0x00df),
    Some(0x00e0), Some(0x00e1), Some(0x00e2), Some(0x00e3),
    Some(0x00e4), Some(0x00e5), Some(0x00e6), Some(0x00e7),
    Some(0x00e8), Some(0x00e9), Some(0x00ea), Some(0x00eb),
    Some(0x00ec), Some(0x00ed), Some(0x00ee), Some(0x00ef),
    Some(0x00f0), Some(0x00f1), Some(0x00f2), Some(0x00f3),
    Some(0x00f4), Some(0x00f5), Some(0x00f6), Some(0x00f7),
    Some(0x00f8), Some(0x00f9), Some(0x00fa), Some(0x00fb),
    Some(0x00fc), Some(0x00fd), Some(0x00fe), Some(0x00ff),
];

/// MacRomanEncoding — the Mac OS standard Roman set.
pub(crate) static MAC_ROMAN_ENCODING: CodeToUnicode = [
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    Some(0x0020), Some(0x0021), Some(0x0022), Some(0x0023),
    Some(0x0024), Some(0x0025), Some(0x0026), Some(0x0027),
    Some(0x0028), Some(0x0029), Some(0x002a), Some(0x002b),
    Some(0x002c), Some(0x002d), Some(0x002e), Some(0x002f),
    Some(0x0030), Some(0x0031), Some(0x0032), Some(0x0033),
    Some(0x0034), Some(0x0035), Some(0x0036), Some(0x0037),
    Some(0x0038), Some(0x0039), Some(0x003a), Some(0x003b),
    Some(0x003c), Some(0x003d), Some(0x003e), Some(0x003f),
    Some(0x0040), Some(0x0041), Some(0x0042), Some(0x0043),
    Some(0x0044), Some(0x0045), Some(0x0046), Some(0x0047),
    Some(0x0048), Some(0x0049), Some(0x004a), Some(0x004b),
    Some(0x004c), Some(0x004d), Some(0x004e), Some(0x004f),
    Some(0x0050), Some(0x0051), Some(0x0052), Some(0x0053),
    Some(0x0054), Some(0x0055), Some(0x0056), Some(0x0057),
    Some(0x0058), Some(0x0059), Some(0x005a), Some(0x005b),
    Some(0x005c), Some(0x005d), Some(0x005e), Some(0x005f),
    Some(0x0060), Some(0x0061), Some(0x0062), Some(0x0063),
    Some(0x0064), Some(0x0065), Some(0x0066), Some(0x0067),
    Some(0x0068), Some(0x0069), Some(0x006a), Some(0x006b),
    Some(0x006c), Some(0x006d), Some(0x006e), Some(0x006f),
    Some(0x0070), Some(0x0071), Some(0x0072), Some(0x0073),
    Some(0x0074), Some(0x0075), Some(0x0076), Some(0x0077),
    Some(0x0078), Some(0x0079), Some(0x007a), Some(0x007b),
    Some(0x007c), Some(0x007d), Some(0x007e), None,
    Some(0x00c4), Some(0x00c5), Some(0x00c7), Some(0x00c9),
    Some(0x00d1), Some(0x00d6), Some(0x00dc), Some(0x00e1),
    Some(0x00e0), Some(0x00e2), Some(0x00e4), Some(0x00e3),
    Some(0x00e5), Some(0x00e7), Some(0x00e9), Some(0x00e8),
    Some(0x00ea), Some(0x00eb), Some(0x00ed), Some(0x00ec),
    Some(0x00ee), Some(0x00ef), Some(0x00f1), Some(0x00f3),
    Some(0x00f2), Some(0x00f4), Some(0x00f6), Some(0x00f5),
    Some(0x00fa), Some(0x00f9), Some(0x00fb), Some(0x00fc),
    Some(0x2020), Some(0x00b0), Some(0x00a2), Some(0x00a3),
    Some(0x00a7), Some(0x2022), Some(0x00b6), Some(0x00df),
    Some(0x00ae), Some(0x00a9), Some(0x2122), Some(0x00b4),
    Some(0x00a8), Some(0x2260), Some(0x00c6), Some(0x00d8),
    Some(0x221e), Some(0x00b1), Some(0x2264), Some(0x2265),
    Some(0x00a5), Some(0x00b5), Some(0x2202), Some(0x2211),
    Some(0x220f), Some(0x03c0), Some(0x222b), Some(0x00aa),
    Some(0x00ba), Some(0x2126), Some(0x00e6), Some(0x00f8),
    Some(0x00bf), Some(0x00a1), Some(0x00ac), Some(0x221a),
    Some(0x0192), Some(0x2248), Some(0x2206), Some(0x00ab),
    Some(0x00bb), Some(0x2026), Some(0x0020), Some(0x00c0),
    Some(0x00c3), Some(0x00d5), Some(0x0152), Some(0x0153),
    Some(0x2013), Some(0x2014), Some(0x201c), Some(0x201d),
    Some(0x2018), Some(0x2019), Some(0x00f7), Some(0x25ca),
    Some(0x00ff), Some(0x0178), Some(0x2044), Some(0x00a4),
    Some(0x2039), Some(0x203a), Some(0xfb01), Some(0xfb02),
    Some(0x2021), Some(0x00b7), Some(0x201a), Some(0x201e),
    Some(0x2030), Some(0x00c2), Some(0x00ca), Some(0x00c1),
    Some(0x00cb), Some(0x00c8), Some(0x00cd), Some(0x00ce),
    Some(0x00cf), Some(0x00cc), Some(0x00d3), Some(0x00d4),
    Some(0xf8ff), Some(0x00d2), Some(0x00da), Some(0x00db),
    Some(0x00d9), Some(0x0131), Some(0x02c6), Some(0x02dc),
    Some(0x00af), Some(0x02d8), Some(0x02d9), Some(0x02da),
    Some(0x00b8), Some(0x02dd), Some(0x02db), Some(0x02c7),
];

/// The built-in encoding of the Symbol font.
pub(crate) static SYMBOL_ENCODING: CodeToUnicode = [
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    Some(0x0020), Some(0x0021), Some(0x2200), Some(0x0023),
    Some(0x2203), Some(0x0025), Some(0x0026), Some(0x220b),
    Some(0x0028), Some(0x0029), Some(0x2217), Some(0x002b),
    Some(0x002c), Some(0x2212), Some(0x002e), Some(0x002f),
    Some(0x0030), Some(0x0031), Some(0x0032), Some(0x0033),
    Some(0x0034), Some(0x0035), Some(0x0036), Some(0x0037),
    Some(0x0038), Some(0x0039), Some(0x003a), Some(0x003b),
    Some(0x003c), Some(0x003d), Some(0x003e), Some(0x003f),
    Some(0x2245), Some(0x0391), Some(0x0392), Some(0x03a7),
    Some(0x2206), Some(0x0395), Some(0x03a6), Some(0x0393),
    Some(0x0397), Some(0x0399), Some(0x03d1), Some(0x039a),
    Some(0x039b), Some(0x039c), Some(0x039d), Some(0x039f),
    Some(0x03a0), Some(0x0398), Some(0x03a1), Some(0x03a3),
    Some(0x03a4), Some(0x03a5), Some(0x03c2), Some(0x2126),
    Some(0x039e), Some(0x03a8), Some(0x0396), Some(0x005b),
    Some(0x2234), Some(0x005d), Some(0x22a5), Some(0x005f),
    Some(0xf8e5), Some(0x03b1), Some(0x03b2), Some(0x03c7),
    Some(0x03b4), Some(0x03b5), Some(0x03c6), Some(0x03b3),
    Some(0x03b7), Some(0x03b9), Some(0x03d5), Some(0x03ba),
    Some(0x03bb), Some(0x00b5), Some(0x03bd), Some(0x03bf),
    Some(0x03c0), Some(0x03b8), Some(0x03c1), Some(0x03c3),
    Some(0x03c4), Some(0x03c5), Some(0x03d6), Some(0x03c9),
    Some(0x03be), Some(0x03c8), Some(0x03b6), Some(0x007b),
    Some(0x007c), Some(0x007d), Some(0x223c), None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, None, None, None,
    None, Some(0x03d2), Some(0x2032), Some(0x2264),
    Some(0x2044), Some(0x221e), Some(0x0192), Some(0x2663),
    Some(0x2662), Some(0x2661), Some(0x2660), Some(0x2194),
    Some(0x2190), Some(0x2191), Some(0x2192), Some(0x2193),
    Some(0x00b0), Some(0x00b1), Some(0x2033), Some(0x2265),
    Some(0x00d7), Some(0x221d), Some(0x2202), Some(0x2022),
    Some(0x00f7), Some(0x2260), Some(0x2261), Some(0x2248),
    Some(0x2026), Some(0x2195), Some(0xf8e7), Some(0x21b5),
    Some(0x2135), Some(0x2111), Some(0x211c), Some(0x2118),
    Some(0x2297), Some(0x2295), Some(0x2205), Some(0x2229),
    Some(0x222a), Some(0x2283), Some(0x2287), Some(0x2284),
    Some(0x2282), Some(0x2286), Some(0x2208), Some(0x2209),
    Some(0x2220), Some(0x2207), Some(0xf6da), Some(0xf6d9),
    Some(0xf6db), Some(0x220f), Some(0x221a), Some(0x22c5),
    Some(0x00ac), Some(0x2227), Some(0x2228), Some(0x21d4),
    Some(0x21d0), Some(0x21d1), Some(0x21d2), Some(0x21d3),
    Some(0x25ca), Some(0x2329), Some(0xf8e8), Some(0xf8e9),
    Some(0xf8ea), Some(0x2211), Some(0xf8eb), Some(0x007c),
    Some(0xf8ed), Some(0xf8ee), Some(0xf8ef), Some(0xf8f0),
    Some(0xf8f1), Some(0x007c), Some(0xf8f3), Some(0x007c),
    None, Some(0x232a), Some(0x222b), Some(0x2320),
    Some(0xf8f5), Some(0x2321), Some(0xf8f6), Some(0x007c),
    Some(0xf8f8), Some(0xf8f9), Some(0xf8fa), Some(0xf8fb),
    Some(0xf8fc), Some(0x2016), Some(0xf8fe), None,
];