exatok 0.1.1

Exact GPU-resident BPE tokenizer whose token ids match tiktoken byte for byte
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
//! GPT-2 byte-to-unicode encoder. OWNER: VocabLoad.
//!
//! Reconstructs the 256-entry byte-to-printable-codepoint map that byte-level
//! BPE uses to avoid invalid UTF-8, and its inverse for decoding.
//!
//! Acceptance: the map is a bijection over all 256 bytes and round-trips every
//! byte value, asserted against the reference mapping rather than merely for
//! self-consistency.
//!
//! # Mapping
//!
//! Bytes in the three printable Latin-1 runs (`0x21..=0x7E`, `0xA1..=0xAC`,
//! `0xAE..=0xFF`) stand for themselves. The other 68 bytes are lifted to
//! `256 + n`, where `n` counts them in ascending byte order. Nothing in the
//! image is a control character, a space, or a surrogate, so an encoded token
//! is always printable UTF-8 and a merge table can be stored as text.
//!
//! The lifted run is therefore exactly `256..=323`, which is why a HuggingFace
//! `vocab.json` shows a leading space as `Ġ` (`U+0120`, 288) and a soft hyphen
//! as `Ń` (`U+0143`, 323).

/// Number of byte values the forward map covers.
pub const ENCODER_LEN: usize = 256;

/// Bytes outside the printable Latin-1 runs, lifted to `256 + n`.
pub const REMAPPED_LEN: usize = 68;

/// Lowest codepoint the encoder emits: byte `0x21`, which maps to itself.
pub const MIN_CODEPOINT: u32 = 0x21;

/// Highest codepoint the encoder emits, `256 + REMAPPED_LEN - 1`.
pub const MAX_CODEPOINT: u32 = 323;

/// Length of the reverse table: every codepoint from `0` to [`MAX_CODEPOINT`].
pub const DECODER_LEN: usize = MAX_CODEPOINT as usize + 1;

/// Reverse-table slot for a codepoint the encoder never emits.
pub const NOT_A_BYTE: u16 = u16::MAX;

/// True when `byte` is printable Latin-1 and so encodes to itself.
///
/// The three runs are the ones GPT-2's `bytes_to_unicode` seeds its list with:
/// `'!'..='~'`, `'\u{A1}'..='\u{AC}'`, `'\u{AE}'..='\u{FF}'`. Widening or
/// narrowing any run renumbers every lifted codepoint and breaks every
/// vocabulary that was written with the reference map.
#[must_use]
#[allow(clippy::manual_range_contains)]
pub const fn maps_to_itself(byte: u8) -> bool {
    (byte >= b'!' && byte <= b'~') || (byte >= 0xA1 && byte <= 0xAC) || byte >= 0xAE
}

const fn build_byte_to_codepoint() -> [u16; ENCODER_LEN] {
    let mut table = [NOT_A_BYTE; ENCODER_LEN];

    let mut byte = 0usize;
    while byte < ENCODER_LEN {
        if maps_to_itself(byte as u8) {
            table[byte] = byte as u16;
        }
        byte += 1;
    }

    // Ascending byte order is load bearing: it fixes which lifted byte gets
    // which codepoint, and the reference map depends on that exact order.
    let mut next = ENCODER_LEN as u16;
    let mut byte = 0usize;
    while byte < ENCODER_LEN {
        if !maps_to_itself(byte as u8) {
            table[byte] = next;
            next += 1;
        }
        byte += 1;
    }

    table
}

const fn build_codepoint_to_byte() -> [u16; DECODER_LEN] {
    let mut table = [NOT_A_BYTE; DECODER_LEN];
    let mut byte = 0usize;
    while byte < ENCODER_LEN {
        table[BYTE_TO_CODEPOINT[byte] as usize] = byte as u16;
        byte += 1;
    }
    table
}

const fn build_byte_to_char() -> [char; ENCODER_LEN] {
    let mut table = ['\u{FFFD}'; ENCODER_LEN];
    let mut byte = 0usize;
    while byte < ENCODER_LEN {
        // Every codepoint here lies in `0x21..=0x143`, well below the surrogate
        // block, so `from_u32` always succeeds. The replacement character is an
        // unreachable placeholder that keeps this `const fn` panic free, and a
        // unit test asserts it never survives into the table.
        table[byte] = match char::from_u32(BYTE_TO_CODEPOINT[byte] as u32) {
            Some(character) => character,
            None => '\u{FFFD}',
        };
        byte += 1;
    }
    table
}

/// Forward map: byte value to its encoded codepoint.
pub const BYTE_TO_CODEPOINT: [u16; ENCODER_LEN] = build_byte_to_codepoint();

/// Forward map: byte value to its encoded character.
pub const BYTE_TO_CHAR: [char; ENCODER_LEN] = build_byte_to_char();

/// Inverse map: codepoint to byte value, [`NOT_A_BYTE`] where undefined.
pub const CODEPOINT_TO_BYTE: [u16; DECODER_LEN] = build_codepoint_to_byte();

/// Encoded codepoint for `byte`.
#[must_use]
pub const fn codepoint_of(byte: u8) -> u16 {
    BYTE_TO_CODEPOINT[byte as usize]
}

/// Encoded character for `byte`.
#[must_use]
pub const fn char_of(byte: u8) -> char {
    BYTE_TO_CHAR[byte as usize]
}

/// Byte that `codepoint` encodes, or `None` if it is outside the image.
#[must_use]
pub const fn byte_of_codepoint(codepoint: u32) -> Option<u8> {
    if codepoint as usize >= DECODER_LEN {
        return None;
    }
    let slot = CODEPOINT_TO_BYTE[codepoint as usize];
    if slot == NOT_A_BYTE {
        None
    } else {
        Some(slot as u8)
    }
}

/// Byte that `character` encodes, or `None` if it is outside the image.
#[must_use]
pub const fn byte_of_char(character: char) -> Option<u8> {
    byte_of_codepoint(character as u32)
}

/// A character in a token string that the GPT-2 alphabet never produces.
///
/// Carries the offending position so an artifact loader can name the line and
/// the character instead of silently dropping the token.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct UnmappableChar {
    /// Position of the offending character, counted in characters not bytes.
    pub char_index: usize,
    /// The character that has no byte preimage.
    pub character: char,
}

/// Encode `bytes` into its printable token-string form.
#[must_use]
pub fn encode(bytes: &[u8]) -> String {
    // Every encoded character is one or two UTF-8 bytes, never more.
    let mut out = String::with_capacity(bytes.len().saturating_mul(2));
    for &byte in bytes {
        out.push(char_of(byte));
    }
    out
}

/// Decode a token string back into the bytes it stands for.
///
/// Fails on the first character with no byte preimage rather than skipping it,
/// so a corrupt `vocab.json` entry cannot quietly become a shorter token.
pub fn decode(text: &str) -> Result<Vec<u8>, UnmappableChar> {
    let mut out = Vec::with_capacity(text.len());
    for (char_index, character) in text.chars().enumerate() {
        match byte_of_char(character) {
            Some(byte) => out.push(byte),
            None => {
                return Err(UnmappableChar {
                    char_index,
                    character,
                });
            }
        }
    }
    Ok(out)
}

#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
mod tests {
    use super::*;
    use std::collections::BTreeSet;

    fn all_bytes() -> Vec<u8> {
        (0..=u8::MAX).collect()
    }

    /// Locks the three self-mapping runs to their reference boundaries.
    ///
    /// Every lifted codepoint is numbered by counting the bytes these runs
    /// reject, so widening a run by one byte shifts `Ġ` off `U+0120` and
    /// renumbers the tail of every published vocabulary.
    #[test]
    fn self_mapping_runs_end_on_the_reference_boundaries() {
        assert!(!maps_to_itself(0x20), "space must be lifted, not self mapped");
        assert!(maps_to_itself(0x21));
        assert!(maps_to_itself(0x7E));
        assert!(!maps_to_itself(0x7F), "DEL must be lifted");
        assert!(!maps_to_itself(0xA0), "NBSP must be lifted");
        assert!(maps_to_itself(0xA1));
        assert!(maps_to_itself(0xAC));
        assert!(
            !maps_to_itself(0xAD),
            "soft hyphen is the hole between the two high runs"
        );
        assert!(maps_to_itself(0xAE));
        assert!(maps_to_itself(0xFF));
    }

    /// Pins specific bytes to the exact codepoints GPT-2 publishes.
    ///
    /// This is the assertion that catches a plausible but wrong map, for
    /// example one that lifts bytes in descending order or starts the lifted
    /// run at 255. Such a map still round-trips, so only absolute values
    /// detect it. A regression here silently retokenizes every input.
    #[test]
    fn known_bytes_encode_to_the_published_codepoints() {
        assert_eq!(codepoint_of(0x00), 256);
        assert_eq!(codepoint_of(0x09), 265, "tab");
        assert_eq!(codepoint_of(0x0A), 266, "newline");
        assert_eq!(codepoint_of(0x20), 288, "space becomes U+0120");
        assert_eq!(codepoint_of(0x21), 0x21, "'!' is self mapped");
        assert_eq!(codepoint_of(0x7E), 0x7E, "'~' is self mapped");
        assert_eq!(codepoint_of(0x7F), 289, "DEL is the byte after space's run");
        assert_eq!(codepoint_of(0xA0), 322, "NBSP becomes U+0142");
        assert_eq!(codepoint_of(0xA1), 0xA1);
        assert_eq!(codepoint_of(0xAC), 0xAC);
        assert_eq!(codepoint_of(0xAD), 323, "soft hyphen is the last lifted byte");
        assert_eq!(codepoint_of(0xAE), 0xAE);
        assert_eq!(codepoint_of(0xFF), 0xFF);

        // Asserted numerically as well as by character: U+0141 and U+0142 are
        // near-identical homoglyphs, so a wrong char literal survives review.
        assert_eq!((codepoint_of(0x20), char_of(0x20)), (0x120, 'Ġ'));
        assert_eq!((codepoint_of(0x0A), char_of(0x0A)), (0x10A, 'Ċ'));
        assert_eq!((codepoint_of(0x9E), char_of(0x9E)), (0x140, 'ŀ'));
        assert_eq!((codepoint_of(0x9F), char_of(0x9F)), (0x141, 'Ł'));
        assert_eq!(
            (codepoint_of(0xA0), char_of(0xA0)),
            (0x142, 'ł'),
            "NBSP is U+0142 LOWERCASE l with stroke, not U+0141"
        );
        assert_eq!((codepoint_of(0xAD), char_of(0xAD)), (0x143, 'Ń'));
        assert_eq!(char_of(b'a'), 'a');
    }

    /// All 256 byte values, pinned to the exact codepoint the reference alphabet
    /// gives each one, as literal values.
    ///
    /// This map IS the initial-state stage of the tokenizer, one step removed. A
    /// byte's codepoint decides which vocabulary entry that byte's single-byte
    /// token is, and that entry's rank is the byte's INITIAL TOKEN ID, which is
    /// where byte-level BPE starts merging: one such id per input byte. So a
    /// single wrong codepoint here does not produce a visibly broken output. It
    /// silently seeds one position with a different id, every merge that touches
    /// that position then takes a different branch, and the encode returns a
    /// perfectly plausible id stream for a string the caller never supplied.
    /// Nothing downstream can detect it: the loop cannot tell a wrong start
    /// state from a different input, and the output still decodes to text.
    ///
    /// Hence literal values for every byte, rather than a count, a round trip, or
    /// a derivation from this module's own [`maps_to_itself`] predicate. The
    /// neighbouring tests are all self-consistent in that sense: they check the
    /// image size, the run boundaries at ten sampled bytes, and thirteen named
    /// codepoints. That leaves a real gap, and here is the witness for it.
    /// Transposing two adjacent self-mapping bytes, `0x4D` and `0x4E`, so that
    /// `M` encodes as `N` and `N` as `M`, keeps the map a bijection, keeps the
    /// image and its bounds identical, keeps the lifted run at `256..=323`, and
    /// touches no sampled byte. Every one of the nine other tests in this module
    /// passes under that mutation and this one fails. It would silently
    /// retokenize every input containing an `M` or an `N`.
    ///
    /// `EXPECTED` is GPT-2's `bytes_to_unicode()` transcribed and run
    /// independently of this module, and it is anchored to published artifacts
    /// rather than only to that transcription: sorting `0..=255` by codepoint
    /// reproduces, exactly, the rank order of the 256 single-byte entries in
    /// `cl100k_base`, `r50k_base`, `p50k_base` and `o200k_base`. That is the
    /// property `tests/vocab_artifacts.rs` and `tests/vocab_real_artifacts.rs`
    /// check against the artifacts themselves, so the numbers below are not a
    /// second opinion from the same source as the code.
    #[test]
    fn every_byte_maps_to_its_published_codepoint_and_back() {
        /// Byte value to codepoint, in ascending byte order, sixteen per row.
        #[rustfmt::skip]
        const EXPECTED: [u16; ENCODER_LEN] = [
            256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271,
            272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287,
            288,  33,  34,  35,  36,  37,  38,  39,  40,  41,  42,  43,  44,  45,  46,  47,
             48,  49,  50,  51,  52,  53,  54,  55,  56,  57,  58,  59,  60,  61,  62,  63,
             64,  65,  66,  67,  68,  69,  70,  71,  72,  73,  74,  75,  76,  77,  78,  79,
             80,  81,  82,  83,  84,  85,  86,  87,  88,  89,  90,  91,  92,  93,  94,  95,
             96,  97,  98,  99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111,
            112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 289,
            290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305,
            306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321,
            322, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 323, 174, 175,
            176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191,
            192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207,
            208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223,
            224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239,
            240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255,
        ];

        for byte in 0..=u8::MAX {
            let expected = EXPECTED[byte as usize];
            assert_eq!(
                codepoint_of(byte),
                expected,
                "byte 0x{byte:02x} must encode to codepoint {expected}"
            );
            // Derived from EXPECTED and not from BYTE_TO_CODEPOINT, so the
            // character table is pinned independently of the numeric one: a build
            // that populated one from a stale copy of the other fails here.
            let character = char::from_u32(u32::from(expected)).unwrap();
            assert_eq!(
                char_of(byte),
                character,
                "byte 0x{byte:02x} must encode to {character:?}"
            );
            assert_eq!(
                byte_of_codepoint(u32::from(expected)),
                Some(byte),
                "codepoint {expected} must decode back to byte 0x{byte:02x}"
            );
            assert_eq!(
                byte_of_char(character),
                Some(byte),
                "{character:?} must decode back to byte 0x{byte:02x}"
            );
        }

        // Exhaustive in the other direction too, so no codepoint outside the
        // image acquires a preimage. Stated over EXPECTED rather than over
        // `maps_to_itself`, for the same reason as above.
        for codepoint in 0..=MAX_CODEPOINT {
            let preimage = EXPECTED
                .iter()
                .position(|&cp| u32::from(cp) == codepoint)
                .map(|index| index as u8);
            assert_eq!(
                byte_of_codepoint(codepoint),
                preimage,
                "codepoint {codepoint} must have exactly the preimage EXPECTED gives it"
            );
        }
    }

    /// Proves injectivity by counting distinct images, not by trusting shape.
    ///
    /// Two bytes sharing a codepoint would make decoding ambiguous and would
    /// map two different tokens onto one vocabulary entry.
    #[test]
    fn forward_map_is_injective_with_the_exact_expected_image() {
        let image: BTreeSet<u16> = all_bytes().iter().map(|&b| codepoint_of(b)).collect();
        assert_eq!(image.len(), ENCODER_LEN, "256 bytes need 256 codepoints");
        assert_eq!(image.iter().copied().min(), Some(MIN_CODEPOINT as u16));
        assert_eq!(image.iter().copied().max(), Some(MAX_CODEPOINT as u16));
    }

    /// Fixes the lifted run at a contiguous `256..=323` of exactly 68 bytes.
    ///
    /// A gap or an extra entry here means the run boundaries drifted, which
    /// changes token strings without changing their length.
    #[test]
    fn lifted_bytes_form_a_contiguous_run_of_sixty_eight() {
        let lifted: Vec<u16> = all_bytes()
            .into_iter()
            .filter(|&b| !maps_to_itself(b))
            .map(codepoint_of)
            .collect();
        assert_eq!(lifted.len(), REMAPPED_LEN);
        let expected: Vec<u16> = (256..=323).collect();
        assert_eq!(lifted, expected, "lifted codepoints must ascend by byte");
    }

    /// Round-trips all 256 bytes through encode then decode.
    ///
    /// Guards the inverse table against being built from a stale forward table:
    /// a single wrong slot would corrupt one byte in every decoded output.
    #[test]
    fn every_byte_round_trips_through_the_inverse() {
        let bytes = all_bytes();
        let encoded = encode(&bytes);
        assert_eq!(encoded.chars().count(), 256);
        assert_eq!(decode(&encoded), Ok(bytes.clone()));

        for byte in bytes {
            assert_eq!(byte_of_char(char_of(byte)), Some(byte));
            assert_eq!(byte_of_codepoint(u32::from(codepoint_of(byte))), Some(byte));
        }
    }

    /// Pins the encoded byte length of the whole alphabet.
    ///
    /// 94 self-mapped ASCII characters cost one byte, the other 162 cost two,
    /// so the alphabet is 418 UTF-8 bytes. A wrong total means some byte
    /// escaped into a three-byte codepoint, which no reference vocabulary has.
    #[test]
    fn encoded_alphabet_is_exactly_418_utf8_bytes() {
        let encoded = encode(&all_bytes());
        assert_eq!(encoded.len(), 418);
        assert_eq!(encoded.chars().filter(char::is_ascii).count(), 94);
    }

    /// Rejects codepoints outside the image instead of aliasing them to bytes.
    ///
    /// Control characters, space, NBSP and the soft hyphen are exactly the
    /// codepoints a naive decoder would accept as themselves, which would let
    /// a raw space in a `vocab.json` token parse as byte `0x20` and diverge
    /// from the reference encoding.
    #[test]
    fn codepoints_outside_the_image_have_no_byte() {
        // The holes, stated from the mapping rather than sampled: these bytes
        // were lifted, so their raw codepoints have no preimage.
        let is_hole = |cp: u32| cp <= 0x20 || (0x7F..=0xA0).contains(&cp) || cp == 0xAD;

        for codepoint in 0..=MAX_CODEPOINT {
            let decoded = byte_of_codepoint(codepoint);
            if is_hole(codepoint) {
                assert_eq!(decoded, None, "codepoint {codepoint} must be a hole");
            } else {
                assert_eq!(
                    decoded.map(codepoint_of).map(u32::from),
                    Some(codepoint),
                    "codepoint {codepoint} must decode and re-encode to itself"
                );
            }
        }

        // Named cases, so the intent survives an edit to the predicate.
        assert_eq!(byte_of_codepoint(0x20), None, "raw space must not decode");
        assert_eq!(byte_of_codepoint(0xA0), None, "raw NBSP must not decode");
        assert_eq!(byte_of_codepoint(0xAD), None, "raw soft hyphen must not decode");
        assert_eq!(byte_of_codepoint(0x21), Some(0x21), "'!' self-maps");
        assert_eq!(byte_of_codepoint(0xFF), Some(0xFF));
        assert_eq!(byte_of_codepoint(MAX_CODEPOINT), Some(0xAD));
        assert_eq!(byte_of_codepoint(MAX_CODEPOINT + 1), None);
        assert_eq!(byte_of_codepoint(0x10_FFFF), None);

        // 324 codepoints up to the ceiling, 256 in the image, so 68 holes:
        // exactly one per lifted byte.
        let holes = (0..=MAX_CODEPOINT)
            .filter(|&cp| byte_of_codepoint(cp).is_none())
            .count();
        assert_eq!(holes, REMAPPED_LEN, "one hole per lifted byte");
    }

    /// Reports the first unmappable character with its exact index.
    ///
    /// A loader needs the position to name the offending artifact line; an
    /// error that only says "bad token" cannot be acted on, and silently
    /// skipping the character would truncate the token.
    ///
    /// The image is NOT a contiguous range. Its holes below [`MAX_CODEPOINT`]
    /// are `0..=0x20`, `0x7F..=0xA0` and `0xAD`, precisely because those bytes
    /// were lifted, so their raw codepoints are absent. Anything in a hole must
    /// be rejected even though it looks like ordinary printable text.
    #[test]
    fn decode_names_the_first_unmappable_character() {
        assert_eq!(
            decode("abĠ\nc"),
            Err(UnmappableChar {
                char_index: 3,
                character: '\n',
            })
        );
        assert_eq!(
            decode(" a"),
            Err(UnmappableChar {
                char_index: 0,
                character: ' ',
            }),
            "a raw space must be rejected: the alphabet writes it as Ġ"
        );
        assert_eq!(
            decode("a\u{AD}"),
            Err(UnmappableChar {
                char_index: 1,
                character: '\u{AD}',
            }),
            "U+00AD is below MAX_CODEPOINT but absent from the image: byte 0xAD \
             was lifted to U+0143, so nothing encodes to U+00AD"
        );
        assert_eq!(
            decode("a\u{7F}"),
            Err(UnmappableChar {
                char_index: 1,
                character: '\u{7F}',
            }),
            "DEL is a hole for the same reason"
        );
        assert_eq!(
            decode("aé"),
            Ok(vec![b'a', 0xE9]),
            "U+00E9 is 233 and byte 233 self-maps, so it IS in the image"
        );
        assert_eq!(decode(""), Ok(Vec::new()));
    }

    /// Proves the unreachable `char::from_u32` fallback never fires.
    ///
    /// If the forward table ever emitted a surrogate the const map would
    /// quietly substitute U+FFFD and two bytes would collide on it.
    #[test]
    fn no_encoded_character_is_the_replacement_char() {
        assert!(
            !BYTE_TO_CHAR.contains(&'\u{FFFD}'),
            "replacement char in the table means a codepoint failed to convert"
        );
        assert_eq!(BYTE_TO_CHAR.len(), ENCODER_LEN);
        assert_eq!(CODEPOINT_TO_BYTE.len(), DECODER_LEN);
        assert_eq!(
            CODEPOINT_TO_BYTE.iter().filter(|&&s| s != NOT_A_BYTE).count(),
            ENCODER_LEN,
            "inverse table must define exactly 256 slots"
        );
    }
}