pub const ENCODER_LEN: usize = 256;
pub const REMAPPED_LEN: usize = 68;
pub const MIN_CODEPOINT: u32 = 0x21;
pub const MAX_CODEPOINT: u32 = 323;
pub const DECODER_LEN: usize = MAX_CODEPOINT as usize + 1;
pub const NOT_A_BYTE: u16 = u16::MAX;
#[must_use]
#[allow(clippy::manual_range_contains)]
pub const fn maps_to_itself(byte: u8) -> bool {
(byte >= b'!' && byte <= b'~') || (byte >= 0xA1 && byte <= 0xAC) || byte >= 0xAE
}
const fn build_byte_to_codepoint() -> [u16; ENCODER_LEN] {
let mut table = [NOT_A_BYTE; ENCODER_LEN];
let mut byte = 0usize;
while byte < ENCODER_LEN {
if maps_to_itself(byte as u8) {
table[byte] = byte as u16;
}
byte += 1;
}
let mut next = ENCODER_LEN as u16;
let mut byte = 0usize;
while byte < ENCODER_LEN {
if !maps_to_itself(byte as u8) {
table[byte] = next;
next += 1;
}
byte += 1;
}
table
}
const fn build_codepoint_to_byte() -> [u16; DECODER_LEN] {
let mut table = [NOT_A_BYTE; DECODER_LEN];
let mut byte = 0usize;
while byte < ENCODER_LEN {
table[BYTE_TO_CODEPOINT[byte] as usize] = byte as u16;
byte += 1;
}
table
}
const fn build_byte_to_char() -> [char; ENCODER_LEN] {
let mut table = ['\u{FFFD}'; ENCODER_LEN];
let mut byte = 0usize;
while byte < ENCODER_LEN {
table[byte] = match char::from_u32(BYTE_TO_CODEPOINT[byte] as u32) {
Some(character) => character,
None => '\u{FFFD}',
};
byte += 1;
}
table
}
pub const BYTE_TO_CODEPOINT: [u16; ENCODER_LEN] = build_byte_to_codepoint();
pub const BYTE_TO_CHAR: [char; ENCODER_LEN] = build_byte_to_char();
pub const CODEPOINT_TO_BYTE: [u16; DECODER_LEN] = build_codepoint_to_byte();
#[must_use]
pub const fn codepoint_of(byte: u8) -> u16 {
BYTE_TO_CODEPOINT[byte as usize]
}
#[must_use]
pub const fn char_of(byte: u8) -> char {
BYTE_TO_CHAR[byte as usize]
}
#[must_use]
pub const fn byte_of_codepoint(codepoint: u32) -> Option<u8> {
if codepoint as usize >= DECODER_LEN {
return None;
}
let slot = CODEPOINT_TO_BYTE[codepoint as usize];
if slot == NOT_A_BYTE {
None
} else {
Some(slot as u8)
}
}
#[must_use]
pub const fn byte_of_char(character: char) -> Option<u8> {
byte_of_codepoint(character as u32)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct UnmappableChar {
pub char_index: usize,
pub character: char,
}
#[must_use]
pub fn encode(bytes: &[u8]) -> String {
let mut out = String::with_capacity(bytes.len().saturating_mul(2));
for &byte in bytes {
out.push(char_of(byte));
}
out
}
pub fn decode(text: &str) -> Result<Vec<u8>, UnmappableChar> {
let mut out = Vec::with_capacity(text.len());
for (char_index, character) in text.chars().enumerate() {
match byte_of_char(character) {
Some(byte) => out.push(byte),
None => {
return Err(UnmappableChar {
char_index,
character,
});
}
}
}
Ok(out)
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
mod tests {
use super::*;
use std::collections::BTreeSet;
fn all_bytes() -> Vec<u8> {
(0..=u8::MAX).collect()
}
#[test]
fn self_mapping_runs_end_on_the_reference_boundaries() {
assert!(!maps_to_itself(0x20), "space must be lifted, not self mapped");
assert!(maps_to_itself(0x21));
assert!(maps_to_itself(0x7E));
assert!(!maps_to_itself(0x7F), "DEL must be lifted");
assert!(!maps_to_itself(0xA0), "NBSP must be lifted");
assert!(maps_to_itself(0xA1));
assert!(maps_to_itself(0xAC));
assert!(
!maps_to_itself(0xAD),
"soft hyphen is the hole between the two high runs"
);
assert!(maps_to_itself(0xAE));
assert!(maps_to_itself(0xFF));
}
#[test]
fn known_bytes_encode_to_the_published_codepoints() {
assert_eq!(codepoint_of(0x00), 256);
assert_eq!(codepoint_of(0x09), 265, "tab");
assert_eq!(codepoint_of(0x0A), 266, "newline");
assert_eq!(codepoint_of(0x20), 288, "space becomes U+0120");
assert_eq!(codepoint_of(0x21), 0x21, "'!' is self mapped");
assert_eq!(codepoint_of(0x7E), 0x7E, "'~' is self mapped");
assert_eq!(codepoint_of(0x7F), 289, "DEL is the byte after space's run");
assert_eq!(codepoint_of(0xA0), 322, "NBSP becomes U+0142");
assert_eq!(codepoint_of(0xA1), 0xA1);
assert_eq!(codepoint_of(0xAC), 0xAC);
assert_eq!(codepoint_of(0xAD), 323, "soft hyphen is the last lifted byte");
assert_eq!(codepoint_of(0xAE), 0xAE);
assert_eq!(codepoint_of(0xFF), 0xFF);
assert_eq!((codepoint_of(0x20), char_of(0x20)), (0x120, 'Ġ'));
assert_eq!((codepoint_of(0x0A), char_of(0x0A)), (0x10A, 'Ċ'));
assert_eq!((codepoint_of(0x9E), char_of(0x9E)), (0x140, 'ŀ'));
assert_eq!((codepoint_of(0x9F), char_of(0x9F)), (0x141, 'Ł'));
assert_eq!(
(codepoint_of(0xA0), char_of(0xA0)),
(0x142, 'ł'),
"NBSP is U+0142 LOWERCASE l with stroke, not U+0141"
);
assert_eq!((codepoint_of(0xAD), char_of(0xAD)), (0x143, 'Ń'));
assert_eq!(char_of(b'a'), 'a');
}
#[test]
fn every_byte_maps_to_its_published_codepoint_and_back() {
#[rustfmt::skip]
const EXPECTED: [u16; ENCODER_LEN] = [
256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271,
272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287,
288, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,
64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79,
80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95,
96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111,
112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 289,
290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305,
306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321,
322, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 323, 174, 175,
176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191,
192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207,
208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223,
224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239,
240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255,
];
for byte in 0..=u8::MAX {
let expected = EXPECTED[byte as usize];
assert_eq!(
codepoint_of(byte),
expected,
"byte 0x{byte:02x} must encode to codepoint {expected}"
);
let character = char::from_u32(u32::from(expected)).unwrap();
assert_eq!(
char_of(byte),
character,
"byte 0x{byte:02x} must encode to {character:?}"
);
assert_eq!(
byte_of_codepoint(u32::from(expected)),
Some(byte),
"codepoint {expected} must decode back to byte 0x{byte:02x}"
);
assert_eq!(
byte_of_char(character),
Some(byte),
"{character:?} must decode back to byte 0x{byte:02x}"
);
}
for codepoint in 0..=MAX_CODEPOINT {
let preimage = EXPECTED
.iter()
.position(|&cp| u32::from(cp) == codepoint)
.map(|index| index as u8);
assert_eq!(
byte_of_codepoint(codepoint),
preimage,
"codepoint {codepoint} must have exactly the preimage EXPECTED gives it"
);
}
}
#[test]
fn forward_map_is_injective_with_the_exact_expected_image() {
let image: BTreeSet<u16> = all_bytes().iter().map(|&b| codepoint_of(b)).collect();
assert_eq!(image.len(), ENCODER_LEN, "256 bytes need 256 codepoints");
assert_eq!(image.iter().copied().min(), Some(MIN_CODEPOINT as u16));
assert_eq!(image.iter().copied().max(), Some(MAX_CODEPOINT as u16));
}
#[test]
fn lifted_bytes_form_a_contiguous_run_of_sixty_eight() {
let lifted: Vec<u16> = all_bytes()
.into_iter()
.filter(|&b| !maps_to_itself(b))
.map(codepoint_of)
.collect();
assert_eq!(lifted.len(), REMAPPED_LEN);
let expected: Vec<u16> = (256..=323).collect();
assert_eq!(lifted, expected, "lifted codepoints must ascend by byte");
}
#[test]
fn every_byte_round_trips_through_the_inverse() {
let bytes = all_bytes();
let encoded = encode(&bytes);
assert_eq!(encoded.chars().count(), 256);
assert_eq!(decode(&encoded), Ok(bytes.clone()));
for byte in bytes {
assert_eq!(byte_of_char(char_of(byte)), Some(byte));
assert_eq!(byte_of_codepoint(u32::from(codepoint_of(byte))), Some(byte));
}
}
#[test]
fn encoded_alphabet_is_exactly_418_utf8_bytes() {
let encoded = encode(&all_bytes());
assert_eq!(encoded.len(), 418);
assert_eq!(encoded.chars().filter(char::is_ascii).count(), 94);
}
#[test]
fn codepoints_outside_the_image_have_no_byte() {
let is_hole = |cp: u32| cp <= 0x20 || (0x7F..=0xA0).contains(&cp) || cp == 0xAD;
for codepoint in 0..=MAX_CODEPOINT {
let decoded = byte_of_codepoint(codepoint);
if is_hole(codepoint) {
assert_eq!(decoded, None, "codepoint {codepoint} must be a hole");
} else {
assert_eq!(
decoded.map(codepoint_of).map(u32::from),
Some(codepoint),
"codepoint {codepoint} must decode and re-encode to itself"
);
}
}
assert_eq!(byte_of_codepoint(0x20), None, "raw space must not decode");
assert_eq!(byte_of_codepoint(0xA0), None, "raw NBSP must not decode");
assert_eq!(byte_of_codepoint(0xAD), None, "raw soft hyphen must not decode");
assert_eq!(byte_of_codepoint(0x21), Some(0x21), "'!' self-maps");
assert_eq!(byte_of_codepoint(0xFF), Some(0xFF));
assert_eq!(byte_of_codepoint(MAX_CODEPOINT), Some(0xAD));
assert_eq!(byte_of_codepoint(MAX_CODEPOINT + 1), None);
assert_eq!(byte_of_codepoint(0x10_FFFF), None);
let holes = (0..=MAX_CODEPOINT)
.filter(|&cp| byte_of_codepoint(cp).is_none())
.count();
assert_eq!(holes, REMAPPED_LEN, "one hole per lifted byte");
}
#[test]
fn decode_names_the_first_unmappable_character() {
assert_eq!(
decode("abĠ\nc"),
Err(UnmappableChar {
char_index: 3,
character: '\n',
})
);
assert_eq!(
decode(" a"),
Err(UnmappableChar {
char_index: 0,
character: ' ',
}),
"a raw space must be rejected: the alphabet writes it as Ġ"
);
assert_eq!(
decode("a\u{AD}"),
Err(UnmappableChar {
char_index: 1,
character: '\u{AD}',
}),
"U+00AD is below MAX_CODEPOINT but absent from the image: byte 0xAD \
was lifted to U+0143, so nothing encodes to U+00AD"
);
assert_eq!(
decode("a\u{7F}"),
Err(UnmappableChar {
char_index: 1,
character: '\u{7F}',
}),
"DEL is a hole for the same reason"
);
assert_eq!(
decode("aé"),
Ok(vec![b'a', 0xE9]),
"U+00E9 is 233 and byte 233 self-maps, so it IS in the image"
);
assert_eq!(decode(""), Ok(Vec::new()));
}
#[test]
fn no_encoded_character_is_the_replacement_char() {
assert!(
!BYTE_TO_CHAR.contains(&'\u{FFFD}'),
"replacement char in the table means a codepoint failed to convert"
);
assert_eq!(BYTE_TO_CHAR.len(), ENCODER_LEN);
assert_eq!(CODEPOINT_TO_BYTE.len(), DECODER_LEN);
assert_eq!(
CODEPOINT_TO_BYTE.iter().filter(|&&s| s != NOT_A_BYTE).count(),
ENCODER_LEN,
"inverse table must define exactly 256 slots"
);
}
}