Skip to main content

card_messages/
encoding.rs

1//! Text codecs used by clearing messages.
2
3use serde::Serialize;
4
5/// Text encoding applied to MTIs, length prefixes, and textual fields.
6#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize)]
7#[non_exhaustive]
8pub enum TextEncoding {
9    /// ISO-8859-1, where every byte maps to the same Unicode code point.
10    Latin1,
11    /// IBM EBCDIC code page 037 (US/Canada).
12    EbcdicCp037,
13    /// IBM EBCDIC code page 500 (International Latin-1).
14    #[default]
15    EbcdicCp500,
16    /// IBM EBCDIC code page 1140 (CP037 with the euro sign).
17    EbcdicCp1140,
18    /// IBM EBCDIC code page 1145 (Latin America and Spain with euro sign).
19    EbcdicCp1145,
20}
21
22impl TextEncoding {
23    /// Decodes bytes deterministically without using a system codec.
24    #[must_use]
25    pub fn decode(self, bytes: &[u8]) -> String {
26        match self {
27            Self::Latin1 => bytes.iter().map(|byte| char::from(*byte)).collect(),
28            Self::EbcdicCp037 => decode_codepoints(bytes, cp037_codepoint),
29            Self::EbcdicCp500 => decode_table(bytes, &CP500_TO_UNICODE),
30            Self::EbcdicCp1140 => decode_codepoints(bytes, cp1140_codepoint),
31            Self::EbcdicCp1145 => decode_table(bytes, &CP1145_TO_UNICODE),
32        }
33    }
34}
35
36fn decode_codepoints(bytes: &[u8], codepoint: fn(u8) -> u16) -> String {
37    bytes
38        .iter()
39        .map(|byte| {
40            char::from_u32(u32::from(codepoint(*byte))).unwrap_or(char::REPLACEMENT_CHARACTER)
41        })
42        .collect()
43}
44
45fn cp037_codepoint(byte: u8) -> u16 {
46    match byte {
47        0x4a => 0x00a2,
48        0x4f => 0x007c,
49        0x5a => 0x0021,
50        0x5f => 0x00ac,
51        0xb0 => 0x005e,
52        0xba => 0x005b,
53        0xbb => 0x005d,
54        _ => CP500_TO_UNICODE[usize::from(byte)],
55    }
56}
57
58fn cp1140_codepoint(byte: u8) -> u16 {
59    if byte == 0x9f {
60        0x20ac
61    } else {
62        cp037_codepoint(byte)
63    }
64}
65
66fn decode_table(bytes: &[u8], table: &[u16; 256]) -> String {
67    bytes
68        .iter()
69        .map(|byte| {
70            char::from_u32(u32::from(table[usize::from(*byte)]))
71                .unwrap_or(char::REPLACEMENT_CHARACTER)
72        })
73        .collect()
74}
75
76// Generated from the standard CP500 mapping. Keeping the table local makes the
77// supported codec deterministic and avoids a system iconv dependency.
78const CP500_TO_UNICODE: [u16; 256] = [
79    0x0000, 0x0001, 0x0002, 0x0003, 0x009C, 0x0009, 0x0086, 0x007F, 0x0097, 0x008D, 0x008E, 0x000B,
80    0x000C, 0x000D, 0x000E, 0x000F, 0x0010, 0x0011, 0x0012, 0x0013, 0x009D, 0x0085, 0x0008, 0x0087,
81    0x0018, 0x0019, 0x0092, 0x008F, 0x001C, 0x001D, 0x001E, 0x001F, 0x0080, 0x0081, 0x0082, 0x0083,
82    0x0084, 0x000A, 0x0017, 0x001B, 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x0005, 0x0006, 0x0007,
83    0x0090, 0x0091, 0x0016, 0x0093, 0x0094, 0x0095, 0x0096, 0x0004, 0x0098, 0x0099, 0x009A, 0x009B,
84    0x0014, 0x0015, 0x009E, 0x001A, 0x0020, 0x00A0, 0x00E2, 0x00E4, 0x00E0, 0x00E1, 0x00E3, 0x00E5,
85    0x00E7, 0x00F1, 0x005B, 0x002E, 0x003C, 0x0028, 0x002B, 0x0021, 0x0026, 0x00E9, 0x00EA, 0x00EB,
86    0x00E8, 0x00ED, 0x00EE, 0x00EF, 0x00EC, 0x00DF, 0x005D, 0x0024, 0x002A, 0x0029, 0x003B, 0x005E,
87    0x002D, 0x002F, 0x00C2, 0x00C4, 0x00C0, 0x00C1, 0x00C3, 0x00C5, 0x00C7, 0x00D1, 0x00A6, 0x002C,
88    0x0025, 0x005F, 0x003E, 0x003F, 0x00F8, 0x00C9, 0x00CA, 0x00CB, 0x00C8, 0x00CD, 0x00CE, 0x00CF,
89    0x00CC, 0x0060, 0x003A, 0x0023, 0x0040, 0x0027, 0x003D, 0x0022, 0x00D8, 0x0061, 0x0062, 0x0063,
90    0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x00AB, 0x00BB, 0x00F0, 0x00FD, 0x00FE, 0x00B1,
91    0x00B0, 0x006A, 0x006B, 0x006C, 0x006D, 0x006E, 0x006F, 0x0070, 0x0071, 0x0072, 0x00AA, 0x00BA,
92    0x00E6, 0x00B8, 0x00C6, 0x00A4, 0x00B5, 0x007E, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078,
93    0x0079, 0x007A, 0x00A1, 0x00BF, 0x00D0, 0x00DD, 0x00DE, 0x00AE, 0x00A2, 0x00A3, 0x00A5, 0x00B7,
94    0x00A9, 0x00A7, 0x00B6, 0x00BC, 0x00BD, 0x00BE, 0x00AC, 0x007C, 0x00AF, 0x00A8, 0x00B4, 0x00D7,
95    0x007B, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x00AD, 0x00F4,
96    0x00F6, 0x00F2, 0x00F3, 0x00F5, 0x007D, 0x004A, 0x004B, 0x004C, 0x004D, 0x004E, 0x004F, 0x0050,
97    0x0051, 0x0052, 0x00B9, 0x00FB, 0x00FC, 0x00F9, 0x00FA, 0x00FF, 0x005C, 0x00F7, 0x0053, 0x0054,
98    0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005A, 0x00B2, 0x00D4, 0x00D6, 0x00D2, 0x00D3, 0x00D5,
99    0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x00B3, 0x00DB,
100    0x00DC, 0x00D9, 0x00DA, 0x009F,
101];
102
103// CP1145 is the Latin America and Spain EBCDIC page. The mapping follows the
104// registered Python codec used for validation, including its line-feed
105// interpretation of byte 0x15.
106const CP1145_TO_UNICODE: [u16; 256] = [
107    0x0000, 0x0001, 0x0002, 0x0003, 0x009C, 0x0009, 0x0086, 0x007F, 0x0097, 0x008D, 0x008E, 0x000B,
108    0x000C, 0x000D, 0x000E, 0x000F, 0x0010, 0x0011, 0x0012, 0x0013, 0x009D, 0x000A, 0x0008, 0x0087,
109    0x0018, 0x0019, 0x0092, 0x008F, 0x001C, 0x001D, 0x001E, 0x001F, 0x0080, 0x0081, 0x0082, 0x0083,
110    0x0084, 0x000A, 0x0017, 0x001B, 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x0005, 0x0006, 0x0007,
111    0x0090, 0x0091, 0x0016, 0x0093, 0x0094, 0x0095, 0x0096, 0x0004, 0x0098, 0x0099, 0x009A, 0x009B,
112    0x0014, 0x0015, 0x009E, 0x001A, 0x0020, 0x00A0, 0x00E2, 0x00E4, 0x00E0, 0x00E1, 0x00E3, 0x00E5,
113    0x00E7, 0x00A6, 0x005B, 0x002E, 0x003C, 0x0028, 0x002B, 0x007C, 0x0026, 0x00E9, 0x00EA, 0x00EB,
114    0x00E8, 0x00ED, 0x00EE, 0x00EF, 0x00EC, 0x00DF, 0x005D, 0x0024, 0x002A, 0x0029, 0x003B, 0x00AC,
115    0x002D, 0x002F, 0x00C2, 0x00C4, 0x00C0, 0x00C1, 0x00C3, 0x00C5, 0x00C7, 0x0023, 0x00F1, 0x002C,
116    0x0025, 0x005F, 0x003E, 0x003F, 0x00F8, 0x00C9, 0x00CA, 0x00CB, 0x00C8, 0x00CD, 0x00CE, 0x00CF,
117    0x00CC, 0x0060, 0x003A, 0x00D1, 0x0040, 0x0027, 0x003D, 0x0022, 0x00D8, 0x0061, 0x0062, 0x0063,
118    0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x00AB, 0x00BB, 0x00F0, 0x00FD, 0x00FE, 0x00B1,
119    0x00B0, 0x006A, 0x006B, 0x006C, 0x006D, 0x006E, 0x006F, 0x0070, 0x0071, 0x0072, 0x00AA, 0x00BA,
120    0x00E6, 0x00B8, 0x00C6, 0x20AC, 0x00B5, 0x00A8, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078,
121    0x0079, 0x007A, 0x00A1, 0x00BF, 0x00D0, 0x00DD, 0x00DE, 0x00AE, 0x00A2, 0x00A3, 0x00A5, 0x00B7,
122    0x00A9, 0x00A7, 0x00B6, 0x00BC, 0x00BD, 0x00BE, 0x005E, 0x0021, 0x00AF, 0x007E, 0x00B4, 0x00D7,
123    0x007B, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x00AD, 0x00F4,
124    0x00F6, 0x00F2, 0x00F3, 0x00F5, 0x007D, 0x004A, 0x004B, 0x004C, 0x004D, 0x004E, 0x004F, 0x0050,
125    0x0051, 0x0052, 0x00B9, 0x00FB, 0x00FC, 0x00F9, 0x00FA, 0x00FF, 0x005C, 0x00F7, 0x0053, 0x0054,
126    0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005A, 0x00B2, 0x00D4, 0x00D6, 0x00D2, 0x00D3, 0x00D5,
127    0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x00B3, 0x00DB,
128    0x00DC, 0x00D9, 0x00DA, 0x009F,
129];
130
131#[cfg(test)]
132mod tests {
133    use super::{CP500_TO_UNICODE, CP1145_TO_UNICODE, TextEncoding};
134
135    #[test]
136    fn cp037_decodes_page_specific_punctuation() {
137        let decoded = TextEncoding::EbcdicCp037.decode(&[0x4a, 0x4f, 0x5a, 0xba, 0xbb]);
138        assert_eq!(decoded, "¢|![]");
139    }
140
141    #[test]
142    fn cp1140_replaces_currency_sign_with_euro() {
143        let decoded = TextEncoding::EbcdicCp1140.decode(&[0x9f]);
144        assert_eq!(decoded, "€");
145    }
146
147    #[test]
148    fn cp500_decodes_digits_and_letters() {
149        let decoded = TextEncoding::EbcdicCp500.decode(&[0xC1, 0xD1, 0xE2, 0xF0]);
150        assert_eq!(decoded, "AJS0");
151    }
152
153    #[test]
154    fn cp1145_decodes_distinctive_printable_characters() {
155        let decoded = TextEncoding::EbcdicCp1145.decode(&[
156            0x49, 0x4F, 0x5F, 0x69, 0x6A, 0x7B, 0x9F, 0xA1, 0xBA, 0xBB, 0xBD,
157        ]);
158        assert_eq!(decoded, "¦|¬#ñÑ€¨^!~");
159    }
160
161    #[test]
162    fn cp1145_decodes_control_15_as_line_feed() {
163        let decoded = TextEncoding::EbcdicCp1145.decode(&[0x15]);
164        assert_eq!(decoded, "\n");
165    }
166
167    #[test]
168    fn cp1145_matches_cp500_except_documented_positions() {
169        const DIFFERENT_BYTES: [u8; 12] = [
170            0x15, 0x49, 0x4F, 0x5F, 0x69, 0x6A, 0x7B, 0x9F, 0xA1, 0xBA, 0xBB, 0xBD,
171        ];
172        for byte in 0_u8..=u8::MAX {
173            if !DIFFERENT_BYTES.contains(&byte) {
174                assert_eq!(
175                    CP1145_TO_UNICODE[usize::from(byte)],
176                    CP500_TO_UNICODE[usize::from(byte)]
177                );
178            }
179        }
180    }
181}