yore/
lib.rs

1#![cfg_attr(not(feature = "std"), no_std)]
2
3#[cfg(feature = "alloc")]
4extern crate alloc;
5
6#[cfg(feature = "alloc")]
7use alloc::borrow::Cow;
8use core::fmt;
9
10pub mod code_pages;
11pub(crate) mod decoder;
12mod encoder;
13pub(crate) use encoder::Encoder;
14
15#[derive(Debug)]
16pub struct EncodeError {}
17
18impl fmt::Display for EncodeError {
19    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
20        f.write_str("Character in UTF-8 string has no mapping defined in code page")
21    }
22}
23
24#[cfg(feature = "std")]
25impl std::error::Error for EncodeError {}
26
27pub trait CodePage: Encoder {
28    /// Encode UTF-8 string into single-byte encoding
29    ///
30    /// Undefined characters will result in [`EncodeError`]
31    ///
32    /// # Examples
33    ///
34    /// ```
35    /// use yore::{CodePage, EncodeError};
36    ///
37    /// // Erase type for example - prefer concrete type over trait object whenever possible
38    /// let cp850: &dyn CodePage = &yore::code_pages::CP850;
39    /// assert_eq!(cp850.encode("text").unwrap(), vec![116, 101, 120, 116]);
40    /// assert!(matches!(cp850.encode("text 🦀"), EncodeError));
41    /// ```
42    #[cfg(feature = "alloc")]
43    #[inline]
44    fn encode<'a>(&self, s: &'a str) -> Result<Cow<'a, [u8]>, EncodeError> {
45        self.encode_helper(s, None)
46    }
47
48    /// Encode UTF-8 string into single-byte encoding
49    ///
50    /// Undefined characters will be replaced with byte `fallback`
51    ///
52    /// # Examples
53    ///
54    /// ```
55    /// use yore::CodePage;
56    ///
57    /// // Erase type for example - prefer concrete type over trait object whenever possible
58    /// let cp850: &dyn CodePage = &yore::code_pages::CP850;
59    /// assert_eq!(cp850.encode_lossy("text 🦀", 168), vec![116, 101, 120, 116, 32, 168])
60    /// ```
61    #[cfg(feature = "alloc")]
62    #[inline]
63    fn encode_lossy<'a>(&self, s: &'a str, fallback: u8) -> Cow<'a, [u8]> {
64        self.encode_helper(s, Some(fallback)).unwrap()
65    }
66
67    /// Decode single-byte encoding into UTF-8 string
68    ///
69    /// Undefined codepoints will result in [`DecodeError`]
70    ///
71    /// # Examples
72    ///
73    /// ```
74    /// use yore::{CodePage, DecodeError};
75    ///
76    /// // Erase types for example - prefer concrete type over trait object whenever possible
77    /// let cp850: &dyn CodePage = &yore::code_pages::CP850;
78    /// let cp857: &dyn CodePage = &yore::code_pages::CP857;
79    /// assert_eq!(cp850.decode(&[116, 101, 120, 116]).unwrap(), "text");
80    ///
81    /// //codepoint 231 is undefined
82    /// assert!(matches!(cp857.decode(&[116, 101, 120, 116, 231]), Err(DecodeError{position: 4, value: 231})));
83    /// ```
84    #[cfg(feature = "alloc")]
85    fn decode<'a>(&self, bytes: &'a [u8]) -> Result<Cow<'a, str>, DecodeError>;
86
87    /// Decode single-byte encoding into UTF-8 string
88    ///
89    /// Undefined codepoints will be replaced with `'�'`
90    ///
91    /// # Examples
92    ///
93    /// ```
94    /// use yore::CodePage;
95    ///
96    /// // Erase type for example - prefer concrete type over trait object whenever possible
97    /// let cp857: &dyn CodePage = &yore::code_pages::CP857;
98    /// //codepoint 231 is undefined
99    /// assert_eq!(cp857.decode_lossy(&[116, 101, 120, 116, 32, 231]), "text �");
100    /// ```
101    #[cfg(feature = "alloc")]
102    #[inline(always)]
103    fn decode_lossy<'a>(&self, bytes: &'a [u8]) -> Cow<'a, str> {
104        self.decode(bytes).unwrap()
105    }
106
107    /// Decode single-byte encoding into UTF-8 string
108    ///
109    /// Undefined codepoints will be replaced with `fallback`
110    ///
111    /// # Examples
112    ///
113    /// ```
114    /// use yore::CodePage;
115    ///
116    /// // Erase type for example - prefer concrete type over trait object whenever possible
117    /// let cp857: &dyn CodePage = &yore::code_pages::CP857;
118    /// //codepoint 231 is undefined
119    /// assert_eq!(cp857.decode_lossy_fallback(&[116, 101, 120, 116, 32, 231], '�'), "text �");
120    /// ```
121    #[cfg(feature = "alloc")]
122    #[inline(always)]
123    fn decode_lossy_fallback<'a>(&self, bytes: &'a [u8], _fallback: char) -> Cow<'a, str> {
124        self.decode(bytes).unwrap()
125    }
126}
127
128#[derive(Debug)]
129pub struct DecodeError {
130    pub position: usize,
131    pub value: u8,
132}
133
134impl fmt::Display for DecodeError {
135    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
136        write!(
137            f,
138            "Undefined codepoint {} at offset {}",
139            self.value, self.position
140        )
141    }
142}
143
144#[cfg(feature = "std")]
145impl std::error::Error for DecodeError {}
146
147#[cfg(all(test, feature = "cp437g"))]
148mod cp437g_tests {
149    use crate::code_pages::{CP437, CP437G};
150
151    #[test]
152    fn ibm_graphics_glyphs_encode_to_low_bytes() {
153        assert_eq!(CP437G.encode_char('☺'), Some(0x01));
154        assert_eq!(CP437G.encode_char('♥'), Some(0x03));
155        assert_eq!(CP437G.encode_char('☼'), Some(0x0F));
156        assert_eq!(CP437G.encode_char('⌂'), Some(0x7F));
157    }
158
159    #[test]
160    fn glyphs_sharing_a_byte_with_ascii_controls_are_representable() {
161        assert_eq!(CP437G.encode_char('○'), Some(0x09));
162        assert_eq!(CP437G.encode_char('◙'), Some(0x0A));
163        assert_eq!(CP437G.encode_char('♪'), Some(0x0D));
164    }
165
166    #[test]
167    fn decode_maps_low_bytes_to_glyphs() {
168        assert_eq!(CP437G.decode_byte(0x01), '☺');
169        assert_eq!(CP437G.decode_byte(0x09), '○');
170        assert_eq!(CP437G.decode_byte(0x0A), '◙');
171        assert_eq!(CP437G.decode_byte(0x0D), '♪');
172        assert_eq!(CP437G.decode_byte(0x7F), '⌂');
173    }
174
175    #[test]
176    fn ascii_control_chars_still_encode_to_their_bytes() {
177        // yore's ASCII fast-path; callers intercept the source char if they
178        // need newline semantics.
179        assert_eq!(CP437G.encode_char('\t'), Some(0x09));
180        assert_eq!(CP437G.encode_char('\n'), Some(0x0A));
181        assert_eq!(CP437G.encode_char('\r'), Some(0x0D));
182    }
183
184    #[test]
185    fn strict_cp437_differs_from_cp437g() {
186        // Strict CP437 has no smiley and keeps the C0 control mapping.
187        assert_eq!(CP437.encode_char('☺'), None);
188        assert_eq!(CP437.decode_byte(0x01), '\u{0001}');
189    }
190}
191
192#[cfg(test)]
193mod no_alloc_tests {
194    use crate::code_pages::{CP437, CP864};
195
196    #[test]
197    fn encode_char_ascii() {
198        assert_eq!(CP437.encode_char('t'), Some(b't'));
199        assert_eq!(CP437.encode_char('\n'), Some(b'\n'));
200    }
201
202    #[test]
203    fn encode_char_high_glyph() {
204        assert_eq!(CP437.encode_char('█'), Some(0xDB));
205        assert_eq!(CP437.encode_char('╔'), Some(0xC9));
206    }
207
208    #[test]
209    fn encode_char_unmapped() {
210        assert_eq!(CP437.encode_char('🦀'), None);
211    }
212
213    #[test]
214    fn decode_byte_complete() {
215        // CP437 is a complete codepage: decode_byte returns `char`.
216        assert_eq!(CP437.decode_byte(b't'), 't');
217        assert_eq!(CP437.decode_byte(0xDB), '█');
218        assert_eq!(CP437.decode_byte(0xC9), '╔');
219    }
220
221    #[test]
222    fn decode_byte_incomplete() {
223        // CP864 is an incomplete codepage: decode_byte returns `Option<char>`,
224        // and has a nonstandard ASCII mapping at 0x25 -> '٪'.
225        assert_eq!(CP864.decode_byte(0x25), Some('٪'));
226        assert_eq!(CP864.decode_byte(b't'), Some('t'));
227    }
228}
229
230#[cfg(all(test, feature = "alloc"))]
231mod tests {
232    use crate::code_pages::{CP1253, CP1255, CP1257, CP857, CP864, CP869, CP874};
233    use crate::CodePage;
234
235    #[test]
236    fn test_nonstandard_ascii() {
237        let bytes = [0x25, 253];
238        //CP864 has nonstandard mapping for 0x25
239        let s = "٪ﻱ";
240        assert_eq!(CP864.decode(&bytes).unwrap(), s);
241        assert_eq!(bytes, *CP864.encode(s).unwrap());
242
243        //Standard '%' should still map to 0x25
244        let s = "%ﻱ";
245        assert_eq!(bytes, *CP864.encode(s).unwrap());
246
247        let s = "AAAAAAA٪";
248        let bytes = [65, 65, 65, 65, 65, 65, 65, 0x25];
249        //Should decode to nonstandard, even if whole usize-len is ascii
250        assert_eq!(CP864.decode(&bytes).unwrap(), s);
251    }
252
253    /// Verify that code pages using the ASCII-optimized decode path
254    /// have standard ASCII mappings for bytes 0-127.
255    #[test]
256    fn verify_ascii_optimized_codepages() {
257        let codepages: &[&dyn CodePage] = &[&CP857, &CP869, &CP874, &CP1253, &CP1255, &CP1257];
258        for cp in codepages {
259            for b in 0u8..128 {
260                let bytes = [b];
261                let expected = core::str::from_utf8(&bytes).unwrap();
262                // undefined byte mappings are fine; only check the ones that decode
263                if let Ok(decoded) = cp.decode(&bytes) {
264                    assert_eq!(
265                        &*decoded, expected,
266                        "byte {b} should decode to ASCII '{expected}'"
267                    );
268                }
269            }
270        }
271    }
272}
yore/lib.rs

yore/
lib.rs
