rama_utils/str/utf8/
mod.rs

1//! Incremental, zero-copy UTF-8 decoding with error handling
2//!
3//! Forked from archived project:
4//! <https://github.com/SimonSapin/rust-utf8/tree/218fea2b57b0e4c3de9fa17a376fcc4a4c0d08f3>
5//!
6//! > Original License:
7//! > <https://github.com/SimonSapin/rust-utf8/blob/218fea2b57b0e4c3de9fa17a376fcc4a4c0d08f3/LICENSE-MIT>
8//! >
9//! > Credits to Simon Sapin (@SimonSapin)
10
11mod lossy;
12
13#[cfg(test)]
14mod tests;
15
16pub use lossy::LossyDecoder;
17
18#[cfg(feature = "std")]
19mod read;
20
21#[cfg(feature = "std")]
22#[cfg_attr(docsrs, doc(cfg(feature = "std")))]
23pub use read::{BufReadDecoder, BufReadDecoderError};
24
25use core::cmp;
26use core::error::Error;
27use core::fmt;
28use core::str;
29
30/// The replacement character, U+FFFD. In lossy decoding, insert it for every decoding error.
31pub const REPLACEMENT_CHARACTER: &str = "\u{FFFD}";
32
33#[derive(Debug, Copy, Clone)]
34pub enum DecodeError<'a> {
35    /// In lossy decoding insert `valid_prefix`, then `"\u{FFFD}"`,
36    /// then call `decode()` again with `remaining_input`.
37    Invalid {
38        valid_prefix: &'a str,
39        invalid_sequence: &'a [u8],
40        remaining_input: &'a [u8],
41    },
42
43    /// Call the `incomplete_suffix.try_complete` method with more input when available.
44    /// If no more input is available, this is an invalid byte sequence.
45    Incomplete {
46        valid_prefix: &'a str,
47        incomplete_suffix: Incomplete,
48    },
49}
50
51impl fmt::Display for DecodeError<'_> {
52    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
53        match *self {
54            DecodeError::Invalid {
55                valid_prefix,
56                invalid_sequence,
57                remaining_input,
58            } => write!(
59                f,
60                "found invalid byte sequence {invalid_sequence:02x?} after \
61                 {valid_byte_count} valid bytes, followed by {unprocessed_byte_count} more \
62                 unprocessed bytes",
63                invalid_sequence = invalid_sequence,
64                valid_byte_count = valid_prefix.len(),
65                unprocessed_byte_count = remaining_input.len()
66            ),
67            DecodeError::Incomplete {
68                valid_prefix,
69                incomplete_suffix,
70            } => write!(
71                f,
72                "found incomplete byte sequence {incomplete_suffix:02x?} after \
73                 {valid_byte_count} bytes",
74                incomplete_suffix = incomplete_suffix,
75                valid_byte_count = valid_prefix.len()
76            ),
77        }
78    }
79}
80
81impl Error for DecodeError<'_> {}
82
83#[derive(Debug, Copy, Clone)]
84pub struct Incomplete {
85    pub buffer: [u8; 4],
86    pub buffer_len: u8,
87}
88
89pub fn decode(input: &[u8]) -> Result<&str, DecodeError<'_>> {
90    let error = match str::from_utf8(input) {
91        Ok(valid) => return Ok(valid),
92        Err(error) => error,
93    };
94
95    // FIXME: separate function from here to guide inlining?
96    let (valid, after_valid) = input.split_at(error.valid_up_to());
97    let valid = unsafe { str::from_utf8_unchecked(valid) };
98
99    match error.error_len() {
100        Some(invalid_sequence_length) => {
101            let (invalid, rest) = after_valid.split_at(invalid_sequence_length);
102            Err(DecodeError::Invalid {
103                valid_prefix: valid,
104                invalid_sequence: invalid,
105                remaining_input: rest,
106            })
107        }
108        None => Err(DecodeError::Incomplete {
109            valid_prefix: valid,
110            incomplete_suffix: Incomplete::new(after_valid),
111        }),
112    }
113}
114
115impl Incomplete {
116    #[must_use]
117    pub fn empty() -> Self {
118        Self {
119            buffer: [0, 0, 0, 0],
120            buffer_len: 0,
121        }
122    }
123
124    #[must_use]
125    pub fn is_empty(&self) -> bool {
126        self.buffer_len == 0
127    }
128
129    #[must_use]
130    pub fn new(bytes: &[u8]) -> Self {
131        let mut buffer = [0, 0, 0, 0];
132        let len = bytes.len();
133        buffer[..len].copy_from_slice(bytes);
134        Self {
135            buffer,
136            buffer_len: len as u8,
137        }
138    }
139
140    /// * `None`: still incomplete, call `try_complete` again with more input.
141    ///   If no more input is available, this is invalid byte sequence.
142    /// * `Some((result, remaining_input))`: We’re done with this `Incomplete`.
143    ///   To keep decoding, pass `remaining_input` to `decode()`.
144    pub fn try_complete<'input>(
145        &mut self,
146        input: &'input [u8],
147    ) -> Option<(Result<&str, &[u8]>, &'input [u8])> {
148        let (consumed, opt_result) = self.try_complete_offsets(input);
149        let result = opt_result?;
150        let remaining_input = &input[consumed..];
151        let result_bytes = self.take_buffer();
152        let result = match result {
153            Ok(()) => Ok(unsafe { str::from_utf8_unchecked(result_bytes) }),
154            Err(()) => Err(result_bytes),
155        };
156        Some((result, remaining_input))
157    }
158
159    fn take_buffer(&mut self) -> &[u8] {
160        let len = self.buffer_len as usize;
161        self.buffer_len = 0;
162        &self.buffer[..len]
163    }
164
165    /// (consumed_from_input, None): not enough input
166    /// (consumed_from_input, Some(Err(()))): error bytes in buffer
167    /// (consumed_from_input, Some(Ok(()))): UTF-8 string in buffer
168    fn try_complete_offsets(&mut self, input: &[u8]) -> (usize, Option<Result<(), ()>>) {
169        let initial_buffer_len = self.buffer_len as usize;
170        let copied_from_input;
171        {
172            let unwritten = &mut self.buffer[initial_buffer_len..];
173            copied_from_input = cmp::min(unwritten.len(), input.len());
174            unwritten[..copied_from_input].copy_from_slice(&input[..copied_from_input]);
175        }
176        let spliced = &self.buffer[..initial_buffer_len + copied_from_input];
177        match str::from_utf8(spliced) {
178            Ok(_) => {
179                self.buffer_len = spliced.len() as u8;
180                (copied_from_input, Some(Ok(())))
181            }
182            Err(error) => {
183                let valid_up_to = error.valid_up_to();
184                if valid_up_to > 0 {
185                    let consumed = valid_up_to.saturating_sub(initial_buffer_len);
186                    self.buffer_len = valid_up_to as u8;
187                    (consumed, Some(Ok(())))
188                } else if let Some(invalid_sequence_length) = error.error_len() {
189                    let consumed = invalid_sequence_length.saturating_sub(initial_buffer_len);
190                    self.buffer_len = invalid_sequence_length as u8;
191                    (consumed, Some(Err(())))
192                } else {
193                    self.buffer_len = spliced.len() as u8;
194                    (copied_from_input, None)
195                }
196            }
197        }
198    }
199}
rama_utils/str/utf8/mod.rs

rama_utils/str/utf8/
mod.rs
