rama_utils/str/utf8/
read.rs

1use super::*;
2
3use core::error::Error;
4use core::fmt;
5use core::str;
6
7use std::io::{self, BufRead};
8
9/// Wraps a `std::io::BufRead` buffered byte stream and decode it as UTF-8.
10pub struct BufReadDecoder<B: BufRead> {
11    buf_read: B,
12    bytes_consumed: usize,
13    incomplete: Incomplete,
14}
15
16#[derive(Debug)]
17pub enum BufReadDecoderError<'a> {
18    /// Represents one UTF-8 error in the byte stream.
19    ///
20    /// In lossy decoding, each such error should be replaced with U+FFFD.
21    /// (See `BufReadDecoder::next_lossy` and `BufReadDecoderError::lossy`.)
22    InvalidByteSequence(&'a [u8]),
23
24    /// An I/O error from the underlying byte stream
25    Io(io::Error),
26}
27
28impl BufReadDecoderError<'_> {
29    /// Replace UTF-8 errors with U+FFFD
30    pub fn lossy(self) -> Result<&'static str, io::Error> {
31        match self {
32            BufReadDecoderError::Io(error) => Err(error),
33            BufReadDecoderError::InvalidByteSequence(_) => Ok(REPLACEMENT_CHARACTER),
34        }
35    }
36}
37
38impl fmt::Display for BufReadDecoderError<'_> {
39    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
40        match *self {
41            BufReadDecoderError::InvalidByteSequence(bytes) => {
42                write!(f, "invalid byte sequence: {bytes:02x?}")
43            }
44            BufReadDecoderError::Io(ref err) => write!(f, "underlying bytestream error: {err}"),
45        }
46    }
47}
48
49impl Error for BufReadDecoderError<'_> {
50    fn source(&self) -> Option<&(dyn Error + 'static)> {
51        match *self {
52            BufReadDecoderError::InvalidByteSequence(_) => None,
53            BufReadDecoderError::Io(ref err) => Some(err),
54        }
55    }
56}
57
58impl<B: BufRead> BufReadDecoder<B> {
59    /// This is to `Read::read_to_string` what `String::from_utf8_lossy` is to `String::from_utf8`.
60    pub fn read_to_string_lossy(buf_read: B) -> io::Result<String> {
61        let mut decoder = Self::new(buf_read);
62        let mut string = String::new();
63        while let Some(result) = decoder.next_lossy() {
64            string.push_str(result?)
65        }
66        Ok(string)
67    }
68
69    pub fn new(buf_read: B) -> Self {
70        Self {
71            buf_read,
72            bytes_consumed: 0,
73            incomplete: Incomplete::empty(),
74        }
75    }
76
77    /// Same as `BufReadDecoder::next_strict`, but replace UTF-8 errors with U+FFFD.
78    pub fn next_lossy(&mut self) -> Option<io::Result<&str>> {
79        self.next_strict()
80            .map(|result| result.or_else(|e| e.lossy()))
81    }
82
83    /// Decode and consume the next chunk of UTF-8 input.
84    ///
85    /// This method is intended to be called repeatedly until it returns `None`,
86    /// which represents EOF from the underlying byte stream.
87    /// This is similar to `Iterator::next`,
88    /// except that decoded chunks borrow the decoder (~iterator)
89    /// so they need to be handled or copied before the next chunk can start decoding.
90    pub fn next_strict(&mut self) -> Option<Result<&str, BufReadDecoderError<'_>>> {
91        enum BytesSource {
92            BufRead(usize),
93            Incomplete,
94        }
95        macro_rules! try_io {
96            ($io_result: expr) => {
97                match $io_result {
98                    Ok(value) => value,
99                    Err(error) => return Some(Err(BufReadDecoderError::Io(error))),
100                }
101            };
102        }
103        let (source, result) = loop {
104            if self.bytes_consumed > 0 {
105                self.buf_read.consume(self.bytes_consumed);
106                self.bytes_consumed = 0;
107            }
108            let buf = try_io!(self.buf_read.fill_buf());
109
110            // Force loop iteration to go through an explicit `continue`
111            enum Unreachable {}
112            let _: Unreachable = if self.incomplete.is_empty() {
113                if buf.is_empty() {
114                    return None; // EOF
115                }
116                match str::from_utf8(buf) {
117                    Ok(_) => break (BytesSource::BufRead(buf.len()), Ok(())),
118                    Err(error) => {
119                        let valid_up_to = error.valid_up_to();
120                        if valid_up_to > 0 {
121                            break (BytesSource::BufRead(valid_up_to), Ok(()));
122                        }
123                        if let Some(invalid_sequence_length) = error.error_len() {
124                            break (BytesSource::BufRead(invalid_sequence_length), Err(()));
125                        } else {
126                            self.bytes_consumed = buf.len();
127                            self.incomplete = Incomplete::new(buf);
128                            // need more input bytes
129                            continue;
130                        }
131                    }
132                }
133            } else {
134                if buf.is_empty() {
135                    break (BytesSource::Incomplete, Err(())); // EOF with incomplete code point
136                }
137                let (consumed, opt_result) = self.incomplete.try_complete_offsets(buf);
138                self.bytes_consumed = consumed;
139                match opt_result {
140                    None => {
141                        // need more input bytes
142                        continue;
143                    }
144                    Some(result) => break (BytesSource::Incomplete, result),
145                }
146            };
147        };
148        let bytes = match source {
149            BytesSource::BufRead(byte_count) => {
150                self.bytes_consumed = byte_count;
151                let buf = try_io!(self.buf_read.fill_buf());
152                &buf[..byte_count]
153            }
154            BytesSource::Incomplete => self.incomplete.take_buffer(),
155        };
156        match result {
157            Ok(()) => Some(Ok(unsafe { str::from_utf8_unchecked(bytes) })),
158            Err(()) => Some(Err(BufReadDecoderError::InvalidByteSequence(bytes))),
159        }
160    }
161}
rama_utils/str/utf8/read.rs

rama_utils/str/utf8/
read.rs
