gproxy_transform/transform/common/
utf8.rs

1//! Incremental UTF-8 decoding for byte streams.
2//!
3//! Network chunks may split a multi-byte UTF-8 sequence across chunk
4//! boundaries. Decoding each chunk independently (e.g. with
5//! `String::from_utf8_lossy`) turns the split character into U+FFFD
6//! replacement characters. This decoder holds back an incomplete trailing
7//! sequence until the next chunk completes it.
8
9/// Streaming UTF-8 decoder: feed byte chunks, get valid `&str` pushed into a
10/// `String`. Truly invalid bytes are replaced with U+FFFD; an *incomplete*
11/// trailing sequence is buffered until more bytes arrive (or [`Self::flush`]).
12#[derive(Debug, Default)]
13pub struct Utf8StreamDecoder {
14    /// Incomplete trailing UTF-8 sequence (at most 3 bytes) held back until
15    /// the next chunk.
16    pending: Vec<u8>,
17}
18
19impl Utf8StreamDecoder {
20    /// Decode `chunk` (prefixed by any held-back bytes) into `out`.
21    pub fn decode_into(&mut self, chunk: &[u8], out: &mut String) {
22        let owned;
23        let mut rest: &[u8] = if self.pending.is_empty() {
24            chunk
25        } else {
26            self.pending.extend_from_slice(chunk);
27            owned = std::mem::take(&mut self.pending);
28            &owned
29        };
30        loop {
31            match std::str::from_utf8(rest) {
32                Ok(s) => {
33                    out.push_str(s);
34                    return;
35                }
36                Err(e) => {
37                    let (valid, after) = rest.split_at(e.valid_up_to());
38                    out.push_str(std::str::from_utf8(valid).expect("valid prefix"));
39                    match e.error_len() {
40                        // Genuinely invalid bytes: replace and continue.
41                        Some(n) => {
42                            out.push(char::REPLACEMENT_CHARACTER);
43                            rest = &after[n..];
44                        }
45                        // Incomplete trailing sequence: hold back for the
46                        // next chunk.
47                        None => {
48                            self.pending = after.to_vec();
49                            return;
50                        }
51                    }
52                }
53            }
54        }
55    }
56
57    /// End of stream: replace any held-back incomplete sequence with U+FFFD.
58    pub fn flush(&mut self, out: &mut String) {
59        if !self.pending.is_empty() {
60            self.pending.clear();
61            out.push(char::REPLACEMENT_CHARACTER);
62        }
63    }
64}
65
66#[cfg(test)]
67mod tests {
68    use super::*;
69
70    #[test]
71    fn multibyte_char_split_across_chunks() {
72        let bytes = "汉字🚀".as_bytes(); // 3+3+4 bytes
73        let mut d = Utf8StreamDecoder::default();
74        let mut out = String::new();
75        // Split inside every character.
76        for b in bytes {
77            d.decode_into(std::slice::from_ref(b), &mut out);
78        }
79        d.flush(&mut out);
80        assert_eq!(out, "汉字🚀");
81    }
82
83    #[test]
84    fn invalid_bytes_replaced_incomplete_tail_flushed() {
85        let mut d = Utf8StreamDecoder::default();
86        let mut out = String::new();
87        d.decode_into(b"a\xffb\xe6\xb1", &mut out); // 0xff invalid, e6 b1 incomplete
88        assert_eq!(out, "a\u{fffd}b");
89        d.flush(&mut out);
90        assert_eq!(out, "a\u{fffd}b\u{fffd}");
91    }
92}
gproxy_transform/transform/common/utf8.rs

gproxy_transform/transform/common/
utf8.rs
