1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
//! UTF-8.
//!
//! The decoder delegates the actual validation to [`core::str::from_utf8`], whose
//! error reporting already follows the Unicode "Best Practices for Using U+FFFD"
//! substitution that the Encoding Standard mandates. That keeps the bulk of the
//! work a `memcpy` and leaves only sequences straddling a call boundary to the
//! byte-at-a-time path.
use crate::result::{DecoderResult, EncoderResult};
use crate::sink::{ByteSink, DECODER_HEADROOM, ENCODER_HEADROOM};
#[derive(Debug, Clone, Copy, Default)]
pub(crate) struct Utf8Decoder {
/// A sequence that is a valid prefix so far but is not yet complete.
buf: [u8; 4],
len: u8,
}
impl Utf8Decoder {
pub(crate) fn new() -> Self {
Self::default()
}
pub(crate) fn decode(
&mut self,
src: &[u8],
sink: &mut ByteSink,
last: bool,
) -> (DecoderResult, usize) {
let mut read = 0usize;
loop {
if !sink.has_room(DECODER_HEADROOM) {
return (DecoderResult::OutputFull, read);
}
// Finish the sequence carried over from a previous call, one byte at a
// time. This runs at most three times per stream position.
if self.len > 0 {
if read == src.len() {
if !last {
return (DecoderResult::InputEmpty, read);
}
let bad = self.len;
self.len = 0;
return (DecoderResult::Malformed(bad), read);
}
self.buf[self.len as usize] = src[read];
self.len += 1;
read += 1;
match core::str::from_utf8(&self.buf[..self.len as usize]) {
Ok(s) => {
sink.write_slice(s.as_bytes());
self.len = 0;
}
Err(e) => match e.error_len() {
// Still incomplete: keep collecting.
None => {}
Some(bad) => {
// The byte just added terminated the sequence without
// completing it, so it belongs to the next one.
debug_assert_eq!(bad, self.len as usize - 1);
self.len = 0;
read -= 1;
return (DecoderResult::Malformed(bad as u8), read);
}
},
}
continue;
}
let rest = &src[read..];
if rest.is_empty() {
return (DecoderResult::InputEmpty, read);
}
// Validate only as far as the output can take, plus enough slack to
// finish the sequence that lands on the boundary. Validating all
// of `rest` every call, and then copying `room` bytes of it, is
// what turns a streaming decode into a quadratic one.
let window = core::cmp::min(rest.len(), sink.room() + 4);
let truncated = window < rest.len();
let chunk = &rest[..window];
let (valid, error_len) = match core::str::from_utf8(chunk) {
Ok(s) => (s.len(), None),
Err(e) => (e.valid_up_to(), Some(e.error_len())),
};
if valid > 0 {
let copied = copy_on_char_boundary(sink, &rest[..valid]);
read += copied;
if copied < valid {
return (DecoderResult::OutputFull, read);
}
}
match error_len {
None => return (DecoderResult::InputEmpty, read),
Some(Some(bad)) => {
if !sink.has_room(DECODER_HEADROOM) {
// Report the error only once there is room for a
// substitution; the next call re-discovers it unchanged.
return (DecoderResult::OutputFull, read);
}
read += bad;
return (DecoderResult::Malformed(bad as u8), read);
}
// A sequence cut off by the window rather than by the end of
// the input: nothing is wrong, so pick it up next time round.
Some(None) if truncated => {}
Some(None) => {
// The input ends in the middle of a sequence.
let tail = &rest[valid..];
debug_assert!(tail.len() < self.buf.len());
self.buf[..tail.len()].copy_from_slice(tail);
self.len = tail.len() as u8;
read += tail.len();
if !last {
return (DecoderResult::InputEmpty, read);
}
}
}
}
}
}
/// Copies as much of `bytes` as fits, truncated to a UTF-8 character boundary so
/// that the output stays valid UTF-8 at every step.
fn copy_on_char_boundary(sink: &mut ByteSink, bytes: &[u8]) -> usize {
let mut n = core::cmp::min(bytes.len(), sink.room());
if n < bytes.len() {
// `bytes` is known-valid UTF-8, so backing up finds a boundary within
// three bytes.
while n > 0 && bytes[n] & 0xC0 == 0x80 {
n -= 1;
}
}
sink.write_slice(&bytes[..n]);
n
}
#[derive(Debug, Clone, Copy, Default)]
pub(crate) struct Utf8Encoder;
impl Utf8Encoder {
pub(crate) fn encode(&mut self, src: &str, sink: &mut ByteSink) -> (EncoderResult, usize) {
if !sink.has_room(ENCODER_HEADROOM) && !src.is_empty() {
return (EncoderResult::OutputFull, 0);
}
let copied = copy_on_char_boundary(sink, src.as_bytes());
if copied < src.len() {
(EncoderResult::OutputFull, copied)
} else {
(EncoderResult::InputEmpty, copied)
}
}
}