1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0
use crate::buffer::ArrayBufferView;
use crate::encoding::Encoder;
use crate::utils::result::ResultExt;
use rquickjs::{function::Opt, CString, Ctx, Exception, Result};
#[derive(rquickjs::class::Trace, rquickjs::JsLifetime)]
#[rquickjs::class]
pub struct StringDecoder {
#[qjs(skip_trace)]
encoder: Encoder,
buffer: Vec<u8>,
buffered_bytes: usize,
missing_bytes: usize,
}
impl StringDecoder {
fn make_string(&self, ctx: &Ctx<'_>, data: &[u8]) -> Result<String> {
self.encoder
.encode_to_string(data, true)
.map_err(|_| Exception::throw_internal(ctx, "Encoding error"))
}
/// Try to decode the given buffer and store the incomplete bytes.
/// The logic was adapted from the [Node implementation].
///
/// [Node implementation]: https://github.com/nodejs/node/blob/ba06c5c509956dc413f91b755c1c93798bb700d4/src/string_decoder.cc#L66
fn decode_data(&mut self, ctx: &Ctx<'_>, mut data: &[u8]) -> Result<String> {
let mut result = String::new();
if matches!(
self.encoder,
Encoder::Utf8 | Encoder::Utf16le | Encoder::Base64
) {
// See if we want bytes to finish a character from the previous
// chunk; if so, copy the new bytes to the missing bytes buffer
// and create a string from it that is to be prepended to the main body.
if self.missing_bytes > 0 {
if matches!(self.encoder, Encoder::Utf8) {
// For UTF-8, we need special alignment treatment:
// If an incomplete character is found at a chunk boundary, we use
// its remainder and try to decode it.
let mut i = 0;
while i < data.len() && i < self.missing_bytes {
if (data[i] & 0xC0) != 0x80 {
// This byte is not a continuation byte even though it should have
// been one. We stop decoding of the incomplete character at this
// point (but still use the rest of the incomplete bytes from this
// chunk) and assume that the new, unexpected byte starts a new one.
self.missing_bytes = 0;
self.buffer.extend_from_slice(&data[..i]);
self.buffered_bytes += i;
data = &data[i..];
break;
}
i += 1;
}
} else if matches!(self.encoder, Encoder::Utf16le) {
// For UTF-16le, we need special alignment treatment:
// If we have a high surrogate we need to extend the missing bytes
// to 3 to get the low surrogate.
let mut i = 0;
while i < data.len() && i < self.missing_bytes {
if (data[i] & 0xFC) == 0xD8 {
self.missing_bytes = 3;
break;
}
i += 1;
}
}
let found_bytes = std::cmp::min(data.len(), self.missing_bytes);
self.buffer.extend_from_slice(&data[..found_bytes]);
data = &data[found_bytes..];
self.missing_bytes -= found_bytes;
self.buffered_bytes += found_bytes;
if self.missing_bytes == 0 {
// We have enough bytes to decode the buffered character
result = self.make_string(ctx, &self.buffer)?;
self.buffer.clear();
self.buffered_bytes = 0;
}
}
// It could be that trying to finish the previous chunk already
// consumed all data that we received in this chunk.
if data.is_empty() {
return Ok(result);
} else {
// If not, that means is no character left to finish at this point.
// See whether there is a character that we may have to cut off and
// finish when receiving the next chunk.
if matches!(self.encoder, Encoder::Utf8) && (data[data.len() - 1] & 0x80) != 0 {
let mut i = data.len() - 1;
loop {
self.buffered_bytes += 1;
if (data[i] & 0xC0) == 0x80 {
// This byte does not start a character (a "trailing" byte).
if self.buffered_bytes >= 4 || i == 0 {
// We either have more then 4 trailing bytes (which means
// the current character would not be inside the range for
// valid Unicode, and in particular cannot be represented
// through JavaScript's UTF-16-based approach to strings), or the
// current buffer does not contain the start of an UTF-8 character
// at all. Either way, this is invalid UTF8 and we can just
// let the engine's decoder handle it.
self.buffer.clear();
self.buffered_bytes = 0;
break;
}
} else {
// Found the first byte of a UTF-8 character. By looking at the
// upper bits we can tell how long the character *should* be.
if (data[i] & 0xE0) == 0xC0 {
self.missing_bytes = 2;
} else if (data[i] & 0xF0) == 0xE0 {
self.missing_bytes = 3;
} else if (data[i] & 0xF8) == 0xF0 {
self.missing_bytes = 4;
} else {
// This lead byte would indicate a character outside of the
// representable range.
self.buffered_bytes = 0;
break;
}
if self.buffered_bytes >= self.missing_bytes {
// Received more or exactly as many trailing bytes than the lead
// character would indicate. In the "==" case, we have valid
// data and don't need to slice anything off;
// in the ">" case, this is invalid UTF-8 anyway.
self.missing_bytes = 0;
self.buffered_bytes = 0;
}
self.missing_bytes -= self.buffered_bytes;
break;
}
i -= 1;
}
} else if matches!(self.encoder, Encoder::Utf16le) {
// WARN: For UTF-16LE we deviate from the specification when an invalid
// high surrogate is found. The spec says we should keep it as is, but
// there no way to encode in UTF-8 (required to interface with quickjs).
// For now, we will replace it with a replacement character.
// See https://github.com/quickjs-ng/quickjs/issues/992
if (data.len() % 2) == 1 {
// We got half a codepoint, and need the second byte of it.
// But we need to avoid rendering high surrogates before we
// have the full character.
if data.len() >= 3 && (data[data.len() - 2] & 0xFC) == 0xD8 {
self.buffered_bytes = 3;
self.missing_bytes = 1;
} else {
self.buffered_bytes = 1;
self.missing_bytes = 1;
}
} else if (data[data.len() - 1] & 0xFC) == 0xD8 {
// Half a split UTF-16 character.
self.buffered_bytes = 2;
self.missing_bytes = 2;
}
} else if matches!(self.encoder, Encoder::Base64) {
self.buffered_bytes = data.len() % 3;
if self.buffered_bytes > 0 {
self.missing_bytes = 3 - self.buffered_bytes;
}
}
if self.buffered_bytes > 0 {
// Copy the requested number of buffered bytes from the end of the
// input into the incomplete character buffer.
self.buffer
.extend_from_slice(&data[data.len() - self.buffered_bytes..]);
data = &data[..data.len() - self.buffered_bytes];
}
if !data.is_empty() {
result.push_str(&self.make_string(ctx, data)?);
}
}
Ok(result)
} else {
// For ASCII, HEX, and LATIN1, we can decode everything directly
self.make_string(ctx, data)
}
}
fn flush(&mut self, ctx: &Ctx<'_>) -> Result<String> {
if matches!(self.encoder, Encoder::Utf16le) && self.buffered_bytes % 2 == 1 {
// Ignore a single trailing byte, like the JS decoder does.
self.missing_bytes -= 1;
self.buffered_bytes -= 1;
}
if self.buffered_bytes == 0 {
return Ok(String::new());
}
let res = self.make_string(ctx, &self.buffer);
self.missing_bytes = 0;
self.buffered_bytes = 0;
self.buffer.clear();
res
}
}
#[rquickjs::methods(rename_all = "camelCase")]
impl StringDecoder {
#[qjs(constructor)]
pub fn new(ctx: Ctx<'_>, encoding: Opt<CString<'_>>) -> Result<Self> {
let encoding = encoding.0.as_ref().map(|e| e.as_str()).unwrap_or("utf-8");
let encoder = Encoder::from_str(encoding).map_err(|_| {
let msg = ["Unknown encoding: ", encoding].concat();
Exception::throw_type(&ctx, &msg)
})?;
Ok(Self {
encoder,
buffer: Vec::new(),
buffered_bytes: 0,
missing_bytes: 0,
})
}
#[qjs(get)]
pub fn encoding(&self) -> &str {
self.encoder.as_label()
}
pub fn end(&mut self, ctx: Ctx<'_>, buffer: Opt<ArrayBufferView<'_>>) -> Result<String> {
let output = if let Some(data) = buffer.0.as_ref().and_then(|b| b.as_bytes()) {
Some(self.decode_data(&ctx, data)?)
} else {
None
};
let flush = self.flush(&ctx)?;
Ok(output
.map(|mut o| {
o.push_str(&flush);
o
})
.unwrap_or(flush))
}
pub fn write(&mut self, ctx: Ctx<'_>, buffer: ArrayBufferView<'_>) -> Result<String> {
let data = buffer
.as_bytes()
.or_throw_msg(&ctx, "Buffer has already been used")?;
self.decode_data(&ctx, data)
}
}