Skip to main content

nodejs/stdlib/
string_decoder.rs

1//! Node `string_decoder` core module: `new StringDecoder(encoding)` with
2//! `.write(buffer)` / `.end([buffer])`. A StringDecoder turns byte chunks into a
3//! string, holding back an incomplete trailing multibyte sequence until the next
4//! chunk completes it.
5//!
6//! Every encoding that has a chunk boundary buffers across it: UTF-8 holds an
7//! incomplete trailing sequence, UTF-16LE holds an odd byte and a trailing high
8//! surrogate, and base64 holds up to two bytes so it only ever emits whole
9//! 3-byte groups. The single-byte encodings consume everything.
10
11use crate::host::{with_host, JsObj};
12use fusevm::Value;
13use indexmap::IndexMap;
14
15/// The methods a `StringDecoder` instance carries. Also the method set its real
16/// prototype object is built from, so a read (`sd.write`), a call (`sd.write(b)`)
17/// and a prototype lookup (`StringDecoder.prototype.write`) cannot disagree.
18pub const INSTANCE_METHODS: &[&str] = &["write", "end", "text"];
19
20/// The prototype getters over the held-back bytes (`lastChar`, `lastNeed`,
21/// `lastTotal`).
22pub const ACCESSORS: &[(&str, bool)] = &[
23    ("lastChar", false),
24    ("lastNeed", false),
25    ("lastTotal", false),
26];
27
28/// Node's `normalizeEncoding`: the `encoding` property reports the CANONICAL
29/// name, not the spelling that was passed — `new StringDecoder('ucs2').encoding`
30/// is `'utf16le'` and `new StringDecoder('UTF-8').encoding` is `'utf8'`. Code
31/// that branches on `decoder.encoding` (iconv-lite does) sees the canonical set.
32///
33/// An encoding outside node's set is `ERR_UNKNOWN_ENCODING` (it used to be
34/// accepted and stored verbatim), and an empty one is `utf8`.
35fn normalize_encoding(enc: &str) -> Result<String, String> {
36    Ok(match enc.to_ascii_lowercase().as_str() {
37        "" | "utf8" | "utf-8" => "utf8",
38        "ucs2" | "ucs-2" | "utf16le" | "utf-16le" => "utf16le",
39        "latin1" | "binary" => "latin1",
40        e @ ("base64" | "base64url" | "hex" | "ascii") => e,
41        _ => {
42            return Err(crate::host::coded_error(
43                "TypeError",
44                "ERR_UNKNOWN_ENCODING",
45                &format!("Unknown encoding: {enc}"),
46            ))
47        }
48    }
49    .to_string())
50}
51
52/// `new StringDecoder([encoding])`.
53pub fn construct(args: &[Value]) -> Result<Value, String> {
54    let enc = match args.first() {
55        None | Some(Value::Undef) => "utf8".to_string(),
56        Some(v) if with_host(|h| h.is_null(v)) => "utf8".to_string(),
57        Some(_) => super::arg_str(args, 0),
58    };
59    let enc = normalize_encoding(&enc)?;
60    Ok(with_host(|h| {
61        let mut m = IndexMap::new();
62        m.insert("@@native".into(), h.new_str("StringDecoder"));
63        m.insert("encoding".into(), h.new_str(enc));
64        // Held-back bytes from a UTF-8 sequence split across chunks.
65        let empty = h.new_array(Vec::new());
66        m.insert("@@pending".into(), empty);
67        h.new_object(m)
68    }))
69}
70
71/// One chunk handed to `write`/`end`: a string passes through untouched (it is
72/// already decoded, and the held-back bytes stay held), and a byte view
73/// contributes the bytes it VIEWS — a `Uint16Array` two per element, a
74/// `DataView` its window. Anything else — an array, a number, an
75/// `ArrayBuffer`, nothing at all — is `ERR_INVALID_ARG_TYPE`, as in node.
76enum Chunk {
77    Text(String),
78    Bytes(Vec<u8>),
79}
80
81fn chunk_of(v: &Value) -> Result<Chunk, String> {
82    if let Some(s) = with_host(|h| h.as_str(v)) {
83        return Ok(Chunk::Text(s));
84    }
85    match crate::stdlib::native_tag(v).as_deref() {
86        Some("Buffer" | "TypedArray" | "DataView") => Ok(Chunk::Bytes(
87            crate::stdlib::buffer::view_bytes(v).unwrap_or_default(),
88        )),
89        _ => Err(crate::host::coded_error(
90            "TypeError",
91            "ERR_INVALID_ARG_TYPE",
92            &format!(
93                "The \"buf\" argument must be an instance of Buffer, TypedArray, or DataView. Received {}",
94                crate::stdlib::received_desc(v)
95            ),
96        )),
97    }
98}
99
100/// `write(buf)`: the text decoded so far, holding back an incomplete tail.
101fn write_chunk(recv: &Value, enc: &str, v: &Value) -> Result<String, String> {
102    match chunk_of(v)? {
103        Chunk::Text(s) => Ok(s),
104        Chunk::Bytes(bytes) => {
105            let mut buf = pending_of(recv);
106            buf.extend(bytes);
107            let (decoded, tail) = decode(enc, &buf);
108            set_pending(recv, &tail);
109            Ok(decoded)
110        }
111    }
112}
113
114fn encoding_of(recv: &Value) -> String {
115    with_host(|h| match h.get(recv) {
116        Some(JsObj::Object(p)) => p
117            .get("encoding")
118            .map(|v| h.str_of(v))
119            .unwrap_or_else(|| "utf8".into()),
120        _ => "utf8".into(),
121    })
122}
123
124fn pending_of(recv: &Value) -> Vec<u8> {
125    with_host(|h| match h.get(recv) {
126        Some(JsObj::Object(p)) => match p.get("@@pending").and_then(|a| h.get(a)) {
127            Some(JsObj::Array(items)) => items.iter().map(|x| h.to_number(x) as u8).collect(),
128            _ => Vec::new(),
129        },
130        _ => Vec::new(),
131    })
132}
133
134fn set_pending(recv: &Value, bytes: &[u8]) {
135    with_host(|h| {
136        let arr = h.new_array(bytes.iter().map(|b| Value::Float(*b as f64)).collect());
137        if let Some(JsObj::Object(p)) = h.get_mut(recv) {
138            p.insert("@@pending".into(), arr);
139        }
140    });
141}
142
143pub fn instance_call(recv: &Value, method: &str, args: &[Value]) -> Result<Value, String> {
144    let enc = encoding_of(recv);
145    match method {
146        "write" => {
147            let s = write_chunk(recv, &enc, &args.first().cloned().unwrap_or(Value::Undef))?;
148            Ok(with_host(|h| h.new_str(s)))
149        }
150        // `text(buf, offset)`: `write` of the bytes from `offset` on.
151        "text" => {
152            let buf = args.first().cloned().unwrap_or(Value::Undef);
153            let offset = with_host(|h| h.to_number(args.get(1).unwrap_or(&Value::Float(0.0))));
154            let bytes = match chunk_of(&buf)? {
155                Chunk::Bytes(b) => b,
156                Chunk::Text(s) => return Ok(with_host(|h| h.new_str(s))),
157            };
158            let from = if offset.is_finite() && offset > 0.0 {
159                offset as usize
160            } else {
161                0
162            };
163            let rest = super::buffer::from_bytes(bytes.get(from..).unwrap_or(&[]));
164            let s = write_chunk(recv, &enc, &rest)?;
165            Ok(with_host(|h| h.new_str(s)))
166        }
167        // The held-back state: `lastNeed` bytes still wanted to finish the
168        // character whose `lastTotal` bytes `lastChar` is gathering.
169        "@get@lastNeed" | "@get@lastTotal" | "@get@lastChar" => {
170            let held = pending_of(recv);
171            let (need, total) = held_shape(&enc, &held);
172            Ok(match method {
173                "@get@lastNeed" => Value::Float(need as f64),
174                "@get@lastTotal" => Value::Float(total as f64),
175                _ => {
176                    let mut four = held.clone();
177                    four.resize(4, 0);
178                    super::buffer::from_bytes(&four)
179                }
180            })
181        }
182        // `end([buf])` is `write(buf)` (when given) and then a flush of
183        // whatever is still held back.
184        "end" => {
185            let mut out = match args.first() {
186                None | Some(Value::Undef) => String::new(),
187                Some(v) => write_chunk(recv, &enc, v)?,
188            };
189            let tail = pending_of(recv);
190            set_pending(recv, &[]);
191            out.push_str(&flush(&enc, &tail));
192            Ok(with_host(|h| h.new_str(out)))
193        }
194        _ => Err(crate::host::type_error(&format!(
195            "{method} is not a function"
196        ))),
197    }
198}
199
200/// `(lastNeed, lastTotal)` for the bytes `held`: a UTF-8 sequence's length
201/// from its lead byte, a UTF-16LE code unit (2) or surrogate pair (4), a
202/// base64 group (3); nothing is held by the single-byte encodings.
203fn held_shape(enc: &str, held: &[u8]) -> (usize, usize) {
204    let n = held.len();
205    if n == 0 {
206        return (0, 0);
207    }
208    let total: usize = match enc {
209        "utf8" => match held[0] {
210            b if b >= 0xF0 => 4,
211            b if b >= 0xE0 => 3,
212            _ => 2,
213        },
214        "utf16le" if n >= 2 => 4,
215        "utf16le" => 2,
216        "base64" | "base64url" => 3,
217        _ => return (0, 0),
218    };
219    (total.saturating_sub(n), total)
220}
221
222/// What `end()` emits for bytes still held when the stream closes. Each encoding
223/// resolves its own remainder, so this is not one blanket replacement char:
224///
225/// * UTF-8 — one `U+FFFD` for the whole truncated sequence (not one per byte).
226/// * UTF-16LE — a held HIGH SURROGATE is emitted as a code unit; a dangling odd
227///   byte is dropped silently, with no replacement char (measured on node
228///   v26.7.0: `d.write(Buffer.from([0x61])); d.end()` is `''`). The lone
229///   surrogate itself becomes `U+FFFD` here — the `utf16` storage boundary.
230/// * base64 — the short group is padded and emitted (`AQ==`), which is the whole
231///   reason the bytes were held rather than encoded early.
232fn flush(enc: &str, tail: &[u8]) -> String {
233    if tail.is_empty() {
234        return String::new();
235    }
236    match enc {
237        "base64" => super::to_base64(tail),
238        "base64url" => super::to_base64url(tail),
239        "utf16le" => {
240            let units: Vec<u16> = tail
241                .chunks_exact(2)
242                .map(|c| u16::from_le_bytes([c[0], c[1]]))
243                .collect();
244            crate::utf16::to_string_lossy(&units)
245        }
246        _ => "\u{FFFD}".to_string(),
247    }
248}
249
250/// Decode `buf` in `enc`, returning (decoded string, held-back trailing bytes).
251/// Single-byte encodings consume everything; the rest hold back the partial tail
252/// that only the next chunk can complete.
253fn decode(enc: &str, buf: &[u8]) -> (String, Vec<u8>) {
254    match enc {
255        // `ascii` masks the high bit, `latin1`/`binary` keep the whole byte.
256        "ascii" => (
257            buf.iter().map(|b| (*b & 0x7f) as char).collect(),
258            Vec::new(),
259        ),
260        "latin1" => (buf.iter().map(|b| *b as char).collect(), Vec::new()),
261        "hex" => (super::to_hex(buf), Vec::new()),
262        // Base64 is 3 bytes → 4 characters. Emitting a short group would pad it
263        // mid-stream (`AQ==` then `AgM=` instead of `AQID`), so the remainder is
264        // held until the group closes or `end()` pads it.
265        "base64" | "base64url" => {
266            let keep = buf.len() % 3;
267            let (head, tail) = buf.split_at(buf.len() - keep);
268            let s = if enc == "base64url" {
269                super::to_base64url(head)
270            } else {
271                super::to_base64(head)
272            };
273            (s, tail.to_vec())
274        }
275        // UTF-16LE: an odd trailing byte is half a code unit, and a trailing HIGH
276        // surrogate is half a code point — both wait for the next chunk.
277        "utf16le" => {
278            let mut keep = buf.len() % 2;
279            let whole = buf.len() - keep;
280            if whole >= 2 {
281                let last = u16::from_le_bytes([buf[whole - 2], buf[whole - 1]]);
282                if (0xD800..0xDC00).contains(&last) {
283                    keep += 2;
284                }
285            }
286            let (head, tail) = buf.split_at(buf.len() - keep);
287            let units: Vec<u16> = head
288                .chunks_exact(2)
289                .map(|c| u16::from_le_bytes([c[0], c[1]]))
290                .collect();
291            (crate::utf16::to_string_lossy(&units), tail.to_vec())
292        }
293        // utf8 / utf-8 (and anything else): keep a split multibyte tail pending.
294        _ => {
295            let split = incomplete_utf8_tail(buf);
296            let (head, tail) = buf.split_at(buf.len() - split);
297            (String::from_utf8_lossy(head).into_owned(), tail.to_vec())
298        }
299    }
300}
301
302/// Number of trailing bytes that form an incomplete UTF-8 sequence (0..=3).
303fn incomplete_utf8_tail(buf: &[u8]) -> usize {
304    // Walk back over continuation bytes (10xxxxxx) to the lead byte.
305    let mut i = buf.len();
306    let mut cont = 0;
307    while i > 0 && buf[i - 1] & 0b1100_0000 == 0b1000_0000 && cont < 3 {
308        i -= 1;
309        cont += 1;
310    }
311    if i == 0 {
312        return 0;
313    }
314    let lead = buf[i - 1];
315    let needed = if lead & 0b1000_0000 == 0 {
316        1
317    } else if lead & 0b1110_0000 == 0b1100_0000 {
318        2
319    } else if lead & 0b1111_0000 == 0b1110_0000 {
320        3
321    } else if lead & 0b1111_1000 == 0b1111_0000 {
322        4
323    } else {
324        1
325    };
326    // If the lead + its continuations are all present, nothing is pending.
327    if cont + 1 >= needed {
328        0
329    } else {
330        cont + 1
331    }
332}