nodejs/stdlib/
string_decoder.rs

1//! Node `string_decoder` core module: `new StringDecoder(encoding)` with
2//! `.write(buffer)` / `.end([buffer])`. A StringDecoder turns byte chunks into a
3//! string, holding back an incomplete trailing multibyte sequence until the next
4//! chunk completes it.
5//!
6//! Every encoding that has a chunk boundary buffers across it: UTF-8 holds an
7//! incomplete trailing sequence, UTF-16LE holds an odd byte and a trailing high
8//! surrogate, and base64 holds up to two bytes so it only ever emits whole
9//! 3-byte groups. The single-byte encodings consume everything.
10
11use crate::host::{with_host, JsObj};
12use fusevm::Value;
13use indexmap::IndexMap;
14
15/// The methods a `StringDecoder` instance carries. Also the method set its real
16/// prototype object is built from, so a read (`sd.write`), a call (`sd.write(b)`)
17/// and a prototype lookup (`StringDecoder.prototype.write`) cannot disagree.
18pub const INSTANCE_METHODS: &[&str] = &["write", "end"];
19
20/// Node's `normalizeEncoding`: the `encoding` property reports the CANONICAL
21/// name, not the spelling that was passed — `new StringDecoder('ucs2').encoding`
22/// is `'utf16le'` and `new StringDecoder('UTF-8').encoding` is `'utf8'`. Code
23/// that branches on `decoder.encoding` (iconv-lite does) sees the canonical set.
24fn normalize_encoding(enc: &str) -> String {
25    match enc.to_ascii_lowercase().as_str() {
26        "utf8" | "utf-8" => "utf8",
27        "ucs2" | "ucs-2" | "utf16le" | "utf-16le" => "utf16le",
28        "latin1" | "binary" => "latin1",
29        other => return other.to_string(),
30    }
31    .to_string()
32}
33
34/// `new StringDecoder([encoding])`.
35pub fn construct(args: &[Value]) -> Result<Value, String> {
36    let enc = if args.is_empty() {
37        "utf8".to_string()
38    } else {
39        super::arg_str(args, 0)
40    };
41    Ok(with_host(|h| {
42        let mut m = IndexMap::new();
43        m.insert("@@native".into(), h.new_str("StringDecoder"));
44        m.insert("encoding".into(), h.new_str(normalize_encoding(&enc)));
45        // Held-back bytes from a UTF-8 sequence split across chunks.
46        let empty = h.new_array(Vec::new());
47        m.insert("@@pending".into(), empty);
48        h.new_object(m)
49    }))
50}
51
52/// The byte content of a Buffer / typed array / array argument.
53fn bytes_of(v: &Value) -> Vec<u8> {
54    with_host(|h| match h.get(v) {
55        Some(JsObj::Object(p)) => {
56            if p.contains_key("@@buffer") {
57                return crate::stdlib::typedarray::elems_with_host(h, v)
58                    .iter()
59                    .map(|x| h.to_number(x) as u8)
60                    .collect();
61            }
62            match p.get("@@bytes").and_then(|a| h.get(a)) {
63                Some(JsObj::Array(items)) => items.iter().map(|x| h.to_number(x) as u8).collect(),
64                _ => Vec::new(),
65            }
66        }
67        Some(JsObj::Array(items)) => items.iter().map(|x| h.to_number(x) as u8).collect(),
68        _ => Vec::new(),
69    })
70}
71
72fn encoding_of(recv: &Value) -> String {
73    with_host(|h| match h.get(recv) {
74        Some(JsObj::Object(p)) => p
75            .get("encoding")
76            .map(|v| h.str_of(v))
77            .unwrap_or_else(|| "utf8".into()),
78        _ => "utf8".into(),
79    })
80}
81
82fn pending_of(recv: &Value) -> Vec<u8> {
83    with_host(|h| match h.get(recv) {
84        Some(JsObj::Object(p)) => match p.get("@@pending").and_then(|a| h.get(a)) {
85            Some(JsObj::Array(items)) => items.iter().map(|x| h.to_number(x) as u8).collect(),
86            _ => Vec::new(),
87        },
88        _ => Vec::new(),
89    })
90}
91
92fn set_pending(recv: &Value, bytes: &[u8]) {
93    with_host(|h| {
94        let arr = h.new_array(bytes.iter().map(|b| Value::Float(*b as f64)).collect());
95        if let Some(JsObj::Object(p)) = h.get_mut(recv) {
96            p.insert("@@pending".into(), arr);
97        }
98    });
99}
100
101pub fn instance_call(recv: &Value, method: &str, args: &[Value]) -> Result<Value, String> {
102    let enc = encoding_of(recv);
103    match method {
104        "write" => {
105            let mut buf = pending_of(recv);
106            buf.extend(bytes_of(&args.first().cloned().unwrap_or(Value::Undef)));
107            let (decoded, tail) = decode(&enc, &buf);
108            set_pending(recv, &tail);
109            Ok(with_host(|h| h.new_str(decoded)))
110        }
111        "end" => {
112            let mut buf = pending_of(recv);
113            if let Some(v) = args.first() {
114                buf.extend(bytes_of(v));
115            }
116            set_pending(recv, &[]);
117            let (mut decoded, tail) = decode(&enc, &buf);
118            decoded.push_str(&flush(&enc, &tail));
119            Ok(with_host(|h| h.new_str(decoded)))
120        }
121        _ => Err(crate::host::type_error(&format!(
122            "{method} is not a function"
123        ))),
124    }
125}
126
127/// What `end()` emits for bytes still held when the stream closes. Each encoding
128/// resolves its own remainder, so this is not one blanket replacement char:
129///
130/// * UTF-8 — one `U+FFFD` for the whole truncated sequence (not one per byte).
131/// * UTF-16LE — a held HIGH SURROGATE is emitted as a code unit; a dangling odd
132///   byte is dropped silently, with no replacement char (measured on node
133///   v26.7.0: `d.write(Buffer.from([0x61])); d.end()` is `''`). The lone
134///   surrogate itself becomes `U+FFFD` here — the `utf16` storage boundary.
135/// * base64 — the short group is padded and emitted (`AQ==`), which is the whole
136///   reason the bytes were held rather than encoded early.
137fn flush(enc: &str, tail: &[u8]) -> String {
138    if tail.is_empty() {
139        return String::new();
140    }
141    match enc {
142        "base64" => super::to_base64(tail),
143        "base64url" => super::to_base64url(tail),
144        "utf16le" => {
145            let units: Vec<u16> = tail
146                .chunks_exact(2)
147                .map(|c| u16::from_le_bytes([c[0], c[1]]))
148                .collect();
149            crate::utf16::to_string_lossy(&units)
150        }
151        _ => "\u{FFFD}".to_string(),
152    }
153}
154
155/// Decode `buf` in `enc`, returning (decoded string, held-back trailing bytes).
156/// Single-byte encodings consume everything; the rest hold back the partial tail
157/// that only the next chunk can complete.
158fn decode(enc: &str, buf: &[u8]) -> (String, Vec<u8>) {
159    match enc {
160        // `ascii` masks the high bit, `latin1`/`binary` keep the whole byte.
161        "ascii" => (
162            buf.iter().map(|b| (*b & 0x7f) as char).collect(),
163            Vec::new(),
164        ),
165        "latin1" => (buf.iter().map(|b| *b as char).collect(), Vec::new()),
166        "hex" => (super::to_hex(buf), Vec::new()),
167        // Base64 is 3 bytes → 4 characters. Emitting a short group would pad it
168        // mid-stream (`AQ==` then `AgM=` instead of `AQID`), so the remainder is
169        // held until the group closes or `end()` pads it.
170        "base64" | "base64url" => {
171            let keep = buf.len() % 3;
172            let (head, tail) = buf.split_at(buf.len() - keep);
173            let s = if enc == "base64url" {
174                super::to_base64url(head)
175            } else {
176                super::to_base64(head)
177            };
178            (s, tail.to_vec())
179        }
180        // UTF-16LE: an odd trailing byte is half a code unit, and a trailing HIGH
181        // surrogate is half a code point — both wait for the next chunk.
182        "utf16le" => {
183            let mut keep = buf.len() % 2;
184            let whole = buf.len() - keep;
185            if whole >= 2 {
186                let last = u16::from_le_bytes([buf[whole - 2], buf[whole - 1]]);
187                if (0xD800..0xDC00).contains(&last) {
188                    keep += 2;
189                }
190            }
191            let (head, tail) = buf.split_at(buf.len() - keep);
192            let units: Vec<u16> = head
193                .chunks_exact(2)
194                .map(|c| u16::from_le_bytes([c[0], c[1]]))
195                .collect();
196            (crate::utf16::to_string_lossy(&units), tail.to_vec())
197        }
198        // utf8 / utf-8 (and anything else): keep a split multibyte tail pending.
199        _ => {
200            let split = incomplete_utf8_tail(buf);
201            let (head, tail) = buf.split_at(buf.len() - split);
202            (String::from_utf8_lossy(head).into_owned(), tail.to_vec())
203        }
204    }
205}
206
207/// Number of trailing bytes that form an incomplete UTF-8 sequence (0..=3).
208fn incomplete_utf8_tail(buf: &[u8]) -> usize {
209    // Walk back over continuation bytes (10xxxxxx) to the lead byte.
210    let mut i = buf.len();
211    let mut cont = 0;
212    while i > 0 && buf[i - 1] & 0b1100_0000 == 0b1000_0000 && cont < 3 {
213        i -= 1;
214        cont += 1;
215    }
216    if i == 0 {
217        return 0;
218    }
219    let lead = buf[i - 1];
220    let needed = if lead & 0b1000_0000 == 0 {
221        1
222    } else if lead & 0b1110_0000 == 0b1100_0000 {
223        2
224    } else if lead & 0b1111_0000 == 0b1110_0000 {
225        3
226    } else if lead & 0b1111_1000 == 0b1111_0000 {
227        4
228    } else {
229        1
230    };
231    // If the lead + its continuations are all present, nothing is pending.
232    if cont + 1 >= needed {
233        0
234    } else {
235        cont + 1
236    }
237}
nodejs/stdlib/string_decoder.rs

nodejs/stdlib/
string_decoder.rs
