mermaid_cli/utils/
ndjson.rs

1//! UTF-8-safe NDJSON line draining for byte-stream readers.
2//!
3//! Newline-delimited JSON over an HTTP byte stream presents two pitfalls:
4//! TCP chunk boundaries don't align with line boundaries, *and* they don't
5//! align with UTF-8 codepoint boundaries (a 3-byte CJK char can straddle
6//! two packets). Decoding each chunk independently with `from_utf8_lossy`
7//! would corrupt those split codepoints into U+FFFD.
8//!
9//! `drain_complete_lines` solves this by buffering raw bytes and only
10//! decoding *complete* lines: splitting on the single byte `b'\n'`
11//! (0x0A) is safe because that byte never appears inside a multi-byte
12//! UTF-8 sequence, and by the time `\n` arrives, every byte of every
13//! preceding codepoint has also arrived.
14
15/// Drain all complete newline-terminated lines out of `buf`, decoding each
16/// as UTF-8 and dropping the trailing `\n`. Any partial trailing line stays
17/// in `buf` so the caller can append more bytes and try again.
18pub fn drain_complete_lines(buf: &mut Vec<u8>) -> Vec<String> {
19    let mut lines = Vec::new();
20    while let Some(newline_pos) = buf.iter().position(|&b| b == b'\n') {
21        // Drain through the `\n` inclusive, then drop it before decoding.
22        let line_bytes: Vec<u8> = buf.drain(..=newline_pos).collect();
23        let content = &line_bytes[..line_bytes.len() - 1];
24        lines.push(String::from_utf8_lossy(content).into_owned());
25    }
26    lines
27}
28
29#[cfg(test)]
30mod tests {
31    use super::drain_complete_lines;
32
33    #[test]
34    fn drain_empty_buf() {
35        let mut buf: Vec<u8> = Vec::new();
36        assert!(drain_complete_lines(&mut buf).is_empty());
37        assert!(buf.is_empty());
38    }
39
40    #[test]
41    fn drain_without_newline_keeps_buf_intact() {
42        let mut buf = b"partial".to_vec();
43        assert!(drain_complete_lines(&mut buf).is_empty());
44        assert_eq!(buf, b"partial");
45    }
46
47    #[test]
48    fn drain_yields_complete_lines_and_keeps_tail() {
49        let mut buf = b"first\nsecond\ntail".to_vec();
50        let lines = drain_complete_lines(&mut buf);
51        assert_eq!(lines, vec!["first".to_string(), "second".to_string()]);
52        assert_eq!(buf, b"tail");
53    }
54
55    /// A 3-byte CJK char ("你" = E4 BD A0) split across chunks must survive
56    /// reassembly intact — no U+FFFD. Simulates the TCP-boundary-splits-a-
57    /// codepoint scenario the helper is guarding against.
58    #[test]
59    fn drain_preserves_codepoints_across_chunks() {
60        let mut buf: Vec<u8> = Vec::new();
61        buf.extend_from_slice(b"hello ");
62        buf.extend_from_slice(&[0xE4, 0xBD]);
63        assert!(drain_complete_lines(&mut buf).is_empty());
64
65        buf.extend_from_slice(&[0xA0]);
66        buf.extend_from_slice("好\n".as_bytes());
67
68        let lines = drain_complete_lines(&mut buf);
69        assert_eq!(lines, vec!["hello 你好".to_string()]);
70        assert!(buf.is_empty());
71    }
72}
mermaid_cli/utils/ndjson.rs

mermaid_cli/utils/
ndjson.rs
