Skip to main content

flatwire/
lib.rs

1//! flatwire - streaming JSON serialization that keeps memory flat and time linear.
2//!
3//! The array helpers are the point: a large collection is written and read one
4//! element at a time, so peak memory is bounded by the largest single element
5//! rather than the whole collection. Wire format is plain JSON via serde_json.
6
7use std::io::{self, Read, Write};
8
9use serde::Serialize;
10use serde_json::Value;
11
12pub mod xml;
13pub mod msgpack;
14pub mod checked;
15pub mod cbor;
16
17/// Encode a whole value to UTF-8 JSON bytes.
18pub fn encode<T: Serialize>(value: &T) -> serde_json::Result<Vec<u8>> {
19    serde_json::to_vec(value)
20}
21
22/// Decode UTF-8 JSON bytes to a `serde_json::Value`.
23pub fn decode(data: &[u8]) -> serde_json::Result<Value> {
24    serde_json::from_slice(data)
25}
26
27/// Stream a value straight to a writer (no intermediate `String`).
28pub fn encode_to<T: Serialize, W: Write>(value: &T, writer: &mut W) -> io::Result<()> {
29    serde_json::to_writer(&mut *writer, value).map_err(io::Error::from)
30}
31
32/// Read a whole value from a reader.
33pub fn decode_from<R: Read>(reader: R) -> serde_json::Result<Value> {
34    serde_json::from_reader(reader)
35}
36
37/// Stream a large collection as a JSON array, one element at a time. Peak memory
38/// is bounded by the largest single element, not the collection length. Returns
39/// the number of elements written.
40pub fn encode_array<T, I, W>(items: I, writer: &mut W) -> io::Result<usize>
41where
42    T: Serialize,
43    I: IntoIterator<Item = T>,
44    W: Write,
45{
46    writer.write_all(b"[")?;
47    let mut count = 0usize;
48    for item in items {
49        if count > 0 {
50            writer.write_all(b",")?;
51        }
52        serde_json::to_writer(&mut *writer, &item).map_err(io::Error::from)?;
53        count += 1;
54    }
55    writer.write_all(b"]")?;
56    Ok(count)
57}
58
59/// A lazy iterator over the elements of a top-level JSON array read from a
60/// reader. Tracks bracket/brace depth and string state to find element
61/// boundaries without materializing the whole array; each element is parsed
62/// individually, so memory stays proportional to the largest element.
63pub struct ArrayIter<R: Read> {
64    reader: R,
65    buf: Vec<u8>,
66    pos: usize,
67    depth: i64,
68    max_depth: i64,
69    in_string: bool,
70    escape: bool,
71    started: bool,
72    finished: bool,
73}
74
75pub fn decode_array<R: Read>(reader: R) -> ArrayIter<R> {
76    ArrayIter {
77        reader,
78        buf: Vec::new(),
79        pos: 0,
80        depth: 0,
81        max_depth: 200,
82        in_string: false,
83        escape: false,
84        started: false,
85        finished: false,
86    }
87}
88
89impl<R: Read> ArrayIter<R> {
90    /// Bound how deeply a single element may nest before the iterator rejects the
91    /// input, guarding against hostile `[[[[...` streams. Use 0 to disable.
92    pub fn with_max_depth(mut self, max_depth: i64) -> Self {
93        self.max_depth = max_depth;
94        self
95    }
96}
97
98impl<R: Read> Iterator for ArrayIter<R> {
99    type Item = io::Result<Value>;
100
101    fn next(&mut self) -> Option<Self::Item> {
102        if self.finished {
103            return None;
104        }
105        // Drop everything already consumed so the buffer never grows with the
106        // array. After this, the current element begins at index 0.
107        if self.pos > 0 {
108            self.buf.drain(0..self.pos);
109            self.pos = 0;
110        }
111        loop {
112            while self.pos < self.buf.len() {
113                let ch = self.buf[self.pos];
114                if !self.started {
115                    if ch.is_ascii_whitespace() {
116                        self.pos += 1;
117                        self.buf.drain(0..self.pos);
118                        self.pos = 0;
119                        continue;
120                    }
121                    if ch != b'[' {
122                        self.finished = true;
123                        return Some(Err(io::Error::new(
124                            io::ErrorKind::InvalidData,
125                            "decode_array expects a top-level JSON array",
126                        )));
127                    }
128                    self.started = true;
129                    self.pos += 1;
130                    self.buf.drain(0..self.pos);
131                    self.pos = 0;
132                    continue;
133                }
134                if self.in_string {
135                    if self.escape {
136                        self.escape = false;
137                    } else if ch == b'\\' {
138                        self.escape = true;
139                    } else if ch == b'"' {
140                        self.in_string = false;
141                    }
142                    self.pos += 1;
143                    continue;
144                }
145                match ch {
146                    b'"' => self.in_string = true,
147                    b'{' | b'[' => {
148                        self.depth += 1;
149                        if self.max_depth > 0 && self.depth > self.max_depth {
150                            self.finished = true;
151                            return Some(Err(io::Error::new(
152                                io::ErrorKind::InvalidData,
153                                "decode_array: nesting depth exceeded",
154                            )));
155                        }
156                    }
157                    b']' if self.depth == 0 => {
158                        let s = std::str::from_utf8(&self.buf[..self.pos])
159                            .unwrap_or("")
160                            .trim();
161                        self.pos += 1;
162                        self.finished = true;
163                        if s.is_empty() {
164                            return None;
165                        }
166                        return Some(serde_json::from_str(s).map_err(io::Error::from));
167                    }
168                    b'}' | b']' => self.depth -= 1,
169                    b',' if self.depth == 0 => {
170                        let s = std::str::from_utf8(&self.buf[..self.pos])
171                            .unwrap_or("")
172                            .trim()
173                            .to_string();
174                        self.pos += 1;
175                        return Some(serde_json::from_str(&s).map_err(io::Error::from));
176                    }
177                    _ => {}
178                }
179                self.pos += 1;
180            }
181            // Ran out of buffered bytes mid-element: read more. elem_start is 0,
182            // so the whole partial element (bounded by element size) is retained.
183            let mut tmp = [0u8; 65536];
184            match self.reader.read(&mut tmp) {
185                Ok(0) => {
186                    self.finished = true;
187                    return Some(Err(io::Error::new(
188                        io::ErrorKind::UnexpectedEof,
189                        "stream ended before the JSON array was closed",
190                    )));
191                }
192                Ok(k) => self.buf.extend_from_slice(&tmp[..k]),
193                Err(e) => {
194                    self.finished = true;
195                    return Some(Err(e));
196                }
197            }
198        }
199    }
200}