document_svg/document/
json_seq.rs

1//! Bounded RFC 7464 JSON Text Sequence and newline-delimited previews.
2
3use std::path::Path;
4use std::str;
5
6use crate::convert::{ConvertOptions, PageConsumer, read_limited_file};
7use crate::document::html::{HtmlBlock, render_blocks_to_pages};
8use crate::error::{Error, Result};
9use crate::ir::Page;
10
11const MAX_JSON_SEQUENCE_BYTES: u64 = 64 * 1024 * 1024;
12const MAX_JSON_SEQUENCE_RECORD_BYTES: usize = 4 * 1024 * 1024;
13const MAX_JSON_SEQUENCE_RECORDS: usize = 100_000;
14const MAX_JSON_SEQUENCE_BLOCKS: usize = 200_000;
15const RFC7464_RECORD_SEPARATOR: u8 = 0x1e;
16
17struct JsonSequencePageSink<'a> {
18    inner: &'a mut dyn PageConsumer,
19    warnings: &'a [String],
20    next_page_number: usize,
21}
22
23impl PageConsumer for JsonSequencePageSink<'_> {
24    fn consume(&mut self, mut page: Page) -> Result<()> {
25        page.number = self.next_page_number;
26        page.source_format = "jsonseq".into();
27        page.title = "JSON Text Sequence".into();
28        self.next_page_number += 1;
29        for warning in self.warnings {
30            page.warn(warning.clone());
31        }
32        self.inner.consume(page)
33    }
34}
35
36pub(crate) fn looks_like_prefix(bytes: &[u8]) -> bool {
37    bytes.first() == Some(&RFC7464_RECORD_SEPARATOR)
38}
39
40pub(crate) fn convert(
41    path: &Path,
42    options: &ConvertOptions,
43    sink: &mut dyn PageConsumer,
44) -> Result<Vec<String>> {
45    let bytes = read_limited_file(
46        path,
47        options.max_input_bytes.min(MAX_JSON_SEQUENCE_BYTES),
48        "JSON Text Sequence input",
49    )?;
50    let newline_delimited = path
51        .extension()
52        .and_then(|extension| extension.to_str())
53        .is_some_and(|extension| extension.eq_ignore_ascii_case("jsonl"));
54    let mut blocks = Vec::new();
55    let mut warning_counts = SequenceWarnings::default();
56    let record_count = if newline_delimited {
57        parse_newline_delimited(&bytes, &mut blocks, &mut warning_counts)?
58    } else {
59        parse_rfc7464(&bytes, &mut blocks, &mut warning_counts)?
60    };
61    if blocks.is_empty() {
62        return Err(Error::InvalidInput(
63            "JSON Text Sequence contains no valid JSON records".into(),
64        ));
65    }
66    let mut warnings = warning_counts.json_warnings.clone();
67    if newline_delimited {
68        warnings.push(
69            "newline-delimited JSON was read in compatibility mode; RFC 7464 uses RS-prefixed records".into(),
70        );
71    }
72    if warning_counts.empty_records > 0 {
73        warnings.push(format!(
74            "{} empty JSON sequence record(s) were skipped",
75            warning_counts.empty_records
76        ));
77    }
78    if warning_counts.invalid_records > 0 {
79        warnings.push(format!(
80            "{} invalid JSON sequence record(s) were skipped; check the sequence before relying on its contents",
81            warning_counts.invalid_records
82        ));
83    }
84    warnings.push(format!(
85        "{record_count} JSON sequence record(s) were laid out in order as inert content"
86    ));
87
88    let mut page_sink = JsonSequencePageSink {
89        inner: sink,
90        warnings: &warnings,
91        next_page_number: 1,
92    };
93    render_blocks_to_pages(&blocks, &mut page_sink, options)?;
94    Ok(warnings)
95}
96
97#[derive(Default)]
98struct SequenceWarnings {
99    empty_records: usize,
100    invalid_records: usize,
101    json_warnings: Vec<String>,
102}
103
104fn parse_rfc7464(
105    bytes: &[u8],
106    blocks: &mut Vec<HtmlBlock>,
107    warnings: &mut SequenceWarnings,
108) -> Result<usize> {
109    if !looks_like_prefix(bytes) {
110        return Err(Error::InvalidInput(
111            "RFC 7464 JSON Text Sequence must begin with an ASCII Record Separator (0x1E)".into(),
112        ));
113    }
114    let mut cursor = 0usize;
115    let mut record_count = 0usize;
116    let mut record_number = 0usize;
117    while cursor < bytes.len() {
118        if bytes[cursor] != RFC7464_RECORD_SEPARATOR {
119            return Err(Error::InvalidInput(
120                "expected an RFC 7464 Record Separator".into(),
121            ));
122        }
123        let body_start = cursor + 1;
124        let next_separator = bytes[body_start..]
125            .iter()
126            .position(|byte| *byte == RFC7464_RECORD_SEPARATOR)
127            .map(|offset| body_start + offset)
128            .unwrap_or(bytes.len());
129        let record = &bytes[body_start..next_separator];
130        record_number += 1;
131        record_count += 1;
132        if record_count > MAX_JSON_SEQUENCE_RECORDS {
133            return Err(Error::LimitExceeded(format!(
134                "JSON Text Sequence exceeds {MAX_JSON_SEQUENCE_RECORDS} records"
135            )));
136        }
137        append_record(record, record_number, blocks, warnings)?;
138        cursor = next_separator;
139    }
140    Ok(record_count)
141}
142
143fn parse_newline_delimited(
144    bytes: &[u8],
145    blocks: &mut Vec<HtmlBlock>,
146    warnings: &mut SequenceWarnings,
147) -> Result<usize> {
148    let mut record_count = 0usize;
149    let mut record_number = 0usize;
150    for line in bytes.split(|byte| *byte == b'\n') {
151        let record = line.strip_suffix(b"\r").unwrap_or(line);
152        if record.iter().all(u8::is_ascii_whitespace) {
153            warnings.empty_records += 1;
154            continue;
155        }
156        if record.first() == Some(&RFC7464_RECORD_SEPARATOR) {
157            return Err(Error::InvalidInput(
158                "newline-delimited JSON cannot contain RFC 7464 Record Separators".into(),
159            ));
160        }
161        record_number += 1;
162        record_count += 1;
163        if record_count > MAX_JSON_SEQUENCE_RECORDS {
164            return Err(Error::LimitExceeded(format!(
165                "newline-delimited JSON exceeds {MAX_JSON_SEQUENCE_RECORDS} records"
166            )));
167        }
168        append_record(record, record_number, blocks, warnings)?;
169    }
170    if record_count == 0 {
171        return Err(Error::InvalidInput(
172            "newline-delimited JSON contains no records".into(),
173        ));
174    }
175    Ok(record_count)
176}
177
178fn append_record(
179    record: &[u8],
180    record_number: usize,
181    blocks: &mut Vec<HtmlBlock>,
182    warnings: &mut SequenceWarnings,
183) -> Result<()> {
184    let mut record = record;
185    while record.last().is_some_and(u8::is_ascii_whitespace) {
186        record = &record[..record.len() - 1];
187    }
188    if record.is_empty() {
189        warnings.empty_records += 1;
190        return Ok(());
191    }
192    if record.len() > MAX_JSON_SEQUENCE_RECORD_BYTES {
193        return Err(Error::LimitExceeded(format!(
194            "JSON sequence record exceeds {MAX_JSON_SEQUENCE_RECORD_BYTES} bytes"
195        )));
196    }
197    let text = match str::from_utf8(record) {
198        Ok(text) => text,
199        Err(_) => {
200            warnings.invalid_records += 1;
201            return Ok(());
202        }
203    };
204    let (mut record_blocks, record_warnings) = match crate::document::json::parse_json_blocks(text)
205    {
206        Ok(result) => result,
207        Err(Error::InvalidInput(_)) => {
208            warnings.invalid_records += 1;
209            return Ok(());
210        }
211        Err(error) => return Err(error),
212    };
213    if let Some(HtmlBlock::Heading { level: 1, text }) = record_blocks.first_mut() {
214        *text = format!("JSON sequence record {record_number}");
215    }
216    let output_blocks = blocks
217        .len()
218        .checked_add(record_blocks.len())
219        .ok_or_else(|| {
220            Error::LimitExceeded("JSON Text Sequence output block count overflowed".into())
221        })?;
222    if output_blocks > MAX_JSON_SEQUENCE_BLOCKS {
223        return Err(Error::LimitExceeded(format!(
224            "JSON Text Sequence exceeds {MAX_JSON_SEQUENCE_BLOCKS} rendered blocks"
225        )));
226    }
227    blocks.append(&mut record_blocks);
228    for warning in record_warnings {
229        if !warnings.json_warnings.contains(&warning) {
230            warnings.json_warnings.push(warning);
231        }
232    }
233    Ok(())
234}
235
236#[cfg(test)]
237mod tests {
238    use super::*;
239
240    #[test]
241    fn detects_json_text_sequence_record_separator() {
242        assert!(looks_like_prefix(b"\x1e{\"event\":\"load\"}\n"));
243        assert!(!looks_like_prefix(b"{\"event\":\"load\"}"));
244    }
245
246    #[test]
247    fn parses_mixed_values_and_labels_each_record() {
248        let bytes = b"\x1e{\"event\":\"load\",\"rows\":3}\n\x1e[1,2,true]\n\x1enull\n";
249        let mut blocks = Vec::new();
250        let mut warnings = SequenceWarnings::default();
251        let records = parse_rfc7464(bytes, &mut blocks, &mut warnings).unwrap();
252        assert_eq!(records, 3);
253        assert!(blocks.iter().any(|block| matches!(
254            block,
255            HtmlBlock::Heading { text, .. } if text == "JSON sequence record 2"
256        )));
257        assert!(blocks.len() > 6);
258    }
259
260    #[test]
261    fn skips_invalid_rfc7464_records_with_a_warning_count() {
262        let mut blocks = Vec::new();
263        let mut warnings = SequenceWarnings::default();
264        let records = parse_rfc7464(
265            b"\x1e{\"ok\":true}\n\x1e{invalid}\n\x1e2\n",
266            &mut blocks,
267            &mut warnings,
268        )
269        .unwrap();
270        assert_eq!(records, 3);
271        assert_eq!(warnings.invalid_records, 1);
272        assert!(!blocks.is_empty());
273    }
274
275    #[test]
276    fn parses_newline_delimited_compatibility_records() {
277        let mut blocks = Vec::new();
278        let mut warnings = SequenceWarnings::default();
279        let records = parse_newline_delimited(
280            b"{\"key\":\"value\"}\n[\"a\",\"b\"]\n",
281            &mut blocks,
282            &mut warnings,
283        )
284        .unwrap();
285        assert_eq!(records, 2);
286        assert!(blocks.len() > 4);
287    }
288
289    #[test]
290    fn skips_empty_and_invalid_rfc7464_records_without_hiding_warnings() {
291        let mut blocks = Vec::new();
292        let mut warnings = SequenceWarnings::default();
293        let records = parse_rfc7464(
294            b"\x1e\x1e{\"event\":\"ok\"}\n\x1e{broken}\n",
295            &mut blocks,
296            &mut warnings,
297        )
298        .unwrap();
299        assert_eq!(records, 3);
300        assert_eq!(warnings.empty_records, 1);
301        assert_eq!(warnings.invalid_records, 1);
302        assert_eq!(blocks.len(), 2);
303    }
304}
document_svg/document/json_seq.rs

document_svg/document/
json_seq.rs
