faucet_core/file_format/
json.rs

1//! JSON-shaped file formats: JSON Lines, a single JSON array, and raw text.
2//!
3//! These need no extra dependency, so they are always compiled — which also
4//! makes them the formats a connector can fall back to when no feature is on.
5
6use crate::error::FaucetError;
7use serde_json::Value;
8
9/// The key a [`RawText`](super::FileFormat::RawText) body lands under.
10pub const RAW_TEXT_FIELD: &str = "text";
11
12/// One JSON value per line.
13///
14/// Blank lines are skipped rather than reported: a trailing newline is the
15/// normal end of an NDJSON file, and a writer that emits `\r\n` should not fail
16/// a reader.
17pub fn decode_lines(bytes: &[u8]) -> Result<Vec<Value>, FaucetError> {
18    let text = as_utf8(bytes, "json_lines")?;
19    let mut out = Vec::new();
20    for (i, line) in text.lines().enumerate() {
21        let line = line.trim();
22        if line.is_empty() {
23            continue;
24        }
25        let v = serde_json::from_str(line)
26            .map_err(|e| FaucetError::Source(format!("json_lines: line {}: {e}", i + 1)))?;
27        out.push(v);
28    }
29    Ok(out)
30}
31
32/// A single JSON array (or a single object, which becomes one record).
33pub fn decode_array(bytes: &[u8]) -> Result<Vec<Value>, FaucetError> {
34    let v: Value = serde_json::from_slice(bytes)
35        .map_err(|e| FaucetError::Source(format!("json_array: {e}")))?;
36    Ok(match v {
37        Value::Array(a) => a,
38        other => vec![other],
39    })
40}
41
42/// The whole body as one record.
43pub fn decode_raw_text(bytes: &[u8]) -> Result<Vec<Value>, FaucetError> {
44    let text = as_utf8(bytes, "raw_text")?;
45    Ok(vec![serde_json::json!({ RAW_TEXT_FIELD: text })])
46}
47
48/// One JSON value per line, newline-terminated.
49pub fn encode_lines(records: &[Value]) -> Result<Vec<u8>, FaucetError> {
50    let mut out = Vec::new();
51    for r in records {
52        serde_json::to_writer(&mut out, r)
53            .map_err(|e| FaucetError::Sink(format!("json_lines: {e}")))?;
54        out.push(b'\n');
55    }
56    Ok(out)
57}
58
59/// A single JSON array.
60pub fn encode_array(records: &[Value]) -> Result<Vec<u8>, FaucetError> {
61    serde_json::to_vec(records).map_err(|e| FaucetError::Sink(format!("json_array: {e}")))
62}
63
64/// The inverse of [`decode_raw_text`]: each record's `text` field, one per line.
65///
66/// A record without a `text` field is written as its JSON form rather than
67/// skipped — dropping a record because it does not match the expected shape is
68/// the silent-data-loss failure this project treats as the worst class of bug.
69pub fn encode_raw_text(records: &[Value]) -> Result<Vec<u8>, FaucetError> {
70    let mut out = Vec::new();
71    for r in records {
72        let line = match r.get(RAW_TEXT_FIELD) {
73            Some(Value::String(s)) => s.clone(),
74            _ => super::cell_text(r),
75        };
76        out.extend_from_slice(line.as_bytes());
77        out.push(b'\n');
78    }
79    Ok(out)
80}
81
82fn as_utf8<'a>(bytes: &'a [u8], what: &str) -> Result<&'a str, FaucetError> {
83    std::str::from_utf8(bytes)
84        .map_err(|e| FaucetError::Source(format!("{what}: body is not UTF-8: {e}")))
85}
86
87#[cfg(test)]
88mod tests {
89    use super::*;
90    use serde_json::json;
91
92    #[test]
93    fn json_lines_round_trips_and_tolerates_blank_lines() {
94        let recs = vec![json!({"a": 1}), json!({"a": 2})];
95        let bytes = encode_lines(&recs).expect("encode");
96        assert_eq!(bytes, b"{\"a\":1}\n{\"a\":2}\n");
97        assert_eq!(decode_lines(&bytes).expect("decode"), recs);
98        // A trailing newline and a \r\n writer must both read back cleanly.
99        assert_eq!(
100            decode_lines(b"{\"a\":1}\r\n\n{\"a\":2}\n").expect("decode"),
101            recs
102        );
103    }
104
105    #[test]
106    fn a_bad_line_names_its_line_number() {
107        let err = decode_lines(b"{\"a\":1}\nnot json\n").expect_err("bad line");
108        assert!(err.to_string().contains("line 2"), "{err}");
109    }
110
111    #[test]
112    fn json_array_accepts_an_array_or_a_bare_object() {
113        assert_eq!(
114            decode_array(br#"[{"a":1},{"a":2}]"#).expect("array"),
115            vec![json!({"a": 1}), json!({"a": 2})]
116        );
117        assert_eq!(
118            decode_array(br#"{"a":1}"#).expect("object"),
119            vec![json!({"a": 1})]
120        );
121        assert_eq!(
122            encode_array(&[json!({"a": 1})]).expect("encode"),
123            br#"[{"a":1}]"#
124        );
125    }
126
127    #[test]
128    fn raw_text_round_trips_through_the_text_field() {
129        let recs = decode_raw_text(b"hello\nworld").expect("decode");
130        assert_eq!(recs, vec![json!({"text": "hello\nworld"})]);
131        assert_eq!(encode_raw_text(&recs).expect("encode"), b"hello\nworld\n");
132    }
133
134    #[test]
135    fn raw_text_encode_never_drops_a_record_of_the_wrong_shape() {
136        let out = encode_raw_text(&[json!({"other": 1})]).expect("encode");
137        assert_eq!(out, b"{\"other\":1}\n");
138    }
139
140    #[test]
141    fn non_utf8_is_reported_not_replaced() {
142        let err = decode_lines(&[0xff, 0xfe]).expect_err("invalid utf-8");
143        assert!(err.to_string().contains("not UTF-8"), "{err}");
144    }
145}
faucet_core/file_format/json.rs

faucet_core/file_format/
json.rs
