faucet_source_rest/
format.rs

1//! Tabular response-body parsing for `response_format: csv | excel` (#497).
2//!
3//! Turns a downloaded file body into a `Vec<Value>` of JSON objects, so an
4//! authenticated file endpoint (a Microsoft Graph `…/content` download, a
5//! signed export URL, …) can be consumed through the same REST source that
6//! already owns auth, retry, and context substitution.
7
8use faucet_core::FaucetError;
9use serde_json::{Map, Value};
10
11/// Parse CSV bytes into records. When `has_headers`, the first row supplies
12/// field names; otherwise fields are named `column_0`, `column_1`, … Values are
13/// strings (matching the `csv` source). Streaming RFC-4180 via `csv-async`, so
14/// quoted fields with embedded newlines are handled correctly.
15pub async fn parse_csv(
16    bytes: &[u8],
17    delimiter: u8,
18    has_headers: bool,
19) -> Result<Vec<Value>, FaucetError> {
20    use futures::StreamExt as _;
21    let mut rdr = csv_async::AsyncReaderBuilder::new()
22        .has_headers(false)
23        .delimiter(delimiter)
24        .flexible(true)
25        .create_reader(bytes);
26    let mut records = rdr.records();
27    let mut headers: Option<Vec<String>> = None;
28    let mut out = Vec::new();
29    while let Some(rec) = records.next().await {
30        let rec = rec.map_err(|e| FaucetError::Source(format!("rest: CSV parse error: {e}")))?;
31        if has_headers && headers.is_none() {
32            headers = Some(rec.iter().map(str::to_string).collect());
33            continue;
34        }
35        let mut obj = Map::new();
36        for (i, field) in rec.iter().enumerate() {
37            let key = headers
38                .as_ref()
39                .and_then(|h| h.get(i).cloned())
40                .unwrap_or_else(|| format!("column_{i}"));
41            obj.insert(key, Value::String(field.to_string()));
42        }
43        out.push(Value::Object(obj));
44    }
45    Ok(out)
46}
47
48/// Parse Excel bytes into records. Requires the `excel` feature.
49#[cfg(feature = "excel")]
50pub fn parse_excel(
51    bytes: &[u8],
52    sheet: Option<&str>,
53    header_row: usize,
54) -> Result<Vec<Value>, FaucetError> {
55    use calamine::{Data, Reader, Xlsx};
56    let cursor = std::io::Cursor::new(bytes.to_vec());
57    let mut wb: Xlsx<_> = calamine::open_workbook_from_rs(cursor)
58        .map_err(|e| FaucetError::Source(format!("rest: opening Excel workbook: {e}")))?;
59    let names = wb.sheet_names().to_vec();
60    let name = match sheet {
61        Some(s) if names.iter().any(|n| n == s) => s.to_string(),
62        Some(s) => match s.parse::<usize>() {
63            Ok(idx) => names.get(idx).cloned().ok_or_else(|| {
64                FaucetError::Source(format!("rest: Excel sheet index {idx} out of range"))
65            })?,
66            Err(_) => {
67                return Err(FaucetError::Source(format!(
68                    "rest: Excel sheet '{s}' not found (available: {})",
69                    names.join(", ")
70                )));
71            }
72        },
73        None => names
74            .first()
75            .cloned()
76            .ok_or_else(|| FaucetError::Source("rest: Excel workbook has no worksheets".into()))?,
77    };
78    let range = wb
79        .worksheet_range(&name)
80        .map_err(|e| FaucetError::Source(format!("rest: reading Excel sheet '{name}': {e}")))?;
81    let rows: Vec<&[Data]> = range.rows().collect();
82    let header = rows.get(header_row).ok_or_else(|| {
83        FaucetError::Source(format!(
84            "rest: Excel header_row {header_row} is beyond the sheet ({} rows)",
85            rows.len()
86        ))
87    })?;
88    let headers: Vec<String> = header.iter().map(cell_to_string).collect();
89    let mut out = Vec::new();
90    for row in rows.iter().skip(header_row + 1) {
91        let mut obj = Map::new();
92        for (i, cell) in row.iter().enumerate() {
93            let key = headers
94                .get(i)
95                .cloned()
96                .filter(|k| !k.is_empty())
97                .unwrap_or_else(|| format!("column_{i}"));
98            obj.insert(key, cell_to_value(cell));
99        }
100        out.push(Value::Object(obj));
101    }
102    Ok(out)
103}
104
105/// Stub when the `excel` feature is off — error loudly rather than mis-parse an
106/// Excel blob as CSV.
107#[cfg(not(feature = "excel"))]
108pub fn parse_excel(
109    _bytes: &[u8],
110    _sheet: Option<&str>,
111    _header_row: usize,
112) -> Result<Vec<Value>, FaucetError> {
113    Err(FaucetError::Config(
114        "rest: `response_format: excel` requires the crate's `excel` feature — rebuild the CLI \
115         with `--features source-rest-excel`"
116            .into(),
117    ))
118}
119
120#[cfg(feature = "excel")]
121fn cell_to_string(cell: &calamine::Data) -> String {
122    use calamine::Data;
123    match cell {
124        Data::String(s) => s.clone(),
125        Data::Empty => String::new(),
126        other => other.to_string(),
127    }
128}
129
130#[cfg(feature = "excel")]
131fn cell_to_value(cell: &calamine::Data) -> Value {
132    use calamine::Data;
133    match cell {
134        Data::Empty => Value::Null,
135        Data::String(s) => Value::String(s.clone()),
136        Data::Bool(b) => Value::Bool(*b),
137        Data::Int(i) => Value::from(*i),
138        Data::Float(f) => serde_json::Number::from_f64(*f)
139            .map(Value::Number)
140            .unwrap_or(Value::Null),
141        Data::DateTime(dt) => Value::String(dt.to_string()),
142        Data::DateTimeIso(s) | Data::DurationIso(s) => Value::String(s.clone()),
143        Data::Error(e) => Value::String(format!("{e:?}")),
144    }
145}
146
147#[cfg(test)]
148mod tests {
149    use super::*;
150
151    #[tokio::test]
152    async fn csv_with_headers() {
153        let recs = parse_csv(b"id,name\n1,Alice\n2,Bob\n", b',', true)
154            .await
155            .unwrap();
156        assert_eq!(recs.len(), 2);
157        assert_eq!(recs[0]["id"], "1");
158        assert_eq!(recs[0]["name"], "Alice");
159        assert_eq!(recs[1]["name"], "Bob");
160    }
161
162    #[tokio::test]
163    async fn csv_without_headers_generates_names() {
164        let recs = parse_csv(b"1,Alice\n", b',', false).await.unwrap();
165        assert_eq!(recs[0]["column_0"], "1");
166        assert_eq!(recs[0]["column_1"], "Alice");
167    }
168
169    #[tokio::test]
170    async fn csv_custom_delimiter_and_embedded_newline() {
171        let recs = parse_csv(b"a;b\n1;\"x\ny\"\n", b';', true).await.unwrap();
172        assert_eq!(recs.len(), 1);
173        assert_eq!(recs[0]["b"], "x\ny");
174    }
175
176    #[cfg(not(feature = "excel"))]
177    #[test]
178    fn excel_without_feature_errors() {
179        assert!(
180            parse_excel(b"x", None, 0)
181                .unwrap_err()
182                .to_string()
183                .contains("excel")
184        );
185    }
186
187    #[cfg(feature = "excel")]
188    #[test]
189    fn cell_conversions_cover_all_variants() {
190        use calamine::Data;
191        assert_eq!(cell_to_value(&Data::Empty), Value::Null);
192        assert_eq!(
193            cell_to_value(&Data::String("s".into())),
194            Value::String("s".into())
195        );
196        assert_eq!(cell_to_value(&Data::Bool(true)), Value::Bool(true));
197        assert_eq!(cell_to_value(&Data::Int(7)), Value::from(7i64));
198        assert_eq!(cell_to_value(&Data::Float(1.5)), Value::from(1.5));
199        assert!(
200            cell_to_value(&Data::DateTime(calamine::ExcelDateTime::new(
201                44_000.0,
202                calamine::ExcelDateTimeType::DateTime,
203                false
204            )))
205            .is_string()
206        );
207        assert!(cell_to_value(&Data::DateTimeIso("2020".into())).is_string());
208        assert!(cell_to_value(&Data::DurationIso("PT1H".into())).is_string());
209        assert!(cell_to_value(&Data::Error(calamine::CellErrorType::Div0)).is_string());
210        assert_eq!(cell_to_string(&Data::String("k".into())), "k");
211        assert_eq!(cell_to_string(&Data::Empty), "");
212        assert_eq!(cell_to_string(&Data::Int(3)), "3");
213    }
214
215    #[cfg(feature = "excel")]
216    #[test]
217    fn excel_sheet_selection_and_error_paths() {
218        let xlsx = include_bytes!("../tests/fixtures/sample.xlsx");
219        // Numeric-index sheet selection (second sheet).
220        let recs = parse_excel(xlsx, Some("1"), 0).unwrap();
221        assert_eq!(recs[0]["k"], "x");
222        // Out-of-range numeric index.
223        assert!(
224            parse_excel(xlsx, Some("99"), 0)
225                .unwrap_err()
226                .to_string()
227                .contains("out of range")
228        );
229        // Named sheet not found.
230        assert!(
231            parse_excel(xlsx, Some("Nope"), 0)
232                .unwrap_err()
233                .to_string()
234                .contains("not found")
235        );
236        // header_row beyond the sheet.
237        assert!(
238            parse_excel(xlsx, None, 9999)
239                .unwrap_err()
240                .to_string()
241                .contains("beyond the sheet")
242        );
243        // Malformed workbook bytes.
244        assert!(parse_excel(b"not-a-workbook", None, 0).is_err());
245    }
246}
faucet_source_rest/format.rs

faucet_source_rest/
format.rs
