sheeter_engine/
canonical.rs

1use std::error::Error;
2use std::io::Cursor;
3
4/// Format-agnostic intermediate value used by the Sheeter hub.
5///
6/// Every reader (CSV, TSV, JSON, YAML, XML) produces a `CanonicalValue`, and
7/// every writer consumes one. Adding a format requires exactly one reader and
8/// one writer, not `N-1` pairwise converters, because every spoke composes
9/// through this hub.
10///
11/// Tabular data is represented as `Array<Object>`. XML documents are
12/// represented as a single-key `Object` whose key is the root element name,
13/// with attributes prefixed by `@` and text stored under `#text`. Repeated
14/// sibling elements become `Array`.
15///
16/// # Examples
17///
18/// ```
19/// use sheeter_engine::CanonicalValue;
20///
21/// let v = CanonicalValue::from_json(r#"[{"a":1}]"#).unwrap();
22/// assert_eq!(v.to_csv().unwrap(), "a\n1\n");
23/// ```
24#[derive(Debug, Clone, PartialEq)]
25pub enum CanonicalValue {
26    Null,
27    Bool(bool),
28    Number(serde_json::Number),
29    String(String),
30    Bytes(Vec<u8>),
31    Array(Vec<CanonicalValue>),
32    Object(Vec<(String, CanonicalValue)>),
33}
34
35type XmlStackFrame = (
36    String,
37    Vec<(String, String)>,
38    Vec<(String, CanonicalValue)>,
39    String,
40);
41
42impl CanonicalValue {
43    pub(crate) fn from_json_value(v: serde_json::Value) -> Self {
44        match v {
45            serde_json::Value::Null => Self::Null,
46            serde_json::Value::Bool(b) => Self::Bool(b),
47            serde_json::Value::Number(n) => Self::Number(n),
48            serde_json::Value::String(s) => Self::String(s),
49            serde_json::Value::Array(a) => {
50                Self::Array(a.into_iter().map(Self::from_json_value).collect())
51            }
52            serde_json::Value::Object(m) => {
53                if m.len() == 1 {
54                    if let Some(serde_json::Value::String(b64)) = m.get("$binary") {
55                        use base64::Engine;
56                        if let Ok(bytes) = base64::engine::general_purpose::STANDARD.decode(b64) {
57                            return Self::Bytes(bytes);
58                        }
59                    }
60                }
61                Self::Object(
62                    m.into_iter()
63                        .map(|(k, v)| (k, Self::from_json_value(v)))
64                        .collect(),
65                )
66            }
67        }
68    }
69
70    pub(crate) fn to_json_value(&self) -> serde_json::Value {
71        match self {
72            Self::Null => serde_json::Value::Null,
73            Self::Bool(b) => serde_json::Value::Bool(*b),
74            Self::Number(n) => serde_json::Value::Number(n.clone()),
75            Self::String(s) => serde_json::Value::String(s.clone()),
76            Self::Bytes(b) => {
77                use base64::Engine;
78                let mut map = serde_json::Map::new();
79                map.insert(
80                    "$binary".to_string(),
81                    serde_json::Value::String(base64::engine::general_purpose::STANDARD.encode(b)),
82                );
83                serde_json::Value::Object(map)
84            }
85            Self::Array(a) => {
86                serde_json::Value::Array(a.iter().map(|v| v.to_json_value()).collect())
87            }
88            Self::Object(o) => {
89                let mut map = serde_json::Map::new();
90                for (k, v) in o {
91                    map.insert(k.clone(), v.to_json_value());
92                }
93                serde_json::Value::Object(map)
94            }
95        }
96    }
97
98    /// Parses JSON into the canonical hub.
99    ///
100    /// # Errors
101    /// Returns an error if the JSON is malformed.
102    ///
103    /// # Examples
104    /// ```
105    /// use sheeter_engine::CanonicalValue;
106    /// assert!(CanonicalValue::from_json(r#"{"a":1}"#).is_ok());
107    /// ```
108    pub fn from_json(input: &str) -> Result<Self, Box<dyn Error>> {
109        Ok(Self::from_json_value(serde_json::from_str(input)?))
110    }
111
112    /// Serializes the canonical hub to JSON.
113    ///
114    /// # Examples
115    /// ```
116    /// use sheeter_engine::CanonicalValue;
117    /// let v = CanonicalValue::String("x".into());
118    /// assert_eq!(v.to_json().unwrap(), r#""x""#);
119    /// ```
120    pub fn to_json(&self) -> Result<String, Box<dyn Error>> {
121        Ok(serde_json::to_string(&self.to_json_value())?)
122    }
123
124    fn from_delimited(input: &str, delim: u8) -> Result<Self, Box<dyn Error>> {
125        let input = input.strip_prefix('\u{feff}').unwrap_or(input);
126        let mut rdr = csv::ReaderBuilder::new()
127            .delimiter(delim)
128            .from_reader(Cursor::new(input));
129        let headers = rdr.headers()?.clone();
130        let mut seen = std::collections::HashSet::new();
131        for h in headers.iter() {
132            if !seen.insert(h) {
133                return Err(format!("duplicate header: {h}").into());
134            }
135        }
136        let mut rows = Vec::new();
137        for result in rdr.records() {
138            let record = result?;
139            let mut obj = Vec::with_capacity(headers.len());
140            for (h, v) in headers.iter().zip(record.iter()) {
141                obj.push((h.to_string(), Self::String(v.to_string())));
142            }
143            rows.push(Self::Object(obj));
144        }
145        Ok(Self::Array(rows))
146    }
147
148    fn to_delimited(&self, delim: u8) -> Result<String, Box<dyn Error>> {
149        let rows = self.as_tabular()?;
150        if rows.is_empty() {
151            return Ok(String::new());
152        }
153        let mut headers: Vec<String> = Vec::new();
154        for row in &rows {
155            let obj = row.as_object()?;
156            for (k, _) in obj {
157                if !headers.contains(k) {
158                    headers.push(k.clone());
159                }
160            }
161        }
162        let mut wtr = csv::WriterBuilder::new()
163            .delimiter(delim)
164            .from_writer(vec![]);
165        wtr.write_record(&headers)?;
166        for row in &rows {
167            let obj = row.as_object()?;
168            let mut record = Vec::with_capacity(headers.len());
169            for h in &headers {
170                let cell = match obj.iter().find(|(k, _)| k == h) {
171                    Some((_, Self::Null)) | None => String::new(),
172                    Some((_, Self::Bool(b))) => b.to_string(),
173                    Some((_, Self::Number(n))) => n.to_string(),
174                    Some((_, Self::String(s))) => s.clone(),
175                    Some((_, Self::Bytes(b))) => {
176                        use base64::Engine;
177                        base64::engine::general_purpose::STANDARD.encode(b)
178                    }
179                    Some((_, Self::Array(_))) | Some((_, Self::Object(_))) => {
180                        return Err(format!(
181                            "nested value at key {h} cannot be represented in tabular output"
182                        )
183                        .into());
184                    }
185                };
186                record.push(cell);
187            }
188            wtr.write_record(&record)?;
189        }
190        let data = wtr.into_inner()?;
191        Ok(String::from_utf8(data)?)
192    }
193
194    pub fn from_toml(input: &str) -> Result<Self, Box<dyn Error>> {
195        let value: toml::Value = toml::from_str(input)?;
196        Ok(Self::from_toml_value(value))
197    }
198
199    fn from_toml_value(v: toml::Value) -> Self {
200        match v {
201            toml::Value::String(s) => Self::String(s),
202            toml::Value::Integer(i) => Self::Number(serde_json::Number::from(i)),
203            toml::Value::Float(f) => match serde_json::Number::from_f64(f) {
204                Some(n) => Self::Number(n),
205                None => Self::String(f.to_string()),
206            },
207            toml::Value::Boolean(b) => Self::Bool(b),
208            toml::Value::Datetime(dt) => Self::String(dt.to_string()),
209            toml::Value::Array(a) => {
210                Self::Array(a.into_iter().map(Self::from_toml_value).collect())
211            }
212            toml::Value::Table(t) => Self::Object(
213                t.into_iter()
214                    .map(|(k, v)| (k, Self::from_toml_value(v)))
215                    .collect(),
216            ),
217        }
218    }
219
220    pub fn to_toml(&self) -> Result<String, Box<dyn Error>> {
221        let value = match self {
222            Self::Array(_) => {
223                let mut table = toml::map::Map::new();
224                table.insert("item".to_string(), self.to_toml_value()?);
225                toml::Value::Table(table)
226            }
227            Self::Object(_) => self.to_toml_value()?,
228            _ => {
229                let mut table = toml::map::Map::new();
230                table.insert("value".to_string(), self.to_toml_value()?);
231                toml::Value::Table(table)
232            }
233        };
234        Ok(toml::to_string_pretty(&value)?)
235    }
236
237    fn to_toml_value(&self) -> Result<toml::Value, Box<dyn Error>> {
238        Ok(match self {
239            Self::Null => return Err("TOML has no null type".into()),
240            Self::Bool(b) => toml::Value::Boolean(*b),
241            Self::Number(n) => {
242                if let Some(i) = n.as_i64() {
243                    toml::Value::Integer(i)
244                } else if let Some(u) = n.as_u64() {
245                    if u <= i64::MAX as u64 {
246                        toml::Value::Integer(u as i64)
247                    } else {
248                        return Err(format!("TOML integer out of range: {u}").into());
249                    }
250                } else if let Some(f) = n.as_f64() {
251                    toml::Value::Float(f)
252                } else {
253                    return Err(format!("cannot represent number {n} in TOML").into());
254                }
255            }
256            Self::String(s) => toml::Value::String(s.clone()),
257            Self::Bytes(b) => {
258                use base64::Engine;
259                toml::Value::String(base64::engine::general_purpose::STANDARD.encode(b))
260            }
261            Self::Array(a) => {
262                let mut items = Vec::with_capacity(a.len());
263                for item in a {
264                    if let Self::Null = item {
265                        return Err("TOML arrays cannot contain null".into());
266                    }
267                    items.push(item.to_toml_value()?);
268                }
269                toml::Value::Array(items)
270            }
271            Self::Object(o) => {
272                let mut table = toml::map::Map::new();
273                for (k, v) in o {
274                    if let Self::Null = v {
275                        continue;
276                    }
277                    table.insert(k.clone(), v.to_toml_value()?);
278                }
279                toml::Value::Table(table)
280            }
281        })
282    }
283
284    pub fn from_xlsx_workbook(bytes: &[u8]) -> Result<Vec<(String, Self)>, Box<dyn Error>> {
285        use calamine::{Reader, Xlsx};
286        let cursor = Cursor::new(bytes.to_vec());
287        let mut workbook: Xlsx<_> = Xlsx::new(cursor)?;
288        let names = workbook.sheet_names().to_owned();
289        let mut out = Vec::with_capacity(names.len());
290        for name in names {
291            let range = workbook
292                .worksheet_range(&name)
293                .map_err(|e| format!("failed to read sheet {name}: {e:?}"))?;
294            out.push((name, Self::from_calamine_range(&range)?));
295        }
296        Ok(out)
297    }
298
299    pub(crate) fn from_calamine_range(
300        range: &calamine::Range<calamine::Data>,
301    ) -> Result<Self, Box<dyn Error>> {
302        use calamine::Data;
303        let mut rows = range.rows();
304        let header = match rows.next() {
305            Some(h) => h.iter().map(cell_to_string).collect::<Vec<String>>(),
306            None => return Ok(Self::Array(Vec::new())),
307        };
308        let mut records = Vec::new();
309        for row in rows {
310            if row.iter().all(|c| matches!(c, Data::Empty)) {
311                continue;
312            }
313            let mut obj = Vec::with_capacity(header.len());
314            for (i, h) in header.iter().enumerate() {
315                let val = match row.get(i) {
316                    Some(Data::Empty) | None => Self::Null,
317                    Some(Data::String(s)) => Self::String(s.clone()),
318                    Some(Data::Float(f)) => {
319                        if f.is_finite() && f.fract() == 0.0 && f.abs() < i64::MAX as f64 {
320                            Self::Number(serde_json::Number::from(*f as i64))
321                        } else {
322                            match serde_json::Number::from_f64(*f) {
323                                Some(n) => Self::Number(n),
324                                None => Self::String(f.to_string()),
325                            }
326                        }
327                    }
328                    Some(Data::Int(i)) => Self::Number(serde_json::Number::from(*i)),
329                    Some(Data::Bool(b)) => Self::Bool(*b),
330                    Some(Data::DateTime(dt)) => Self::String(dt.to_string()),
331                    Some(Data::DateTimeIso(s)) => Self::String(s.clone()),
332                    Some(Data::DurationIso(s)) => Self::String(s.clone()),
333                    Some(Data::Error(e)) => Self::String(format!("{e:?}")),
334                };
335                obj.push((h.clone(), val));
336            }
337            records.push(Self::Object(obj));
338        }
339        Ok(Self::Array(records))
340    }
341
342    pub fn to_xlsx_workbook(sheets: &[(String, Self)]) -> Result<Vec<u8>, Box<dyn Error>> {
343        use rust_xlsxwriter::Workbook;
344        let mut wb = Workbook::new();
345        for (name, value) in sheets {
346            let ws = wb.add_worksheet();
347            ws.set_name(name.as_str())?;
348            let rows = value.as_tabular()?;
349            if rows.is_empty() {
350                continue;
351            }
352            let mut headers: Vec<String> = Vec::new();
353            for row in &rows {
354                let obj = row.as_object()?;
355                for (k, _) in obj {
356                    if !headers.contains(k) {
357                        headers.push(k.clone());
358                    }
359                }
360            }
361            for (col, h) in headers.iter().enumerate() {
362                ws.write_string(0, col as u16, h)?;
363            }
364            for (i, row) in rows.iter().enumerate() {
365                let obj = row.as_object()?;
366                for (col, h) in headers.iter().enumerate() {
367                    let row_idx = (i + 1) as u32;
368                    let col_idx = col as u16;
369                    match obj.iter().find(|(k, _)| k == h) {
370                        Some((_, Self::String(s))) => {
371                            ws.write_string(row_idx, col_idx, s)?;
372                        }
373                        Some((_, Self::Number(n))) => {
374                            if let Some(f) = n.as_f64() {
375                                ws.write_number(row_idx, col_idx, f)?;
376                            } else {
377                                ws.write_string(row_idx, col_idx, n.to_string())?;
378                            }
379                        }
380                        Some((_, Self::Bool(b))) => {
381                            ws.write_boolean(row_idx, col_idx, *b)?;
382                        }
383                        Some((_, Self::Null)) | None => {}
384                        Some((_, Self::Bytes(b))) => {
385                            use base64::Engine;
386                            let s = base64::engine::general_purpose::STANDARD.encode(b);
387                            ws.write_string(row_idx, col_idx, &s)?;
388                        }
389                        Some((_, Self::Array(_))) | Some((_, Self::Object(_))) => {
390                            return Err(format!(
391                                "nested value at key {h} cannot be represented in XLSX"
392                            )
393                            .into());
394                        }
395                    }
396                }
397            }
398        }
399        Ok(wb.save_to_buffer()?)
400    }
401
402    pub fn from_sql_workbook(
403        input: &str,
404        dialect: crate::SqlDialect,
405    ) -> Result<Vec<(String, Self)>, Box<dyn Error>> {
406        use sqlparser::ast::{
407            Expr, SetExpr, Statement, TableObject, Value as SqlValue, ValueWithSpan, Values,
408        };
409        use sqlparser::dialect::{GenericDialect, MySqlDialect, PostgreSqlDialect, SQLiteDialect};
410        use sqlparser::parser::Parser;
411
412        let statements = match dialect {
413            crate::SqlDialect::Generic => Parser::parse_sql(&GenericDialect {}, input)?,
414            crate::SqlDialect::PostgreSql => Parser::parse_sql(&PostgreSqlDialect {}, input)?,
415            crate::SqlDialect::Sqlite => Parser::parse_sql(&SQLiteDialect {}, input)?,
416            crate::SqlDialect::MySql => Parser::parse_sql(&MySqlDialect {}, input)?,
417        };
418
419        let mut schemas: std::collections::HashMap<String, Vec<String>> =
420            std::collections::HashMap::new();
421        let mut tables: Vec<SqlTable> = Vec::new();
422
423        for statement in statements {
424            match statement {
425                Statement::CreateTable(create) => {
426                    let name = clean_ident(&create.name.to_string());
427                    let cols: Vec<String> = create
428                        .columns
429                        .iter()
430                        .map(|c| clean_ident(&c.name.to_string()))
431                        .collect();
432                    schemas.insert(name, cols);
433                }
434                Statement::Insert(insert) => {
435                    let table = match &insert.table {
436                        TableObject::TableName(n) => clean_ident(&n.to_string()),
437                        _ => {
438                            return Err(
439                                "INSERT into table functions or queries is not supported".into()
440                            );
441                        }
442                    };
443                    let source = insert
444                        .source
445                        .as_ref()
446                        .ok_or("INSERT without a source is not supported")?;
447                    let rows = match source.body.as_ref() {
448                        SetExpr::Values(Values { rows, .. }) => rows,
449                        _ => {
450                            return Err(
451                                "only INSERT ... VALUES is supported for data extraction".into()
452                            );
453                        }
454                    };
455                    let explicit_cols: Vec<String> = insert
456                        .columns
457                        .iter()
458                        .map(|c| clean_ident(&c.to_string()))
459                        .collect();
460                    let cols = if explicit_cols.is_empty() {
461                        schemas.get(&table).cloned().unwrap_or_else(|| {
462                            (0..rows.first().map(|r| r.len()).unwrap_or(0))
463                                .map(|i| format!("column_{}", i + 1))
464                                .collect()
465                        })
466                    } else {
467                        explicit_cols
468                    };
469                    let slot = match tables.iter_mut().find(|(n, _)| n == &table) {
470                        Some((_, rows)) => rows,
471                        None => {
472                            tables.push((table.clone(), Vec::new()));
473                            &mut tables.last_mut().expect("just pushed").1
474                        }
475                    };
476                    for row in rows {
477                        let mut obj = Vec::with_capacity(cols.len());
478                        for (i, expr) in row.iter().enumerate() {
479                            let key = cols
480                                .get(i)
481                                .cloned()
482                                .unwrap_or_else(|| format!("column_{}", i + 1));
483                            let value = match expr {
484                                Expr::Value(ValueWithSpan { value, .. }) => match value {
485                                    SqlValue::Number(n, _) => {
486                                        if let Ok(i) = n.parse::<i64>() {
487                                            Self::Number(serde_json::Number::from(i))
488                                        } else if let Ok(f) = n.parse::<f64>() {
489                                            match serde_json::Number::from_f64(f) {
490                                                Some(n) => Self::Number(n),
491                                                None => Self::String(n.clone()),
492                                            }
493                                        } else {
494                                            Self::String(n.clone())
495                                        }
496                                    }
497                                    SqlValue::SingleQuotedString(s)
498                                    | SqlValue::DoubleQuotedString(s)
499                                    | SqlValue::EscapedStringLiteral(s) => Self::String(s.clone()),
500                                    SqlValue::Boolean(b) => Self::Bool(*b),
501                                    SqlValue::Null => Self::Null,
502                                    other => Self::String(other.to_string()),
503                                },
504                                Expr::Identifier(id) => Self::String(id.value.clone()),
505                                other => Self::String(other.to_string()),
506                            };
507                            obj.push((key, value));
508                        }
509                        slot.push(obj);
510                    }
511                }
512                _ => {}
513            }
514        }
515
516        if tables.is_empty() {
517            return Err("no INSERT statements with literal VALUES found".into());
518        }
519
520        Ok(tables
521            .into_iter()
522            .map(|(name, rows)| {
523                let records = rows.into_iter().map(Self::Object).collect();
524                (name, Self::Array(records))
525            })
526            .collect())
527    }
528
529    pub fn to_sql_workbook(sheets: &[(String, Self)]) -> Result<String, Box<dyn Error>> {
530        let mut out = String::new();
531        for (table, value) in sheets {
532            out.push_str(&value.to_sql_with_table(table)?);
533        }
534        Ok(out)
535    }
536
537    pub fn to_sql_with_table(&self, table: &str) -> Result<String, Box<dyn Error>> {
538        let rows = self.as_tabular()?;
539        if rows.is_empty() {
540            return Ok(String::new());
541        }
542        let mut headers: Vec<String> = Vec::new();
543        for row in &rows {
544            let obj = row.as_object()?;
545            for (k, _) in obj {
546                if !headers.contains(k) {
547                    headers.push(k.clone());
548                }
549            }
550        }
551        let cols = headers
552            .iter()
553            .map(|h| quote_ident(h))
554            .collect::<Vec<_>>()
555            .join(", ");
556        let mut out = String::new();
557        for row in &rows {
558            let obj = row.as_object()?;
559            let mut values = Vec::with_capacity(headers.len());
560            for h in &headers {
561                let cell = match obj.iter().find(|(k, _)| k == h) {
562                    Some((_, Self::Null)) | None => "NULL".to_string(),
563                    Some((_, Self::Bool(b))) => b.to_string().to_uppercase(),
564                    Some((_, Self::Number(n))) => n.to_string(),
565                    Some((_, Self::String(s))) => format!("'{}'", s.replace('\'', "''")),
566                    Some((_, Self::Bytes(b))) => {
567                        format!(
568                            "X'{}'",
569                            b.iter()
570                                .map(|byte| format!("{byte:02x}"))
571                                .collect::<String>()
572                        )
573                    }
574                    Some((_, Self::Array(_))) | Some((_, Self::Object(_))) => {
575                        return Err(format!(
576                            "nested value at key {h} cannot be represented in SQL"
577                        )
578                        .into());
579                    }
580                };
581                values.push(cell);
582            }
583            out.push_str(&format!(
584                "INSERT INTO {} ({}) VALUES ({});\n",
585                quote_ident(table),
586                cols,
587                values.join(", ")
588            ));
589        }
590        Ok(out)
591    }
592
593    fn as_object(&self) -> Result<&Vec<(String, CanonicalValue)>, Box<dyn Error>> {
594        match self {
595            Self::Object(o) => Ok(o),
596            _ => Err("expected object".into()),
597        }
598    }
599
600    pub(crate) fn as_tabular(&self) -> Result<Vec<&CanonicalValue>, Box<dyn Error>> {
601        if let Some(arr) = find_first_object_array(self) {
602            return Ok(arr.iter().collect());
603        }
604        let mut current = self;
605        while let Self::Object(o) = current {
606            if o.len() != 1 {
607                break;
608            }
609            match &o[0].1 {
610                Self::Object(_) => current = &o[0].1,
611                _ => break,
612            }
613        }
614        if let Self::Object(o) = current {
615            if o.iter().all(|(_, v)| is_scalar(v)) {
616                return Ok(vec![current]);
617            }
618        }
619        Err("cannot determine tabular structure".into())
620    }
621
622    /// Parses CSV into the canonical hub.
623    ///
624    /// # Examples
625    /// ```
626    /// use sheeter_engine::CanonicalValue;
627    /// let v = CanonicalValue::from_csv("a\n1").unwrap();
628    /// assert_eq!(v.to_json().unwrap(), r#"[{"a":"1"}]"#);
629    /// ```
630    pub fn from_csv(input: &str) -> Result<Self, Box<dyn Error>> {
631        Self::from_delimited(input, b',')
632    }
633
634    /// Serializes the canonical hub to CSV.
635    ///
636    /// # Examples
637    /// ```
638    /// use sheeter_engine::CanonicalValue;
639    /// let v = CanonicalValue::from_json(r#"[{"a":1}]"#).unwrap();
640    /// assert_eq!(v.to_csv().unwrap(), "a\n1\n");
641    /// ```
642    pub fn to_csv(&self) -> Result<String, Box<dyn Error>> {
643        self.to_delimited(b',')
644    }
645
646    /// Parses TSV into the canonical hub.
647    ///
648    /// # Examples
649    /// ```
650    /// use sheeter_engine::CanonicalValue;
651    /// let v = CanonicalValue::from_tsv("a\tb\n1\t2").unwrap();
652    /// assert_eq!(v.to_json().unwrap(), r#"[{"a":"1","b":"2"}]"#);
653    /// ```
654    pub fn from_tsv(input: &str) -> Result<Self, Box<dyn Error>> {
655        Self::from_delimited(input, b'\t')
656    }
657
658    /// Serializes the canonical hub to TSV.
659    ///
660    /// # Examples
661    /// ```
662    /// use sheeter_engine::CanonicalValue;
663    /// let v = CanonicalValue::from_json(r#"[{"a":1}]"#).unwrap();
664    /// assert_eq!(v.to_tsv().unwrap(), "a\n1\n");
665    /// ```
666    pub fn to_tsv(&self) -> Result<String, Box<dyn Error>> {
667        self.to_delimited(b'\t')
668    }
669
670    /// Parses YAML into the canonical hub.
671    ///
672    /// # Examples
673    /// ```
674    /// use sheeter_engine::CanonicalValue;
675    /// let v = CanonicalValue::from_yaml("- a: 1\n").unwrap();
676    /// assert_eq!(v.to_json().unwrap(), r#"[{"a":1}]"#);
677    /// ```
678    pub fn from_yaml(input: &str) -> Result<Self, Box<dyn Error>> {
679        let opts = serde_saphyr::options! {
680            strict_booleans: true,
681            ignore_binary_tag_for_string: true,
682        };
683        let v: serde_json::Value = serde_saphyr::from_str_with_options(input, opts)?;
684        Ok(Self::from_json_value(v))
685    }
686
687    /// Serializes the canonical hub to YAML.
688    ///
689    /// # Examples
690    /// ```
691    /// use sheeter_engine::CanonicalValue;
692    /// let v = CanonicalValue::from_json(r#"[{"a":1}]"#).unwrap();
693    /// assert!(v.to_yaml().unwrap().contains("a: 1"));
694    /// ```
695    pub fn to_yaml(&self) -> Result<String, Box<dyn Error>> {
696        Ok(serde_saphyr::to_string(&self.to_json_value())?)
697    }
698
699    /// Parses XML into the canonical hub.
700    ///
701    /// Attributes become `@name` keys, text content becomes `#text`, and
702    /// repeated sibling elements become arrays. The returned value is an
703    /// object with exactly one key: the root element name.
704    ///
705    /// # Examples
706    /// ```
707    /// use sheeter_engine::CanonicalValue;
708    /// let v = CanonicalValue::from_xml("<r><a>1</a><a>2</a></r>").unwrap();
709    /// assert_eq!(v.to_json().unwrap(), r#"{"r":{"a":["1","2"]}}"#);
710    /// ```
711    pub fn from_xml(input: &str) -> Result<Self, Box<dyn Error>> {
712        use quick_xml::events::Event;
713        use quick_xml::Reader;
714
715        let mut reader = Reader::from_str(input);
716        reader.config_mut().trim_text(true);
717        let mut stack: Vec<XmlStackFrame> = Vec::new();
718        let mut root: Option<(String, CanonicalValue)> = None;
719
720        loop {
721            match reader.read_event() {
722                Ok(Event::Start(e)) => {
723                    let name = String::from_utf8_lossy(e.name().as_ref()).to_string();
724                    let mut attrs = Vec::new();
725                    for a in e.attributes() {
726                        let a = a?;
727                        attrs.push((
728                            String::from_utf8_lossy(a.key.as_ref()).to_string(),
729                            a.unescape_value()?.to_string(),
730                        ));
731                    }
732                    stack.push((name, attrs, Vec::new(), String::new()));
733                }
734                Ok(Event::Text(e)) => {
735                    if let Some(top) = stack.last_mut() {
736                        top.3.push_str(&e.unescape()?);
737                    }
738                }
739                Ok(Event::End(_)) => {
740                    let (name, attributes, children, text) = stack.pop().ok_or("unbalanced XML")?;
741                    let trimmed = text.trim().to_string();
742                    let value = if attributes.is_empty() && children.is_empty() {
743                        Self::String(trimmed)
744                    } else {
745                        let mut obj: Vec<(String, CanonicalValue)> = Vec::new();
746                        for (k, v) in attributes {
747                            obj.push((format!("@{k}"), Self::String(v)));
748                        }
749                        if !trimmed.is_empty() {
750                            obj.push(("#text".to_string(), Self::String(trimmed)));
751                        }
752                        for (cname, cval) in children {
753                            if let Some(slot) = obj.iter_mut().find(|(k, _)| k == &cname) {
754                                match &mut slot.1 {
755                                    Self::Array(a) => a.push(cval),
756                                    _ => {
757                                        let old = slot.1.clone();
758                                        slot.1 = Self::Array(vec![old, cval]);
759                                    }
760                                }
761                            } else {
762                                obj.push((cname, cval));
763                            }
764                        }
765                        Self::Object(obj)
766                    };
767                    if let Some(parent) = stack.last_mut() {
768                        parent.2.push((name, value));
769                    } else {
770                        root = Some((name, value));
771                    }
772                }
773                Ok(Event::Eof) => break,
774                Ok(_) => {}
775                Err(e) => return Err(e.into()),
776            }
777        }
778
779        let (root_name, root_value) = root.ok_or("empty XML")?;
780        Ok(Self::Object(vec![(root_name, root_value)]))
781    }
782
783    /// Serializes the canonical hub to XML.
784    ///
785    /// # Examples
786    /// ```
787    /// use sheeter_engine::CanonicalValue;
788    /// let v = CanonicalValue::from_json(r#"[{"a":"1"}]"#).unwrap();
789    /// let xml = v.to_xml().unwrap();
790    /// assert!(xml.contains("<root>"));
791    /// assert!(xml.contains("<a>1</a>"));
792    /// ```
793    pub fn to_xml(&self) -> Result<String, Box<dyn Error>> {
794        let mut writer = quick_xml::Writer::new_with_indent(Vec::new(), b' ', 2);
795        match self {
796            Self::Object(o) if o.len() == 1 && !o[0].0.starts_with('@') && o[0].0 != "#text" => {
797                write_element(&mut writer, &o[0].0, &o[0].1)?;
798            }
799            Self::Array(a) => {
800                write_open(&mut writer, "root", &[])?;
801                for item in a {
802                    write_element(&mut writer, "item", item)?;
803                }
804                write_close(&mut writer, "root")?;
805            }
806            _ => write_element(&mut writer, "root", self)?,
807        }
808        Ok(String::from_utf8(writer.into_inner())?)
809    }
810}
811
812fn write_open<W: std::io::Write>(
813    writer: &mut quick_xml::Writer<W>,
814    name: &str,
815    attrs: &[(String, String)],
816) -> Result<(), Box<dyn Error>> {
817    use quick_xml::events::{BytesStart, Event};
818    let mut start = BytesStart::new(name);
819    for (k, v) in attrs {
820        start.push_attribute((k.as_str(), v.as_str()));
821    }
822    writer.write_event(Event::Start(start))?;
823    Ok(())
824}
825
826fn write_close<W: std::io::Write>(
827    writer: &mut quick_xml::Writer<W>,
828    name: &str,
829) -> Result<(), Box<dyn Error>> {
830    use quick_xml::events::{BytesEnd, Event};
831    writer.write_event(Event::End(BytesEnd::new(name)))?;
832    Ok(())
833}
834
835fn write_element<W: std::io::Write>(
836    writer: &mut quick_xml::Writer<W>,
837    name: &str,
838    value: &CanonicalValue,
839) -> Result<(), Box<dyn Error>> {
840    use quick_xml::events::{BytesText, Event};
841
842    match value {
843        CanonicalValue::Array(items) => {
844            for item in items {
845                write_element(writer, name, item)?;
846            }
847        }
848        CanonicalValue::Object(o) => {
849            let attrs: Vec<(String, String)> = o
850                .iter()
851                .filter_map(|(k, v)| {
852                    k.strip_prefix('@').and_then(|attr| match v {
853                        CanonicalValue::String(s) => Some((attr.to_string(), s.clone())),
854                        _ => None,
855                    })
856                })
857                .collect();
858            write_open(writer, name, &attrs)?;
859            for (k, v) in o {
860                if k.starts_with('@') {
861                    continue;
862                }
863                if k == "#text" {
864                    if let CanonicalValue::String(s) = v {
865                        writer.write_event(Event::Text(BytesText::new(s)))?;
866                    }
867                    continue;
868                }
869                write_element(writer, k, v)?;
870            }
871            write_close(writer, name)?;
872        }
873        CanonicalValue::String(s) => {
874            write_open(writer, name, &[])?;
875            writer.write_event(Event::Text(BytesText::new(s)))?;
876            write_close(writer, name)?;
877        }
878        CanonicalValue::Bytes(b) => {
879            use base64::Engine;
880            let s = base64::engine::general_purpose::STANDARD.encode(b);
881            write_open(writer, name, &[])?;
882            writer.write_event(Event::Text(BytesText::new(&s)))?;
883            write_close(writer, name)?;
884        }
885        CanonicalValue::Number(n) => {
886            write_open(writer, name, &[])?;
887            writer.write_event(Event::Text(BytesText::new(&n.to_string())))?;
888            write_close(writer, name)?;
889        }
890        CanonicalValue::Bool(b) => {
891            write_open(writer, name, &[])?;
892            writer.write_event(Event::Text(BytesText::new(&b.to_string())))?;
893            write_close(writer, name)?;
894        }
895        CanonicalValue::Null => {
896            write_open(writer, name, &[])?;
897            write_close(writer, name)?;
898        }
899    }
900    Ok(())
901}
902
903type SqlRow = Vec<(String, CanonicalValue)>;
904type SqlTable = (String, Vec<SqlRow>);
905
906impl CanonicalValue {
907    /// Parses an ODS (OpenDocument Spreadsheet) file into a value.
908    ///
909    /// A single-sheet ODS becomes an array of row objects; a multi-sheet
910    /// ODS becomes an object keyed by sheet name.
911    ///
912    /// # Examples
913    ///
914    /// ```ignore
915    /// // Fixture covered in tests/ods.rs.
916    /// ```
917    pub fn from_ods(bytes: &[u8]) -> Result<Self, Box<dyn Error>> {
918        use calamine::{Ods, Reader};
919        let cursor = Cursor::new(bytes.to_vec());
920        let mut workbook: Ods<_> = Ods::new(cursor)?;
921        let names = workbook.sheet_names().to_owned();
922        let mut out: Vec<(String, CanonicalValue)> = Vec::with_capacity(names.len());
923        for name in names {
924            let range = workbook
925                .worksheet_range(&name)
926                .map_err(|e| format!("failed to read sheet {name}: {e:?}"))?;
927            out.push((name, Self::from_calamine_range(&range)?));
928        }
929        if out.len() == 1 {
930            Ok(out.remove(0).1)
931        } else {
932            Ok(Self::Object(out))
933        }
934    }
935}
936
937/// Writes a workbook as an ODS file.
938///
939/// Each entry of `sheets` becomes a `table:table` inside the resulting
940/// `content.xml`. The `mimetype` entry is written first and stored
941/// uncompressed, as required by the ODF spec.
942///
943/// # Errors
944///
945/// Returns an error if any row contains nested values or if ZIP/XML I/O fails.
946pub fn to_ods_workbook(sheets: &[(String, CanonicalValue)]) -> Result<Vec<u8>, Box<dyn Error>> {
947    use std::io::Write as IoWrite;
948    use zip::write::SimpleFileOptions;
949    use zip::{CompressionMethod, ZipWriter};
950
951    let content = ods_content_xml(sheets)?;
952    let manifest = ods_manifest_xml();
953    let mimetype = "application/vnd.oasis.opendocument.spreadsheet";
954
955    let cursor = Cursor::new(Vec::new());
956    let mut zip = ZipWriter::new(cursor);
957    let stored = SimpleFileOptions::default().compression_method(CompressionMethod::Stored);
958    let deflated = SimpleFileOptions::default().compression_method(CompressionMethod::Deflated);
959
960    zip.start_file("mimetype", stored)?;
961    zip.write_all(mimetype.as_bytes())?;
962
963    zip.start_file("META-INF/manifest.xml", deflated)?;
964    zip.write_all(manifest.as_bytes())?;
965
966    zip.start_file("content.xml", deflated)?;
967    zip.write_all(content.as_bytes())?;
968
969    let cursor = zip.finish()?;
970    Ok(cursor.into_inner())
971}
972
973fn ods_manifest_xml() -> String {
974    r#"<?xml version="1.0" encoding="UTF-8"?>
975<manifest:manifest xmlns:manifest="urn:oasis:names:tc:opendocument:xmlns:manifest:1.0" manifest:version="1.2">
976  <manifest:file-entry manifest:full-path="/" manifest:version="1.2" manifest:media-type="application/vnd.oasis.opendocument.spreadsheet"/>
977  <manifest:file-entry manifest:full-path="content.xml" manifest:media-type="text/xml"/>
978</manifest:manifest>
979"#
980    .to_string()
981}
982
983fn ods_content_xml(sheets: &[(String, CanonicalValue)]) -> Result<String, Box<dyn Error>> {
984    use quick_xml::escape::escape;
985    let mut out = String::new();
986    out.push_str(r#"<?xml version="1.0" encoding="UTF-8"?>"#);
987    out.push_str(
988        r#"<office:document-content xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" xmlns:table="urn:oasis:names:tc:opendocument:xmlns:table:1.0" xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0" office:version="1.2">"#,
989    );
990    out.push_str("<office:body><office:spreadsheet>");
991
992    for (name, value) in sheets {
993        out.push_str(&format!("<table:table table:name=\"{}\">", escape(name)));
994        let rows = value.as_tabular()?;
995        let mut headers: Vec<String> = Vec::new();
996        for row in &rows {
997            if let CanonicalValue::Object(o) = row {
998                for (k, _) in o {
999                    if !headers.contains(k) {
1000                        headers.push(k.clone());
1001                    }
1002                }
1003            }
1004        }
1005        out.push_str("<table:table-row>");
1006        for h in &headers {
1007            out.push_str("<table:table-cell office:value-type=\"string\">");
1008            out.push_str(&format!("<text:p>{}</text:p>", escape(h)));
1009            out.push_str("</table:table-cell>");
1010        }
1011        out.push_str("</table:table-row>");
1012
1013        for row in &rows {
1014            if let CanonicalValue::Object(o) = row {
1015                out.push_str("<table:table-row>");
1016                for h in &headers {
1017                    match o.iter().find(|(k, _)| k == h).map(|(_, v)| v) {
1018                        Some(CanonicalValue::String(s)) => {
1019                            out.push_str("<table:table-cell office:value-type=\"string\">");
1020                            out.push_str(&format!("<text:p>{}</text:p>", escape(s)));
1021                            out.push_str("</table:table-cell>");
1022                        }
1023                        Some(CanonicalValue::Number(n)) => {
1024                            out.push_str(&format!(
1025                                "<table:table-cell office:value-type=\"float\" office:value=\"{}\">",
1026                                n
1027                            ));
1028                            out.push_str(&format!("<text:p>{}</text:p>", n));
1029                            out.push_str("</table:table-cell>");
1030                        }
1031                        Some(CanonicalValue::Bool(b)) => {
1032                            out.push_str(&format!(
1033                                "<table:table-cell office:value-type=\"boolean\" office:boolean-value=\"{}\">",
1034                                b
1035                            ));
1036                            out.push_str(&format!(
1037                                "<text:p>{}</text:p>",
1038                                if *b { "TRUE" } else { "FALSE" }
1039                            ));
1040                            out.push_str("</table:table-cell>");
1041                        }
1042                        Some(CanonicalValue::Bytes(b)) => {
1043                            use base64::Engine;
1044                            let s = base64::engine::general_purpose::STANDARD.encode(b);
1045                            out.push_str("<table:table-cell office:value-type=\"string\">");
1046                            out.push_str(&format!("<text:p>{}</text:p>", s));
1047                            out.push_str("</table:table-cell>");
1048                        }
1049                        Some(CanonicalValue::Null) | None => {
1050                            out.push_str("<table:table-cell/>");
1051                        }
1052                        Some(CanonicalValue::Array(_)) | Some(CanonicalValue::Object(_)) => {
1053                            return Err(format!(
1054                                "nested value at key {h} cannot be represented in ODS"
1055                            )
1056                            .into());
1057                        }
1058                    }
1059                }
1060                out.push_str("</table:table-row>");
1061            }
1062        }
1063        out.push_str("</table:table>");
1064    }
1065
1066    out.push_str("</office:spreadsheet></office:body></office:document-content>");
1067    Ok(out)
1068}
1069
1070fn is_scalar(v: &CanonicalValue) -> bool {
1071    matches!(
1072        v,
1073        CanonicalValue::Null
1074            | CanonicalValue::Bool(_)
1075            | CanonicalValue::Number(_)
1076            | CanonicalValue::String(_)
1077    )
1078}
1079
1080fn find_first_object_array(v: &CanonicalValue) -> Option<&Vec<CanonicalValue>> {
1081    match v {
1082        CanonicalValue::Array(a)
1083            if !a.is_empty() && a.iter().all(|x| matches!(x, CanonicalValue::Object(_))) =>
1084        {
1085            Some(a)
1086        }
1087        CanonicalValue::Object(o) => {
1088            for (_, v) in o {
1089                if let Some(arr) = find_first_object_array(v) {
1090                    return Some(arr);
1091                }
1092            }
1093            None
1094        }
1095        _ => None,
1096    }
1097}
1098
1099fn cell_to_string(c: &calamine::Data) -> String {
1100    use calamine::Data;
1101    match c {
1102        Data::Empty => String::new(),
1103        Data::String(s) => s.clone(),
1104        Data::Float(f) => f.to_string(),
1105        Data::Int(i) => i.to_string(),
1106        Data::Bool(b) => b.to_string(),
1107        Data::DateTime(dt) => dt.to_string(),
1108        Data::DateTimeIso(s) => s.clone(),
1109        Data::DurationIso(s) => s.clone(),
1110        Data::Error(e) => format!("{e:?}"),
1111    }
1112}
1113
1114fn clean_ident(s: &str) -> String {
1115    s.trim_matches('"').trim_matches('`').to_string()
1116}
1117
1118fn quote_ident(s: &str) -> String {
1119    format!("\"{}\"", s.replace('"', "\"\""))
1120}
1121
1122impl CanonicalValue {
1123    /// Parses NDJSON (newline-delimited JSON) into an array.
1124    ///
1125    /// Each non-empty line must be a complete JSON value. The result is a
1126    /// `CanonicalValue::Array` with one element per line, preserving order.
1127    ///
1128    /// # Errors
1129    /// Returns an error if any non-empty line is malformed JSON.
1130    ///
1131    /// # Examples
1132    /// ```
1133    /// use sheeter_engine::CanonicalValue;
1134    /// let v = CanonicalValue::from_ndjson("{\"a\":1}\n{\"a\":2}\n").unwrap();
1135    /// assert_eq!(v.to_json().unwrap(), r#"[{"a":1},{"a":2}]"#);
1136    /// ```
1137    pub fn from_ndjson(input: &str) -> Result<Self, Box<dyn std::error::Error>> {
1138        let mut values = Vec::new();
1139        for line in input.lines() {
1140            let trimmed = line.trim();
1141            if trimmed.is_empty() {
1142                continue;
1143            }
1144            let v: serde_json::Value = serde_json::from_str(trimmed)?;
1145            values.push(Self::from_json_value(v));
1146        }
1147        Ok(Self::Array(values))
1148    }
1149
1150    /// Serializes the value as NDJSON (one JSON value per line).
1151    ///
1152    /// Arrays become one line per element. Non-array values become a
1153    /// single line.
1154    ///
1155    /// # Errors
1156    /// Returns an error if serialization fails.
1157    ///
1158    /// # Examples
1159    /// ```
1160    /// use sheeter_engine::CanonicalValue;
1161    /// let v = CanonicalValue::from_json(r#"[{"a":1},{"a":2}]"#).unwrap();
1162    /// assert_eq!(v.to_ndjson().unwrap(), "{\"a\":1}\n{\"a\":2}\n");
1163    /// ```
1164    pub fn to_ndjson(&self) -> Result<String, Box<dyn std::error::Error>> {
1165        let mut out = String::new();
1166        match self {
1167            Self::Array(items) => {
1168                for item in items {
1169                    out.push_str(&serde_json::to_string(&item.to_json_value())?);
1170                    out.push('\n');
1171                }
1172            }
1173            other => {
1174                out.push_str(&serde_json::to_string(&other.to_json_value())?);
1175                out.push('\n');
1176            }
1177        }
1178        Ok(out)
1179    }
1180}
sheeter_engine/canonical.rs

sheeter_engine/
canonical.rs
