sheeter_engine/
canonical.rs

1use std::error::Error;
2use std::io::Cursor;
3
4/// Format-agnostic intermediate value used by the Sheeter hub.
5///
6/// Every reader (CSV, TSV, JSON, YAML, XML) produces a `CanonicalValue`, and
7/// every writer consumes one. Adding a format requires exactly one reader and
8/// one writer, not `N-1` pairwise converters, because every spoke composes
9/// through this hub.
10///
11/// Tabular data is represented as `Array<Object>`. XML documents are
12/// represented as a single-key `Object` whose key is the root element name,
13/// with attributes prefixed by `@` and text stored under `#text`. Repeated
14/// sibling elements become `Array`.
15///
16/// # Examples
17///
18/// ```
19/// use sheeter_engine::CanonicalValue;
20///
21/// let v = CanonicalValue::from_json(r#"[{"a":1}]"#).unwrap();
22/// assert_eq!(v.to_csv().unwrap(), "a\n1\n");
23/// ```
24#[derive(Debug, Clone, PartialEq)]
25#[non_exhaustive]
26pub enum CanonicalValue {
27    Null,
28    Bool(bool),
29    Number(serde_json::Number),
30    String(String),
31    Bytes(Vec<u8>),
32    Array(Vec<CanonicalValue>),
33    Object(Vec<(String, CanonicalValue)>),
34}
35
36type XmlStackFrame = (
37    String,
38    Vec<(String, String)>,
39    Vec<(String, CanonicalValue)>,
40    String,
41);
42
43impl CanonicalValue {
44    pub(crate) fn from_json_value(v: serde_json::Value) -> Self {
45        match v {
46            serde_json::Value::Null => Self::Null,
47            serde_json::Value::Bool(b) => Self::Bool(b),
48            serde_json::Value::Number(n) => Self::Number(n),
49            serde_json::Value::String(s) => Self::String(s),
50            serde_json::Value::Array(a) => {
51                Self::Array(a.into_iter().map(Self::from_json_value).collect())
52            }
53            serde_json::Value::Object(m) => {
54                if m.len() == 1 {
55                    if let Some(serde_json::Value::String(b64)) = m.get("$binary") {
56                        use base64::Engine;
57                        if let Ok(bytes) = base64::engine::general_purpose::STANDARD.decode(b64) {
58                            return Self::Bytes(bytes);
59                        }
60                    }
61                }
62                Self::Object(
63                    m.into_iter()
64                        .map(|(k, v)| (k, Self::from_json_value(v)))
65                        .collect(),
66                )
67            }
68        }
69    }
70
71    pub(crate) fn to_json_value(&self) -> serde_json::Value {
72        match self {
73            Self::Null => serde_json::Value::Null,
74            Self::Bool(b) => serde_json::Value::Bool(*b),
75            Self::Number(n) => serde_json::Value::Number(n.clone()),
76            Self::String(s) => serde_json::Value::String(s.clone()),
77            Self::Bytes(b) => {
78                use base64::Engine;
79                let mut map = serde_json::Map::new();
80                map.insert(
81                    "$binary".to_string(),
82                    serde_json::Value::String(base64::engine::general_purpose::STANDARD.encode(b)),
83                );
84                serde_json::Value::Object(map)
85            }
86            Self::Array(a) => {
87                serde_json::Value::Array(a.iter().map(|v| v.to_json_value()).collect())
88            }
89            Self::Object(o) => {
90                let mut map = serde_json::Map::new();
91                for (k, v) in o {
92                    map.insert(k.clone(), v.to_json_value());
93                }
94                serde_json::Value::Object(map)
95            }
96        }
97    }
98
99    /// Parses JSON into the canonical hub.
100    ///
101    /// # Errors
102    /// Returns an error if the JSON is malformed.
103    ///
104    /// # Examples
105    /// ```
106    /// use sheeter_engine::CanonicalValue;
107    /// assert!(CanonicalValue::from_json(r#"{"a":1}"#).is_ok());
108    /// ```
109    pub fn from_json(input: &str) -> Result<Self, Box<dyn Error>> {
110        Ok(Self::from_json_value(serde_json::from_str(input)?))
111    }
112
113    /// Serializes the canonical hub to JSON.
114    ///
115    /// # Examples
116    /// ```
117    /// use sheeter_engine::CanonicalValue;
118    /// let v = CanonicalValue::String("x".into());
119    /// assert_eq!(v.to_json().unwrap(), r#""x""#);
120    /// ```
121    pub fn to_json(&self) -> Result<String, Box<dyn Error>> {
122        Ok(serde_json::to_string(&self.to_json_value())?)
123    }
124
125    fn from_delimited(input: &str, delim: u8) -> Result<Self, Box<dyn Error>> {
126        let input = input.strip_prefix('\u{feff}').unwrap_or(input);
127        let mut rdr = csv::ReaderBuilder::new()
128            .delimiter(delim)
129            .from_reader(Cursor::new(input));
130        let headers = rdr.headers()?.clone();
131        let mut seen = std::collections::HashSet::new();
132        for h in headers.iter() {
133            if !seen.insert(h) {
134                return Err(format!("duplicate header: {h}").into());
135            }
136        }
137        let mut rows = Vec::new();
138        for result in rdr.records() {
139            let record = result?;
140            let mut obj = Vec::with_capacity(headers.len());
141            for (h, v) in headers.iter().zip(record.iter()) {
142                obj.push((h.to_string(), Self::String(v.to_string())));
143            }
144            rows.push(Self::Object(obj));
145        }
146        Ok(Self::Array(rows))
147    }
148
149    fn to_delimited(&self, delim: u8) -> Result<String, Box<dyn Error>> {
150        let rows = self.as_tabular()?;
151        if rows.is_empty() {
152            return Ok(String::new());
153        }
154        let mut headers: Vec<String> = Vec::new();
155        for row in &rows {
156            let obj = row.as_object()?;
157            for (k, _) in obj {
158                if !headers.contains(k) {
159                    headers.push(k.clone());
160                }
161            }
162        }
163        let mut wtr = csv::WriterBuilder::new()
164            .delimiter(delim)
165            .from_writer(vec![]);
166        wtr.write_record(&headers)?;
167        for row in &rows {
168            let obj = row.as_object()?;
169            let mut record = Vec::with_capacity(headers.len());
170            for h in &headers {
171                let cell = match obj.iter().find(|(k, _)| k == h) {
172                    Some((_, Self::Null)) | None => String::new(),
173                    Some((_, Self::Bool(b))) => b.to_string(),
174                    Some((_, Self::Number(n))) => n.to_string(),
175                    Some((_, Self::String(s))) => s.clone(),
176                    Some((_, Self::Bytes(b))) => {
177                        use base64::Engine;
178                        base64::engine::general_purpose::STANDARD.encode(b)
179                    }
180                    Some((_, Self::Array(_))) | Some((_, Self::Object(_))) => {
181                        return Err(format!(
182                            "nested value at key {h} cannot be represented in tabular output"
183                        )
184                        .into());
185                    }
186                };
187                record.push(cell);
188            }
189            wtr.write_record(&record)?;
190        }
191        let data = wtr.into_inner()?;
192        Ok(String::from_utf8(data)?)
193    }
194
195    pub fn from_toml(input: &str) -> Result<Self, Box<dyn Error>> {
196        let value: toml::Value = toml::from_str(input)?;
197        Ok(Self::from_toml_value(value))
198    }
199
200    fn from_toml_value(v: toml::Value) -> Self {
201        match v {
202            toml::Value::String(s) => Self::String(s),
203            toml::Value::Integer(i) => Self::Number(serde_json::Number::from(i)),
204            toml::Value::Float(f) => match serde_json::Number::from_f64(f) {
205                Some(n) => Self::Number(n),
206                None => Self::String(f.to_string()),
207            },
208            toml::Value::Boolean(b) => Self::Bool(b),
209            toml::Value::Datetime(dt) => Self::String(dt.to_string()),
210            toml::Value::Array(a) => {
211                Self::Array(a.into_iter().map(Self::from_toml_value).collect())
212            }
213            toml::Value::Table(t) => Self::Object(
214                t.into_iter()
215                    .map(|(k, v)| (k, Self::from_toml_value(v)))
216                    .collect(),
217            ),
218        }
219    }
220
221    pub fn to_toml(&self) -> Result<String, Box<dyn Error>> {
222        let value = match self {
223            Self::Array(_) => {
224                let mut table = toml::map::Map::new();
225                table.insert("item".to_string(), self.to_toml_value()?);
226                toml::Value::Table(table)
227            }
228            Self::Object(_) => self.to_toml_value()?,
229            _ => {
230                let mut table = toml::map::Map::new();
231                table.insert("value".to_string(), self.to_toml_value()?);
232                toml::Value::Table(table)
233            }
234        };
235        Ok(toml::to_string_pretty(&value)?)
236    }
237
238    fn to_toml_value(&self) -> Result<toml::Value, Box<dyn Error>> {
239        Ok(match self {
240            Self::Null => return Err("TOML has no null type".into()),
241            Self::Bool(b) => toml::Value::Boolean(*b),
242            Self::Number(n) => {
243                if let Some(i) = n.as_i64() {
244                    toml::Value::Integer(i)
245                } else if let Some(u) = n.as_u64() {
246                    if u <= i64::MAX as u64 {
247                        toml::Value::Integer(u as i64)
248                    } else {
249                        return Err(format!("TOML integer out of range: {u}").into());
250                    }
251                } else if let Some(f) = n.as_f64() {
252                    toml::Value::Float(f)
253                } else {
254                    return Err(format!("cannot represent number {n} in TOML").into());
255                }
256            }
257            Self::String(s) => toml::Value::String(s.clone()),
258            Self::Bytes(b) => {
259                use base64::Engine;
260                toml::Value::String(base64::engine::general_purpose::STANDARD.encode(b))
261            }
262            Self::Array(a) => {
263                let mut items = Vec::with_capacity(a.len());
264                for item in a {
265                    if let Self::Null = item {
266                        return Err("TOML arrays cannot contain null".into());
267                    }
268                    items.push(item.to_toml_value()?);
269                }
270                toml::Value::Array(items)
271            }
272            Self::Object(o) => {
273                let mut table = toml::map::Map::new();
274                for (k, v) in o {
275                    if let Self::Null = v {
276                        continue;
277                    }
278                    table.insert(k.clone(), v.to_toml_value()?);
279                }
280                toml::Value::Table(table)
281            }
282        })
283    }
284
285    pub fn from_xlsx_workbook(bytes: &[u8]) -> Result<Vec<(String, Self)>, Box<dyn Error>> {
286        use calamine::{Reader, Xlsx};
287        let cursor = Cursor::new(bytes.to_vec());
288        let mut workbook: Xlsx<_> = Xlsx::new(cursor)?;
289        let names = workbook.sheet_names().to_owned();
290        let mut out = Vec::with_capacity(names.len());
291        for name in names {
292            let range = workbook
293                .worksheet_range(&name)
294                .map_err(|e| format!("failed to read sheet {name}: {e:?}"))?;
295            out.push((name, Self::from_calamine_range(&range)?));
296        }
297        Ok(out)
298    }
299
300    pub(crate) fn from_calamine_range(
301        range: &calamine::Range<calamine::Data>,
302    ) -> Result<Self, Box<dyn Error>> {
303        use calamine::Data;
304        let mut rows = range.rows();
305        let header = match rows.next() {
306            Some(h) => h.iter().map(cell_to_string).collect::<Vec<String>>(),
307            None => return Ok(Self::Array(Vec::new())),
308        };
309        let mut records = Vec::new();
310        for row in rows {
311            if row.iter().all(|c| matches!(c, Data::Empty)) {
312                continue;
313            }
314            let mut obj = Vec::with_capacity(header.len());
315            for (i, h) in header.iter().enumerate() {
316                let val = match row.get(i) {
317                    Some(Data::Empty) | None => Self::Null,
318                    Some(Data::String(s)) => Self::String(s.clone()),
319                    Some(Data::Float(f)) => {
320                        if f.is_finite() && f.fract() == 0.0 && f.abs() < i64::MAX as f64 {
321                            Self::Number(serde_json::Number::from(*f as i64))
322                        } else {
323                            match serde_json::Number::from_f64(*f) {
324                                Some(n) => Self::Number(n),
325                                None => Self::String(f.to_string()),
326                            }
327                        }
328                    }
329                    Some(Data::Int(i)) => Self::Number(serde_json::Number::from(*i)),
330                    Some(Data::Bool(b)) => Self::Bool(*b),
331                    Some(Data::DateTime(dt)) => Self::String(dt.to_string()),
332                    Some(Data::DateTimeIso(s)) => Self::String(s.clone()),
333                    Some(Data::DurationIso(s)) => Self::String(s.clone()),
334                    Some(Data::Error(e)) => Self::String(format!("{e:?}")),
335                };
336                obj.push((h.clone(), val));
337            }
338            records.push(Self::Object(obj));
339        }
340        Ok(Self::Array(records))
341    }
342
343    pub fn to_xlsx_workbook(sheets: &[(String, Self)]) -> Result<Vec<u8>, Box<dyn Error>> {
344        use rust_xlsxwriter::Workbook;
345        let mut wb = Workbook::new();
346        for (name, value) in sheets {
347            let ws = wb.add_worksheet();
348            ws.set_name(name.as_str())?;
349            let rows = value.as_tabular()?;
350            if rows.is_empty() {
351                continue;
352            }
353            let mut headers: Vec<String> = Vec::new();
354            for row in &rows {
355                let obj = row.as_object()?;
356                for (k, _) in obj {
357                    if !headers.contains(k) {
358                        headers.push(k.clone());
359                    }
360                }
361            }
362            for (col, h) in headers.iter().enumerate() {
363                ws.write_string(0, col as u16, h)?;
364            }
365            for (i, row) in rows.iter().enumerate() {
366                let obj = row.as_object()?;
367                for (col, h) in headers.iter().enumerate() {
368                    let row_idx = (i + 1) as u32;
369                    let col_idx = col as u16;
370                    match obj.iter().find(|(k, _)| k == h) {
371                        Some((_, Self::String(s))) => {
372                            ws.write_string(row_idx, col_idx, s)?;
373                        }
374                        Some((_, Self::Number(n))) => {
375                            if let Some(f) = n.as_f64() {
376                                ws.write_number(row_idx, col_idx, f)?;
377                            } else {
378                                ws.write_string(row_idx, col_idx, n.to_string())?;
379                            }
380                        }
381                        Some((_, Self::Bool(b))) => {
382                            ws.write_boolean(row_idx, col_idx, *b)?;
383                        }
384                        Some((_, Self::Null)) | None => {}
385                        Some((_, Self::Bytes(b))) => {
386                            use base64::Engine;
387                            let s = base64::engine::general_purpose::STANDARD.encode(b);
388                            ws.write_string(row_idx, col_idx, &s)?;
389                        }
390                        Some((_, Self::Array(_))) | Some((_, Self::Object(_))) => {
391                            return Err(format!(
392                                "nested value at key {h} cannot be represented in XLSX"
393                            )
394                            .into());
395                        }
396                    }
397                }
398            }
399        }
400        Ok(wb.save_to_buffer()?)
401    }
402
403    pub fn from_sql_workbook(
404        input: &str,
405        dialect: crate::SqlDialect,
406    ) -> Result<Vec<(String, Self)>, Box<dyn Error>> {
407        use sqlparser::ast::{
408            Expr, SetExpr, Statement, TableObject, Value as SqlValue, ValueWithSpan, Values,
409        };
410        use sqlparser::dialect::{GenericDialect, MySqlDialect, PostgreSqlDialect, SQLiteDialect};
411        use sqlparser::parser::Parser;
412
413        let statements = match dialect {
414            crate::SqlDialect::Generic => Parser::parse_sql(&GenericDialect {}, input)?,
415            crate::SqlDialect::PostgreSql => Parser::parse_sql(&PostgreSqlDialect {}, input)?,
416            crate::SqlDialect::Sqlite => Parser::parse_sql(&SQLiteDialect {}, input)?,
417            crate::SqlDialect::MySql => Parser::parse_sql(&MySqlDialect {}, input)?,
418        };
419
420        let mut schemas: std::collections::HashMap<String, Vec<String>> =
421            std::collections::HashMap::new();
422        let mut tables: Vec<SqlTable> = Vec::new();
423
424        for statement in statements {
425            match statement {
426                Statement::CreateTable(create) => {
427                    let name = clean_ident(&create.name.to_string());
428                    let cols: Vec<String> = create
429                        .columns
430                        .iter()
431                        .map(|c| clean_ident(&c.name.to_string()))
432                        .collect();
433                    schemas.insert(name, cols);
434                }
435                Statement::Insert(insert) => {
436                    let table = match &insert.table {
437                        TableObject::TableName(n) => clean_ident(&n.to_string()),
438                        _ => {
439                            return Err(
440                                "INSERT into table functions or queries is not supported".into()
441                            );
442                        }
443                    };
444                    let source = insert
445                        .source
446                        .as_ref()
447                        .ok_or("INSERT without a source is not supported")?;
448                    let rows = match source.body.as_ref() {
449                        SetExpr::Values(Values { rows, .. }) => rows,
450                        _ => {
451                            return Err(
452                                "only INSERT ... VALUES is supported for data extraction".into()
453                            );
454                        }
455                    };
456                    let explicit_cols: Vec<String> = insert
457                        .columns
458                        .iter()
459                        .map(|c| clean_ident(&c.to_string()))
460                        .collect();
461                    let cols = if explicit_cols.is_empty() {
462                        schemas.get(&table).cloned().unwrap_or_else(|| {
463                            (0..rows.first().map(|r| r.len()).unwrap_or(0))
464                                .map(|i| format!("column_{}", i + 1))
465                                .collect()
466                        })
467                    } else {
468                        explicit_cols
469                    };
470                    let slot = match tables.iter_mut().find(|(n, _)| n == &table) {
471                        Some((_, rows)) => rows,
472                        None => {
473                            tables.push((table.clone(), Vec::new()));
474                            &mut tables.last_mut().expect("just pushed").1
475                        }
476                    };
477                    for row in rows {
478                        let mut obj = Vec::with_capacity(cols.len());
479                        for (i, expr) in row.iter().enumerate() {
480                            let key = cols
481                                .get(i)
482                                .cloned()
483                                .unwrap_or_else(|| format!("column_{}", i + 1));
484                            let value = match expr {
485                                Expr::Value(ValueWithSpan { value, .. }) => match value {
486                                    SqlValue::Number(n, _) => {
487                                        if let Ok(i) = n.parse::<i64>() {
488                                            Self::Number(serde_json::Number::from(i))
489                                        } else if let Ok(f) = n.parse::<f64>() {
490                                            match serde_json::Number::from_f64(f) {
491                                                Some(n) => Self::Number(n),
492                                                None => Self::String(n.clone()),
493                                            }
494                                        } else {
495                                            Self::String(n.clone())
496                                        }
497                                    }
498                                    SqlValue::SingleQuotedString(s)
499                                    | SqlValue::DoubleQuotedString(s)
500                                    | SqlValue::EscapedStringLiteral(s) => Self::String(s.clone()),
501                                    SqlValue::Boolean(b) => Self::Bool(*b),
502                                    SqlValue::Null => Self::Null,
503                                    other => Self::String(other.to_string()),
504                                },
505                                Expr::Identifier(id) => Self::String(id.value.clone()),
506                                other => Self::String(other.to_string()),
507                            };
508                            obj.push((key, value));
509                        }
510                        slot.push(obj);
511                    }
512                }
513                _ => {}
514            }
515        }
516
517        if tables.is_empty() {
518            return Err("no INSERT statements with literal VALUES found".into());
519        }
520
521        Ok(tables
522            .into_iter()
523            .map(|(name, rows)| {
524                let records = rows.into_iter().map(Self::Object).collect();
525                (name, Self::Array(records))
526            })
527            .collect())
528    }
529
530    pub fn to_sql_workbook(sheets: &[(String, Self)]) -> Result<String, Box<dyn Error>> {
531        let mut out = String::new();
532        for (table, value) in sheets {
533            out.push_str(&value.to_sql_with_table(table)?);
534        }
535        Ok(out)
536    }
537
538    pub fn to_sql_with_table(&self, table: &str) -> Result<String, Box<dyn Error>> {
539        let rows = self.as_tabular()?;
540        if rows.is_empty() {
541            return Ok(String::new());
542        }
543        let mut headers: Vec<String> = Vec::new();
544        for row in &rows {
545            let obj = row.as_object()?;
546            for (k, _) in obj {
547                if !headers.contains(k) {
548                    headers.push(k.clone());
549                }
550            }
551        }
552        let cols = headers
553            .iter()
554            .map(|h| quote_ident(h))
555            .collect::<Vec<_>>()
556            .join(", ");
557        let mut out = String::new();
558        for row in &rows {
559            let obj = row.as_object()?;
560            let mut values = Vec::with_capacity(headers.len());
561            for h in &headers {
562                let cell = match obj.iter().find(|(k, _)| k == h) {
563                    Some((_, Self::Null)) | None => "NULL".to_string(),
564                    Some((_, Self::Bool(b))) => b.to_string().to_uppercase(),
565                    Some((_, Self::Number(n))) => n.to_string(),
566                    Some((_, Self::String(s))) => format!("'{}'", s.replace('\'', "''")),
567                    Some((_, Self::Bytes(b))) => {
568                        format!(
569                            "X'{}'",
570                            b.iter()
571                                .map(|byte| format!("{byte:02x}"))
572                                .collect::<String>()
573                        )
574                    }
575                    Some((_, Self::Array(_))) | Some((_, Self::Object(_))) => {
576                        return Err(format!(
577                            "nested value at key {h} cannot be represented in SQL"
578                        )
579                        .into());
580                    }
581                };
582                values.push(cell);
583            }
584            out.push_str(&format!(
585                "INSERT INTO {} ({}) VALUES ({});\n",
586                quote_ident(table),
587                cols,
588                values.join(", ")
589            ));
590        }
591        Ok(out)
592    }
593
594    fn as_object(&self) -> Result<&Vec<(String, CanonicalValue)>, Box<dyn Error>> {
595        match self {
596            Self::Object(o) => Ok(o),
597            _ => Err("expected object".into()),
598        }
599    }
600
601    pub(crate) fn as_tabular(&self) -> Result<Vec<&CanonicalValue>, Box<dyn Error>> {
602        if let Some(arr) = find_first_object_array(self) {
603            return Ok(arr.iter().collect());
604        }
605        let mut current = self;
606        while let Self::Object(o) = current {
607            if o.len() != 1 {
608                break;
609            }
610            match &o[0].1 {
611                Self::Object(_) => current = &o[0].1,
612                _ => break,
613            }
614        }
615        if let Self::Object(o) = current {
616            if o.iter().all(|(_, v)| is_scalar(v)) {
617                return Ok(vec![current]);
618            }
619        }
620        Err("cannot determine tabular structure".into())
621    }
622
623    /// Parses CSV into the canonical hub.
624    ///
625    /// # Examples
626    /// ```
627    /// use sheeter_engine::CanonicalValue;
628    /// let v = CanonicalValue::from_csv("a\n1").unwrap();
629    /// assert_eq!(v.to_json().unwrap(), r#"[{"a":"1"}]"#);
630    /// ```
631    pub fn from_csv(input: &str) -> Result<Self, Box<dyn Error>> {
632        Self::from_delimited(input, b',')
633    }
634
635    /// Serializes the canonical hub to CSV.
636    ///
637    /// # Examples
638    /// ```
639    /// use sheeter_engine::CanonicalValue;
640    /// let v = CanonicalValue::from_json(r#"[{"a":1}]"#).unwrap();
641    /// assert_eq!(v.to_csv().unwrap(), "a\n1\n");
642    /// ```
643    pub fn to_csv(&self) -> Result<String, Box<dyn Error>> {
644        self.to_delimited(b',')
645    }
646
647    /// Parses TSV into the canonical hub.
648    ///
649    /// # Examples
650    /// ```
651    /// use sheeter_engine::CanonicalValue;
652    /// let v = CanonicalValue::from_tsv("a\tb\n1\t2").unwrap();
653    /// assert_eq!(v.to_json().unwrap(), r#"[{"a":"1","b":"2"}]"#);
654    /// ```
655    pub fn from_tsv(input: &str) -> Result<Self, Box<dyn Error>> {
656        Self::from_delimited(input, b'\t')
657    }
658
659    /// Serializes the canonical hub to TSV.
660    ///
661    /// # Examples
662    /// ```
663    /// use sheeter_engine::CanonicalValue;
664    /// let v = CanonicalValue::from_json(r#"[{"a":1}]"#).unwrap();
665    /// assert_eq!(v.to_tsv().unwrap(), "a\n1\n");
666    /// ```
667    pub fn to_tsv(&self) -> Result<String, Box<dyn Error>> {
668        self.to_delimited(b'\t')
669    }
670
671    /// Parses YAML into the canonical hub.
672    ///
673    /// # Examples
674    /// ```
675    /// use sheeter_engine::CanonicalValue;
676    /// let v = CanonicalValue::from_yaml("- a: 1\n").unwrap();
677    /// assert_eq!(v.to_json().unwrap(), r#"[{"a":1}]"#);
678    /// ```
679    pub fn from_yaml(input: &str) -> Result<Self, Box<dyn Error>> {
680        let opts = serde_saphyr::options! {
681            strict_booleans: true,
682            ignore_binary_tag_for_string: true,
683        };
684        let v: serde_json::Value = serde_saphyr::from_str_with_options(input, opts)?;
685        Ok(Self::from_json_value(v))
686    }
687
688    /// Serializes the canonical hub to YAML.
689    ///
690    /// # Examples
691    /// ```
692    /// use sheeter_engine::CanonicalValue;
693    /// let v = CanonicalValue::from_json(r#"[{"a":1}]"#).unwrap();
694    /// assert!(v.to_yaml().unwrap().contains("a: 1"));
695    /// ```
696    pub fn to_yaml(&self) -> Result<String, Box<dyn Error>> {
697        Ok(serde_saphyr::to_string(&self.to_json_value())?)
698    }
699
700    /// Parses XML into the canonical hub.
701    ///
702    /// Attributes become `@name` keys, text content becomes `#text`, and
703    /// repeated sibling elements become arrays. The returned value is an
704    /// object with exactly one key: the root element name.
705    ///
706    /// # Examples
707    /// ```
708    /// use sheeter_engine::CanonicalValue;
709    /// let v = CanonicalValue::from_xml("<r><a>1</a><a>2</a></r>").unwrap();
710    /// assert_eq!(v.to_json().unwrap(), r#"{"r":{"a":["1","2"]}}"#);
711    /// ```
712    pub fn from_xml(input: &str) -> Result<Self, Box<dyn Error>> {
713        use quick_xml::events::Event;
714        use quick_xml::Reader;
715
716        let mut reader = Reader::from_str(input);
717        reader.config_mut().trim_text(true);
718        let mut stack: Vec<XmlStackFrame> = Vec::new();
719        let mut root: Option<(String, CanonicalValue)> = None;
720
721        loop {
722            match reader.read_event() {
723                Ok(Event::Start(e)) => {
724                    let name = String::from_utf8_lossy(e.name().as_ref()).to_string();
725                    let mut attrs = Vec::new();
726                    for a in e.attributes() {
727                        let a = a?;
728                        attrs.push((
729                            String::from_utf8_lossy(a.key.as_ref()).to_string(),
730                            a.unescape_value()?.to_string(),
731                        ));
732                    }
733                    stack.push((name, attrs, Vec::new(), String::new()));
734                }
735                Ok(Event::Text(e)) => {
736                    if let Some(top) = stack.last_mut() {
737                        top.3.push_str(&e.unescape()?);
738                    }
739                }
740                Ok(Event::End(_)) => {
741                    let (name, attributes, children, text) = stack.pop().ok_or("unbalanced XML")?;
742                    let trimmed = text.trim().to_string();
743                    let value = if attributes.is_empty() && children.is_empty() {
744                        Self::String(trimmed)
745                    } else {
746                        let mut obj: Vec<(String, CanonicalValue)> = Vec::new();
747                        for (k, v) in attributes {
748                            obj.push((format!("@{k}"), Self::String(v)));
749                        }
750                        if !trimmed.is_empty() {
751                            obj.push(("#text".to_string(), Self::String(trimmed)));
752                        }
753                        for (cname, cval) in children {
754                            if let Some(slot) = obj.iter_mut().find(|(k, _)| k == &cname) {
755                                match &mut slot.1 {
756                                    Self::Array(a) => a.push(cval),
757                                    _ => {
758                                        let old = slot.1.clone();
759                                        slot.1 = Self::Array(vec![old, cval]);
760                                    }
761                                }
762                            } else {
763                                obj.push((cname, cval));
764                            }
765                        }
766                        Self::Object(obj)
767                    };
768                    if let Some(parent) = stack.last_mut() {
769                        parent.2.push((name, value));
770                    } else {
771                        root = Some((name, value));
772                    }
773                }
774                Ok(Event::Eof) => break,
775                Ok(_) => {}
776                Err(e) => return Err(e.into()),
777            }
778        }
779
780        let (root_name, root_value) = root.ok_or("empty XML")?;
781        Ok(Self::Object(vec![(root_name, root_value)]))
782    }
783
784    /// Serializes the canonical hub to XML.
785    ///
786    /// # Examples
787    /// ```
788    /// use sheeter_engine::CanonicalValue;
789    /// let v = CanonicalValue::from_json(r#"[{"a":"1"}]"#).unwrap();
790    /// let xml = v.to_xml().unwrap();
791    /// assert!(xml.contains("<root>"));
792    /// assert!(xml.contains("<a>1</a>"));
793    /// ```
794    pub fn to_xml(&self) -> Result<String, Box<dyn Error>> {
795        let mut writer = quick_xml::Writer::new_with_indent(Vec::new(), b' ', 2);
796        match self {
797            Self::Object(o) if o.len() == 1 && !o[0].0.starts_with('@') && o[0].0 != "#text" => {
798                write_element(&mut writer, &o[0].0, &o[0].1)?;
799            }
800            Self::Array(a) => {
801                write_open(&mut writer, "root", &[])?;
802                for item in a {
803                    write_element(&mut writer, "item", item)?;
804                }
805                write_close(&mut writer, "root")?;
806            }
807            _ => write_element(&mut writer, "root", self)?,
808        }
809        Ok(String::from_utf8(writer.into_inner())?)
810    }
811}
812
813fn write_open<W: std::io::Write>(
814    writer: &mut quick_xml::Writer<W>,
815    name: &str,
816    attrs: &[(String, String)],
817) -> Result<(), Box<dyn Error>> {
818    use quick_xml::events::{BytesStart, Event};
819    let mut start = BytesStart::new(name);
820    for (k, v) in attrs {
821        start.push_attribute((k.as_str(), v.as_str()));
822    }
823    writer.write_event(Event::Start(start))?;
824    Ok(())
825}
826
827fn write_close<W: std::io::Write>(
828    writer: &mut quick_xml::Writer<W>,
829    name: &str,
830) -> Result<(), Box<dyn Error>> {
831    use quick_xml::events::{BytesEnd, Event};
832    writer.write_event(Event::End(BytesEnd::new(name)))?;
833    Ok(())
834}
835
836fn write_element<W: std::io::Write>(
837    writer: &mut quick_xml::Writer<W>,
838    name: &str,
839    value: &CanonicalValue,
840) -> Result<(), Box<dyn Error>> {
841    use quick_xml::events::{BytesText, Event};
842
843    match value {
844        CanonicalValue::Array(items) => {
845            for item in items {
846                write_element(writer, name, item)?;
847            }
848        }
849        CanonicalValue::Object(o) => {
850            let attrs: Vec<(String, String)> = o
851                .iter()
852                .filter_map(|(k, v)| {
853                    k.strip_prefix('@').and_then(|attr| match v {
854                        CanonicalValue::String(s) => Some((attr.to_string(), s.clone())),
855                        _ => None,
856                    })
857                })
858                .collect();
859            write_open(writer, name, &attrs)?;
860            for (k, v) in o {
861                if k.starts_with('@') {
862                    continue;
863                }
864                if k == "#text" {
865                    if let CanonicalValue::String(s) = v {
866                        writer.write_event(Event::Text(BytesText::new(s)))?;
867                    }
868                    continue;
869                }
870                write_element(writer, k, v)?;
871            }
872            write_close(writer, name)?;
873        }
874        CanonicalValue::String(s) => {
875            write_open(writer, name, &[])?;
876            writer.write_event(Event::Text(BytesText::new(s)))?;
877            write_close(writer, name)?;
878        }
879        CanonicalValue::Bytes(b) => {
880            use base64::Engine;
881            let s = base64::engine::general_purpose::STANDARD.encode(b);
882            write_open(writer, name, &[])?;
883            writer.write_event(Event::Text(BytesText::new(&s)))?;
884            write_close(writer, name)?;
885        }
886        CanonicalValue::Number(n) => {
887            write_open(writer, name, &[])?;
888            writer.write_event(Event::Text(BytesText::new(&n.to_string())))?;
889            write_close(writer, name)?;
890        }
891        CanonicalValue::Bool(b) => {
892            write_open(writer, name, &[])?;
893            writer.write_event(Event::Text(BytesText::new(&b.to_string())))?;
894            write_close(writer, name)?;
895        }
896        CanonicalValue::Null => {
897            write_open(writer, name, &[])?;
898            write_close(writer, name)?;
899        }
900    }
901    Ok(())
902}
903
904type SqlRow = Vec<(String, CanonicalValue)>;
905type SqlTable = (String, Vec<SqlRow>);
906
907impl CanonicalValue {
908    /// Parses an ODS (OpenDocument Spreadsheet) file into a value.
909    ///
910    /// A single-sheet ODS becomes an array of row objects; a multi-sheet
911    /// ODS becomes an object keyed by sheet name.
912    ///
913    /// # Examples
914    ///
915    /// ```ignore
916    /// // Fixture covered in tests/ods.rs.
917    /// ```
918    pub fn from_ods(bytes: &[u8]) -> Result<Self, Box<dyn Error>> {
919        use calamine::{Ods, Reader};
920        let cursor = Cursor::new(bytes.to_vec());
921        let mut workbook: Ods<_> = Ods::new(cursor)?;
922        let names = workbook.sheet_names().to_owned();
923        let mut out: Vec<(String, CanonicalValue)> = Vec::with_capacity(names.len());
924        for name in names {
925            let range = workbook
926                .worksheet_range(&name)
927                .map_err(|e| format!("failed to read sheet {name}: {e:?}"))?;
928            out.push((name, Self::from_calamine_range(&range)?));
929        }
930        if out.len() == 1 {
931            Ok(out.remove(0).1)
932        } else {
933            Ok(Self::Object(out))
934        }
935    }
936}
937
938/// Writes a workbook as an ODS file.
939///
940/// Each entry of `sheets` becomes a `table:table` inside the resulting
941/// `content.xml`. The `mimetype` entry is written first and stored
942/// uncompressed, as required by the ODF spec.
943///
944/// # Errors
945///
946/// Returns an error if any row contains nested values or if ZIP/XML I/O fails.
947pub fn to_ods_workbook(sheets: &[(String, CanonicalValue)]) -> Result<Vec<u8>, Box<dyn Error>> {
948    use std::io::Write as IoWrite;
949    use zip::write::SimpleFileOptions;
950    use zip::{CompressionMethod, ZipWriter};
951
952    let content = ods_content_xml(sheets)?;
953    let manifest = ods_manifest_xml();
954    let mimetype = "application/vnd.oasis.opendocument.spreadsheet";
955
956    let cursor = Cursor::new(Vec::new());
957    let mut zip = ZipWriter::new(cursor);
958    let stored = SimpleFileOptions::default().compression_method(CompressionMethod::Stored);
959    let deflated = SimpleFileOptions::default().compression_method(CompressionMethod::Deflated);
960
961    zip.start_file("mimetype", stored)?;
962    zip.write_all(mimetype.as_bytes())?;
963
964    zip.start_file("META-INF/manifest.xml", deflated)?;
965    zip.write_all(manifest.as_bytes())?;
966
967    zip.start_file("content.xml", deflated)?;
968    zip.write_all(content.as_bytes())?;
969
970    let cursor = zip.finish()?;
971    Ok(cursor.into_inner())
972}
973
974fn ods_manifest_xml() -> String {
975    r#"<?xml version="1.0" encoding="UTF-8"?>
976<manifest:manifest xmlns:manifest="urn:oasis:names:tc:opendocument:xmlns:manifest:1.0" manifest:version="1.2">
977  <manifest:file-entry manifest:full-path="/" manifest:version="1.2" manifest:media-type="application/vnd.oasis.opendocument.spreadsheet"/>
978  <manifest:file-entry manifest:full-path="content.xml" manifest:media-type="text/xml"/>
979</manifest:manifest>
980"#
981    .to_string()
982}
983
984fn ods_content_xml(sheets: &[(String, CanonicalValue)]) -> Result<String, Box<dyn Error>> {
985    use quick_xml::escape::escape;
986    let mut out = String::new();
987    out.push_str(r#"<?xml version="1.0" encoding="UTF-8"?>"#);
988    out.push_str(
989        r#"<office:document-content xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" xmlns:table="urn:oasis:names:tc:opendocument:xmlns:table:1.0" xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0" office:version="1.2">"#,
990    );
991    out.push_str("<office:body><office:spreadsheet>");
992
993    for (name, value) in sheets {
994        out.push_str(&format!("<table:table table:name=\"{}\">", escape(name)));
995        let rows = value.as_tabular()?;
996        let mut headers: Vec<String> = Vec::new();
997        for row in &rows {
998            if let CanonicalValue::Object(o) = row {
999                for (k, _) in o {
1000                    if !headers.contains(k) {
1001                        headers.push(k.clone());
1002                    }
1003                }
1004            }
1005        }
1006        out.push_str("<table:table-row>");
1007        for h in &headers {
1008            out.push_str("<table:table-cell office:value-type=\"string\">");
1009            out.push_str(&format!("<text:p>{}</text:p>", escape(h)));
1010            out.push_str("</table:table-cell>");
1011        }
1012        out.push_str("</table:table-row>");
1013
1014        for row in &rows {
1015            if let CanonicalValue::Object(o) = row {
1016                out.push_str("<table:table-row>");
1017                for h in &headers {
1018                    match o.iter().find(|(k, _)| k == h).map(|(_, v)| v) {
1019                        Some(CanonicalValue::String(s)) => {
1020                            out.push_str("<table:table-cell office:value-type=\"string\">");
1021                            out.push_str(&format!("<text:p>{}</text:p>", escape(s)));
1022                            out.push_str("</table:table-cell>");
1023                        }
1024                        Some(CanonicalValue::Number(n)) => {
1025                            out.push_str(&format!(
1026                                "<table:table-cell office:value-type=\"float\" office:value=\"{}\">",
1027                                n
1028                            ));
1029                            out.push_str(&format!("<text:p>{}</text:p>", n));
1030                            out.push_str("</table:table-cell>");
1031                        }
1032                        Some(CanonicalValue::Bool(b)) => {
1033                            out.push_str(&format!(
1034                                "<table:table-cell office:value-type=\"boolean\" office:boolean-value=\"{}\">",
1035                                b
1036                            ));
1037                            out.push_str(&format!(
1038                                "<text:p>{}</text:p>",
1039                                if *b { "TRUE" } else { "FALSE" }
1040                            ));
1041                            out.push_str("</table:table-cell>");
1042                        }
1043                        Some(CanonicalValue::Bytes(b)) => {
1044                            use base64::Engine;
1045                            let s = base64::engine::general_purpose::STANDARD.encode(b);
1046                            out.push_str("<table:table-cell office:value-type=\"string\">");
1047                            out.push_str(&format!("<text:p>{}</text:p>", s));
1048                            out.push_str("</table:table-cell>");
1049                        }
1050                        Some(CanonicalValue::Null) | None => {
1051                            out.push_str("<table:table-cell/>");
1052                        }
1053                        Some(CanonicalValue::Array(_)) | Some(CanonicalValue::Object(_)) => {
1054                            return Err(format!(
1055                                "nested value at key {h} cannot be represented in ODS"
1056                            )
1057                            .into());
1058                        }
1059                    }
1060                }
1061                out.push_str("</table:table-row>");
1062            }
1063        }
1064        out.push_str("</table:table>");
1065    }
1066
1067    out.push_str("</office:spreadsheet></office:body></office:document-content>");
1068    Ok(out)
1069}
1070
1071fn is_scalar(v: &CanonicalValue) -> bool {
1072    matches!(
1073        v,
1074        CanonicalValue::Null
1075            | CanonicalValue::Bool(_)
1076            | CanonicalValue::Number(_)
1077            | CanonicalValue::String(_)
1078    )
1079}
1080
1081fn find_first_object_array(v: &CanonicalValue) -> Option<&Vec<CanonicalValue>> {
1082    match v {
1083        CanonicalValue::Array(a)
1084            if !a.is_empty() && a.iter().all(|x| matches!(x, CanonicalValue::Object(_))) =>
1085        {
1086            Some(a)
1087        }
1088        CanonicalValue::Object(o) => {
1089            for (_, v) in o {
1090                if let Some(arr) = find_first_object_array(v) {
1091                    return Some(arr);
1092                }
1093            }
1094            None
1095        }
1096        _ => None,
1097    }
1098}
1099
1100fn cell_to_string(c: &calamine::Data) -> String {
1101    use calamine::Data;
1102    match c {
1103        Data::Empty => String::new(),
1104        Data::String(s) => s.clone(),
1105        Data::Float(f) => f.to_string(),
1106        Data::Int(i) => i.to_string(),
1107        Data::Bool(b) => b.to_string(),
1108        Data::DateTime(dt) => dt.to_string(),
1109        Data::DateTimeIso(s) => s.clone(),
1110        Data::DurationIso(s) => s.clone(),
1111        Data::Error(e) => format!("{e:?}"),
1112    }
1113}
1114
1115fn clean_ident(s: &str) -> String {
1116    s.trim_matches('"').trim_matches('`').to_string()
1117}
1118
1119fn quote_ident(s: &str) -> String {
1120    format!("\"{}\"", s.replace('"', "\"\""))
1121}
1122
1123impl CanonicalValue {
1124    /// Parses NDJSON (newline-delimited JSON) into an array.
1125    ///
1126    /// Each non-empty line must be a complete JSON value. The result is a
1127    /// `CanonicalValue::Array` with one element per line, preserving order.
1128    ///
1129    /// # Errors
1130    /// Returns an error if any non-empty line is malformed JSON.
1131    ///
1132    /// # Examples
1133    /// ```
1134    /// use sheeter_engine::CanonicalValue;
1135    /// let v = CanonicalValue::from_ndjson("{\"a\":1}\n{\"a\":2}\n").unwrap();
1136    /// assert_eq!(v.to_json().unwrap(), r#"[{"a":1},{"a":2}]"#);
1137    /// ```
1138    pub fn from_ndjson(input: &str) -> Result<Self, Box<dyn std::error::Error>> {
1139        let mut values = Vec::new();
1140        for line in input.lines() {
1141            let trimmed = line.trim();
1142            if trimmed.is_empty() {
1143                continue;
1144            }
1145            let v: serde_json::Value = serde_json::from_str(trimmed)?;
1146            values.push(Self::from_json_value(v));
1147        }
1148        Ok(Self::Array(values))
1149    }
1150
1151    /// Serializes the value as NDJSON (one JSON value per line).
1152    ///
1153    /// Arrays become one line per element. Non-array values become a
1154    /// single line.
1155    ///
1156    /// # Errors
1157    /// Returns an error if serialization fails.
1158    ///
1159    /// # Examples
1160    /// ```
1161    /// use sheeter_engine::CanonicalValue;
1162    /// let v = CanonicalValue::from_json(r#"[{"a":1},{"a":2}]"#).unwrap();
1163    /// assert_eq!(v.to_ndjson().unwrap(), "{\"a\":1}\n{\"a\":2}\n");
1164    /// ```
1165    pub fn to_ndjson(&self) -> Result<String, Box<dyn std::error::Error>> {
1166        let mut out = String::new();
1167        match self {
1168            Self::Array(items) => {
1169                for item in items {
1170                    out.push_str(&serde_json::to_string(&item.to_json_value())?);
1171                    out.push('\n');
1172                }
1173            }
1174            other => {
1175                out.push_str(&serde_json::to_string(&other.to_json_value())?);
1176                out.push('\n');
1177            }
1178        }
1179        Ok(out)
1180    }
1181}
sheeter_engine/canonical.rs

sheeter_engine/
canonical.rs
