sheeter_engine/
canonical.rs

1use std::error::Error;
2use std::io::Cursor;
3
4/// Format-agnostic intermediate value used by the Sheeter hub.
5///
6/// Every reader (CSV, TSV, JSON, YAML, XML) produces a `CanonicalValue`, and
7/// every writer consumes one. Adding a format requires exactly one reader and
8/// one writer, not `N-1` pairwise converters, because every spoke composes
9/// through this hub.
10///
11/// Tabular data is represented as `Array<Object>`. XML documents are
12/// represented as a single-key `Object` whose key is the root element name,
13/// with attributes prefixed by `@` and text stored under `#text`. Repeated
14/// sibling elements become `Array`.
15///
16/// # Examples
17///
18/// ```
19/// use sheeter_engine::CanonicalValue;
20///
21/// let v = CanonicalValue::from_json(r#"[{"a":1}]"#).unwrap();
22/// assert_eq!(v.to_csv().unwrap(), "a\n1\n");
23/// ```
24#[derive(Debug, Clone, PartialEq)]
25pub enum CanonicalValue {
26    Null,
27    Bool(bool),
28    Number(serde_json::Number),
29    String(String),
30    Array(Vec<CanonicalValue>),
31    Object(Vec<(String, CanonicalValue)>),
32}
33
34type XmlStackFrame = (
35    String,
36    Vec<(String, String)>,
37    Vec<(String, CanonicalValue)>,
38    String,
39);
40
41impl CanonicalValue {
42    pub(crate) fn from_json_value(v: serde_json::Value) -> Self {
43        match v {
44            serde_json::Value::Null => Self::Null,
45            serde_json::Value::Bool(b) => Self::Bool(b),
46            serde_json::Value::Number(n) => Self::Number(n),
47            serde_json::Value::String(s) => Self::String(s),
48            serde_json::Value::Array(a) => {
49                Self::Array(a.into_iter().map(Self::from_json_value).collect())
50            }
51            serde_json::Value::Object(m) => Self::Object(
52                m.into_iter()
53                    .map(|(k, v)| (k, Self::from_json_value(v)))
54                    .collect(),
55            ),
56        }
57    }
58
59    pub(crate) fn to_json_value(&self) -> serde_json::Value {
60        match self {
61            Self::Null => serde_json::Value::Null,
62            Self::Bool(b) => serde_json::Value::Bool(*b),
63            Self::Number(n) => serde_json::Value::Number(n.clone()),
64            Self::String(s) => serde_json::Value::String(s.clone()),
65            Self::Array(a) => {
66                serde_json::Value::Array(a.iter().map(|v| v.to_json_value()).collect())
67            }
68            Self::Object(o) => {
69                let mut map = serde_json::Map::new();
70                for (k, v) in o {
71                    map.insert(k.clone(), v.to_json_value());
72                }
73                serde_json::Value::Object(map)
74            }
75        }
76    }
77
78    /// Parses JSON into the canonical hub.
79    ///
80    /// # Errors
81    /// Returns an error if the JSON is malformed.
82    ///
83    /// # Examples
84    /// ```
85    /// use sheeter_engine::CanonicalValue;
86    /// assert!(CanonicalValue::from_json(r#"{"a":1}"#).is_ok());
87    /// ```
88    pub fn from_json(input: &str) -> Result<Self, Box<dyn Error>> {
89        Ok(Self::from_json_value(serde_json::from_str(input)?))
90    }
91
92    /// Serializes the canonical hub to JSON.
93    ///
94    /// # Examples
95    /// ```
96    /// use sheeter_engine::CanonicalValue;
97    /// let v = CanonicalValue::String("x".into());
98    /// assert_eq!(v.to_json().unwrap(), r#""x""#);
99    /// ```
100    pub fn to_json(&self) -> Result<String, Box<dyn Error>> {
101        Ok(serde_json::to_string(&self.to_json_value())?)
102    }
103
104    fn from_delimited(input: &str, delim: u8) -> Result<Self, Box<dyn Error>> {
105        let input = input.strip_prefix('\u{feff}').unwrap_or(input);
106        let mut rdr = csv::ReaderBuilder::new()
107            .delimiter(delim)
108            .from_reader(Cursor::new(input));
109        let headers = rdr.headers()?.clone();
110        let mut seen = std::collections::HashSet::new();
111        for h in headers.iter() {
112            if !seen.insert(h) {
113                return Err(format!("duplicate header: {h}").into());
114            }
115        }
116        let mut rows = Vec::new();
117        for result in rdr.records() {
118            let record = result?;
119            let mut obj = Vec::with_capacity(headers.len());
120            for (h, v) in headers.iter().zip(record.iter()) {
121                obj.push((h.to_string(), Self::String(v.to_string())));
122            }
123            rows.push(Self::Object(obj));
124        }
125        Ok(Self::Array(rows))
126    }
127
128    fn to_delimited(&self, delim: u8) -> Result<String, Box<dyn Error>> {
129        let rows = self.as_tabular()?;
130        if rows.is_empty() {
131            return Ok(String::new());
132        }
133        let mut headers: Vec<String> = Vec::new();
134        for row in &rows {
135            let obj = row.as_object()?;
136            for (k, _) in obj {
137                if !headers.contains(k) {
138                    headers.push(k.clone());
139                }
140            }
141        }
142        let mut wtr = csv::WriterBuilder::new()
143            .delimiter(delim)
144            .from_writer(vec![]);
145        wtr.write_record(&headers)?;
146        for row in &rows {
147            let obj = row.as_object()?;
148            let mut record = Vec::with_capacity(headers.len());
149            for h in &headers {
150                let cell = match obj.iter().find(|(k, _)| k == h) {
151                    Some((_, Self::Null)) | None => String::new(),
152                    Some((_, Self::Bool(b))) => b.to_string(),
153                    Some((_, Self::Number(n))) => n.to_string(),
154                    Some((_, Self::String(s))) => s.clone(),
155                    Some((_, Self::Array(_))) | Some((_, Self::Object(_))) => {
156                        return Err(format!(
157                            "nested value at key {h} cannot be represented in tabular output"
158                        )
159                        .into());
160                    }
161                };
162                record.push(cell);
163            }
164            wtr.write_record(&record)?;
165        }
166        let data = wtr.into_inner()?;
167        Ok(String::from_utf8(data)?)
168    }
169
170    pub fn from_toml(input: &str) -> Result<Self, Box<dyn Error>> {
171        let value: toml::Value = toml::from_str(input)?;
172        Ok(Self::from_toml_value(value))
173    }
174
175    fn from_toml_value(v: toml::Value) -> Self {
176        match v {
177            toml::Value::String(s) => Self::String(s),
178            toml::Value::Integer(i) => Self::Number(serde_json::Number::from(i)),
179            toml::Value::Float(f) => match serde_json::Number::from_f64(f) {
180                Some(n) => Self::Number(n),
181                None => Self::String(f.to_string()),
182            },
183            toml::Value::Boolean(b) => Self::Bool(b),
184            toml::Value::Datetime(dt) => Self::String(dt.to_string()),
185            toml::Value::Array(a) => {
186                Self::Array(a.into_iter().map(Self::from_toml_value).collect())
187            }
188            toml::Value::Table(t) => Self::Object(
189                t.into_iter()
190                    .map(|(k, v)| (k, Self::from_toml_value(v)))
191                    .collect(),
192            ),
193        }
194    }
195
196    pub fn to_toml(&self) -> Result<String, Box<dyn Error>> {
197        let value = match self {
198            Self::Array(_) => {
199                let mut table = toml::map::Map::new();
200                table.insert("item".to_string(), self.to_toml_value()?);
201                toml::Value::Table(table)
202            }
203            Self::Object(_) => self.to_toml_value()?,
204            _ => {
205                let mut table = toml::map::Map::new();
206                table.insert("value".to_string(), self.to_toml_value()?);
207                toml::Value::Table(table)
208            }
209        };
210        Ok(toml::to_string_pretty(&value)?)
211    }
212
213    fn to_toml_value(&self) -> Result<toml::Value, Box<dyn Error>> {
214        Ok(match self {
215            Self::Null => return Err("TOML has no null type".into()),
216            Self::Bool(b) => toml::Value::Boolean(*b),
217            Self::Number(n) => {
218                if let Some(i) = n.as_i64() {
219                    toml::Value::Integer(i)
220                } else if let Some(u) = n.as_u64() {
221                    if u <= i64::MAX as u64 {
222                        toml::Value::Integer(u as i64)
223                    } else {
224                        return Err(format!("TOML integer out of range: {u}").into());
225                    }
226                } else if let Some(f) = n.as_f64() {
227                    toml::Value::Float(f)
228                } else {
229                    return Err(format!("cannot represent number {n} in TOML").into());
230                }
231            }
232            Self::String(s) => toml::Value::String(s.clone()),
233            Self::Array(a) => {
234                let mut items = Vec::with_capacity(a.len());
235                for item in a {
236                    if let Self::Null = item {
237                        return Err("TOML arrays cannot contain null".into());
238                    }
239                    items.push(item.to_toml_value()?);
240                }
241                toml::Value::Array(items)
242            }
243            Self::Object(o) => {
244                let mut table = toml::map::Map::new();
245                for (k, v) in o {
246                    if let Self::Null = v {
247                        continue;
248                    }
249                    table.insert(k.clone(), v.to_toml_value()?);
250                }
251                toml::Value::Table(table)
252            }
253        })
254    }
255
256    pub fn from_xlsx_workbook(bytes: &[u8]) -> Result<Vec<(String, Self)>, Box<dyn Error>> {
257        use calamine::{Reader, Xlsx};
258        let cursor = Cursor::new(bytes.to_vec());
259        let mut workbook: Xlsx<_> = Xlsx::new(cursor)?;
260        let names = workbook.sheet_names().to_owned();
261        let mut out = Vec::with_capacity(names.len());
262        for name in names {
263            let range = workbook
264                .worksheet_range(&name)
265                .map_err(|e| format!("failed to read sheet {name}: {e:?}"))?;
266            out.push((name, Self::from_calamine_range(&range)?));
267        }
268        Ok(out)
269    }
270
271    fn from_calamine_range(
272        range: &calamine::Range<calamine::Data>,
273    ) -> Result<Self, Box<dyn Error>> {
274        use calamine::Data;
275        let mut rows = range.rows();
276        let header = match rows.next() {
277            Some(h) => h.iter().map(cell_to_string).collect::<Vec<String>>(),
278            None => return Ok(Self::Array(Vec::new())),
279        };
280        let mut records = Vec::new();
281        for row in rows {
282            if row.iter().all(|c| matches!(c, Data::Empty)) {
283                continue;
284            }
285            let mut obj = Vec::with_capacity(header.len());
286            for (i, h) in header.iter().enumerate() {
287                let val = match row.get(i) {
288                    Some(Data::Empty) | None => Self::Null,
289                    Some(Data::String(s)) => Self::String(s.clone()),
290                    Some(Data::Float(f)) => {
291                        if f.is_finite() && f.fract() == 0.0 && f.abs() < i64::MAX as f64 {
292                            Self::Number(serde_json::Number::from(*f as i64))
293                        } else {
294                            match serde_json::Number::from_f64(*f) {
295                                Some(n) => Self::Number(n),
296                                None => Self::String(f.to_string()),
297                            }
298                        }
299                    }
300                    Some(Data::Int(i)) => Self::Number(serde_json::Number::from(*i)),
301                    Some(Data::Bool(b)) => Self::Bool(*b),
302                    Some(Data::DateTime(dt)) => Self::String(dt.to_string()),
303                    Some(Data::DateTimeIso(s)) => Self::String(s.clone()),
304                    Some(Data::DurationIso(s)) => Self::String(s.clone()),
305                    Some(Data::Error(e)) => Self::String(format!("{e:?}")),
306                };
307                obj.push((h.clone(), val));
308            }
309            records.push(Self::Object(obj));
310        }
311        Ok(Self::Array(records))
312    }
313
314    pub fn to_xlsx_workbook(sheets: &[(String, Self)]) -> Result<Vec<u8>, Box<dyn Error>> {
315        use rust_xlsxwriter::Workbook;
316        let mut wb = Workbook::new();
317        for (name, value) in sheets {
318            let ws = wb.add_worksheet();
319            ws.set_name(name.as_str())?;
320            let rows = value.as_tabular()?;
321            if rows.is_empty() {
322                continue;
323            }
324            let mut headers: Vec<String> = Vec::new();
325            for row in &rows {
326                let obj = row.as_object()?;
327                for (k, _) in obj {
328                    if !headers.contains(k) {
329                        headers.push(k.clone());
330                    }
331                }
332            }
333            for (col, h) in headers.iter().enumerate() {
334                ws.write_string(0, col as u16, h)?;
335            }
336            for (i, row) in rows.iter().enumerate() {
337                let obj = row.as_object()?;
338                for (col, h) in headers.iter().enumerate() {
339                    let row_idx = (i + 1) as u32;
340                    let col_idx = col as u16;
341                    match obj.iter().find(|(k, _)| k == h) {
342                        Some((_, Self::String(s))) => {
343                            ws.write_string(row_idx, col_idx, s)?;
344                        }
345                        Some((_, Self::Number(n))) => {
346                            if let Some(f) = n.as_f64() {
347                                ws.write_number(row_idx, col_idx, f)?;
348                            } else {
349                                ws.write_string(row_idx, col_idx, n.to_string())?;
350                            }
351                        }
352                        Some((_, Self::Bool(b))) => {
353                            ws.write_boolean(row_idx, col_idx, *b)?;
354                        }
355                        Some((_, Self::Null)) | None => {}
356                        Some((_, Self::Array(_))) | Some((_, Self::Object(_))) => {
357                            return Err(format!(
358                                "nested value at key {h} cannot be represented in XLSX"
359                            )
360                            .into());
361                        }
362                    }
363                }
364            }
365        }
366        Ok(wb.save_to_buffer()?)
367    }
368
369    pub fn from_sql_workbook(
370        input: &str,
371        dialect: crate::SqlDialect,
372    ) -> Result<Vec<(String, Self)>, Box<dyn Error>> {
373        use sqlparser::ast::{
374            Expr, SetExpr, Statement, TableObject, Value as SqlValue, ValueWithSpan, Values,
375        };
376        use sqlparser::dialect::{GenericDialect, MySqlDialect, PostgreSqlDialect, SQLiteDialect};
377        use sqlparser::parser::Parser;
378
379        let statements = match dialect {
380            crate::SqlDialect::Generic => Parser::parse_sql(&GenericDialect {}, input)?,
381            crate::SqlDialect::PostgreSql => Parser::parse_sql(&PostgreSqlDialect {}, input)?,
382            crate::SqlDialect::Sqlite => Parser::parse_sql(&SQLiteDialect {}, input)?,
383            crate::SqlDialect::MySql => Parser::parse_sql(&MySqlDialect {}, input)?,
384        };
385
386        let mut schemas: std::collections::HashMap<String, Vec<String>> =
387            std::collections::HashMap::new();
388        let mut tables: Vec<SqlTable> = Vec::new();
389
390        for statement in statements {
391            match statement {
392                Statement::CreateTable(create) => {
393                    let name = clean_ident(&create.name.to_string());
394                    let cols: Vec<String> = create
395                        .columns
396                        .iter()
397                        .map(|c| clean_ident(&c.name.to_string()))
398                        .collect();
399                    schemas.insert(name, cols);
400                }
401                Statement::Insert(insert) => {
402                    let table = match &insert.table {
403                        TableObject::TableName(n) => clean_ident(&n.to_string()),
404                        _ => {
405                            return Err(
406                                "INSERT into table functions or queries is not supported".into()
407                            );
408                        }
409                    };
410                    let source = insert
411                        .source
412                        .as_ref()
413                        .ok_or("INSERT without a source is not supported")?;
414                    let rows = match source.body.as_ref() {
415                        SetExpr::Values(Values { rows, .. }) => rows,
416                        _ => {
417                            return Err(
418                                "only INSERT ... VALUES is supported for data extraction".into()
419                            );
420                        }
421                    };
422                    let explicit_cols: Vec<String> = insert
423                        .columns
424                        .iter()
425                        .map(|c| clean_ident(&c.to_string()))
426                        .collect();
427                    let cols = if explicit_cols.is_empty() {
428                        schemas.get(&table).cloned().unwrap_or_else(|| {
429                            (0..rows.first().map(|r| r.len()).unwrap_or(0))
430                                .map(|i| format!("column_{}", i + 1))
431                                .collect()
432                        })
433                    } else {
434                        explicit_cols
435                    };
436                    let slot = match tables.iter_mut().find(|(n, _)| n == &table) {
437                        Some((_, rows)) => rows,
438                        None => {
439                            tables.push((table.clone(), Vec::new()));
440                            &mut tables.last_mut().expect("just pushed").1
441                        }
442                    };
443                    for row in rows {
444                        let mut obj = Vec::with_capacity(cols.len());
445                        for (i, expr) in row.iter().enumerate() {
446                            let key = cols
447                                .get(i)
448                                .cloned()
449                                .unwrap_or_else(|| format!("column_{}", i + 1));
450                            let value = match expr {
451                                Expr::Value(ValueWithSpan { value, .. }) => match value {
452                                    SqlValue::Number(n, _) => {
453                                        if let Ok(i) = n.parse::<i64>() {
454                                            Self::Number(serde_json::Number::from(i))
455                                        } else if let Ok(f) = n.parse::<f64>() {
456                                            match serde_json::Number::from_f64(f) {
457                                                Some(n) => Self::Number(n),
458                                                None => Self::String(n.clone()),
459                                            }
460                                        } else {
461                                            Self::String(n.clone())
462                                        }
463                                    }
464                                    SqlValue::SingleQuotedString(s)
465                                    | SqlValue::DoubleQuotedString(s)
466                                    | SqlValue::EscapedStringLiteral(s) => Self::String(s.clone()),
467                                    SqlValue::Boolean(b) => Self::Bool(*b),
468                                    SqlValue::Null => Self::Null,
469                                    other => Self::String(other.to_string()),
470                                },
471                                Expr::Identifier(id) => Self::String(id.value.clone()),
472                                other => Self::String(other.to_string()),
473                            };
474                            obj.push((key, value));
475                        }
476                        slot.push(obj);
477                    }
478                }
479                _ => {}
480            }
481        }
482
483        if tables.is_empty() {
484            return Err("no INSERT statements with literal VALUES found".into());
485        }
486
487        Ok(tables
488            .into_iter()
489            .map(|(name, rows)| {
490                let records = rows.into_iter().map(Self::Object).collect();
491                (name, Self::Array(records))
492            })
493            .collect())
494    }
495
496    pub fn to_sql_workbook(sheets: &[(String, Self)]) -> Result<String, Box<dyn Error>> {
497        let mut out = String::new();
498        for (table, value) in sheets {
499            out.push_str(&value.to_sql_with_table(table)?);
500        }
501        Ok(out)
502    }
503
504    pub fn to_sql_with_table(&self, table: &str) -> Result<String, Box<dyn Error>> {
505        let rows = self.as_tabular()?;
506        if rows.is_empty() {
507            return Ok(String::new());
508        }
509        let mut headers: Vec<String> = Vec::new();
510        for row in &rows {
511            let obj = row.as_object()?;
512            for (k, _) in obj {
513                if !headers.contains(k) {
514                    headers.push(k.clone());
515                }
516            }
517        }
518        let cols = headers
519            .iter()
520            .map(|h| quote_ident(h))
521            .collect::<Vec<_>>()
522            .join(", ");
523        let mut out = String::new();
524        for row in &rows {
525            let obj = row.as_object()?;
526            let mut values = Vec::with_capacity(headers.len());
527            for h in &headers {
528                let cell = match obj.iter().find(|(k, _)| k == h) {
529                    Some((_, Self::Null)) | None => "NULL".to_string(),
530                    Some((_, Self::Bool(b))) => b.to_string().to_uppercase(),
531                    Some((_, Self::Number(n))) => n.to_string(),
532                    Some((_, Self::String(s))) => format!("'{}'", s.replace('\'', "''")),
533                    Some((_, Self::Array(_))) | Some((_, Self::Object(_))) => {
534                        return Err(format!(
535                            "nested value at key {h} cannot be represented in SQL"
536                        )
537                        .into());
538                    }
539                };
540                values.push(cell);
541            }
542            out.push_str(&format!(
543                "INSERT INTO {} ({}) VALUES ({});\n",
544                quote_ident(table),
545                cols,
546                values.join(", ")
547            ));
548        }
549        Ok(out)
550    }
551
552    fn as_object(&self) -> Result<&Vec<(String, CanonicalValue)>, Box<dyn Error>> {
553        match self {
554            Self::Object(o) => Ok(o),
555            _ => Err("expected object".into()),
556        }
557    }
558
559    fn as_tabular(&self) -> Result<Vec<&CanonicalValue>, Box<dyn Error>> {
560        if let Some(arr) = find_first_object_array(self) {
561            return Ok(arr.iter().collect());
562        }
563        let mut current = self;
564        while let Self::Object(o) = current {
565            if o.len() != 1 {
566                break;
567            }
568            match &o[0].1 {
569                Self::Object(_) => current = &o[0].1,
570                _ => break,
571            }
572        }
573        if let Self::Object(o) = current {
574            if o.iter().all(|(_, v)| is_scalar(v)) {
575                return Ok(vec![current]);
576            }
577        }
578        Err("cannot determine tabular structure".into())
579    }
580
581    /// Parses CSV into the canonical hub.
582    ///
583    /// # Examples
584    /// ```
585    /// use sheeter_engine::CanonicalValue;
586    /// let v = CanonicalValue::from_csv("a\n1").unwrap();
587    /// assert_eq!(v.to_json().unwrap(), r#"[{"a":"1"}]"#);
588    /// ```
589    pub fn from_csv(input: &str) -> Result<Self, Box<dyn Error>> {
590        Self::from_delimited(input, b',')
591    }
592
593    /// Serializes the canonical hub to CSV.
594    ///
595    /// # Examples
596    /// ```
597    /// use sheeter_engine::CanonicalValue;
598    /// let v = CanonicalValue::from_json(r#"[{"a":1}]"#).unwrap();
599    /// assert_eq!(v.to_csv().unwrap(), "a\n1\n");
600    /// ```
601    pub fn to_csv(&self) -> Result<String, Box<dyn Error>> {
602        self.to_delimited(b',')
603    }
604
605    /// Parses TSV into the canonical hub.
606    ///
607    /// # Examples
608    /// ```
609    /// use sheeter_engine::CanonicalValue;
610    /// let v = CanonicalValue::from_tsv("a\tb\n1\t2").unwrap();
611    /// assert_eq!(v.to_json().unwrap(), r#"[{"a":"1","b":"2"}]"#);
612    /// ```
613    pub fn from_tsv(input: &str) -> Result<Self, Box<dyn Error>> {
614        Self::from_delimited(input, b'\t')
615    }
616
617    /// Serializes the canonical hub to TSV.
618    ///
619    /// # Examples
620    /// ```
621    /// use sheeter_engine::CanonicalValue;
622    /// let v = CanonicalValue::from_json(r#"[{"a":1}]"#).unwrap();
623    /// assert_eq!(v.to_tsv().unwrap(), "a\n1\n");
624    /// ```
625    pub fn to_tsv(&self) -> Result<String, Box<dyn Error>> {
626        self.to_delimited(b'\t')
627    }
628
629    /// Parses YAML into the canonical hub.
630    ///
631    /// # Examples
632    /// ```
633    /// use sheeter_engine::CanonicalValue;
634    /// let v = CanonicalValue::from_yaml("- a: 1\n").unwrap();
635    /// assert_eq!(v.to_json().unwrap(), r#"[{"a":1}]"#);
636    /// ```
637    pub fn from_yaml(input: &str) -> Result<Self, Box<dyn Error>> {
638        let opts = serde_saphyr::options! {
639            strict_booleans: true,
640            ignore_binary_tag_for_string: true,
641        };
642        let v: serde_json::Value = serde_saphyr::from_str_with_options(input, opts)?;
643        Ok(Self::from_json_value(v))
644    }
645
646    /// Serializes the canonical hub to YAML.
647    ///
648    /// # Examples
649    /// ```
650    /// use sheeter_engine::CanonicalValue;
651    /// let v = CanonicalValue::from_json(r#"[{"a":1}]"#).unwrap();
652    /// assert!(v.to_yaml().unwrap().contains("a: 1"));
653    /// ```
654    pub fn to_yaml(&self) -> Result<String, Box<dyn Error>> {
655        Ok(serde_saphyr::to_string(&self.to_json_value())?)
656    }
657
658    /// Parses XML into the canonical hub.
659    ///
660    /// Attributes become `@name` keys, text content becomes `#text`, and
661    /// repeated sibling elements become arrays. The returned value is an
662    /// object with exactly one key: the root element name.
663    ///
664    /// # Examples
665    /// ```
666    /// use sheeter_engine::CanonicalValue;
667    /// let v = CanonicalValue::from_xml("<r><a>1</a><a>2</a></r>").unwrap();
668    /// assert_eq!(v.to_json().unwrap(), r#"{"r":{"a":["1","2"]}}"#);
669    /// ```
670    pub fn from_xml(input: &str) -> Result<Self, Box<dyn Error>> {
671        use quick_xml::events::Event;
672        use quick_xml::Reader;
673
674        let mut reader = Reader::from_str(input);
675        reader.config_mut().trim_text(true);
676        let mut stack: Vec<XmlStackFrame> = Vec::new();
677        let mut root: Option<(String, CanonicalValue)> = None;
678
679        loop {
680            match reader.read_event() {
681                Ok(Event::Start(e)) => {
682                    let name = String::from_utf8_lossy(e.name().as_ref()).to_string();
683                    let mut attrs = Vec::new();
684                    for a in e.attributes() {
685                        let a = a?;
686                        attrs.push((
687                            String::from_utf8_lossy(a.key.as_ref()).to_string(),
688                            a.unescape_value()?.to_string(),
689                        ));
690                    }
691                    stack.push((name, attrs, Vec::new(), String::new()));
692                }
693                Ok(Event::Text(e)) => {
694                    if let Some(top) = stack.last_mut() {
695                        top.3.push_str(&e.unescape()?);
696                    }
697                }
698                Ok(Event::End(_)) => {
699                    let (name, attributes, children, text) = stack.pop().ok_or("unbalanced XML")?;
700                    let trimmed = text.trim().to_string();
701                    let value = if attributes.is_empty() && children.is_empty() {
702                        Self::String(trimmed)
703                    } else {
704                        let mut obj: Vec<(String, CanonicalValue)> = Vec::new();
705                        for (k, v) in attributes {
706                            obj.push((format!("@{k}"), Self::String(v)));
707                        }
708                        if !trimmed.is_empty() {
709                            obj.push(("#text".to_string(), Self::String(trimmed)));
710                        }
711                        for (cname, cval) in children {
712                            if let Some(slot) = obj.iter_mut().find(|(k, _)| k == &cname) {
713                                match &mut slot.1 {
714                                    Self::Array(a) => a.push(cval),
715                                    _ => {
716                                        let old = slot.1.clone();
717                                        slot.1 = Self::Array(vec![old, cval]);
718                                    }
719                                }
720                            } else {
721                                obj.push((cname, cval));
722                            }
723                        }
724                        Self::Object(obj)
725                    };
726                    if let Some(parent) = stack.last_mut() {
727                        parent.2.push((name, value));
728                    } else {
729                        root = Some((name, value));
730                    }
731                }
732                Ok(Event::Eof) => break,
733                Ok(_) => {}
734                Err(e) => return Err(e.into()),
735            }
736        }
737
738        let (root_name, root_value) = root.ok_or("empty XML")?;
739        Ok(Self::Object(vec![(root_name, root_value)]))
740    }
741
742    /// Serializes the canonical hub to XML.
743    ///
744    /// # Examples
745    /// ```
746    /// use sheeter_engine::CanonicalValue;
747    /// let v = CanonicalValue::from_json(r#"[{"a":"1"}]"#).unwrap();
748    /// let xml = v.to_xml().unwrap();
749    /// assert!(xml.contains("<root>"));
750    /// assert!(xml.contains("<a>1</a>"));
751    /// ```
752    pub fn to_xml(&self) -> Result<String, Box<dyn Error>> {
753        let mut writer = quick_xml::Writer::new_with_indent(Vec::new(), b' ', 2);
754        match self {
755            Self::Object(o) if o.len() == 1 && !o[0].0.starts_with('@') && o[0].0 != "#text" => {
756                write_element(&mut writer, &o[0].0, &o[0].1)?;
757            }
758            Self::Array(a) => {
759                write_open(&mut writer, "root", &[])?;
760                for item in a {
761                    write_element(&mut writer, "item", item)?;
762                }
763                write_close(&mut writer, "root")?;
764            }
765            _ => write_element(&mut writer, "root", self)?,
766        }
767        Ok(String::from_utf8(writer.into_inner())?)
768    }
769}
770
771fn write_open<W: std::io::Write>(
772    writer: &mut quick_xml::Writer<W>,
773    name: &str,
774    attrs: &[(String, String)],
775) -> Result<(), Box<dyn Error>> {
776    use quick_xml::events::{BytesStart, Event};
777    let mut start = BytesStart::new(name);
778    for (k, v) in attrs {
779        start.push_attribute((k.as_str(), v.as_str()));
780    }
781    writer.write_event(Event::Start(start))?;
782    Ok(())
783}
784
785fn write_close<W: std::io::Write>(
786    writer: &mut quick_xml::Writer<W>,
787    name: &str,
788) -> Result<(), Box<dyn Error>> {
789    use quick_xml::events::{BytesEnd, Event};
790    writer.write_event(Event::End(BytesEnd::new(name)))?;
791    Ok(())
792}
793
794fn write_element<W: std::io::Write>(
795    writer: &mut quick_xml::Writer<W>,
796    name: &str,
797    value: &CanonicalValue,
798) -> Result<(), Box<dyn Error>> {
799    use quick_xml::events::{BytesText, Event};
800
801    match value {
802        CanonicalValue::Array(items) => {
803            for item in items {
804                write_element(writer, name, item)?;
805            }
806        }
807        CanonicalValue::Object(o) => {
808            let attrs: Vec<(String, String)> = o
809                .iter()
810                .filter_map(|(k, v)| {
811                    k.strip_prefix('@').and_then(|attr| match v {
812                        CanonicalValue::String(s) => Some((attr.to_string(), s.clone())),
813                        _ => None,
814                    })
815                })
816                .collect();
817            write_open(writer, name, &attrs)?;
818            for (k, v) in o {
819                if k.starts_with('@') {
820                    continue;
821                }
822                if k == "#text" {
823                    if let CanonicalValue::String(s) = v {
824                        writer.write_event(Event::Text(BytesText::new(s)))?;
825                    }
826                    continue;
827                }
828                write_element(writer, k, v)?;
829            }
830            write_close(writer, name)?;
831        }
832        CanonicalValue::String(s) => {
833            write_open(writer, name, &[])?;
834            writer.write_event(Event::Text(BytesText::new(s)))?;
835            write_close(writer, name)?;
836        }
837        CanonicalValue::Number(n) => {
838            write_open(writer, name, &[])?;
839            writer.write_event(Event::Text(BytesText::new(&n.to_string())))?;
840            write_close(writer, name)?;
841        }
842        CanonicalValue::Bool(b) => {
843            write_open(writer, name, &[])?;
844            writer.write_event(Event::Text(BytesText::new(&b.to_string())))?;
845            write_close(writer, name)?;
846        }
847        CanonicalValue::Null => {
848            write_open(writer, name, &[])?;
849            write_close(writer, name)?;
850        }
851    }
852    Ok(())
853}
854
855type SqlRow = Vec<(String, CanonicalValue)>;
856type SqlTable = (String, Vec<SqlRow>);
857
858fn is_scalar(v: &CanonicalValue) -> bool {
859    matches!(
860        v,
861        CanonicalValue::Null
862            | CanonicalValue::Bool(_)
863            | CanonicalValue::Number(_)
864            | CanonicalValue::String(_)
865    )
866}
867
868fn find_first_object_array(v: &CanonicalValue) -> Option<&Vec<CanonicalValue>> {
869    match v {
870        CanonicalValue::Array(a)
871            if !a.is_empty() && a.iter().all(|x| matches!(x, CanonicalValue::Object(_))) =>
872        {
873            Some(a)
874        }
875        CanonicalValue::Object(o) => {
876            for (_, v) in o {
877                if let Some(arr) = find_first_object_array(v) {
878                    return Some(arr);
879                }
880            }
881            None
882        }
883        _ => None,
884    }
885}
886
887fn cell_to_string(c: &calamine::Data) -> String {
888    use calamine::Data;
889    match c {
890        Data::Empty => String::new(),
891        Data::String(s) => s.clone(),
892        Data::Float(f) => f.to_string(),
893        Data::Int(i) => i.to_string(),
894        Data::Bool(b) => b.to_string(),
895        Data::DateTime(dt) => dt.to_string(),
896        Data::DateTimeIso(s) => s.clone(),
897        Data::DurationIso(s) => s.clone(),
898        Data::Error(e) => format!("{e:?}"),
899    }
900}
901
902fn clean_ident(s: &str) -> String {
903    s.trim_matches('"').trim_matches('`').to_string()
904}
905
906fn quote_ident(s: &str) -> String {
907    format!("\"{}\"", s.replace('"', "\"\""))
908}
909
910impl CanonicalValue {
911    /// Parses NDJSON (newline-delimited JSON) into an array.
912    ///
913    /// Each non-empty line must be a complete JSON value. The result is a
914    /// `CanonicalValue::Array` with one element per line, preserving order.
915    ///
916    /// # Errors
917    /// Returns an error if any non-empty line is malformed JSON.
918    ///
919    /// # Examples
920    /// ```
921    /// use sheeter_engine::CanonicalValue;
922    /// let v = CanonicalValue::from_ndjson("{\"a\":1}\n{\"a\":2}\n").unwrap();
923    /// assert_eq!(v.to_json().unwrap(), r#"[{"a":1},{"a":2}]"#);
924    /// ```
925    pub fn from_ndjson(input: &str) -> Result<Self, Box<dyn std::error::Error>> {
926        let mut values = Vec::new();
927        for line in input.lines() {
928            let trimmed = line.trim();
929            if trimmed.is_empty() {
930                continue;
931            }
932            let v: serde_json::Value = serde_json::from_str(trimmed)?;
933            values.push(Self::from_json_value(v));
934        }
935        Ok(Self::Array(values))
936    }
937
938    /// Serializes the value as NDJSON (one JSON value per line).
939    ///
940    /// Arrays become one line per element. Non-array values become a
941    /// single line.
942    ///
943    /// # Errors
944    /// Returns an error if serialization fails.
945    ///
946    /// # Examples
947    /// ```
948    /// use sheeter_engine::CanonicalValue;
949    /// let v = CanonicalValue::from_json(r#"[{"a":1},{"a":2}]"#).unwrap();
950    /// assert_eq!(v.to_ndjson().unwrap(), "{\"a\":1}\n{\"a\":2}\n");
951    /// ```
952    pub fn to_ndjson(&self) -> Result<String, Box<dyn std::error::Error>> {
953        let mut out = String::new();
954        match self {
955            Self::Array(items) => {
956                for item in items {
957                    out.push_str(&serde_json::to_string(&item.to_json_value())?);
958                    out.push('\n');
959                }
960            }
961            other => {
962                out.push_str(&serde_json::to_string(&other.to_json_value())?);
963                out.push('\n');
964            }
965        }
966        Ok(out)
967    }
968}
sheeter_engine/canonical.rs

sheeter_engine/
canonical.rs
