sheeter_engine/

lib.rs

//! Sheeter is a data file conversion library and CLI tool.
//!
//! This crate provides functions to convert between CSV and JSON formats.
//! It is designed to be small, fast, and portable, suitable for use on
//! low-end devices such as Android phones. The core idea is simple: take
//! data in one format, understand its structure, and emit it in another
//! format without unnecessary overhead.
//!
//! # Examples
//!
//! Convert CSV to JSON:
//!
//! ```
//! use sheeter_engine::csv_to_json;
//!
//! let csv = "name,age\nAlice,30\nBob,25";
//! let json = csv_to_json(csv).unwrap();
//! assert_eq!(json, r#"[{"name":"Alice","age":"30"},{"name":"Bob","age":"25"}]"#);
//! ```
//!
//! Convert JSON to CSV:
//!
//! ```
//! use sheeter_engine::json_to_csv;
//!
//! let json = r#"[{"name":"Alice","age":"30"},{"name":"Bob","age":"25"}]"#;
//! let csv = json_to_csv(json).unwrap();
//! assert_eq!(csv, "name,age\nAlice,30\nBob,25\n");
//! ```

pub mod canonical;
pub mod detect;
pub mod filter;
pub mod inspect;
pub mod json_tools;
pub mod ndjson;
pub mod parquet_io;
pub mod query;
pub mod sort;
pub mod transform;
pub mod view;
pub mod xlsx_tools;

pub use canonical::{to_ods_workbook, CanonicalValue};
pub use detect::{detect_content, detect_extension};
pub use filter::Filter;
pub use inspect::inspect;
pub use ndjson::{stream_delimited_to_ndjson, stream_ndjson_to_delimited, stream_ndjson_to_ndjson};
pub use parquet_io::from_parquet;
pub use parquet_io::to_parquet;
pub use sort::{external_sort_delimited, SortSpec};
pub use transform::Transforms;

use std::io::Cursor;

/// Converts a delimited string into a JSON array of objects.
///
/// This is the shared engine behind `csv_to_json` and `tsv_to_json`. The first
/// line is read as the header row; every subsequent line becomes a JSON object
/// mapping header names to string cell values. Duplicate header names are
/// rejected because they would silently overwrite data.
///
/// # Errors
///
/// Returns an error on malformed input, duplicate headers, or JSON
/// serialization failure.
///
/// # Examples
///
/// ```
/// use sheeter_engine::csv_to_json;
///
/// let csv = "name,age\nAlice,30";
/// assert_eq!(
///     csv_to_json(csv).unwrap(),
///     r#"[{"name":"Alice","age":"30"}]"#
/// );
/// ```
fn delimited_to_json(input: &str, delim: u8) -> Result<String, Box<dyn std::error::Error>> {
    let input = strip_bom(input);
    let mut rdr = csv::ReaderBuilder::new()
        .delimiter(delim)
        .from_reader(Cursor::new(input));
    let headers = rdr.headers()?.clone();
    let mut seen = std::collections::HashSet::new();
    for header in headers.iter() {
        if !seen.insert(header) {
            return Err(format!("duplicate header: {header}").into());
        }
    }
    let mut records = Vec::new();
    for result in rdr.records() {
        let record = result?;
        let mut map = serde_json::Map::new();
        for (header, value) in headers.iter().zip(record.iter()) {
            map.insert(
                header.to_string(),
                serde_json::Value::String(value.to_string()),
            );
        }
        records.push(serde_json::Value::Object(map));
    }
    Ok(serde_json::to_string(&records)?)
}

/// Converts a JSON array of objects into a delimited string.
///
/// This is the shared engine behind `json_to_csv` and `json_to_tsv`. The header
/// row is the union of all keys across all objects, in first-appearance order.
/// Missing keys become empty cells. Strings are written as-is; numbers and
/// booleans use their JSON literal forms; null becomes an empty cell. Nested
/// objects and arrays cannot be represented losslessly in flat output and cause
/// an error.
///
/// # Errors
///
/// Returns an error if the JSON is not an array of objects, contains nested
/// values, or if writing fails.
///
/// # Examples
///
/// ```
/// use sheeter_engine::json_to_csv;
///
/// let json = r#"[{"a":1},{"a":2,"b":3}]"#;
/// assert_eq!(json_to_csv(json).unwrap(), "a,b\n1,\n2,3\n");
/// ```
fn json_to_delimited(input: &str, delim: u8) -> Result<String, Box<dyn std::error::Error>> {
    let values: Vec<serde_json::Value> = serde_json::from_str(input)?;
    if values.is_empty() {
        return Ok(String::new());
    }
    let mut headers: Vec<String> = Vec::new();
    for value in &values {
        let obj = value.as_object().ok_or("JSON array must contain objects")?;
        for key in obj.keys() {
            if !headers.contains(key) {
                headers.push(key.clone());
            }
        }
    }
    let mut wtr = csv::WriterBuilder::new()
        .delimiter(delim)
        .from_writer(vec![]);
    wtr.write_record(&headers)?;
    for value in values {
        let obj = value.as_object().ok_or("JSON array must contain objects")?;
        let mut record: Vec<String> = Vec::with_capacity(headers.len());
        for h in &headers {
            let cell = match obj.get(h) {
                Some(serde_json::Value::String(s)) => s.clone(),
                Some(serde_json::Value::Number(n)) => n.to_string(),
                Some(serde_json::Value::Bool(b)) => b.to_string(),
                Some(serde_json::Value::Null) => String::new(),
                Some(serde_json::Value::Object(_)) | Some(serde_json::Value::Array(_)) => {
                    return Err(format!(
                        "nested value at key {h} cannot be represented in flat output"
                    )
                    .into());
                }
                None => String::new(),
            };
            record.push(cell);
        }
        wtr.write_record(&record)?;
    }
    let data = wtr.into_inner()?;
    Ok(String::from_utf8(data)?)
}

/// Converts a CSV string into a JSON array of objects.
///
/// CSV (Comma-Separated Values) is a simple tabular format. This function reads
/// the first row as the header and each subsequent row as a record, producing a
/// JSON array of objects. All values are emitted as JSON strings.
///
/// # Errors
///
/// Returns an error on malformed CSV, duplicate headers, or JSON failure.
///
/// # Examples
///
/// ```
/// use sheeter_engine::csv_to_json;
///
/// let csv = "name,age\nAlice,30\nBob,25";
/// let json = csv_to_json(csv).unwrap();
/// assert_eq!(json, r#"[{"name":"Alice","age":"30"},{"name":"Bob","age":"25"}]"#);
/// ```
pub fn csv_to_json(csv: &str) -> Result<String, Box<dyn std::error::Error>> {
    delimited_to_json(csv, b',')
}

/// Converts a TSV string into a JSON array of objects.
///
/// TSV (Tab-Separated Values) is the same tabular model as CSV but uses a tab
/// character as the field separator. This is useful for data exported from
/// spreadsheet tools that prefer tabs over commas.
///
/// # Errors
///
/// Returns an error on malformed TSV, duplicate headers, or JSON failure.
///
/// # Examples
///
/// ```
/// use sheeter_engine::tsv_to_json;
///
/// let tsv = "name\tage\nAlice\t30";
/// let json = tsv_to_json(tsv).unwrap();
/// assert_eq!(json, r#"[{"name":"Alice","age":"30"}]"#);
/// ```
pub fn tsv_to_json(tsv: &str) -> Result<String, Box<dyn std::error::Error>> {
    delimited_to_json(tsv, b'\t')
}

/// Converts a JSON array of objects into a CSV string.
///
/// The header row is the union of all keys across all objects. Missing keys
/// become empty cells. Nested objects or arrays cause an error to prevent
/// silent corruption of round-trips.
///
/// # Errors
///
/// Returns an error if the input is not an array of objects, contains nested
/// values, or if writing fails.
///
/// # Examples
///
/// ```
/// use sheeter_engine::json_to_csv;
///
/// let json = r#"[{"name":"Alice","age":"30"},{"name":"Bob","age":"25"}]"#;
/// assert_eq!(json_to_csv(json).unwrap(), "name,age\nAlice,30\nBob,25\n");
/// ```
pub fn json_to_csv(json: &str) -> Result<String, Box<dyn std::error::Error>> {
    json_to_delimited(json, b',')
}

/// Converts a JSON array of objects into a TSV string.
///
/// Same semantics as `json_to_csv`, but fields are separated by a tab
/// character. The output uses `\n` line endings.
///
/// # Errors
///
/// Returns an error if the input is not an array of objects, contains nested
/// values, or if writing fails.
///
/// # Examples
///
/// ```
/// use sheeter_engine::json_to_tsv;
///
/// let json = r#"[{"name":"Alice","age":"30"},{"name":"Bob","age":"25"}]"#;
/// assert_eq!(json_to_tsv(json).unwrap(), "name\tage\nAlice\t30\nBob\t25\n");
/// ```
pub fn json_to_tsv(json: &str) -> Result<String, Box<dyn std::error::Error>> {
    json_to_delimited(json, b'\t')
}

/// Removes the UTF-8 BOM (`\u{feff}`) from the start of a string if present.
///
/// Some tools (notably Excel) prepend a BOM to CSV/TSV exports. Without
/// stripping, the first header name would include the invisible BOM character
/// and any downstream lookup by name would silently fail. This explicit
/// preprocessing guarantees clean headers regardless of parser behavior.
fn strip_bom(input: &str) -> &str {
    input.strip_prefix('\u{feff}').unwrap_or(input)
}

/// Converts a YAML sequence of mappings into a JSON array of objects.
///
/// Non-string YAML keys (e.g. `1:`) are converted to their string form
/// (`"1"`), matching the convention of every mainstream YAML-to-JSON
/// converter. Collisions are not detected; if the source YAML contains both
/// `1:` and `"1":`, one value wins.
///
/// This function parses YAML with a strict schema to avoid the "Norway problem":
/// `yes`, `no`, `on`, `off`, `y`, `n` are treated as strings under YAML 1.2
/// semantics. Nested sequences or mappings cause an error because they cannot
/// round-trip losslessly through flat CSV/TSV.
///
/// # Errors
///
/// Returns an error if the YAML is not a sequence of mappings, contains nested
/// values, or if JSON serialization fails.
///
/// # Examples
///
/// ```
/// use sheeter_engine::yaml_to_json;
///
/// let yaml = "- name: Alice\n  age: '30'\n- name: Bob\n  age: '25'\n";
/// let json = yaml_to_json(yaml).unwrap();
/// assert_eq!(json, r#"[{"name":"Alice","age":"30"},{"name":"Bob","age":"25"}]"#);
/// ```
pub fn yaml_to_json(yaml: &str) -> Result<String, Box<dyn std::error::Error>> {
    let opts = serde_saphyr::options! {
        strict_booleans: true,
        ignore_binary_tag_for_string: true,
    };
    let values: Vec<serde_json::Value> = serde_saphyr::from_str_with_options(yaml, opts)?;
    for value in &values {
        let obj = value
            .as_object()
            .ok_or("YAML sequence must contain mappings")?;
        for (key, val) in obj {
            if val.is_object() || val.is_array() {
                return Err(format!(
                    "nested YAML value at key {key} cannot be represented in flat output"
                )
                .into());
            }
        }
    }
    Ok(serde_json::to_string(&values)?)
}

/// Converts a JSON array of objects into a YAML sequence of mappings.
///
/// This is the inverse of `yaml_to_json`. The output is a YAML 1.2 document
/// where each object becomes a mapping in a sequence. Key order is preserved.
/// Nested objects or arrays cause an error for the same reason as CSV: they
/// cannot round-trip losslessly through a flat format.
///
/// # Errors
///
/// Returns an error if the JSON is not an array of objects, contains nested
/// values, or if YAML serialization fails.
///
/// # Examples
///
/// ```
/// use sheeter_engine::json_to_yaml;
///
/// let json = r#"[{"name":"Alice","age":"30"}]"#;
/// let yaml = json_to_yaml(json).unwrap();
/// assert!(yaml.contains("name: Alice"));
/// ```
pub fn json_to_yaml(json: &str) -> Result<String, Box<dyn std::error::Error>> {
    let values: Vec<serde_json::Value> = serde_json::from_str(json)?;
    for value in &values {
        let obj = value.as_object().ok_or("JSON array must contain objects")?;
        for (key, val) in obj {
            if val.is_object() || val.is_array() {
                return Err(format!(
                    "nested JSON value at key {key} cannot be represented in flat YAML"
                )
                .into());
            }
        }
    }
    Ok(serde_saphyr::to_string(&values)?)
}

/// Converts CSV to YAML by composing `csv_to_json` and `json_to_yaml`.
///
/// # Examples
///
/// ```
/// use sheeter_engine::csv_to_yaml;
///
/// let csv = "name,age\nAlice,30";
/// let yaml = csv_to_yaml(csv).unwrap();
/// assert!(yaml.contains("name: Alice"));
/// ```
pub fn csv_to_yaml(csv: &str) -> Result<String, Box<dyn std::error::Error>> {
    json_to_yaml(&csv_to_json(csv)?)
}

/// Converts TSV to YAML by composing `tsv_to_json` and `json_to_yaml`.
///
/// # Examples
///
/// ```
/// use sheeter_engine::tsv_to_yaml;
///
/// let tsv = "name\tage\nAlice\t30";
/// let yaml = tsv_to_yaml(tsv).unwrap();
/// assert!(yaml.contains("name: Alice"));
/// ```
pub fn tsv_to_yaml(tsv: &str) -> Result<String, Box<dyn std::error::Error>> {
    json_to_yaml(&tsv_to_json(tsv)?)
}

/// Converts YAML to CSV by composing `yaml_to_json` and `json_to_csv`.
///
/// # Examples
///
/// ```
/// use sheeter_engine::yaml_to_csv;
///
/// let yaml = "- name: Alice\n  age: '30'\n";
/// let csv = yaml_to_csv(yaml).unwrap();
/// assert_eq!(csv, "name,age\nAlice,30\n");
/// ```
pub fn yaml_to_csv(yaml: &str) -> Result<String, Box<dyn std::error::Error>> {
    json_to_csv(&yaml_to_json(yaml)?)
}

/// Converts YAML to TSV by composing `yaml_to_json` and `json_to_tsv`.
///
/// # Examples
///
/// ```
/// use sheeter_engine::yaml_to_tsv;
///
/// let yaml = "- name: Alice\n  age: '30'\n";
/// let tsv = yaml_to_tsv(yaml).unwrap();
/// assert_eq!(tsv, "name\tage\nAlice\t30\n");
/// ```
pub fn yaml_to_tsv(yaml: &str) -> Result<String, Box<dyn std::error::Error>> {
    json_to_tsv(&yaml_to_json(yaml)?)
}

/// Converts CSV to TSV by composing `csv_to_json` and `json_to_tsv`.
///
/// # Examples
///
/// ```
/// use sheeter_engine::csv_to_tsv;
///
/// let csv = "name,age\nAlice,30";
/// let tsv = csv_to_tsv(csv).unwrap();
/// assert_eq!(tsv, "name\tage\nAlice\t30\n");
/// ```
pub fn csv_to_tsv(csv: &str) -> Result<String, Box<dyn std::error::Error>> {
    json_to_tsv(&csv_to_json(csv)?)
}

/// Converts TSV to CSV by composing `tsv_to_json` and `json_to_csv`.
///
/// # Examples
///
/// ```
/// use sheeter_engine::tsv_to_csv;
///
/// let tsv = "name\tage\nAlice\t30";
/// let csv = tsv_to_csv(tsv).unwrap();
/// assert_eq!(csv, "name,age\nAlice,30\n");
/// ```
pub fn tsv_to_csv(tsv: &str) -> Result<String, Box<dyn std::error::Error>> {
    json_to_csv(&tsv_to_json(tsv)?)
}

/// Converts an XML document to a JSON representation.
///
/// Attributes become `@name` keys, text becomes `#text`, and repeated
/// sibling elements become arrays. The output has exactly one top-level key:
/// the root element name.
///
/// # Errors
/// Returns an error if the XML is malformed or empty.
///
/// # Examples
/// ```
/// use sheeter_engine::xml_to_json;
/// let json = xml_to_json("<r><a>1</a><a>2</a></r>").unwrap();
/// assert_eq!(json, r#"{"r":{"a":["1","2"]}}"#);
/// ```
pub fn xml_to_json(xml: &str) -> Result<String, Box<dyn std::error::Error>> {
    CanonicalValue::from_xml(xml)?.to_json()
}

/// Converts a JSON value to XML.
///
/// Tabular JSON (array of objects) is wrapped in `<root><item>...`. Other
/// JSON shapes are wrapped in `<root>` unless they are a single-key object,
/// which becomes the root element directly. Keys starting with `@` become
/// attributes and `#text` becomes text content.
///
/// # Errors
/// Returns an error if the JSON is malformed.
///
/// # Examples
/// ```
/// use sheeter_engine::json_to_xml;
/// let xml = json_to_xml(r#"[{"a":"1"}]"#).unwrap();
/// assert!(xml.contains("<root>"));
/// ```
pub fn json_to_xml(json: &str) -> Result<String, Box<dyn std::error::Error>> {
    CanonicalValue::from_json(json)?.to_xml()
}

/// Converts CSV to XML via the canonical hub.
///
/// # Examples
/// ```
/// use sheeter_engine::csv_to_xml;
/// let xml = csv_to_xml("a\n1").unwrap();
/// assert!(xml.contains("<item>"));
/// ```
pub fn csv_to_xml(input: &str) -> Result<String, Box<dyn std::error::Error>> {
    CanonicalValue::from_csv(input)?.to_xml()
}

/// Converts TSV to XML via the canonical hub.
///
/// # Examples
/// ```
/// use sheeter_engine::tsv_to_xml;
/// let xml = tsv_to_xml("a\tb\n1\t2").unwrap();
/// assert!(xml.contains("<item>"));
/// ```
pub fn tsv_to_xml(input: &str) -> Result<String, Box<dyn std::error::Error>> {
    CanonicalValue::from_tsv(input)?.to_xml()
}

/// Converts YAML to XML via the canonical hub.
///
/// # Examples
/// ```
/// use sheeter_engine::yaml_to_xml;
/// let xml = yaml_to_xml("- a: 1\n").unwrap();
/// assert!(xml.contains("<item>"));
/// ```
pub fn yaml_to_xml(input: &str) -> Result<String, Box<dyn std::error::Error>> {
    CanonicalValue::from_yaml(input)?.to_xml()
}

/// Converts XML to CSV, extracting the tabular array if present.
///
/// Expects the XML to follow `<root><item><col>value</col></item></root>`
/// (or any single-level root whose single child key holds the rows). Errors
/// if the structure is not tabular.
///
/// # Errors
/// Returns an error if the XML is malformed or not tabular.
///
/// # Examples
/// ```
/// use sheeter_engine::xml_to_csv;
/// let csv = xml_to_csv("<root><item><a>1</a></item></root>").unwrap();
/// assert_eq!(csv, "a\n1\n");
/// ```
pub fn xml_to_csv(xml: &str) -> Result<String, Box<dyn std::error::Error>> {
    CanonicalValue::from_xml(xml)?.to_csv()
}

/// Converts XML to TSV, extracting the tabular array if present.
///
/// # Examples
/// ```
/// use sheeter_engine::xml_to_tsv;
/// let tsv = xml_to_tsv("<root><item><a>1</a></item></root>").unwrap();
/// assert_eq!(tsv, "a\n1\n");
/// ```
pub fn xml_to_tsv(xml: &str) -> Result<String, Box<dyn std::error::Error>> {
    CanonicalValue::from_xml(xml)?.to_tsv()
}

/// Converts XML to YAML, extracting the tabular array if present.
///
/// # Examples
/// ```
/// use sheeter_engine::xml_to_yaml;
/// let yaml = xml_to_yaml("<root><item><a>1</a></item></root>").unwrap();
/// assert!(yaml.contains("a:"));
/// ```
pub fn xml_to_yaml(xml: &str) -> Result<String, Box<dyn std::error::Error>> {
    CanonicalValue::from_xml(xml)?.to_yaml()
}

/// Converts a TOML document to a JSON value.
///
/// TOML datetimes are stringified (`2024-01-15T10:30:00Z` becomes the string
/// `"2024-01-15T10:30:00Z"`), matching the behavior of YAML. Nested tables
/// and arrays of tables map directly to JSON objects and arrays.
///
/// # Errors
/// Returns an error if the TOML is malformed.
///
/// # Examples
/// ```
/// use sheeter_engine::toml_to_json;
/// let json = toml_to_json("a = 1\n").unwrap();
/// assert_eq!(json, r#"{"a":1}"#);
/// ```
pub fn toml_to_json(toml: &str) -> Result<String, Box<dyn std::error::Error>> {
    CanonicalValue::from_toml(toml)?.to_json()
}

/// Converts a TOML document to CSV, extracting the first array of tables.
///
/// Flat TOML documents (all scalar values) become a single-row CSV. Documents
/// with an array of tables at any depth use that array as the row source.
///
/// # Errors
/// Returns an error if the TOML is malformed or has no tabular structure.
///
/// # Examples
/// ```
/// use sheeter_engine::toml_to_csv;
/// assert_eq!(toml_to_csv("a = 1\n").unwrap(), "a\n1\n");
/// ```
pub fn toml_to_csv(toml: &str) -> Result<String, Box<dyn std::error::Error>> {
    CanonicalValue::from_toml(toml)?.to_csv()
}

/// Converts a TOML document to TSV.
///
/// # Errors
/// Returns an error if the TOML is malformed or has no tabular structure.
///
/// # Examples
/// ```
/// use sheeter_engine::toml_to_tsv;
/// assert_eq!(toml_to_tsv("a = 1\n").unwrap(), "a\n1\n");
/// ```
pub fn toml_to_tsv(toml: &str) -> Result<String, Box<dyn std::error::Error>> {
    CanonicalValue::from_toml(toml)?.to_tsv()
}

/// Converts a TOML document to YAML.
///
/// # Errors
/// Returns an error if the TOML is malformed.
///
/// # Examples
/// ```
/// use sheeter_engine::toml_to_yaml;
/// let yaml = toml_to_yaml("a = 1\n").unwrap();
/// assert!(yaml.contains("a:"));
/// ```
pub fn toml_to_yaml(toml: &str) -> Result<String, Box<dyn std::error::Error>> {
    CanonicalValue::from_toml(toml)?.to_yaml()
}

/// Converts a TOML document to XML.
///
/// # Errors
/// Returns an error if the TOML is malformed.
///
/// # Examples
/// ```
/// use sheeter_engine::toml_to_xml;
/// let xml = toml_to_xml("a = 1\n").unwrap();
/// assert!(xml.contains("<a>"));
/// ```
pub fn toml_to_xml(toml: &str) -> Result<String, Box<dyn std::error::Error>> {
    CanonicalValue::from_toml(toml)?.to_xml()
}

/// Converts a JSON value to TOML.
///
/// JSON arrays at the root are wrapped in `{item: [...]}` because TOML
/// requires a table at the document root. Null values are skipped, since
/// TOML has no null type; JSON arrays containing null error out.
///
/// # Errors
/// Returns an error if the JSON is malformed or contains values TOML cannot
/// represent (null inside arrays, integers outside i64 range).
///
/// # Examples
/// ```
/// use sheeter_engine::json_to_toml;
/// let toml = json_to_toml(r#"{"a":1}"#).unwrap();
/// assert!(toml.contains("a = 1"));
/// ```
pub fn json_to_toml(json: &str) -> Result<String, Box<dyn std::error::Error>> {
    CanonicalValue::from_json(json)?.to_toml()
}

/// Converts CSV to TOML via the canonical hub.
///
/// The resulting TOML document is `[[item]]` array-of-tables, one item per
/// CSV row.
///
/// # Errors
/// Returns an error if the CSV is malformed.
///
/// # Examples
/// ```
/// use sheeter_engine::csv_to_toml;
/// let toml = csv_to_toml("a\n1\n").unwrap();
/// assert!(toml.contains("[[item]]"));
/// ```
pub fn csv_to_toml(csv: &str) -> Result<String, Box<dyn std::error::Error>> {
    CanonicalValue::from_csv(csv)?.to_toml()
}

/// Converts TSV to TOML via the canonical hub.
///
/// # Errors
/// Returns an error if the TSV is malformed.
///
/// # Examples
/// ```
/// use sheeter_engine::tsv_to_toml;
/// let toml = tsv_to_toml("a\tb\n1\t2\n").unwrap();
/// assert!(toml.contains("[[item]]"));
/// ```
pub fn tsv_to_toml(tsv: &str) -> Result<String, Box<dyn std::error::Error>> {
    CanonicalValue::from_tsv(tsv)?.to_toml()
}

/// Converts YAML to TOML via the canonical hub.
///
/// # Errors
/// Returns an error if the YAML is malformed or cannot be represented in TOML.
///
/// # Examples
/// ```
/// use sheeter_engine::yaml_to_toml;
/// let toml = yaml_to_toml("- a: 1\n").unwrap();
/// assert!(toml.contains("[[item]]"));
/// ```
pub fn yaml_to_toml(yaml: &str) -> Result<String, Box<dyn std::error::Error>> {
    CanonicalValue::from_yaml(yaml)?.to_toml()
}

/// Converts XML to TOML via the canonical hub.
///
/// Attribute markers (`@name`) and text markers (`#text`) are preserved as
/// quoted TOML keys, allowing round-trip back to XML.
///
/// # Errors
/// Returns an error if the XML is malformed or cannot be represented in TOML.
///
/// # Examples
/// ```
/// use sheeter_engine::xml_to_toml;
/// let toml = xml_to_toml("<r><a>1</a></r>").unwrap();
/// assert!(toml.contains("a = \"1\""));
/// ```
pub fn xml_to_toml(xml: &str) -> Result<String, Box<dyn std::error::Error>> {
    CanonicalValue::from_xml(xml)?.to_toml()
}

/// Supported data formats.
///
/// Each variant corresponds to a reader/writer pair on
/// [`CanonicalValue`](crate::CanonicalValue).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum Format {
    Csv,
    Tsv,
    Json,
    Yaml,
    Xml,
    Toml,
    Xlsx,
    Sql,
    Ndjson,
    Ods,
    Parquet,
}

/// SQL dialect used when parsing input. The writer emits ANSI-compatible
/// `INSERT` statements that SQLite, PostgreSQL, and MySQL all accept.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
#[non_exhaustive]
pub enum SqlDialect {
    #[default]
    Generic,
    PostgreSql,
    Sqlite,
    MySql,
}

impl std::str::FromStr for SqlDialect {
    type Err = String;
    fn from_str(s: &str) -> Result<Self, Self::Err> {
        match s.to_ascii_lowercase().as_str() {
            "generic" | "ansi" => Ok(Self::Generic),
            "postgres" | "postgresql" | "pg" => Ok(Self::PostgreSql),
            "sqlite" => Ok(Self::Sqlite),
            "mysql" | "mariadb" => Ok(Self::MySql),
            other => Err(format!(
                "unknown SQL dialect {other:?}; expected generic, postgresql, sqlite, or mysql"
            )),
        }
    }
}

/// One output document: `(sheet_name, bytes)`. Returned by
/// [`convert_workbook`] because multi-sheet inputs can produce more than one
/// output.
pub type OutputDoc = (String, Vec<u8>);

impl Format {
    /// Every format the Sheeter engine can read and write.
    pub const ALL: [Format; 11] = [
        Format::Csv,
        Format::Tsv,
        Format::Json,
        Format::Yaml,
        Format::Xml,
        Format::Toml,
        Format::Xlsx,
        Format::Sql,
        Format::Ndjson,
        Format::Ods,
        Format::Parquet,
    ];
}

/// Converts `input` from one format to another via the canonical hub.
///
/// If `from` equals `to`, the input is returned unchanged. That preserves
/// BOM markers, formatting, and other byte-level details that a round-trip
/// through the hub would normalize away.
///
/// # Errors
///
/// Returns an error if the input cannot be parsed as `from` or serialized
/// as `to`, or if the document cannot be represented in the target format
/// (for example, nested values when converting to CSV).
///
/// # Examples
///
/// ```
/// use sheeter_engine::{convert, Format};
///
/// let json = convert(Format::Csv, Format::Json, "a\n1\n").unwrap();
/// assert_eq!(json, r#"[{"a":"1"}]"#);
/// ```
pub fn convert(
    from: Format,
    to: Format,
    input: &str,
) -> Result<String, Box<dyn std::error::Error>> {
    if from == to {
        return Ok(input.to_string());
    }
    if from == Format::Xlsx || to == Format::Xlsx {
        return Err("XLSX is binary; use convert_workbook instead".into());
    }
    if from == Format::Ods || to == Format::Ods {
        return Err("ODS is binary; use convert_workbook instead".into());
    }
    if from == Format::Parquet || to == Format::Parquet {
        return Err("Parquet is binary; use convert_workbook instead".into());
    }
    let hub = match from {
        Format::Csv => CanonicalValue::from_csv(input)?,
        Format::Tsv => CanonicalValue::from_tsv(input)?,
        Format::Json => CanonicalValue::from_json(input)?,
        Format::Yaml => CanonicalValue::from_yaml(input)?,
        Format::Xml => CanonicalValue::from_xml(input)?,
        Format::Toml => CanonicalValue::from_toml(input)?,
        Format::Sql => {
            let mut sheets = CanonicalValue::from_sql_workbook(input, SqlDialect::Generic)?;
            if sheets.len() != 1 {
                return Err("multi-table SQL requires convert_workbook".into());
            }
            sheets.remove(0).1
        }
        Format::Ndjson => CanonicalValue::from_ndjson(input)?,
        Format::Ods => unreachable!(),
        Format::Xlsx => unreachable!(),
        Format::Parquet => unreachable!(),
    };
    match to {
        Format::Csv => hub.to_csv(),
        Format::Tsv => hub.to_tsv(),
        Format::Json => hub.to_json(),
        Format::Yaml => hub.to_yaml(),
        Format::Xml => hub.to_xml(),
        Format::Toml => hub.to_toml(),
        Format::Sql => hub.to_sql_with_table("data"),
        Format::Ndjson => hub.to_ndjson(),

        Format::Ods => unreachable!(),
        Format::Xlsx => unreachable!(),
        Format::Parquet => unreachable!(),
    }
}

/// Converts a byte buffer between formats, supporting multi-sheet workbooks.
///
/// Returns one entry per output document. For XLSX inputs with N sheets and
/// CSV/TSV outputs, the result has N entries. For JSON/YAML/XML/TOML/XLSX
/// outputs, the result has exactly one entry containing all sheets merged
/// into a single document.
///
/// The returned string is the sheet name (from XLSX) or `"Sheet1"` for
/// single-document inputs. Callers decide how to map these names to file
/// paths.
///
/// # Errors
///
/// Returns an error if the input cannot be parsed or the output cannot be
/// represented in the target format.
///
/// # Examples
///
/// ```
/// use sheeter_engine::{convert_workbook, Format};
///
/// let out = convert_workbook(Format::Csv, Format::Json, b"a\n1\n").unwrap();
/// assert_eq!(out.len(), 1);
/// assert_eq!(out[0].0, "Sheet1");
/// ```
pub fn convert_workbook(
    from: Format,
    to: Format,
    input: &[u8],
) -> Result<Vec<OutputDoc>, Box<dyn std::error::Error>> {
    convert_workbook_with_options(from, to, input, SqlDialect::Generic, None)
}

/// Like [`convert_workbook`], but lets the caller pick the SQL dialect and
/// the table name used when writing SQL output.
pub fn convert_workbook_with_options(
    from: Format,
    to: Format,
    input: &[u8],
    dialect: SqlDialect,
    table_name: Option<&str>,
) -> Result<Vec<OutputDoc>, Box<dyn std::error::Error>> {
    if from == to {
        return match from {
            Format::Xlsx | Format::Ods | Format::Parquet => {
                Ok(vec![("Sheet1".to_string(), input.to_vec())])
            }
            _ => {
                let text = std::str::from_utf8(input)?;
                Ok(vec![("Sheet1".to_string(), text.as_bytes().to_vec())])
            }
        };
    }

    let sheets: Vec<(String, CanonicalValue)> = match from {
        Format::Xlsx => CanonicalValue::from_xlsx_workbook(input)?,
        Format::Parquet => vec![("Sheet1".to_string(), crate::from_parquet(input)?)],
        Format::Ods => {
            let v = CanonicalValue::from_ods(input)?;
            match v {
                CanonicalValue::Object(o) => o,
                other => vec![("Sheet1".to_string(), other)],
            }
        }
        Format::Sql => {
            let text = std::str::from_utf8(input)?;
            CanonicalValue::from_sql_workbook(text, dialect)?
        }
        Format::Ndjson => {
            let text = std::str::from_utf8(input)?;
            vec![("Sheet1".to_string(), CanonicalValue::from_ndjson(text)?)]
        }
        _ => {
            let text = std::str::from_utf8(input)?;
            let hub = match from {
                Format::Csv => CanonicalValue::from_csv(text)?,
                Format::Tsv => CanonicalValue::from_tsv(text)?,
                Format::Json => CanonicalValue::from_json(text)?,
                Format::Yaml => CanonicalValue::from_yaml(text)?,
                Format::Xml => CanonicalValue::from_xml(text)?,
                Format::Toml => CanonicalValue::from_toml(text)?,
                Format::Sql | Format::Ndjson | Format::Xlsx | Format::Ods | Format::Parquet => {
                    unreachable!()
                }
            };
            vec![("Sheet1".to_string(), hub)]
        }
    };

    if to == Format::Sql {
        let text = if sheets.len() == 1 {
            let table = table_name.unwrap_or("data");
            sheets[0].1.to_sql_with_table(table)?
        } else {
            CanonicalValue::to_sql_workbook(&sheets)?
        };
        return Ok(vec![("Sheet1".to_string(), text.into_bytes())]);
    }

    if to == Format::Ods {
        let bytes = to_ods_workbook(&sheets)?;
        return Ok(vec![("Sheet1".to_string(), bytes)]);
    }

    if to == Format::Ndjson {
        let text = if sheets.len() == 1 {
            sheets[0].1.to_ndjson()?
        } else {
            let merged = CanonicalValue::Object(sheets);
            merged.to_ndjson()?
        };
        return Ok(vec![("Sheet1".to_string(), text.into_bytes())]);
    }

    if to == Format::Xlsx {
        let bytes = CanonicalValue::to_xlsx_workbook(&sheets)?;
        return Ok(vec![("Sheet1".to_string(), bytes)]);
    }

    if to == Format::Parquet {
        if sheets.len() != 1 {
            return Err("Parquet output supports exactly one table".into());
        }
        let bytes = crate::parquet_io::to_parquet(&sheets[0].1)?;
        return Ok(vec![("Sheet1".to_string(), bytes)]);
    }

    if sheets.len() == 1 {
        let text = write_single(to, &sheets[0].1)?;
        return Ok(vec![(sheets[0].0.clone(), text.into_bytes())]);
    }

    match to {
        Format::Json | Format::Yaml | Format::Xml | Format::Toml => {
            let merged = CanonicalValue::Object(sheets);
            Ok(vec![(
                "Sheet1".to_string(),
                write_single(to, &merged)?.into_bytes(),
            )])
        }
        Format::Csv | Format::Tsv => {
            let mut out = Vec::with_capacity(sheets.len());
            for (name, value) in &sheets {
                out.push((name.clone(), write_single(to, value)?.into_bytes()));
            }
            Ok(out)
        }
        Format::Sql => unreachable!(),
        Format::Ndjson => unreachable!(),
        Format::Ods => unreachable!(),
        Format::Xlsx => unreachable!(),
        Format::Parquet => unreachable!(),
    }
}

fn write_single(
    format: Format,
    value: &CanonicalValue,
) -> Result<String, Box<dyn std::error::Error>> {
    match format {
        Format::Csv => value.to_csv(),
        Format::Tsv => value.to_tsv(),
        Format::Json => value.to_json(),
        Format::Yaml => value.to_yaml(),
        Format::Xml => value.to_xml(),
        Format::Toml => value.to_toml(),
        Format::Sql => value.to_sql_with_table("data"),
        Format::Ndjson => value.to_ndjson(),
        Format::Ods => Err("ODS must go through to_ods_workbook".into()),
        Format::Xlsx => Err("XLSX must go through to_xlsx_workbook".into()),
        Format::Parquet => Err("Parquet must go through parquet_io::to_parquet".into()),
    }
}

/// Converts CSV or TSV input to CSV or TSV output using constant memory.
///
/// Reads row by row and writes row by row, so peak memory is bounded by the
/// largest single row, not by file size. Applies filter, select, rename, and
/// limit transforms on the fly. Sorting and other buffering transforms are
/// not supported here.
///
/// # Errors
/// Returns an error if the input is malformed, if a transform fails, or if
/// the output writer fails.
///
/// # Examples
///
/// ```
/// use sheeter_engine::{stream_delimited, Transforms};
///
/// let input = b"a\n1\n2\n3\n";
/// let mut output = Vec::new();
/// stream_delimited(input.as_slice(), b',', &mut output, b',', &Transforms::default()).unwrap();
/// assert_eq!(output, b"a\n1\n2\n3\n");
/// ```
pub fn stream_delimited(
    reader: impl std::io::Read,
    from_delim: u8,
    writer: impl std::io::Write,
    to_delim: u8,
    transforms: &Transforms,
) -> Result<(), Box<dyn std::error::Error>> {
    let mut rdr = csv::ReaderBuilder::new()
        .delimiter(from_delim)
        .from_reader(reader);
    let raw_headers: Vec<String> = rdr.headers()?.iter().map(|s| s.to_string()).collect();

    let mut renamed_headers = raw_headers.clone();
    for (from, to) in &transforms.rename {
        for h in renamed_headers.iter_mut() {
            if h == from {
                *h = to.clone();
                break;
            }
        }
    }
    let final_headers: Vec<String> = match &transforms.select {
        Some(sel) => sel.clone(),
        None => renamed_headers,
    };

    let mut wtr = csv::WriterBuilder::new()
        .delimiter(to_delim)
        .from_writer(writer);
    wtr.write_record(&final_headers)?;

    let limit = transforms.limit.unwrap_or(usize::MAX);
    let mut emitted = 0usize;
    let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();
    let dedup_cols = transforms.dedup.as_ref().cloned();
    for result in rdr.records() {
        if emitted >= limit {
            break;
        }
        let record = result?;
        let mut row: Vec<(String, CanonicalValue)> = Vec::with_capacity(raw_headers.len());
        for (i, h) in raw_headers.iter().enumerate() {
            let v = record.get(i).unwrap_or("").to_string();
            row.push((h.clone(), CanonicalValue::String(v)));
        }
        if let Some(new_row) = transforms.apply_to_row(row) {
            if let Some(cols) = &dedup_cols {
                let key: String = if cols.is_empty() {
                    new_row
                        .iter()
                        .map(|(_, v)| match v {
                            CanonicalValue::String(s) => s.clone(),
                            _ => String::new(),
                        })
                        .collect::<Vec<_>>()
                        .join("\x1f")
                } else {
                    cols.iter()
                        .map(|c| {
                            new_row
                                .iter()
                                .find(|(k, _)| k == c)
                                .map(|(_, v)| match v {
                                    CanonicalValue::String(s) => s.clone(),
                                    _ => String::new(),
                                })
                                .unwrap_or_default()
                        })
                        .collect::<Vec<_>>()
                        .join("\x1f")
                };
                if !seen.insert(key) {
                    continue;
                }
            }
            let values: Vec<String> = new_row
                .iter()
                .map(|(_, v)| match v {
                    CanonicalValue::Null => String::new(),
                    CanonicalValue::String(s) => s.clone(),
                    CanonicalValue::Number(n) => n.to_string(),
                    CanonicalValue::Bool(b) => b.to_string(),
                    _ => String::new(),
                })
                .collect();
            wtr.write_record(&values)?;
            emitted += 1;
        }
    }
    wtr.flush()?;
    Ok(())
}