Skip to main content

mcd_query/
lib.rs

1//! Read-only SQL querying for MCD package tables.
2
3use std::path::Path;
4
5use anyhow::{Context, Result, bail};
6use mcd_core::{
7    Manifest, McdPackage,
8    schema::{ColumnType, TableColumnSchema},
9    tables::{DataTable, TypedValue, load_manifest_tables},
10};
11use rusqlite::{
12    Connection, params, params_from_iter,
13    types::{Value, ValueRef},
14};
15use serde::{Deserialize, Serialize};
16use serde_json::{Map, Value as JsonValue, json};
17
18/// Run a read-only SQL query against manifest-declared package tables.
19pub fn query_package(package: &McdPackage, sql: &str) -> Result<QueryResult> {
20    validate_read_only_sql(sql)?;
21
22    let manifest = package.manifest()?;
23    let tables = load_manifest_tables(package, &manifest)?;
24    let mut connection = Connection::open_in_memory()?;
25    connection.execute_batch("PRAGMA query_only = OFF;")?;
26    load_tables_into_sqlite(&mut connection, &manifest, tables.values())?;
27    connection.execute_batch("PRAGMA query_only = ON;")?;
28
29    let mut statement = connection
30        .prepare(sql)
31        .with_context(|| "prepare SQL query")?;
32    if !statement.readonly() {
33        bail!("query must be read-only");
34    }
35
36    query_rows(&mut statement)
37}
38
39/// Open an MCD package from disk and run a read-only SQL query against its tables.
40pub fn query_path(path: impl AsRef<Path>, sql: &str) -> Result<QueryResult> {
41    let package = McdPackage::open_path(path)?;
42    query_package(&package, sql)
43}
44
45/// Structured result of an SQL query.
46#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
47pub struct QueryResult {
48    /// Result column names in query order.
49    pub columns: Vec<String>,
50    /// Result rows in column order.
51    pub rows: Vec<Vec<QueryValue>>,
52}
53
54impl QueryResult {
55    /// Number of returned rows.
56    #[must_use]
57    pub fn row_count(&self) -> usize {
58        self.rows.len()
59    }
60
61    /// Return result rows as JSON objects keyed by column name.
62    #[must_use]
63    pub fn rows_as_json(&self) -> JsonValue {
64        JsonValue::Array(
65            self.rows
66                .iter()
67                .map(|row| {
68                    let mut object = Map::new();
69                    for (index, value) in row.iter().enumerate() {
70                        object.insert(self.columns[index].clone(), value.as_json());
71                    }
72                    JsonValue::Object(object)
73                })
74                .collect(),
75        )
76    }
77
78    /// Return a JSON object with columns, rows, and row count.
79    #[must_use]
80    pub fn as_json(&self) -> JsonValue {
81        json!({
82            "columns": self.columns,
83            "rows": self.rows_as_json(),
84            "rowCount": self.row_count(),
85        })
86    }
87
88    /// Serialize the result as pretty JSON.
89    pub fn to_json_pretty(&self) -> Result<String> {
90        Ok(serde_json::to_string_pretty(&self.as_json())?)
91    }
92
93    /// Serialize the result as CSV.
94    #[must_use]
95    pub fn to_csv(&self) -> String {
96        let mut lines = vec![
97            self.columns
98                .iter()
99                .map(|cell| csv_escape(cell))
100                .collect::<Vec<_>>()
101                .join(","),
102        ];
103        for row in &self.rows {
104            lines.push(
105                row.iter()
106                    .map(|value| csv_escape(&value.display()))
107                    .collect::<Vec<_>>()
108                    .join(","),
109            );
110        }
111        lines.join("\n") + "\n"
112    }
113
114    /// Serialize the result as a simple ASCII table.
115    #[must_use]
116    pub fn to_table(&self) -> String {
117        let mut widths = self.columns.iter().map(String::len).collect::<Vec<_>>();
118        for row in &self.rows {
119            for (index, value) in row.iter().enumerate() {
120                widths[index] = widths[index].max(value.display().len());
121            }
122        }
123
124        let mut lines = Vec::new();
125        push_separator(&mut lines, &widths);
126        push_row(&mut lines, &self.columns, &widths);
127        push_separator(&mut lines, &widths);
128        for row in &self.rows {
129            let cells = row.iter().map(QueryValue::display).collect::<Vec<_>>();
130            push_row(&mut lines, &cells, &widths);
131        }
132        push_separator(&mut lines, &widths);
133        lines.push(format!("{} row(s)", self.rows.len()));
134        lines.join("\n") + "\n"
135    }
136}
137
138/// One SQL result cell value.
139#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
140#[serde(tag = "type", content = "value", rename_all = "snake_case")]
141pub enum QueryValue {
142    /// SQL NULL.
143    Null,
144    /// Signed integer value.
145    Integer(i64),
146    /// Floating point value.
147    Real(f64),
148    /// UTF-8 text value.
149    Text(String),
150    /// Binary blob value.
151    Blob(Vec<u8>),
152}
153
154impl QueryValue {
155    /// Convert this value to a plain JSON scalar.
156    #[must_use]
157    pub fn as_json(&self) -> JsonValue {
158        match self {
159            Self::Null => JsonValue::Null,
160            Self::Integer(value) => json!(value),
161            Self::Real(value) => json!(value),
162            Self::Text(value) => json!(value),
163            Self::Blob(value) => json!(value),
164        }
165    }
166
167    /// Return a display string suitable for table and CSV output.
168    #[must_use]
169    pub fn display(&self) -> String {
170        match self {
171            Self::Null => String::new(),
172            Self::Integer(value) => value.to_string(),
173            Self::Real(value) => value.to_string(),
174            Self::Text(value) => value.clone(),
175            Self::Blob(value) => format!("<{} bytes>", value.len()),
176        }
177    }
178}
179
180fn validate_read_only_sql(sql: &str) -> Result<()> {
181    let trimmed = sql.trim();
182    if trimmed.is_empty() {
183        bail!("SQL query cannot be empty");
184    }
185    let without_final_semicolon = trimmed.trim_end_matches(';').trim_end();
186    if without_final_semicolon.contains(';') {
187        bail!("query must contain exactly one SQL statement");
188    }
189
190    let lowercase = trimmed.to_ascii_lowercase();
191    if !(lowercase.starts_with("select") || lowercase.starts_with("with")) {
192        bail!("query must be a SELECT statement");
193    }
194    Ok(())
195}
196
197const METADATA_TABLES: &[&str] = &[
198    "mcd_tables",
199    "mcd_columns",
200    "mcd_primary_keys",
201    "mcd_foreign_keys",
202    "mcd_units",
203];
204
205fn load_tables_into_sqlite<'a>(
206    connection: &mut Connection,
207    manifest: &Manifest,
208    tables: impl IntoIterator<Item = &'a mcd_core::tables::DataTable>,
209) -> Result<()> {
210    let tables = tables.into_iter().collect::<Vec<_>>();
211    reject_metadata_name_collisions(&tables)?;
212
213    connection.execute_batch("PRAGMA foreign_keys = ON;")?;
214    let transaction = connection.transaction()?;
215    create_metadata_tables(&transaction)?;
216    insert_metadata(&transaction, manifest, &tables)?;
217
218    for table in &tables {
219        create_data_table(&transaction, table)?;
220    }
221
222    for table in &tables {
223        let placeholders = std::iter::repeat_n("?", table.schema.columns.len())
224            .collect::<Vec<_>>()
225            .join(", ");
226        let insert_sql = format!(
227            "INSERT INTO {} VALUES ({placeholders})",
228            quote_identifier(&table.id)
229        );
230        let mut insert = transaction.prepare(&insert_sql)?;
231        for row in &table.rows {
232            let values = table
233                .schema
234                .columns
235                .iter()
236                .map(|column| {
237                    let value = row
238                        .cells
239                        .get(&column.name)
240                        .with_context(|| format!("missing cell '{}'", column.name))?;
241                    Ok(sqlite_value(value))
242                })
243                .collect::<Result<Vec<_>>>()?;
244            insert.execute(params_from_iter(values))?;
245        }
246    }
247    transaction.commit()?;
248    Ok(())
249}
250
251fn reject_metadata_name_collisions(tables: &[&DataTable]) -> Result<()> {
252    for table in tables {
253        if METADATA_TABLES.contains(&table.id.as_str()) {
254            bail!(
255                "table id '{}' is reserved for MCD SQL metadata introspection",
256                table.id
257            );
258        }
259    }
260    Ok(())
261}
262
263fn create_data_table(transaction: &rusqlite::Transaction<'_>, table: &DataTable) -> Result<()> {
264    let mut definitions = table
265        .schema
266        .columns
267        .iter()
268        .map(|column| {
269            format!(
270                "{} {}",
271                quote_identifier(&column.name),
272                sqlite_type(column.value_type)
273            )
274        })
275        .collect::<Vec<_>>();
276
277    if !table.schema.primary_key.is_empty() {
278        definitions.push(format!(
279            "PRIMARY KEY ({})",
280            quote_identifiers(&table.schema.primary_key)
281        ));
282    }
283
284    for foreign_key in &table.schema.foreign_keys {
285        definitions.push(format!(
286            "FOREIGN KEY ({}) REFERENCES {} ({})",
287            quote_identifiers(&foreign_key.columns),
288            quote_identifier(&foreign_key.references.table),
289            quote_identifiers(&foreign_key.references.columns)
290        ));
291    }
292
293    transaction.execute(
294        &format!(
295            "CREATE TABLE {} ({})",
296            quote_identifier(&table.id),
297            definitions.join(", ")
298        ),
299        [],
300    )?;
301    Ok(())
302}
303
304fn create_metadata_tables(transaction: &rusqlite::Transaction<'_>) -> Result<()> {
305    transaction.execute_batch(
306        r#"
307        CREATE TABLE mcd_tables (
308            table_id TEXT PRIMARY KEY,
309            data_path TEXT NOT NULL,
310            schema_path TEXT NOT NULL
311        );
312        CREATE TABLE mcd_columns (
313            table_id TEXT NOT NULL,
314            column_name TEXT NOT NULL,
315            ordinal INTEGER NOT NULL,
316            type TEXT NOT NULL,
317            label TEXT,
318            nullable INTEGER NOT NULL,
319            enum_values TEXT,
320            unit_code TEXT,
321            unit_label TEXT,
322            unit_custom INTEGER NOT NULL,
323            PRIMARY KEY (table_id, column_name)
324        );
325        CREATE TABLE mcd_primary_keys (
326            table_id TEXT NOT NULL,
327            column_name TEXT NOT NULL,
328            ordinal INTEGER NOT NULL,
329            PRIMARY KEY (table_id, ordinal)
330        );
331        CREATE TABLE mcd_foreign_keys (
332            table_id TEXT NOT NULL,
333            column_name TEXT NOT NULL,
334            ordinal INTEGER NOT NULL,
335            ref_table_id TEXT NOT NULL,
336            ref_column_name TEXT NOT NULL,
337            PRIMARY KEY (table_id, column_name, ref_table_id, ref_column_name)
338        );
339        CREATE TABLE mcd_units (
340            table_id TEXT NOT NULL,
341            column_name TEXT NOT NULL,
342            unit_code TEXT,
343            unit_label TEXT,
344            unit_custom INTEGER NOT NULL,
345            PRIMARY KEY (table_id, column_name)
346        );
347        "#,
348    )?;
349    Ok(())
350}
351
352fn insert_metadata(
353    transaction: &rusqlite::Transaction<'_>,
354    manifest: &Manifest,
355    tables: &[&DataTable],
356) -> Result<()> {
357    for table in tables {
358        let entry = manifest
359            .tables
360            .iter()
361            .find(|entry| entry.id == table.id)
362            .with_context(|| format!("missing manifest entry for table '{}'", table.id))?;
363        transaction.execute(
364            "INSERT INTO mcd_tables (table_id, data_path, schema_path) VALUES (?, ?, ?)",
365            params![entry.id, entry.data, entry.schema],
366        )?;
367
368        for (index, column) in table.schema.columns.iter().enumerate() {
369            insert_column_metadata(transaction, &table.id, index, column)?;
370        }
371
372        for (index, column) in table.schema.primary_key.iter().enumerate() {
373            transaction.execute(
374                "INSERT INTO mcd_primary_keys (table_id, column_name, ordinal) VALUES (?, ?, ?)",
375                params![table.id, column, index as i64 + 1],
376            )?;
377        }
378
379        for foreign_key in &table.schema.foreign_keys {
380            for (index, (column, referenced_column)) in foreign_key
381                .columns
382                .iter()
383                .zip(foreign_key.references.columns.iter())
384                .enumerate()
385            {
386                transaction.execute(
387                    "INSERT INTO mcd_foreign_keys (table_id, column_name, ordinal, ref_table_id, ref_column_name) VALUES (?, ?, ?, ?, ?)",
388                    params![
389                        table.id,
390                        column,
391                        index as i64 + 1,
392                        foreign_key.references.table,
393                        referenced_column
394                    ],
395                )?;
396            }
397        }
398    }
399    Ok(())
400}
401
402fn insert_column_metadata(
403    transaction: &rusqlite::Transaction<'_>,
404    table_id: &str,
405    index: usize,
406    column: &TableColumnSchema,
407) -> Result<()> {
408    let enum_values = if column.enum_values.is_empty() {
409        None
410    } else {
411        Some(serde_json::to_string(&column.enum_values)?)
412    };
413    let unit_code = column.unit.as_ref().and_then(|unit| unit.code.as_deref());
414    let unit_label = column.unit.as_ref().and_then(|unit| unit.label.as_deref());
415    let unit_custom = column.unit.as_ref().is_some_and(|unit| unit.custom);
416
417    transaction.execute(
418        "INSERT INTO mcd_columns (table_id, column_name, ordinal, type, label, nullable, enum_values, unit_code, unit_label, unit_custom) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
419        params![
420            table_id,
421            column.name,
422            index as i64 + 1,
423            column_type_name(column.value_type),
424            column.label,
425            i64::from(column.nullable),
426            enum_values,
427            unit_code,
428            unit_label,
429            i64::from(unit_custom),
430        ],
431    )?;
432
433    if column.unit.is_some() {
434        transaction.execute(
435            "INSERT INTO mcd_units (table_id, column_name, unit_code, unit_label, unit_custom) VALUES (?, ?, ?, ?, ?)",
436            params![table_id, column.name, unit_code, unit_label, i64::from(unit_custom)],
437        )?;
438    }
439    Ok(())
440}
441
442fn sqlite_type(column_type: ColumnType) -> &'static str {
443    match column_type {
444        ColumnType::Integer | ColumnType::Boolean => "INTEGER",
445        ColumnType::Decimal => "REAL",
446        ColumnType::String
447        | ColumnType::Date
448        | ColumnType::Datetime
449        | ColumnType::Time
450        | ColumnType::Enum => "TEXT",
451    }
452}
453
454fn column_type_name(column_type: ColumnType) -> &'static str {
455    match column_type {
456        ColumnType::String => "string",
457        ColumnType::Integer => "integer",
458        ColumnType::Decimal => "decimal",
459        ColumnType::Boolean => "boolean",
460        ColumnType::Date => "date",
461        ColumnType::Datetime => "datetime",
462        ColumnType::Time => "time",
463        ColumnType::Enum => "enum",
464    }
465}
466
467fn sqlite_value(value: &TypedValue) -> Value {
468    match value {
469        TypedValue::Null => Value::Null,
470        TypedValue::String(value)
471        | TypedValue::Decimal(value)
472        | TypedValue::Date(value)
473        | TypedValue::Datetime(value)
474        | TypedValue::Time(value)
475        | TypedValue::Enum(value) => Value::Text(value.clone()),
476        TypedValue::Integer(value) => Value::Integer(*value),
477        TypedValue::Boolean(value) => Value::Integer(i64::from(*value)),
478    }
479}
480
481fn quote_identifier(identifier: &str) -> String {
482    format!("\"{}\"", identifier.replace('"', "\"\""))
483}
484
485fn quote_identifiers(identifiers: &[String]) -> String {
486    identifiers
487        .iter()
488        .map(|identifier| quote_identifier(identifier))
489        .collect::<Vec<_>>()
490        .join(", ")
491}
492
493fn query_rows(statement: &mut rusqlite::Statement<'_>) -> Result<QueryResult> {
494    let columns = statement
495        .column_names()
496        .into_iter()
497        .map(str::to_owned)
498        .collect::<Vec<_>>();
499    let column_count = columns.len();
500    let mut rows = Vec::new();
501    let mut query = statement.query([])?;
502    while let Some(row) = query.next()? {
503        let mut values = Vec::with_capacity(column_count);
504        for index in 0..column_count {
505            values.push(query_value(row.get_ref(index)?));
506        }
507        rows.push(values);
508    }
509    Ok(QueryResult { columns, rows })
510}
511
512fn query_value(value: ValueRef<'_>) -> QueryValue {
513    match value {
514        ValueRef::Null => QueryValue::Null,
515        ValueRef::Integer(value) => QueryValue::Integer(value),
516        ValueRef::Real(value) => QueryValue::Real(value),
517        ValueRef::Text(value) => QueryValue::Text(String::from_utf8_lossy(value).into_owned()),
518        ValueRef::Blob(value) => QueryValue::Blob(value.to_vec()),
519    }
520}
521
522fn push_separator(lines: &mut Vec<String>, widths: &[usize]) {
523    let parts = widths
524        .iter()
525        .map(|width| "-".repeat(width + 2))
526        .collect::<Vec<_>>();
527    lines.push(format!("+{}+", parts.join("+")));
528}
529
530fn push_row(lines: &mut Vec<String>, cells: &[String], widths: &[usize]) {
531    let cells = cells
532        .iter()
533        .enumerate()
534        .map(|(index, cell)| format!(" {:width$} ", cell, width = widths[index]))
535        .collect::<Vec<_>>();
536    lines.push(format!("|{}|", cells.join("|")));
537}
538
539fn csv_escape(value: &str) -> String {
540    if value.contains([',', '"', '\n', '\r']) {
541        format!("\"{}\"", value.replace('"', "\"\""))
542    } else {
543        value.to_owned()
544    }
545}
546
547#[cfg(test)]
548mod tests {
549    use mcd_core::package::MCD_MIMETYPE;
550
551    use super::*;
552
553    #[test]
554    fn queries_aggregate_values() {
555        let package = package();
556        let result = query_package(
557            &package,
558            "select count(*) as rows, max(revenue_gbp) as max_revenue from revenue",
559        )
560        .expect("query succeeds");
561
562        assert_eq!(result.columns, ["rows", "max_revenue"]);
563        assert_eq!(result.row_count(), 1);
564        assert_eq!(result.rows[0][0], QueryValue::Integer(2));
565        assert_eq!(result.rows[0][1], QueryValue::Real(142500.0));
566    }
567
568    #[test]
569    fn rejects_writes() {
570        let package = package();
571        let err = query_package(&package, "delete from revenue").expect_err("write rejected");
572
573        assert!(err.to_string().contains("query must be a SELECT statement"));
574    }
575
576    #[test]
577    fn exposes_mcd_schema_metadata_to_sql() {
578        let package = related_package();
579
580        let primary_keys = query_package(
581            &package,
582            "select table_id, column_name, ordinal from mcd_primary_keys order by table_id",
583        )
584        .expect("primary keys query succeeds");
585        assert_eq!(
586            primary_keys.rows,
587            vec![
588                vec![
589                    QueryValue::Text("customers".to_owned()),
590                    QueryValue::Text("customer_id".to_owned()),
591                    QueryValue::Integer(1),
592                ],
593                vec![
594                    QueryValue::Text("orders".to_owned()),
595                    QueryValue::Text("order_id".to_owned()),
596                    QueryValue::Integer(1),
597                ],
598            ]
599        );
600
601        let foreign_keys = query_package(
602            &package,
603            "select table_id, column_name, ref_table_id, ref_column_name from mcd_foreign_keys",
604        )
605        .expect("foreign keys query succeeds");
606        assert_eq!(
607            foreign_keys.rows,
608            vec![vec![
609                QueryValue::Text("orders".to_owned()),
610                QueryValue::Text("customer_id".to_owned()),
611                QueryValue::Text("customers".to_owned()),
612                QueryValue::Text("customer_id".to_owned()),
613            ]]
614        );
615
616        let units = query_package(
617            &package,
618            "select table_id, column_name, unit_code, unit_label from mcd_units",
619        )
620        .expect("units query succeeds");
621        assert_eq!(
622            units.rows,
623            vec![vec![
624                QueryValue::Text("orders".to_owned()),
625                QueryValue::Text("amount".to_owned()),
626                QueryValue::Text("GBP".to_owned()),
627                QueryValue::Text("GBP".to_owned()),
628            ]]
629        );
630    }
631
632    #[test]
633    fn creates_sqlite_key_constraints_for_pragma_introspection() {
634        let package = related_package();
635
636        let table_info = query_package(
637            &package,
638            "select name, pk from pragma_table_info('customers') where pk > 0",
639        )
640        .expect("pragma table_info query succeeds");
641        assert_eq!(
642            table_info.rows,
643            vec![vec![
644                QueryValue::Text("customer_id".to_owned()),
645                QueryValue::Integer(1),
646            ]]
647        );
648
649        let foreign_key_info = query_package(
650            &package,
651            "select [table], [from], [to] from pragma_foreign_key_list('orders')",
652        )
653        .expect("pragma foreign_key_list query succeeds");
654        assert_eq!(
655            foreign_key_info.rows,
656            vec![vec![
657                QueryValue::Text("customers".to_owned()),
658                QueryValue::Text("customer_id".to_owned()),
659                QueryValue::Text("customer_id".to_owned()),
660            ]]
661        );
662    }
663
664    fn package() -> McdPackage {
665        McdPackage::from_bytes(&zip_bytes(&[
666            ("mimetype", MCD_MIMETYPE),
667            (
668                "manifest.json",
669                r#"{"format":"MCD","version":"0.1","profile":"MCD-Core","entrypoint":"content/main.md","tables":[{"id":"revenue","data":"tables/revenue.csv","schema":"tables/revenue.schema.json"}]}"#,
670            ),
671            ("content/main.md", "# Report\n"),
672            (
673                "tables/revenue.schema.json",
674                r#"{"id":"revenue","columns":[{"name":"quarter","type":"string"},{"name":"revenue_gbp","type":"decimal"}]}"#,
675            ),
676            ("tables/revenue.csv", "quarter,revenue_gbp\nQ1,125000.00\nQ2,142500.00\n"),
677        ]))
678        .expect("package opens")
679    }
680
681    fn related_package() -> McdPackage {
682        McdPackage::from_bytes(&zip_bytes(&[
683            ("mimetype", MCD_MIMETYPE),
684            (
685                "manifest.json",
686                r#"{"format":"MCD","version":"0.1","profile":"MCD-Core","entrypoint":"content/main.md","tables":[
687                    {"id":"customers","data":"tables/customers.csv","schema":"tables/customers.schema.json"},
688                    {"id":"orders","data":"tables/orders.csv","schema":"tables/orders.schema.json"}
689                ]}"#,
690            ),
691            ("content/main.md", "# Orders\n"),
692            (
693                "tables/customers.schema.json",
694                r#"{"id":"customers","primaryKey":["customer_id"],"columns":[
695                    {"name":"customer_id","type":"string"},
696                    {"name":"name","type":"string"}
697                ]}"#,
698            ),
699            ("tables/customers.csv", "customer_id,name\nc1,Alice\n"),
700            (
701                "tables/orders.schema.json",
702                r#"{"id":"orders","primaryKey":["order_id"],"foreignKeys":[{
703                    "columns":["customer_id"],
704                    "references":{"table":"customers","columns":["customer_id"]}
705                }],"columns":[
706                    {"name":"order_id","type":"string"},
707                    {"name":"customer_id","type":"string"},
708                    {"name":"amount","type":"decimal","unit":{"code":"GBP","label":"GBP"}}
709                ]}"#,
710            ),
711            ("tables/orders.csv", "order_id,customer_id,amount\no1,c1,12.50\n"),
712        ]))
713        .expect("package opens")
714    }
715
716    fn zip_bytes(entries: &[(&str, &str)]) -> Vec<u8> {
717        use std::io::{Cursor, Write};
718        use zip::{CompressionMethod, ZipWriter, write::SimpleFileOptions};
719
720        let cursor = Cursor::new(Vec::new());
721        let mut writer = ZipWriter::new(cursor);
722        let options = SimpleFileOptions::default().compression_method(CompressionMethod::Stored);
723
724        for (path, content) in entries {
725            writer.start_file(*path, options).expect("start file");
726            writer.write_all(content.as_bytes()).expect("write file");
727        }
728
729        writer.finish().expect("finish zip").into_inner()
730    }
731}