1use std::path::Path;
4
5use anyhow::{Context, Result, bail};
6use mcd_core::{
7 Manifest, McdPackage,
8 schema::{ColumnType, TableColumnSchema},
9 tables::{DataTable, TypedValue, load_manifest_tables},
10};
11use rusqlite::{
12 Connection, params, params_from_iter,
13 types::{Value, ValueRef},
14};
15use serde::{Deserialize, Serialize};
16use serde_json::{Map, Value as JsonValue, json};
17
18pub fn query_package(package: &McdPackage, sql: &str) -> Result<QueryResult> {
20 validate_read_only_sql(sql)?;
21
22 let manifest = package.manifest()?;
23 let tables = load_manifest_tables(package, &manifest)?;
24 let mut connection = Connection::open_in_memory()?;
25 connection.execute_batch("PRAGMA query_only = OFF;")?;
26 load_tables_into_sqlite(&mut connection, &manifest, tables.values())?;
27 connection.execute_batch("PRAGMA query_only = ON;")?;
28
29 let mut statement = connection
30 .prepare(sql)
31 .with_context(|| "prepare SQL query")?;
32 if !statement.readonly() {
33 bail!("query must be read-only");
34 }
35
36 query_rows(&mut statement)
37}
38
39pub fn query_path(path: impl AsRef<Path>, sql: &str) -> Result<QueryResult> {
41 let package = McdPackage::open_path(path)?;
42 query_package(&package, sql)
43}
44
45#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
47pub struct QueryResult {
48 pub columns: Vec<String>,
50 pub rows: Vec<Vec<QueryValue>>,
52}
53
54impl QueryResult {
55 #[must_use]
57 pub fn row_count(&self) -> usize {
58 self.rows.len()
59 }
60
61 #[must_use]
63 pub fn rows_as_json(&self) -> JsonValue {
64 JsonValue::Array(
65 self.rows
66 .iter()
67 .map(|row| {
68 let mut object = Map::new();
69 for (index, value) in row.iter().enumerate() {
70 object.insert(self.columns[index].clone(), value.as_json());
71 }
72 JsonValue::Object(object)
73 })
74 .collect(),
75 )
76 }
77
78 #[must_use]
80 pub fn as_json(&self) -> JsonValue {
81 json!({
82 "columns": self.columns,
83 "rows": self.rows_as_json(),
84 "rowCount": self.row_count(),
85 })
86 }
87
88 pub fn to_json_pretty(&self) -> Result<String> {
90 Ok(serde_json::to_string_pretty(&self.as_json())?)
91 }
92
93 #[must_use]
95 pub fn to_csv(&self) -> String {
96 let mut lines = vec![
97 self.columns
98 .iter()
99 .map(|cell| csv_escape(cell))
100 .collect::<Vec<_>>()
101 .join(","),
102 ];
103 for row in &self.rows {
104 lines.push(
105 row.iter()
106 .map(|value| csv_escape(&value.display()))
107 .collect::<Vec<_>>()
108 .join(","),
109 );
110 }
111 lines.join("\n") + "\n"
112 }
113
114 #[must_use]
116 pub fn to_table(&self) -> String {
117 let mut widths = self.columns.iter().map(String::len).collect::<Vec<_>>();
118 for row in &self.rows {
119 for (index, value) in row.iter().enumerate() {
120 widths[index] = widths[index].max(value.display().len());
121 }
122 }
123
124 let mut lines = Vec::new();
125 push_separator(&mut lines, &widths);
126 push_row(&mut lines, &self.columns, &widths);
127 push_separator(&mut lines, &widths);
128 for row in &self.rows {
129 let cells = row.iter().map(QueryValue::display).collect::<Vec<_>>();
130 push_row(&mut lines, &cells, &widths);
131 }
132 push_separator(&mut lines, &widths);
133 lines.push(format!("{} row(s)", self.rows.len()));
134 lines.join("\n") + "\n"
135 }
136}
137
138#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
140#[serde(tag = "type", content = "value", rename_all = "snake_case")]
141pub enum QueryValue {
142 Null,
144 Integer(i64),
146 Real(f64),
148 Text(String),
150 Blob(Vec<u8>),
152}
153
154impl QueryValue {
155 #[must_use]
157 pub fn as_json(&self) -> JsonValue {
158 match self {
159 Self::Null => JsonValue::Null,
160 Self::Integer(value) => json!(value),
161 Self::Real(value) => json!(value),
162 Self::Text(value) => json!(value),
163 Self::Blob(value) => json!(value),
164 }
165 }
166
167 #[must_use]
169 pub fn display(&self) -> String {
170 match self {
171 Self::Null => String::new(),
172 Self::Integer(value) => value.to_string(),
173 Self::Real(value) => value.to_string(),
174 Self::Text(value) => value.clone(),
175 Self::Blob(value) => format!("<{} bytes>", value.len()),
176 }
177 }
178}
179
180fn validate_read_only_sql(sql: &str) -> Result<()> {
181 let trimmed = sql.trim();
182 if trimmed.is_empty() {
183 bail!("SQL query cannot be empty");
184 }
185 let without_final_semicolon = trimmed.trim_end_matches(';').trim_end();
186 if without_final_semicolon.contains(';') {
187 bail!("query must contain exactly one SQL statement");
188 }
189
190 let lowercase = trimmed.to_ascii_lowercase();
191 if !(lowercase.starts_with("select") || lowercase.starts_with("with")) {
192 bail!("query must be a SELECT statement");
193 }
194 Ok(())
195}
196
197const METADATA_TABLES: &[&str] = &[
198 "mcd_tables",
199 "mcd_columns",
200 "mcd_primary_keys",
201 "mcd_foreign_keys",
202 "mcd_units",
203];
204
205fn load_tables_into_sqlite<'a>(
206 connection: &mut Connection,
207 manifest: &Manifest,
208 tables: impl IntoIterator<Item = &'a mcd_core::tables::DataTable>,
209) -> Result<()> {
210 let tables = tables.into_iter().collect::<Vec<_>>();
211 reject_metadata_name_collisions(&tables)?;
212
213 connection.execute_batch("PRAGMA foreign_keys = ON;")?;
214 let transaction = connection.transaction()?;
215 create_metadata_tables(&transaction)?;
216 insert_metadata(&transaction, manifest, &tables)?;
217
218 for table in &tables {
219 create_data_table(&transaction, table)?;
220 }
221
222 for table in &tables {
223 let placeholders = std::iter::repeat_n("?", table.schema.columns.len())
224 .collect::<Vec<_>>()
225 .join(", ");
226 let insert_sql = format!(
227 "INSERT INTO {} VALUES ({placeholders})",
228 quote_identifier(&table.id)
229 );
230 let mut insert = transaction.prepare(&insert_sql)?;
231 for row in &table.rows {
232 let values = table
233 .schema
234 .columns
235 .iter()
236 .map(|column| {
237 let value = row
238 .cells
239 .get(&column.name)
240 .with_context(|| format!("missing cell '{}'", column.name))?;
241 Ok(sqlite_value(value))
242 })
243 .collect::<Result<Vec<_>>>()?;
244 insert.execute(params_from_iter(values))?;
245 }
246 }
247 transaction.commit()?;
248 Ok(())
249}
250
251fn reject_metadata_name_collisions(tables: &[&DataTable]) -> Result<()> {
252 for table in tables {
253 if METADATA_TABLES.contains(&table.id.as_str()) {
254 bail!(
255 "table id '{}' is reserved for MCD SQL metadata introspection",
256 table.id
257 );
258 }
259 }
260 Ok(())
261}
262
263fn create_data_table(transaction: &rusqlite::Transaction<'_>, table: &DataTable) -> Result<()> {
264 let mut definitions = table
265 .schema
266 .columns
267 .iter()
268 .map(|column| {
269 format!(
270 "{} {}",
271 quote_identifier(&column.name),
272 sqlite_type(column.value_type)
273 )
274 })
275 .collect::<Vec<_>>();
276
277 if !table.schema.primary_key.is_empty() {
278 definitions.push(format!(
279 "PRIMARY KEY ({})",
280 quote_identifiers(&table.schema.primary_key)
281 ));
282 }
283
284 for foreign_key in &table.schema.foreign_keys {
285 definitions.push(format!(
286 "FOREIGN KEY ({}) REFERENCES {} ({})",
287 quote_identifiers(&foreign_key.columns),
288 quote_identifier(&foreign_key.references.table),
289 quote_identifiers(&foreign_key.references.columns)
290 ));
291 }
292
293 transaction.execute(
294 &format!(
295 "CREATE TABLE {} ({})",
296 quote_identifier(&table.id),
297 definitions.join(", ")
298 ),
299 [],
300 )?;
301 Ok(())
302}
303
304fn create_metadata_tables(transaction: &rusqlite::Transaction<'_>) -> Result<()> {
305 transaction.execute_batch(
306 r#"
307 CREATE TABLE mcd_tables (
308 table_id TEXT PRIMARY KEY,
309 data_path TEXT NOT NULL,
310 schema_path TEXT NOT NULL
311 );
312 CREATE TABLE mcd_columns (
313 table_id TEXT NOT NULL,
314 column_name TEXT NOT NULL,
315 ordinal INTEGER NOT NULL,
316 type TEXT NOT NULL,
317 label TEXT,
318 nullable INTEGER NOT NULL,
319 enum_values TEXT,
320 unit_code TEXT,
321 unit_label TEXT,
322 unit_custom INTEGER NOT NULL,
323 PRIMARY KEY (table_id, column_name)
324 );
325 CREATE TABLE mcd_primary_keys (
326 table_id TEXT NOT NULL,
327 column_name TEXT NOT NULL,
328 ordinal INTEGER NOT NULL,
329 PRIMARY KEY (table_id, ordinal)
330 );
331 CREATE TABLE mcd_foreign_keys (
332 table_id TEXT NOT NULL,
333 column_name TEXT NOT NULL,
334 ordinal INTEGER NOT NULL,
335 ref_table_id TEXT NOT NULL,
336 ref_column_name TEXT NOT NULL,
337 PRIMARY KEY (table_id, column_name, ref_table_id, ref_column_name)
338 );
339 CREATE TABLE mcd_units (
340 table_id TEXT NOT NULL,
341 column_name TEXT NOT NULL,
342 unit_code TEXT,
343 unit_label TEXT,
344 unit_custom INTEGER NOT NULL,
345 PRIMARY KEY (table_id, column_name)
346 );
347 "#,
348 )?;
349 Ok(())
350}
351
352fn insert_metadata(
353 transaction: &rusqlite::Transaction<'_>,
354 manifest: &Manifest,
355 tables: &[&DataTable],
356) -> Result<()> {
357 for table in tables {
358 let entry = manifest
359 .tables
360 .iter()
361 .find(|entry| entry.id == table.id)
362 .with_context(|| format!("missing manifest entry for table '{}'", table.id))?;
363 transaction.execute(
364 "INSERT INTO mcd_tables (table_id, data_path, schema_path) VALUES (?, ?, ?)",
365 params![entry.id, entry.data, entry.schema],
366 )?;
367
368 for (index, column) in table.schema.columns.iter().enumerate() {
369 insert_column_metadata(transaction, &table.id, index, column)?;
370 }
371
372 for (index, column) in table.schema.primary_key.iter().enumerate() {
373 transaction.execute(
374 "INSERT INTO mcd_primary_keys (table_id, column_name, ordinal) VALUES (?, ?, ?)",
375 params![table.id, column, index as i64 + 1],
376 )?;
377 }
378
379 for foreign_key in &table.schema.foreign_keys {
380 for (index, (column, referenced_column)) in foreign_key
381 .columns
382 .iter()
383 .zip(foreign_key.references.columns.iter())
384 .enumerate()
385 {
386 transaction.execute(
387 "INSERT INTO mcd_foreign_keys (table_id, column_name, ordinal, ref_table_id, ref_column_name) VALUES (?, ?, ?, ?, ?)",
388 params![
389 table.id,
390 column,
391 index as i64 + 1,
392 foreign_key.references.table,
393 referenced_column
394 ],
395 )?;
396 }
397 }
398 }
399 Ok(())
400}
401
402fn insert_column_metadata(
403 transaction: &rusqlite::Transaction<'_>,
404 table_id: &str,
405 index: usize,
406 column: &TableColumnSchema,
407) -> Result<()> {
408 let enum_values = if column.enum_values.is_empty() {
409 None
410 } else {
411 Some(serde_json::to_string(&column.enum_values)?)
412 };
413 let unit_code = column.unit.as_ref().and_then(|unit| unit.code.as_deref());
414 let unit_label = column.unit.as_ref().and_then(|unit| unit.label.as_deref());
415 let unit_custom = column.unit.as_ref().is_some_and(|unit| unit.custom);
416
417 transaction.execute(
418 "INSERT INTO mcd_columns (table_id, column_name, ordinal, type, label, nullable, enum_values, unit_code, unit_label, unit_custom) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
419 params![
420 table_id,
421 column.name,
422 index as i64 + 1,
423 column_type_name(column.value_type),
424 column.label,
425 i64::from(column.nullable),
426 enum_values,
427 unit_code,
428 unit_label,
429 i64::from(unit_custom),
430 ],
431 )?;
432
433 if column.unit.is_some() {
434 transaction.execute(
435 "INSERT INTO mcd_units (table_id, column_name, unit_code, unit_label, unit_custom) VALUES (?, ?, ?, ?, ?)",
436 params![table_id, column.name, unit_code, unit_label, i64::from(unit_custom)],
437 )?;
438 }
439 Ok(())
440}
441
442fn sqlite_type(column_type: ColumnType) -> &'static str {
443 match column_type {
444 ColumnType::Integer | ColumnType::Boolean => "INTEGER",
445 ColumnType::Decimal => "REAL",
446 ColumnType::String
447 | ColumnType::Date
448 | ColumnType::Datetime
449 | ColumnType::Time
450 | ColumnType::Enum => "TEXT",
451 }
452}
453
454fn column_type_name(column_type: ColumnType) -> &'static str {
455 match column_type {
456 ColumnType::String => "string",
457 ColumnType::Integer => "integer",
458 ColumnType::Decimal => "decimal",
459 ColumnType::Boolean => "boolean",
460 ColumnType::Date => "date",
461 ColumnType::Datetime => "datetime",
462 ColumnType::Time => "time",
463 ColumnType::Enum => "enum",
464 }
465}
466
467fn sqlite_value(value: &TypedValue) -> Value {
468 match value {
469 TypedValue::Null => Value::Null,
470 TypedValue::String(value)
471 | TypedValue::Decimal(value)
472 | TypedValue::Date(value)
473 | TypedValue::Datetime(value)
474 | TypedValue::Time(value)
475 | TypedValue::Enum(value) => Value::Text(value.clone()),
476 TypedValue::Integer(value) => Value::Integer(*value),
477 TypedValue::Boolean(value) => Value::Integer(i64::from(*value)),
478 }
479}
480
481fn quote_identifier(identifier: &str) -> String {
482 format!("\"{}\"", identifier.replace('"', "\"\""))
483}
484
485fn quote_identifiers(identifiers: &[String]) -> String {
486 identifiers
487 .iter()
488 .map(|identifier| quote_identifier(identifier))
489 .collect::<Vec<_>>()
490 .join(", ")
491}
492
493fn query_rows(statement: &mut rusqlite::Statement<'_>) -> Result<QueryResult> {
494 let columns = statement
495 .column_names()
496 .into_iter()
497 .map(str::to_owned)
498 .collect::<Vec<_>>();
499 let column_count = columns.len();
500 let mut rows = Vec::new();
501 let mut query = statement.query([])?;
502 while let Some(row) = query.next()? {
503 let mut values = Vec::with_capacity(column_count);
504 for index in 0..column_count {
505 values.push(query_value(row.get_ref(index)?));
506 }
507 rows.push(values);
508 }
509 Ok(QueryResult { columns, rows })
510}
511
512fn query_value(value: ValueRef<'_>) -> QueryValue {
513 match value {
514 ValueRef::Null => QueryValue::Null,
515 ValueRef::Integer(value) => QueryValue::Integer(value),
516 ValueRef::Real(value) => QueryValue::Real(value),
517 ValueRef::Text(value) => QueryValue::Text(String::from_utf8_lossy(value).into_owned()),
518 ValueRef::Blob(value) => QueryValue::Blob(value.to_vec()),
519 }
520}
521
522fn push_separator(lines: &mut Vec<String>, widths: &[usize]) {
523 let parts = widths
524 .iter()
525 .map(|width| "-".repeat(width + 2))
526 .collect::<Vec<_>>();
527 lines.push(format!("+{}+", parts.join("+")));
528}
529
530fn push_row(lines: &mut Vec<String>, cells: &[String], widths: &[usize]) {
531 let cells = cells
532 .iter()
533 .enumerate()
534 .map(|(index, cell)| format!(" {:width$} ", cell, width = widths[index]))
535 .collect::<Vec<_>>();
536 lines.push(format!("|{}|", cells.join("|")));
537}
538
539fn csv_escape(value: &str) -> String {
540 if value.contains([',', '"', '\n', '\r']) {
541 format!("\"{}\"", value.replace('"', "\"\""))
542 } else {
543 value.to_owned()
544 }
545}
546
547#[cfg(test)]
548mod tests {
549 use mcd_core::package::MCD_MIMETYPE;
550
551 use super::*;
552
553 #[test]
554 fn queries_aggregate_values() {
555 let package = package();
556 let result = query_package(
557 &package,
558 "select count(*) as rows, max(revenue_gbp) as max_revenue from revenue",
559 )
560 .expect("query succeeds");
561
562 assert_eq!(result.columns, ["rows", "max_revenue"]);
563 assert_eq!(result.row_count(), 1);
564 assert_eq!(result.rows[0][0], QueryValue::Integer(2));
565 assert_eq!(result.rows[0][1], QueryValue::Real(142500.0));
566 }
567
568 #[test]
569 fn rejects_writes() {
570 let package = package();
571 let err = query_package(&package, "delete from revenue").expect_err("write rejected");
572
573 assert!(err.to_string().contains("query must be a SELECT statement"));
574 }
575
576 #[test]
577 fn exposes_mcd_schema_metadata_to_sql() {
578 let package = related_package();
579
580 let primary_keys = query_package(
581 &package,
582 "select table_id, column_name, ordinal from mcd_primary_keys order by table_id",
583 )
584 .expect("primary keys query succeeds");
585 assert_eq!(
586 primary_keys.rows,
587 vec![
588 vec![
589 QueryValue::Text("customers".to_owned()),
590 QueryValue::Text("customer_id".to_owned()),
591 QueryValue::Integer(1),
592 ],
593 vec![
594 QueryValue::Text("orders".to_owned()),
595 QueryValue::Text("order_id".to_owned()),
596 QueryValue::Integer(1),
597 ],
598 ]
599 );
600
601 let foreign_keys = query_package(
602 &package,
603 "select table_id, column_name, ref_table_id, ref_column_name from mcd_foreign_keys",
604 )
605 .expect("foreign keys query succeeds");
606 assert_eq!(
607 foreign_keys.rows,
608 vec![vec![
609 QueryValue::Text("orders".to_owned()),
610 QueryValue::Text("customer_id".to_owned()),
611 QueryValue::Text("customers".to_owned()),
612 QueryValue::Text("customer_id".to_owned()),
613 ]]
614 );
615
616 let units = query_package(
617 &package,
618 "select table_id, column_name, unit_code, unit_label from mcd_units",
619 )
620 .expect("units query succeeds");
621 assert_eq!(
622 units.rows,
623 vec![vec![
624 QueryValue::Text("orders".to_owned()),
625 QueryValue::Text("amount".to_owned()),
626 QueryValue::Text("GBP".to_owned()),
627 QueryValue::Text("GBP".to_owned()),
628 ]]
629 );
630 }
631
632 #[test]
633 fn creates_sqlite_key_constraints_for_pragma_introspection() {
634 let package = related_package();
635
636 let table_info = query_package(
637 &package,
638 "select name, pk from pragma_table_info('customers') where pk > 0",
639 )
640 .expect("pragma table_info query succeeds");
641 assert_eq!(
642 table_info.rows,
643 vec![vec![
644 QueryValue::Text("customer_id".to_owned()),
645 QueryValue::Integer(1),
646 ]]
647 );
648
649 let foreign_key_info = query_package(
650 &package,
651 "select [table], [from], [to] from pragma_foreign_key_list('orders')",
652 )
653 .expect("pragma foreign_key_list query succeeds");
654 assert_eq!(
655 foreign_key_info.rows,
656 vec![vec![
657 QueryValue::Text("customers".to_owned()),
658 QueryValue::Text("customer_id".to_owned()),
659 QueryValue::Text("customer_id".to_owned()),
660 ]]
661 );
662 }
663
664 fn package() -> McdPackage {
665 McdPackage::from_bytes(&zip_bytes(&[
666 ("mimetype", MCD_MIMETYPE),
667 (
668 "manifest.json",
669 r#"{"format":"MCD","version":"0.1","profile":"MCD-Core","entrypoint":"content/main.md","tables":[{"id":"revenue","data":"tables/revenue.csv","schema":"tables/revenue.schema.json"}]}"#,
670 ),
671 ("content/main.md", "# Report\n"),
672 (
673 "tables/revenue.schema.json",
674 r#"{"id":"revenue","columns":[{"name":"quarter","type":"string"},{"name":"revenue_gbp","type":"decimal"}]}"#,
675 ),
676 ("tables/revenue.csv", "quarter,revenue_gbp\nQ1,125000.00\nQ2,142500.00\n"),
677 ]))
678 .expect("package opens")
679 }
680
681 fn related_package() -> McdPackage {
682 McdPackage::from_bytes(&zip_bytes(&[
683 ("mimetype", MCD_MIMETYPE),
684 (
685 "manifest.json",
686 r#"{"format":"MCD","version":"0.1","profile":"MCD-Core","entrypoint":"content/main.md","tables":[
687 {"id":"customers","data":"tables/customers.csv","schema":"tables/customers.schema.json"},
688 {"id":"orders","data":"tables/orders.csv","schema":"tables/orders.schema.json"}
689 ]}"#,
690 ),
691 ("content/main.md", "# Orders\n"),
692 (
693 "tables/customers.schema.json",
694 r#"{"id":"customers","primaryKey":["customer_id"],"columns":[
695 {"name":"customer_id","type":"string"},
696 {"name":"name","type":"string"}
697 ]}"#,
698 ),
699 ("tables/customers.csv", "customer_id,name\nc1,Alice\n"),
700 (
701 "tables/orders.schema.json",
702 r#"{"id":"orders","primaryKey":["order_id"],"foreignKeys":[{
703 "columns":["customer_id"],
704 "references":{"table":"customers","columns":["customer_id"]}
705 }],"columns":[
706 {"name":"order_id","type":"string"},
707 {"name":"customer_id","type":"string"},
708 {"name":"amount","type":"decimal","unit":{"code":"GBP","label":"GBP"}}
709 ]}"#,
710 ),
711 ("tables/orders.csv", "order_id,customer_id,amount\no1,c1,12.50\n"),
712 ]))
713 .expect("package opens")
714 }
715
716 fn zip_bytes(entries: &[(&str, &str)]) -> Vec<u8> {
717 use std::io::{Cursor, Write};
718 use zip::{CompressionMethod, ZipWriter, write::SimpleFileOptions};
719
720 let cursor = Cursor::new(Vec::new());
721 let mut writer = ZipWriter::new(cursor);
722 let options = SimpleFileOptions::default().compression_method(CompressionMethod::Stored);
723
724 for (path, content) in entries {
725 writer.start_file(*path, options).expect("start file");
726 writer.write_all(content.as_bytes()).expect("write file");
727 }
728
729 writer.finish().expect("finish zip").into_inner()
730 }
731}