opys_engine/commands/
stats.rs

1//! `opys stats` — render the project's configured `[[stats]]` sections. Each
2//! stat is a single SQL query (`sql`) run against an in-memory, throwaway
3//! relational view of the corpus (tables `docs`, `tags`, `sections`, `fields`);
4//! its result set is rendered as a markdown table. Pure read; no feature/type
5//! special-casing. The SQL engine is GlueSQL (pure Rust, in-memory).
6
7use std::io::IsTerminal;
8
9use futures::executor::block_on;
10use gluesql::prelude::{Glue, MemoryStorage, Payload, Value as GlueValue};
11use serde_json::{json, Map, Value};
12
13use crate::body;
14use crate::doc::Doc;
15use crate::error::{usage, Result};
16use crate::project_config::{ProjectConfig, SectionKind, StatSpec};
17use crate::Ctx;
18
19/// Split a tag into its key and optional value at the first `:` or `=`. A plain
20/// tag has no value (`osc` → `("osc", None)`); a keyed tag splits at the first
21/// separator (`area:parsing` → `("area", Some("parsing"))`).
22fn split_tag(t: &str) -> (&str, Option<&str>) {
23    match t.find([':', '=']) {
24        Some(i) => (&t[..i], Some(&t[i + 1..])),
25        None => (t, None),
26    }
27}
28
29/// Sum the lengths of all top-level array fields in a JSON object, giving the
30/// total number of items extracted from a structured section.
31fn count_array_items(data: &Value) -> usize {
32    match data {
33        Value::Object(map) => map
34            .values()
35            .filter_map(|v| v.as_array())
36            .map(|arr| arr.len())
37            .sum(),
38        Value::Array(arr) => arr.len(),
39        _ => 0,
40    }
41}
42
43/// A YAML frontmatter value converted to JSON (best-effort; unrepresentable
44/// values become `null`).
45pub(crate) fn yaml_to_json(v: &serde_norway::Value) -> Value {
46    serde_json::to_value(v).unwrap_or(Value::Null)
47}
48
49/// The projected JSON for a non-structured section present in `d`, or `None`
50/// when the section is absent/empty. Mirrors the countable-item logic behind
51/// coverage stats. Structured sections are handled by `structured_section_json`.
52pub(crate) fn section_json(d: &Doc, kind: SectionKind, heading: &str) -> Option<Value> {
53    match kind {
54        SectionKind::Checklist => {
55            let items = body::checklist_items(&d.body, heading);
56            if items.is_empty() {
57                return None;
58            }
59            let unchecked = items.iter().filter(|i| !i.checked).count();
60            Some(json!({ "kind": "checklist", "items": items.len(), "unchecked": unchecked }))
61        }
62        SectionKind::Log => {
63            let content = body::section(&d.body, heading);
64            let count = content.lines().filter(|l| l.starts_with("- ")).count();
65            if count == 0 {
66                return None;
67            }
68            Some(json!({ "kind": "log", "items": count }))
69        }
70        SectionKind::Structured | SectionKind::Prose => None,
71    }
72}
73
74/// Countable JSON for a `structured` section: parse its schema, extract the
75/// section body, and count the extracted array items.
76pub(crate) fn structured_section_json(
77    d: &Doc,
78    structure: Option<&str>,
79    heading: &str,
80) -> Option<Value> {
81    if !body::has_section(&d.body, heading) {
82        return None;
83    }
84    let src = structure?;
85    let schema = crate::mdprism::parse_schema(src).ok()?;
86    let content = body::section(&d.body, heading);
87    let data = schema.extract(&content).ok()?;
88    let count = count_array_items(&data);
89    if count == 0 {
90        return None;
91    }
92    Some(json!({ "kind": "structured", "items": count }))
93}
94
95/// Project one document to the JSON object the corpus tables are built from.
96fn doc_json(pcfg: &ProjectConfig, d: &Doc) -> Value {
97    let id = d.id().unwrap_or("");
98    let tname = d.id().and_then(|i| pcfg.type_name_for_id(i));
99    let tags = d.frontmatter.tags().unwrap_or_default();
100
101    let mut fields = Map::new();
102    let mut sections = Map::new();
103    if let Some(tn) = tname {
104        let t = &pcfg.types[tn];
105        for fname in t.fields.keys() {
106            if let Some(v) = d.frontmatter.get(fname) {
107                fields.insert(fname.clone(), yaml_to_json(v));
108            }
109        }
110        for sec in &t.sections {
111            let s = if sec.kind == SectionKind::Structured {
112                structured_section_json(d, sec.structure.as_deref(), &sec.heading)
113            } else {
114                section_json(d, sec.kind, &sec.heading)
115            };
116            if let Some(s) = s {
117                sections.insert(sec.heading.clone(), s);
118            }
119        }
120    }
121
122    json!({
123        "id": id,
124        "num": if id.is_empty() { 0 } else { crate::refs::id_number(id) },
125        "type": tname,
126        "status": d.status(),
127        "title": d.title,
128        "created": d.frontmatter.get_str("created"),
129        "updated": d.frontmatter.get_str("updated"),
130        "tags": tags,
131        "fields": Value::Object(fields),
132        "sections": Value::Object(sections),
133    })
134}
135
136/// The corpus projection: a JSON array with one object per live document. This
137/// is the intermediate the relational tables are materialized from.
138pub fn corpus_json(pcfg: &ProjectConfig, docs: &[&Doc]) -> Value {
139    Value::Array(docs.iter().map(|d| doc_json(pcfg, d)).collect())
140}
141
142/// A SQL single-quoted string literal (doubles embedded quotes).
143fn sql_lit(s: &str) -> String {
144    format!("'{}'", s.replace('\'', "''"))
145}
146
147/// A SQL literal for an optional string: the quoted text, or `NULL`.
148fn opt_lit(v: Option<&str>) -> String {
149    v.map(sql_lit).unwrap_or_else(|| "NULL".to_string())
150}
151
152/// Stringify a scalar JSON value for a `fields`/text column (objects/arrays are
153/// compacted to JSON text; that shape is rare for frontmatter scalars).
154pub(crate) fn json_scalar(v: &Value) -> String {
155    match v {
156        Value::String(s) => s.clone(),
157        Value::Null => String::new(),
158        Value::Bool(b) => b.to_string(),
159        Value::Number(n) => n.to_string(),
160        other => other.to_string(),
161    }
162}
163
164/// Build the `CREATE TABLE` + `INSERT` DDL/DML that materializes the corpus into
165/// the four relational tables the stats SQL queries. Empty tables get no INSERT.
166fn materialize(corpus: &Value) -> String {
167    let mut docs_rows: Vec<String> = Vec::new();
168    let mut tag_rows: Vec<String> = Vec::new();
169    let mut section_rows: Vec<String> = Vec::new();
170    let mut field_rows: Vec<String> = Vec::new();
171
172    for d in corpus.as_array().into_iter().flatten() {
173        let id = d["id"].as_str().unwrap_or("");
174        let num = d["num"].as_i64().unwrap_or(0);
175        docs_rows.push(format!(
176            "({}, {}, {}, {}, {}, {}, {})",
177            sql_lit(id),
178            num,
179            opt_lit(d["type"].as_str()),
180            opt_lit(d["status"].as_str()),
181            sql_lit(d["title"].as_str().unwrap_or("")),
182            opt_lit(d["created"].as_str()),
183            opt_lit(d["updated"].as_str()),
184        ));
185
186        for t in d["tags"].as_array().into_iter().flatten() {
187            if let Some(tag) = t.as_str() {
188                let (key, value) = split_tag(tag);
189                tag_rows.push(format!(
190                    "({}, {}, {}, {})",
191                    sql_lit(id),
192                    sql_lit(tag),
193                    sql_lit(key),
194                    opt_lit(value),
195                ));
196            }
197        }
198
199        if let Some(obj) = d["sections"].as_object() {
200            for (heading, s) in obj {
201                section_rows.push(format!(
202                    "({}, {}, {}, {}, {})",
203                    sql_lit(id),
204                    sql_lit(heading),
205                    opt_lit(s["kind"].as_str()),
206                    s["items"].as_i64().unwrap_or(0),
207                    s["unchecked"].as_i64().unwrap_or(0),
208                ));
209            }
210        }
211
212        if let Some(obj) = d["fields"].as_object() {
213            for (key, v) in obj {
214                // A list field contributes one row per element; a scalar, one row.
215                match v {
216                    Value::Array(items) => {
217                        for it in items {
218                            field_rows.push(format!(
219                                "({}, {}, {})",
220                                sql_lit(id),
221                                sql_lit(key),
222                                sql_lit(&json_scalar(it)),
223                            ));
224                        }
225                    }
226                    _ => field_rows.push(format!(
227                        "({}, {}, {})",
228                        sql_lit(id),
229                        sql_lit(key),
230                        sql_lit(&json_scalar(v)),
231                    )),
232                }
233            }
234        }
235    }
236
237    let mut sql = String::new();
238    sql.push_str(
239        "CREATE TABLE docs (id TEXT, num INTEGER, type TEXT, status TEXT, title TEXT, created TEXT, updated TEXT);\n",
240    );
241    sql.push_str("CREATE TABLE tags (doc_id TEXT, tag TEXT, key TEXT, value TEXT);\n");
242    sql.push_str(
243        "CREATE TABLE sections (doc_id TEXT, heading TEXT, kind TEXT, items INTEGER, unchecked INTEGER);\n",
244    );
245    sql.push_str("CREATE TABLE fields (doc_id TEXT, key TEXT, value TEXT);\n");
246
247    let mut insert = |table: &str, rows: &[String]| {
248        if !rows.is_empty() {
249            sql.push_str(&format!(
250                "INSERT INTO {table} VALUES {};\n",
251                rows.join(", ")
252            ));
253        }
254    };
255    insert("docs", &docs_rows);
256    insert("tags", &tag_rows);
257    insert("sections", &section_rows);
258    insert("fields", &field_rows);
259    sql
260}
261
262/// Render one GlueSQL value as a markdown-table cell string.
263pub(crate) fn cell(v: &GlueValue) -> String {
264    match v {
265        GlueValue::Str(s) => s.clone(),
266        GlueValue::Bool(b) => b.to_string(),
267        GlueValue::I8(n) => n.to_string(),
268        GlueValue::I16(n) => n.to_string(),
269        GlueValue::I32(n) => n.to_string(),
270        GlueValue::I64(n) => n.to_string(),
271        GlueValue::I128(n) => n.to_string(),
272        GlueValue::U8(n) => n.to_string(),
273        GlueValue::U16(n) => n.to_string(),
274        GlueValue::U32(n) => n.to_string(),
275        GlueValue::U64(n) => n.to_string(),
276        GlueValue::U128(n) => n.to_string(),
277        GlueValue::F32(x) => fmt_float(*x as f64),
278        GlueValue::F64(x) => fmt_float(*x),
279        GlueValue::Null => String::new(),
280        other => format!("{other:?}"),
281    }
282}
283
284/// Format a float without a trailing `.0` (so `ROUND(...)` reads as `67`, not
285/// `67.0`), but keep real fractions.
286fn fmt_float(x: f64) -> String {
287    if x.is_finite() && x.fract() == 0.0 {
288        format!("{}", x as i64)
289    } else {
290        format!("{x}")
291    }
292}
293
294/// Escape a cell for a GFM table (pipes and newlines would break the row).
295fn esc(s: &str) -> String {
296    s.replace('|', "\\|").replace('\n', " ")
297}
298
299/// An in-memory corpus database. Built once per `opys stats` / `verify` run and
300/// reused across every stat — materializing (and re-parsing the INSERT DML) once
301/// per stat would be needless O(stats × corpus) work.
302pub type CorpusDb = Glue<MemoryStorage>;
303
304/// Materialize `corpus` into a fresh in-memory database (the four corpus tables).
305pub fn build_db(corpus: &Value) -> std::result::Result<CorpusDb, String> {
306    let setup = materialize(corpus);
307    let mut glue = Glue::new(MemoryStorage::default());
308    block_on(glue.execute(&setup)).map_err(|e| format!("corpus build failed ({e})"))?;
309    Ok(glue)
310}
311
312/// Run one stat's `sql` over an already-built corpus DB, returning the SELECT
313/// result as (column labels, string rows). `Err` carries a human-readable
314/// problem (a query that fails to run, or is not a SELECT).
315fn run_sql(
316    db: &mut CorpusDb,
317    sql: &str,
318) -> std::result::Result<(Vec<String>, Vec<Vec<String>>), String> {
319    let payloads = block_on(db.execute(sql)).map_err(|e| format!("query failed ({e})"))?;
320    let last = payloads
321        .last()
322        .ok_or_else(|| "query produced no result set".to_string())?;
323    match last {
324        Payload::Select { labels, rows } => {
325            let rows = rows.iter().map(|r| r.iter().map(cell).collect()).collect();
326            Ok((labels.clone(), rows))
327        }
328        other => Err(format!(
329            "query must end in a SELECT (got {})",
330            payload_kind(other)
331        )),
332    }
333}
334
335fn payload_kind(p: &Payload) -> &'static str {
336    match p {
337        Payload::Select { .. } | Payload::SelectMap(_) => "a projection",
338        Payload::Insert(_) => "INSERT",
339        Payload::Update(_) => "UPDATE",
340        Payload::Delete(_) => "DELETE",
341        _ => "a non-select statement",
342    }
343}
344
345/// A markdown table (with an `## name` heading) for a stat's result set. An
346/// empty result renders as a note rather than a malformed (header-only) table.
347pub(crate) fn render_table(name: &str, labels: &[String], rows: &[Vec<String>]) -> String {
348    format!("## {name}\n\n{}", table_body(labels, rows))
349}
350
351/// The bare markdown table (no heading) — shared with `opys query`.
352pub(crate) fn table_body(labels: &[String], rows: &[Vec<String>]) -> String {
353    let mut out = String::new();
354    if labels.is_empty() || rows.is_empty() {
355        out.push_str("_(no rows)_\n");
356        return out;
357    }
358    let header: Vec<String> = labels.iter().map(|l| esc(l)).collect();
359    out.push_str(&format!("| {} |\n", header.join(" | ")));
360    out.push_str(&format!("| {} |\n", vec!["---"; labels.len()].join(" | ")));
361    for r in rows {
362        let cells: Vec<String> = (0..labels.len())
363            .map(|i| esc(r.get(i).map(String::as_str).unwrap_or("")))
364            .collect();
365        out.push_str(&format!("| {} |\n", cells.join(" | ")));
366    }
367    out
368}
369
370/// Print rendered markdown: styled for a terminal, raw when piped, `plain`,
371/// or `NO_COLOR` — shared by `stats` and `query`.
372pub(crate) fn print_markdown(out: &str, plain: bool) {
373    let styled =
374        !plain && std::io::stdout().is_terminal() && std::env::var_os("NO_COLOR").is_none();
375    if styled {
376        termimad::MadSkin::default().print_text(out);
377    } else {
378        println!("{out}");
379    }
380}
381
382/// Expand a stat `template`: substitute `{column}` with `resolve(column)`,
383/// treating `{{`/`}}` as literal braces. `Err` on an unclosed/unmatched brace or
384/// a placeholder `resolve` rejects (an unknown column).
385fn expand_template(
386    tmpl: &str,
387    resolve: &dyn Fn(&str) -> Option<String>,
388) -> std::result::Result<String, String> {
389    let mut out = String::new();
390    let mut chars = tmpl.chars().peekable();
391    while let Some(c) = chars.next() {
392        match c {
393            '{' if chars.peek() == Some(&'{') => {
394                chars.next();
395                out.push('{');
396            }
397            '}' if chars.peek() == Some(&'}') => {
398                chars.next();
399                out.push('}');
400            }
401            '{' => {
402                let mut name = String::new();
403                loop {
404                    match chars.next() {
405                        Some('}') => break,
406                        Some(ch) => name.push(ch),
407                        None => return Err("unclosed '{' in template".to_string()),
408                    }
409                }
410                let key = name.trim();
411                match resolve(key) {
412                    Some(v) => out.push_str(&v),
413                    None => return Err(format!("template references unknown column '{{{key}}}'")),
414                }
415            }
416            '}' => return Err("unmatched '}' in template (use '}}' for a literal)".to_string()),
417            _ => out.push(c),
418        }
419    }
420    Ok(out)
421}
422
423/// Render a stat's result set through its row `template` (each row → one rendered
424/// block, joined under the `## name` heading). Validates placeholders against the
425/// column labels first, so an unknown `{column}` fails even with zero rows.
426fn render_templated(
427    name: &str,
428    labels: &[String],
429    rows: &[Vec<String>],
430    tmpl: &str,
431) -> std::result::Result<String, String> {
432    let known = |k: &str| labels.iter().any(|l| l == k);
433    // Column/brace check independent of row count (catches config errors early).
434    expand_template(tmpl, &|k| known(k).then(String::new))?;
435
436    let mut out = format!("## {name}\n\n");
437    if rows.is_empty() {
438        out.push_str("_(no rows)_\n");
439        return Ok(out);
440    }
441    for row in rows {
442        let cell_for = |k: &str| {
443            labels
444                .iter()
445                .position(|l| l == k)
446                .map(|i| row.get(i).cloned().unwrap_or_default())
447        };
448        out.push_str(&expand_template(tmpl, &cell_for)?);
449        out.push('\n');
450    }
451    Ok(out)
452}
453
454/// Run one stat against an already-built corpus DB and format the result — as a
455/// markdown table, or through the stat's row `template` when set. Returns a
456/// human-readable problem (prefixed with the stat name) on failure. Used by
457/// `render_all` and by `verify`.
458pub fn render_stat_on(db: &mut CorpusDb, spec: &StatSpec) -> std::result::Result<String, String> {
459    let (labels, rows) =
460        run_sql(db, &spec.sql).map_err(|e| format!("stats '{}': {e}", spec.name))?;
461    match &spec.template {
462        Some(tmpl) => render_templated(&spec.name, &labels, &rows, tmpl)
463            .map_err(|e| format!("stats '{}': {e}", spec.name)),
464        None => Ok(render_table(&spec.name, &labels, &rows)),
465    }
466}
467
468/// Render one stat over `corpus` (builds a DB just for it). Convenience for
469/// config-time validation over an empty corpus.
470pub fn render_stat(spec: &StatSpec, corpus: &Value) -> std::result::Result<String, String> {
471    let mut db = build_db(corpus)?;
472    render_stat_on(&mut db, spec)
473}
474
475/// Render every configured stat over `docs`, concatenated with a blank line
476/// between sections. The corpus DB is built once and reused. `Err` carries the
477/// first failing stat's problem message.
478pub fn render_all(pcfg: &ProjectConfig, docs: &[&Doc]) -> std::result::Result<String, String> {
479    let corpus = corpus_json(pcfg, docs);
480    let mut db = build_db(&corpus)?;
481    let mut sections = Vec::new();
482    for spec in &pcfg.stats {
483        sections.push(render_stat_on(&mut db, spec)?.trim_end().to_string());
484    }
485    Ok(sections.join("\n\n"))
486}
487
488pub fn run(ctx: &Ctx, plain: bool) -> Result<()> {
489    let prj = ctx.open()?;
490    let (docs, _) = ctx.backend.load_docs(&prj);
491    let doc_refs: Vec<&Doc> = docs.iter().collect();
492    let out = render_all(&prj.pcfg, &doc_refs).map_err(usage)?;
493    if out.is_empty() {
494        return Ok(());
495    }
496    print_markdown(&out, plain);
497    Ok(())
498}
opys_engine/commands/stats.rs

opys_engine/commands/
stats.rs
