mariadb-cli 0.1.1

One-shot agent-native command line client for MariaDB servers
Documentation
//! The profile of a set, per column, instead of the rows themselves.

use super::value::scalar_text;
use serde_json::{Map, Value};

/// How many of the most frequent values a profile shows when nobody has said.
///
/// A profile exists to be read at a glance, and it must not grow with the set
/// it describes — a proof in this file measures exactly that. So the built-in
/// is a small number and `None` means the built-in, never "all of them".
const MOST_FREQUENT_SHOWN: usize = 3;

/// The profile of a set, per column, instead of the rows themselves.
///
/// Every count is over the elements that reached this point, so filtering and
/// paging are already reflected: the profile describes the answer the caller
/// would have received, not the set the server holds.
pub(super) fn profile_of(elements: &[Value], most_frequent_shown: Option<usize>) -> Value {
    // Insertion order, so two runs of the same request profile the columns in
    // the same order and produce the same bytes.
    let mut columns: Vec<String> = Vec::new();
    let mut seen = std::collections::BTreeSet::new();
    for element in elements {
        if let Some(fields) = element.as_object() {
            for name in fields.keys() {
                if seen.insert(name.clone()) {
                    columns.push(name.clone());
                }
            }
        }
    }

    let profiles: Vec<Value> = columns
        .iter()
        .map(|name| column_profile(name, elements, most_frequent_shown))
        .collect();

    Value::Object(Map::from_iter([
        ("rows".to_owned(), Value::from(elements.len())),
        ("columns".to_owned(), Value::Array(profiles)),
    ]))
}

/// The profile of one column across every element.
fn column_profile(name: &str, elements: &[Value], most_frequent_shown: Option<usize>) -> Value {
    let mut absent = 0usize;
    let mut present = 0usize;
    let mut kinds = std::collections::BTreeSet::new();
    let mut tally: std::collections::BTreeMap<String, usize> = std::collections::BTreeMap::new();

    for element in elements {
        match element.get(name) {
            None | Some(Value::Null) => absent += 1,
            Some(found) => {
                present += 1;
                kinds.insert(shape_name(found));
                *tally.entry(scalar_text(found).into_owned()).or_default() += 1;
            }
        }
    }

    // Ordered by how often each value appeared, and by the value itself when
    // two appeared equally often, so the sample never varies between runs.
    let mut ranked: Vec<(&String, &usize)> = tally.iter().collect();
    ranked.sort_by(|left, right| right.1.cmp(left.1).then_with(|| left.0.cmp(right.0)));
    let sample: Vec<Value> = ranked
        .into_iter()
        .take(most_frequent_shown.unwrap_or(MOST_FREQUENT_SHOWN))
        .map(|(value, count)| {
            Value::Object(Map::from_iter([
                ("value".to_owned(), Value::from(value.clone())),
                ("rows".to_owned(), Value::from(*count)),
            ]))
        })
        .collect();

    Value::Object(Map::from_iter([
        ("name".to_owned(), Value::from(name)),
        (
            "kind".to_owned(),
            Value::from(kinds.into_iter().collect::<Vec<_>>().join("|")),
        ),
        ("present".to_owned(), Value::from(present)),
        ("absent".to_owned(), Value::from(absent)),
        ("distinct".to_owned(), Value::from(tally.len())),
        ("most_frequent".to_owned(), Value::Array(sample)),
    ]))
}

/// The published name of the shape one value took.
fn shape_name(value: &Value) -> &'static str {
    match value {
        Value::Null => "absent",
        Value::Bool(_) => "boolean",
        Value::Number(_) => "number",
        Value::String(_) => "text",
        Value::Array(_) => "list",
        Value::Object(_) => "structure",
    }
}

#[cfg(test)]
mod tests {
    use super::super::Pipeline;
    use crate::output::envelope::Envelope;
    use serde_json::{json, Value};

    fn sample() -> Envelope {
        Envelope::new(
            "schema tables",
            json!({"rows": [
                {"name": "alpha", "rows": 10, "engine": "InnoDB"},
                {"name": "beta",  "rows":  2, "engine": "Aria"},
                {"name": "gamma", "rows": 30, "engine": "InnoDB"}
            ]}),
        )
    }

    /// A profile describes the answer rather than reproducing it, so it has to
    /// be smaller than the set and has to carry what the set was made of.
    #[test]
    fn a_profile_is_smaller_than_the_set_and_describes_it() {
        let pipeline = Pipeline {
            summary: true,
            ..Pipeline::default()
        };
        let whole = Pipeline::default().apply(sample()).expect("fits");
        let out = pipeline.apply(sample()).expect("fits");

        assert_eq!(out.data["rows"], 3);
        let columns = out.data["columns"].as_array().expect("columns array");
        assert_eq!(columns.len(), 3, "one entry per column of the set");

        let engine = columns
            .iter()
            .find(|column| column["name"] == "engine")
            .expect("the column is profiled");
        assert_eq!(engine["present"], 3);
        assert_eq!(engine["absent"], 0);
        assert_eq!(engine["distinct"], 2);
        assert_eq!(engine["kind"], "text");
        assert_eq!(engine["most_frequent"][0]["value"], "InnoDB");
        assert_eq!(engine["most_frequent"][0]["rows"], 2);

        // On three rows a profile is naturally larger than the set it
        // describes, and saying otherwise would be arranging the test to agree
        // with the requirement. What actually has to hold is below.
        let _ = whole;
    }

    /// The requirement is that a profile is smaller than the set, and what
    /// makes that true is that its size follows the number of COLUMNS and not
    /// the number of rows. Asserting it on a three-row set would prove nothing,
    /// so it is asserted where it means something.
    #[test]
    fn a_profile_does_not_grow_with_the_set_it_describes() {
        /// What a profile may grow by between ten rows and ten thousand.
        const DIGITS_AND_SAMPLE: usize = 64;

        let profile_of_rows = |count: usize| {
            let rows: Vec<Value> = (0..count)
                .map(|index| json!({"name": format!("row-{index}"), "engine": "InnoDB"}))
                .collect();
            let envelope = Envelope::new("query run", json!({ "rows": rows }));
            let out = Pipeline {
                summary: true,
                ..Pipeline::default()
            }
            .apply(envelope)
            .expect("fits");
            (
                serde_json::to_string(&out.data).expect("serialises").len(),
                serde_json::to_string(&json!({ "rows": (0..count)
                    .map(|index| json!({"name": format!("row-{index}"), "engine": "InnoDB"}))
                    .collect::<Vec<_>>() }))
                .expect("serialises")
                .len(),
            )
        };

        let (small_profile, _) = profile_of_rows(10);
        let (large_profile, large_set) = profile_of_rows(10_000);

        // Not byte-identical, and it must not be claimed to be: the counts
        // themselves grow a few digits, and the sample carries longer values.
        // What matters is that a thousand times the rows costs a handful of
        // bytes rather than a thousand times the bytes.
        assert!(
            large_profile < small_profile + DIGITS_AND_SAMPLE,
            "a profile follows the columns: {small_profile} bytes for ten rows and \
             {large_profile} for ten thousand"
        );
        assert!(
            large_profile < large_set,
            "a profile of a set worth summarising must be smaller than the set"
        );
    }

    /// A column absent from some rows must be counted as absent rather than
    /// silently ignored, or a caller would read a partial column as a full one.
    #[test]
    fn a_profile_counts_what_was_absent() {
        let envelope = Envelope::new(
            "query run",
            json!({"rows": [{"a": 1, "b": "x"}, {"a": 2}, {"a": 3, "b": null}]}),
        );
        let out = Pipeline {
            summary: true,
            ..Pipeline::default()
        }
        .apply(envelope)
        .expect("fits");
        let columns = out.data["columns"].as_array().expect("columns array");
        let b = columns
            .iter()
            .find(|column| column["name"] == "b")
            .expect("profiled");
        assert_eq!(b["present"], 1);
        assert_eq!(
            b["absent"], 2,
            "a missing field and an absent value both count"
        );
    }
}