use super::value::scalar_text;
use serde_json::{Map, Value};
const MOST_FREQUENT_SHOWN: usize = 3;
pub(super) fn profile_of(elements: &[Value], most_frequent_shown: Option<usize>) -> Value {
let mut columns: Vec<String> = Vec::new();
let mut seen = std::collections::BTreeSet::new();
for element in elements {
if let Some(fields) = element.as_object() {
for name in fields.keys() {
if seen.insert(name.clone()) {
columns.push(name.clone());
}
}
}
}
let profiles: Vec<Value> = columns
.iter()
.map(|name| column_profile(name, elements, most_frequent_shown))
.collect();
Value::Object(Map::from_iter([
("rows".to_owned(), Value::from(elements.len())),
("columns".to_owned(), Value::Array(profiles)),
]))
}
fn column_profile(name: &str, elements: &[Value], most_frequent_shown: Option<usize>) -> Value {
let mut absent = 0usize;
let mut present = 0usize;
let mut kinds = std::collections::BTreeSet::new();
let mut tally: std::collections::BTreeMap<String, usize> = std::collections::BTreeMap::new();
for element in elements {
match element.get(name) {
None | Some(Value::Null) => absent += 1,
Some(found) => {
present += 1;
kinds.insert(shape_name(found));
*tally.entry(scalar_text(found).into_owned()).or_default() += 1;
}
}
}
let mut ranked: Vec<(&String, &usize)> = tally.iter().collect();
ranked.sort_by(|left, right| right.1.cmp(left.1).then_with(|| left.0.cmp(right.0)));
let sample: Vec<Value> = ranked
.into_iter()
.take(most_frequent_shown.unwrap_or(MOST_FREQUENT_SHOWN))
.map(|(value, count)| {
Value::Object(Map::from_iter([
("value".to_owned(), Value::from(value.clone())),
("rows".to_owned(), Value::from(*count)),
]))
})
.collect();
Value::Object(Map::from_iter([
("name".to_owned(), Value::from(name)),
(
"kind".to_owned(),
Value::from(kinds.into_iter().collect::<Vec<_>>().join("|")),
),
("present".to_owned(), Value::from(present)),
("absent".to_owned(), Value::from(absent)),
("distinct".to_owned(), Value::from(tally.len())),
("most_frequent".to_owned(), Value::Array(sample)),
]))
}
fn shape_name(value: &Value) -> &'static str {
match value {
Value::Null => "absent",
Value::Bool(_) => "boolean",
Value::Number(_) => "number",
Value::String(_) => "text",
Value::Array(_) => "list",
Value::Object(_) => "structure",
}
}
#[cfg(test)]
mod tests {
use super::super::Pipeline;
use crate::output::envelope::Envelope;
use serde_json::{json, Value};
fn sample() -> Envelope {
Envelope::new(
"schema tables",
json!({"rows": [
{"name": "alpha", "rows": 10, "engine": "InnoDB"},
{"name": "beta", "rows": 2, "engine": "Aria"},
{"name": "gamma", "rows": 30, "engine": "InnoDB"}
]}),
)
}
#[test]
fn a_profile_is_smaller_than_the_set_and_describes_it() {
let pipeline = Pipeline {
summary: true,
..Pipeline::default()
};
let whole = Pipeline::default().apply(sample()).expect("fits");
let out = pipeline.apply(sample()).expect("fits");
assert_eq!(out.data["rows"], 3);
let columns = out.data["columns"].as_array().expect("columns array");
assert_eq!(columns.len(), 3, "one entry per column of the set");
let engine = columns
.iter()
.find(|column| column["name"] == "engine")
.expect("the column is profiled");
assert_eq!(engine["present"], 3);
assert_eq!(engine["absent"], 0);
assert_eq!(engine["distinct"], 2);
assert_eq!(engine["kind"], "text");
assert_eq!(engine["most_frequent"][0]["value"], "InnoDB");
assert_eq!(engine["most_frequent"][0]["rows"], 2);
let _ = whole;
}
#[test]
fn a_profile_does_not_grow_with_the_set_it_describes() {
const DIGITS_AND_SAMPLE: usize = 64;
let profile_of_rows = |count: usize| {
let rows: Vec<Value> = (0..count)
.map(|index| json!({"name": format!("row-{index}"), "engine": "InnoDB"}))
.collect();
let envelope = Envelope::new("query run", json!({ "rows": rows }));
let out = Pipeline {
summary: true,
..Pipeline::default()
}
.apply(envelope)
.expect("fits");
(
serde_json::to_string(&out.data).expect("serialises").len(),
serde_json::to_string(&json!({ "rows": (0..count)
.map(|index| json!({"name": format!("row-{index}"), "engine": "InnoDB"}))
.collect::<Vec<_>>() }))
.expect("serialises")
.len(),
)
};
let (small_profile, _) = profile_of_rows(10);
let (large_profile, large_set) = profile_of_rows(10_000);
assert!(
large_profile < small_profile + DIGITS_AND_SAMPLE,
"a profile follows the columns: {small_profile} bytes for ten rows and \
{large_profile} for ten thousand"
);
assert!(
large_profile < large_set,
"a profile of a set worth summarising must be smaller than the set"
);
}
#[test]
fn a_profile_counts_what_was_absent() {
let envelope = Envelope::new(
"query run",
json!({"rows": [{"a": 1, "b": "x"}, {"a": 2}, {"a": 3, "b": null}]}),
);
let out = Pipeline {
summary: true,
..Pipeline::default()
}
.apply(envelope)
.expect("fits");
let columns = out.data["columns"].as_array().expect("columns array");
let b = columns
.iter()
.find(|column| column["name"] == "b")
.expect("profiled");
assert_eq!(b["present"], 1);
assert_eq!(
b["absent"], 2,
"a missing field and an absent value both count"
);
}
}