faucet-cli 1.13.0

//! Pure case generation for a template test suite (#648).
//!
//! Everything here is a function of the suite spec plus the template's
//! `params:` declaration — no I/O, no template store, no pipeline. That is
//! deliberate: the interesting failure modes (a cartesian explosion, an
//! `exclude` that matches nothing, an all-pairs reduction that drops a pair)
//! are all decidable from the inputs, and a pure generator lets them be tested
//! without materializing a single config.

use serde_json::Value;
use std::collections::{BTreeMap, BTreeSet};

use super::spec::{Auto, Combine, Expect, MAX_CASES, Suite};
use crate::error::{CliError, CliResult};
use crate::params::ParamsSpec;

/// One case to run, after generation.
#[derive(Debug, Clone, PartialEq)]
pub struct GeneratedCase {
    pub name: String,
    pub params: BTreeMap<String, Value>,
    pub expect: Expect,
    /// Where the case came from, for the report — an operator seeing a red
    /// `auto` case should not have to guess why a case they never wrote exists.
    pub origin: Origin,
}

/// How a case came to exist.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Origin {
    /// Written out in `cases:`.
    Explicit,
    /// Generated by `combine:`.
    Combined,
    /// Derived from the template's `params:` by `auto:`.
    Auto,
}

impl Origin {
    pub fn as_str(self) -> &'static str {
        match self {
            Self::Explicit => "explicit",
            Self::Combined => "combine",
            Self::Auto => "auto",
        }
    }
}

/// Build the ordered case list: explicit, then generated, then derived.
///
/// Order is stable (and the generated part is sorted) so a CI diff shows a
/// real change rather than map iteration order.
pub fn generate(suite: &Suite, params: &ParamsSpec) -> CliResult<Vec<GeneratedCase>> {
    let mut out: Vec<GeneratedCase> = suite
        .cases
        .iter()
        .map(|c| GeneratedCase {
            name: c.name.clone(),
            params: c.params.clone(),
            expect: c.expect.clone(),
            origin: Origin::Explicit,
        })
        .collect();

    if let Some(cb) = &suite.combine {
        out.extend(from_combine(cb, params)?);
    }
    if let Some(auto) = &suite.auto {
        out.extend(from_auto(auto, params));
    }

    if out.len() > MAX_CASES {
        return Err(CliError::Config(format!(
            "template test suite generated {} cases, over the {MAX_CASES} ceiling. Narrow \
             `combine.params`, add `exclude:` entries, or set `pairwise: true` (all-pairs keeps \
             the two-param interactions that hold most bugs). This is an error rather than a \
             truncation on purpose: a report covering part of the space would read green.",
            out.len()
        )));
    }

    // A duplicate name makes `--filter` ambiguous and the report unreadable.
    let mut seen = BTreeSet::new();
    for c in &out {
        if !seen.insert(c.name.clone()) {
            return Err(CliError::Config(format!(
                "template test suite: duplicate case name '{}'",
                c.name
            )));
        }
    }
    Ok(out)
}

/// Cartesian product (or all-pairs), minus the excluded combinations.
///
/// Required params the sweep does not name are filled with a placeholder, the
/// same way `auto:` fills them. Without that, a `combine:` over any template
/// with a required param would report every case red for a reason that has
/// nothing to do with the axes under test.
fn from_combine(cb: &Combine, params: &ParamsSpec) -> CliResult<Vec<GeneratedCase>> {
    let axes: Vec<(&String, &Vec<Value>)> = cb.params.iter().collect();
    let full = cartesian(&axes);
    let selected = if cb.pairwise {
        all_pairs(&axes, full)
    } else {
        full
    };

    let kept: Vec<BTreeMap<String, Value>> = selected
        .into_iter()
        .filter(|combo| !cb.exclude.iter().any(|ex| matches_partial(combo, ex)))
        .collect();

    // An `exclude` that matches nothing is almost always a typo in a param
    // name or value, and it silently widens the tested space rather than
    // narrowing it — the opposite of what the author asked for.
    for ex in &cb.exclude {
        if !ex.is_empty() && !cb.params.keys().any(|k| ex.contains_key(k)) {
            return Err(CliError::Config(format!(
                "template test suite: `exclude` entry {} names no param in `combine.params` — \
                 it can never match",
                serde_json::to_string(ex).unwrap_or_default()
            )));
        }
    }

    let filler: BTreeMap<String, Value> = params
        .iter()
        .filter(|(n, p)| p.required && p.computed.is_none() && !cb.params.contains_key(*n))
        .map(|(n, p)| (n.clone(), placeholder_for(p)))
        .collect();

    Ok(kept
        .into_iter()
        .map(|combo| {
            // The name shows only the swept axes — the filler is scaffolding,
            // not part of what the case asserts.
            let name = combo_name(&combo);
            let mut params = filler.clone();
            params.extend(combo);
            GeneratedCase {
                name,
                params,
                expect: cb.expect.clone(),
                origin: Origin::Combined,
            }
        })
        .collect())
}

/// Full cartesian product, in a deterministic order.
fn cartesian(axes: &[(&String, &Vec<Value>)]) -> Vec<BTreeMap<String, Value>> {
    let mut out: Vec<BTreeMap<String, Value>> = vec![BTreeMap::new()];
    for (name, values) in axes {
        let mut next = Vec::with_capacity(out.len() * values.len());
        for base in &out {
            for v in values.iter() {
                let mut c = base.clone();
                c.insert((*name).clone(), v.clone());
                next.push(c);
            }
        }
        out = next;
    }
    out
}

/// Greedy all-pairs reduction: keep the fewest combinations that still cover
/// every (param-a=value, param-b=value) pair.
///
/// Greedy rather than optimal because the optimal set is NP-hard to find and
/// the difference is a handful of cases — while the *coverage* property, which
/// is what the suite promises, is exact either way (asserted in the tests).
fn all_pairs(
    axes: &[(&String, &Vec<Value>)],
    full: Vec<BTreeMap<String, Value>>,
) -> Vec<BTreeMap<String, Value>> {
    if axes.len() < 2 {
        return full;
    }
    // Every pair that must be covered.
    let mut needed: BTreeSet<(String, String, String, String)> = BTreeSet::new();
    for i in 0..axes.len() {
        for j in (i + 1)..axes.len() {
            for a in axes[i].1 {
                for b in axes[j].1 {
                    needed.insert((axes[i].0.clone(), key_of(a), axes[j].0.clone(), key_of(b)));
                }
            }
        }
    }

    let mut chosen: Vec<BTreeMap<String, Value>> = Vec::new();
    while !needed.is_empty() {
        // Pick the candidate covering the most still-needed pairs. Ties break
        // on the first candidate, which keeps the output deterministic.
        let best = full
            .iter()
            .max_by_key(|combo| pairs_of(combo).intersection(&needed).count())
            .cloned();
        let Some(best) = best else { break };
        let covered = pairs_of(&best);
        if covered.intersection(&needed).count() == 0 {
            break;
        }
        for p in covered {
            needed.remove(&p);
        }
        chosen.push(best);
    }
    chosen
}

/// The pairs one combination covers.
fn pairs_of(combo: &BTreeMap<String, Value>) -> BTreeSet<(String, String, String, String)> {
    let entries: Vec<(&String, &Value)> = combo.iter().collect();
    let mut out = BTreeSet::new();
    for i in 0..entries.len() {
        for j in (i + 1)..entries.len() {
            out.insert((
                entries[i].0.clone(),
                key_of(entries[i].1),
                entries[j].0.clone(),
                key_of(entries[j].1),
            ));
        }
    }
    out
}

/// A stable string key for a param value, so pairs compare structurally
/// (`1` and `1.0` are different values and stay different pairs).
fn key_of(v: &Value) -> String {
    match v {
        Value::String(s) => s.clone(),
        other => other.to_string(),
    }
}

/// Whether `combo` matches every key/value in the partial assignment `part`.
fn matches_partial(combo: &BTreeMap<String, Value>, part: &BTreeMap<String, Value>) -> bool {
    part.iter()
        .all(|(k, v)| combo.get(k).is_some_and(|got| key_of(got) == key_of(v)))
}

/// `sink=jsonl,object=Account` — readable in a report and stable across runs.
fn combo_name(params: &BTreeMap<String, Value>) -> String {
    params
        .iter()
        .map(|(k, v)| format!("{k}={}", key_of(v)))
        .collect::<Vec<_>>()
        .join(",")
}

/// Cases derived from the template's own `params:` declaration.
fn from_auto(auto: &Auto, params: &ParamsSpec) -> Vec<GeneratedCase> {
    let mut out = Vec::new();

    if auto.defaults_baseline {
        // Every required param still has to be supplied — the baseline is
        // "everything else defaulted", not "nothing supplied", which would
        // just be the required-omitted case wearing a different name.
        let mut supplied = BTreeMap::new();
        for (name, p) in params {
            if p.required && p.computed.is_none() {
                supplied.insert(name.clone(), placeholder_for(p));
            }
        }
        out.push(GeneratedCase {
            name: "auto:defaults".into(),
            params: supplied,
            expect: Expect::default(),
            origin: Origin::Auto,
        });
    }

    if auto.enum_coverage {
        let required: BTreeMap<String, Value> = params
            .iter()
            .filter(|(_, p)| p.required && p.computed.is_none())
            .map(|(n, p)| (n.clone(), placeholder_for(p)))
            .collect();
        for (name, p) in params {
            if p.computed.is_some() {
                continue;
            }
            for v in &p.values {
                let mut supplied = required.clone();
                supplied.insert(name.clone(), v.clone());
                out.push(GeneratedCase {
                    name: format!("auto:{name}={}", key_of(v)),
                    params: supplied,
                    expect: Expect::default(),
                    origin: Origin::Auto,
                });
            }
        }
    }

    if auto.required_omitted {
        let required: Vec<&String> = params
            .iter()
            .filter(|(_, p)| p.required && p.computed.is_none())
            .map(|(n, _)| n)
            .collect();
        for omitted in &required {
            let supplied: BTreeMap<String, Value> = params
                .iter()
                .filter(|(n, p)| p.required && p.computed.is_none() && n != omitted)
                .map(|(n, p)| (n.clone(), placeholder_for(p)))
                .collect();
            out.push(GeneratedCase {
                name: format!("auto:missing-{omitted}"),
                params: supplied,
                expect: Expect {
                    valid: false,
                    // The bind reports a missing required param by name, so
                    // the assertion can be specific rather than "it failed
                    // somehow" — which would also pass on an unrelated break.
                    error: Some((*omitted).clone()),
                },
                origin: Origin::Auto,
            });
        }
    }

    out
}

/// A type-shaped stand-in for a required param with no declared values.
///
/// Prefers a declared value when there is one: a param with a closed `values:`
/// set would reject an invented string, turning every derived case into a
/// false failure.
fn placeholder_for(p: &crate::params::ParamSpec) -> Value {
    if let Some(first) = p.values.first() {
        return first.clone();
    }
    p.kind.placeholder()
}

#[cfg(test)]
mod tests {
    use super::super::spec::Case;
    use super::*;
    use crate::params::{ParamSpec, ParamType};
    use serde_json::json;

    fn combine(params: &[(&str, Vec<Value>)], pairwise: bool) -> Combine {
        Combine {
            params: params
                .iter()
                .map(|(k, v)| ((*k).to_string(), v.clone()))
                .collect(),
            exclude: Vec::new(),
            pairwise,
            expect: Expect::default(),
        }
    }

    fn suite_of(cb: Combine) -> Suite {
        Suite {
            cases: Vec::new(),
            combine: Some(cb),
            auto: None,
            behavioral: Vec::new(),
        }
    }

    #[test]
    fn cartesian_covers_every_combination() {
        let cb = combine(
            &[
                ("sink", vec![json!("jsonl"), json!("bigquery")]),
                ("object", vec![json!("A"), json!("B"), json!("C")]),
            ],
            false,
        );
        let cases = generate(&suite_of(cb), &ParamsSpec::new()).expect("generate");
        assert_eq!(cases.len(), 6, "2 × 3");
        assert!(cases.iter().all(|c| c.origin == Origin::Combined));
        // Names are stable and readable.
        assert!(cases.iter().any(|c| c.name == "object=A,sink=jsonl"));
    }

    #[test]
    fn combine_fills_required_params_it_does_not_sweep() {
        // Otherwise every generated case fails on a missing required param —
        // a red suite that says nothing about the axes under test.
        let mut spec = ParamsSpec::new();
        spec.insert(
            "tenant_id".into(),
            ParamSpec {
                kind: ParamType::String,
                required: true,
                default: None,
                secret: false,
                description: None,
                computed: None,
                values: Vec::new(),
            },
        );
        let cb = combine(&[("region", vec![json!("us"), json!("eu")])], false);
        let cases = generate(&suite_of(cb), &spec).expect("generate");
        assert_eq!(cases.len(), 2);
        for c in &cases {
            assert!(
                c.params.contains_key("tenant_id"),
                "required param must be filled: {:?}",
                c.params
            );
            assert!(
                !c.name.contains("tenant_id"),
                "the filler is scaffolding, not an axis: {}",
                c.name
            );
        }
    }

    #[test]
    fn exclude_drops_only_the_matching_combinations() {
        let mut cb = combine(
            &[
                ("sink", vec![json!("jsonl"), json!("bigquery")]),
                ("object", vec![json!("A"), json!("B")]),
            ],
            false,
        );
        cb.exclude = vec![
            [("sink".to_string(), json!("bigquery"))]
                .into_iter()
                .collect(),
        ];
        let cases = generate(&suite_of(cb), &ParamsSpec::new()).expect("generate");
        assert_eq!(cases.len(), 2, "both bigquery rows dropped");
        assert!(cases.iter().all(|c| c.params["sink"] == json!("jsonl")));
    }

    #[test]
    fn an_exclude_that_can_never_match_is_an_error() {
        // Almost always a typo'd param name — and it silently *widens* the
        // tested space rather than narrowing it, which is the opposite of
        // what the author asked for.
        let mut cb = combine(&[("sink", vec![json!("jsonl")])], false);
        cb.exclude = vec![[("snik".to_string(), json!("jsonl"))].into_iter().collect()];
        let err = generate(&suite_of(cb), &ParamsSpec::new()).expect_err("typo");
        assert!(err.to_string().contains("never match"), "{err}");
    }

    #[test]
    fn pairwise_covers_every_pair_with_fewer_cases() {
        let axes = [
            ("a", vec![json!(1), json!(2), json!(3)]),
            ("b", vec![json!("x"), json!("y"), json!("z")]),
            ("c", vec![json!(true), json!(false)]),
        ];
        let full = generate(&suite_of(combine(&axes, false)), &ParamsSpec::new()).unwrap();
        let paired = generate(&suite_of(combine(&axes, true)), &ParamsSpec::new()).unwrap();
        assert_eq!(full.len(), 18);
        assert!(
            paired.len() < full.len(),
            "all-pairs must reduce the count: {} vs {}",
            paired.len(),
            full.len()
        );

        // The property that matters: every pair is still covered.
        let mut covered = BTreeSet::new();
        for c in &paired {
            covered.extend(pairs_of(&c.params));
        }
        for c in &full {
            for p in pairs_of(&c.params) {
                assert!(covered.contains(&p), "all-pairs dropped the pair {p:?}");
            }
        }
    }

    #[test]
    fn pairwise_is_a_no_op_below_two_params() {
        let cases = generate(
            &suite_of(combine(&[("a", vec![json!(1), json!(2)])], true)),
            &ParamsSpec::new(),
        )
        .unwrap();
        assert_eq!(cases.len(), 2, "one axis has no pairs to reduce");
    }

    fn params_spec() -> ParamsSpec {
        let mut spec = ParamsSpec::new();
        spec.insert(
            "tenant".into(),
            ParamSpec {
                kind: ParamType::String,
                required: true,
                default: None,
                secret: false,
                description: None,
                computed: None,
                values: Vec::new(),
            },
        );
        spec.insert(
            "sink".into(),
            ParamSpec {
                kind: ParamType::String,
                required: false,
                default: Some(json!("jsonl")),
                secret: false,
                description: None,
                computed: None,
                values: vec![json!("jsonl"), json!("bigquery")],
            },
        );
        spec
    }

    #[test]
    fn auto_enum_coverage_walks_every_declared_value() {
        let suite = Suite {
            cases: Vec::new(),
            combine: None,
            auto: Some(Auto {
                enum_coverage: true,
                required_omitted: false,
                defaults_baseline: false,
            }),
            behavioral: Vec::new(),
        };
        let cases = generate(&suite, &params_spec()).expect("generate");
        let names: Vec<&str> = cases.iter().map(|c| c.name.as_str()).collect();
        assert!(names.contains(&"auto:sink=jsonl"), "{names:?}");
        assert!(names.contains(&"auto:sink=bigquery"), "{names:?}");
        // Required params are still supplied, or every derived case would fail
        // for a reason that has nothing to do with the value under test.
        assert!(cases.iter().all(|c| c.params.contains_key("tenant")));
    }

    #[test]
    fn auto_required_omitted_expects_a_named_failure() {
        let suite = Suite {
            cases: Vec::new(),
            combine: None,
            auto: Some(Auto {
                enum_coverage: false,
                required_omitted: true,
                defaults_baseline: false,
            }),
            behavioral: Vec::new(),
        };
        let cases = generate(&suite, &params_spec()).expect("generate");
        assert_eq!(cases.len(), 1, "one required param");
        let c = &cases[0];
        assert_eq!(c.name, "auto:missing-tenant");
        assert!(!c.params.contains_key("tenant"));
        assert!(c.expect.expects_failure());
        assert_eq!(
            c.expect.error.as_deref(),
            Some("tenant"),
            "the assertion names the param, so an unrelated break does not pass it"
        );
    }

    #[test]
    fn auto_defaults_baseline_supplies_only_the_required() {
        let suite = Suite {
            cases: Vec::new(),
            combine: None,
            auto: Some(Auto {
                enum_coverage: false,
                required_omitted: false,
                defaults_baseline: true,
            }),
            behavioral: Vec::new(),
        };
        let cases = generate(&suite, &params_spec()).expect("generate");
        assert_eq!(cases.len(), 1);
        assert_eq!(cases[0].name, "auto:defaults");
        assert!(cases[0].params.contains_key("tenant"));
        assert!(
            !cases[0].params.contains_key("sink"),
            "an optional param must be left to its default — that is the case"
        );
    }

    #[test]
    fn a_required_param_with_declared_values_uses_one_of_them() {
        // An invented placeholder would be rejected by the param's own
        // `values:` constraint, turning every derived case into a false red.
        let mut spec = ParamsSpec::new();
        spec.insert(
            "region".into(),
            ParamSpec {
                kind: ParamType::String,
                required: true,
                default: None,
                secret: false,
                description: None,
                computed: None,
                values: vec![json!("us"), json!("eu")],
            },
        );
        let suite = Suite {
            cases: Vec::new(),
            combine: None,
            auto: Some(Auto {
                enum_coverage: false,
                required_omitted: false,
                defaults_baseline: true,
            }),
            behavioral: Vec::new(),
        };
        let cases = generate(&suite, &spec).expect("generate");
        assert_eq!(cases[0].params["region"], json!("us"));
    }

    #[test]
    fn a_computed_param_is_never_supplied() {
        // Supplying one is refused server-side, so a derived case that did
        // would fail for a reason unrelated to what it tests.
        let mut spec = ParamsSpec::new();
        spec.insert(
            "domain".into(),
            ParamSpec {
                kind: ParamType::String,
                required: false,
                default: None,
                secret: false,
                description: None,
                computed: Some("${map:region|us=com|*=eu}".into()),
                values: Vec::new(),
            },
        );
        let suite = Suite {
            cases: Vec::new(),
            combine: None,
            auto: Some(Auto {
                enum_coverage: true,
                required_omitted: true,
                defaults_baseline: true,
            }),
            behavioral: Vec::new(),
        };
        let cases = generate(&suite, &spec).expect("generate");
        assert!(cases.iter().all(|c| !c.params.contains_key("domain")));
    }

    #[test]
    fn an_explosion_is_an_error_not_a_truncation() {
        // Silently testing part of the space reports green, which is worse
        // than reporting nothing.
        let big: Vec<Value> = (0..10).map(|i| json!(i)).collect();
        let cb = combine(
            &[("a", big.clone()), ("b", big.clone()), ("c", big.clone())],
            false,
        );
        let err = generate(&suite_of(cb), &ParamsSpec::new()).expect_err("1000 cases");
        let msg = err.to_string();
        assert!(msg.contains("1000"), "{msg}");
        assert!(msg.contains("pairwise"), "must name the way out: {msg}");
    }

    #[test]
    fn duplicate_case_names_are_rejected() {
        let suite = Suite {
            cases: vec![
                Case {
                    name: "a".into(),
                    params: BTreeMap::new(),
                    expect: Expect::default(),
                },
                Case {
                    name: "a".into(),
                    params: BTreeMap::new(),
                    expect: Expect::default(),
                },
            ],
            combine: None,
            auto: None,
            behavioral: Vec::new(),
        };
        let err = generate(&suite, &ParamsSpec::new()).expect_err("dupe");
        assert!(err.to_string().contains("duplicate case name"), "{err}");
    }

    #[test]
    fn explicit_cases_come_first_and_keep_their_order() {
        let suite = Suite {
            cases: vec![
                Case {
                    name: "z".into(),
                    params: BTreeMap::new(),
                    expect: Expect::default(),
                },
                Case {
                    name: "a".into(),
                    params: BTreeMap::new(),
                    expect: Expect::default(),
                },
            ],
            combine: Some(combine(&[("s", vec![json!(1)])], false)),
            auto: None,
            behavioral: Vec::new(),
        };
        let cases = generate(&suite, &ParamsSpec::new()).expect("generate");
        assert_eq!(cases[0].name, "z", "author order is preserved");
        assert_eq!(cases[1].name, "a");
        assert_eq!(cases[2].origin, Origin::Combined);
    }
}