use crate::result::RunResult;
pub trait Comparable {
fn recipe_name(&self) -> &str;
fn renderer(&self) -> &str;
fn passed(&self) -> bool;
}
impl Comparable for RunResult {
fn recipe_name(&self) -> &str {
&self.recipe_name
}
fn renderer(&self) -> &str {
&self.renderer
}
fn passed(&self) -> bool {
self.passed
}
}
fn outcome<R: Comparable>(result: &R) -> &'static str {
if result.passed() {
PASS
} else {
FAIL
}
}
pub const PASS: &str = "PASS";
pub const FAIL: &str = "FAIL";
pub const SKIP: &str = "SKIP";
fn key<R: Comparable>(result: &R) -> (String, String) {
(
result.recipe_name().to_string(),
result.renderer().to_string(),
)
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BehaviorDelta {
pub recipe: String,
pub renderer: String,
pub before_outcome: String,
pub after_outcome: String,
}
impl BehaviorDelta {
pub fn explanation(&self) -> String {
format!(
"{} [{}]: {} -> {}",
self.recipe, self.renderer, self.before_outcome, self.after_outcome
)
}
}
#[derive(Debug, Clone)]
pub struct BeforeAfterResult<R = RunResult> {
pub before: Vec<R>,
pub after: Vec<R>,
pub deltas: Vec<BehaviorDelta>,
}
impl<R> BeforeAfterResult<R> {
pub fn to_markdown(&self) -> String {
if self.deltas.is_empty() {
return "**Behavioral deltas:** none — before/after outcomes match.\n".to_string();
}
let mut lines = vec!["**Behavioral deltas:**".to_string(), String::new()];
for delta in &self.deltas {
lines.push(format!("- {}", delta.explanation()));
}
format!("{}\n", lines.join("\n"))
}
}
pub fn compute_deltas<R: Comparable>(before: &[R], after: &[R]) -> Vec<BehaviorDelta> {
let before_keys: Vec<(String, String)> = before.iter().map(key).collect();
let after_keys: Vec<(String, String)> = after.iter().map(key).collect();
let mut ordered_keys: Vec<(String, String)> = before_keys.clone();
for k in &after_keys {
if !before_keys.contains(k) {
ordered_keys.push(k.clone());
}
}
let mut deltas = Vec::new();
for k in ordered_keys {
let b = before.iter().find(|r| key(*r) == k);
let a = after.iter().find(|r| key(*r) == k);
let before_outcome = b.map(outcome).unwrap_or(SKIP);
let after_outcome = a.map(outcome).unwrap_or(SKIP);
if before_outcome != after_outcome {
deltas.push(BehaviorDelta {
recipe: k.0.clone(),
renderer: k.1.clone(),
before_outcome: before_outcome.to_string(),
after_outcome: after_outcome.to_string(),
});
}
}
deltas
}
pub fn build_before_after<R: Comparable>(before: Vec<R>, after: Vec<R>) -> BeforeAfterResult<R> {
let deltas = compute_deltas(&before, &after);
BeforeAfterResult {
before,
after,
deltas,
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::BTreeMap;
fn run(name: &str, renderer: &str, passed: bool) -> RunResult {
RunResult {
result_version: Some(crate::result::RESULT_SCHEMA_VERSION),
recipe_name: name.to_string(),
passed,
exit_code: None,
duration_seconds: 0.0,
priority: "P0".to_string(),
execution: "scripted".to_string(),
renderer: renderer.to_string(),
score: if passed { 1.0 } else { 0.0 },
steps: vec![],
assertions: vec![],
artifacts: BTreeMap::new(),
}
}
#[test]
fn identical_runs_have_no_deltas() {
let a = vec![run("login", "default", true)];
assert!(compute_deltas(&a, &a).is_empty());
}
#[test]
fn a_regression_is_reported() {
let deltas = compute_deltas(
&[run("login", "default", true)],
&[run("login", "default", false)],
);
assert_eq!(deltas.len(), 1);
assert_eq!(deltas[0].explanation(), "login [default]: PASS -> FAIL");
}
#[test]
fn a_fix_is_reported_too() {
let deltas = compute_deltas(
&[run("login", "default", false)],
&[run("login", "default", true)],
);
assert_eq!(deltas[0].explanation(), "login [default]: FAIL -> PASS");
}
#[test]
fn the_same_recipe_under_two_renderers_is_two_entries() {
let deltas = compute_deltas(
&[run("login", "alpha", true), run("login", "beta", true)],
&[run("login", "alpha", true), run("login", "beta", false)],
);
assert_eq!(deltas.len(), 1);
assert_eq!(deltas[0].renderer, "beta");
}
#[test]
fn a_recipe_that_stopped_running_is_a_delta_not_a_gap() {
let deltas = compute_deltas(&[run("login", "default", true)], &[]);
assert_eq!(deltas[0].explanation(), "login [default]: PASS -> SKIP");
}
#[test]
fn a_newly_added_recipe_is_reported() {
let deltas = compute_deltas(&[], &[run("login", "default", true)]);
assert_eq!(deltas[0].explanation(), "login [default]: SKIP -> PASS");
}
#[test]
fn ordering_follows_before_then_new_arrivals() {
let deltas = compute_deltas(
&[run("a", "d", true), run("b", "d", true)],
&[
run("b", "d", false),
run("a", "d", false),
run("c", "d", true),
],
);
let names: Vec<&str> = deltas.iter().map(|d| d.recipe.as_str()).collect();
assert_eq!(names, ["a", "b", "c"]);
}
#[test]
fn no_deltas_says_so_rather_than_rendering_nothing() {
let r = build_before_after(
vec![run("login", "default", true)],
vec![run("login", "default", true)],
);
assert!(r.to_markdown().contains("none"), "{}", r.to_markdown());
}
#[test]
fn markdown_lists_every_delta() {
let r = build_before_after(
vec![run("a", "d", true), run("b", "d", true)],
vec![run("a", "d", false), run("b", "d", false)],
);
let md = r.to_markdown();
assert!(md.contains("- a [d]: PASS -> FAIL"), "{md}");
assert!(md.contains("- b [d]: PASS -> FAIL"), "{md}");
}
#[test]
fn build_keeps_both_sides_alongside_the_deltas() {
let r = build_before_after(vec![run("a", "d", true)], vec![run("a", "d", false)]);
assert_eq!(r.before.len(), 1);
assert_eq!(r.after.len(), 1);
assert_eq!(r.deltas.len(), 1);
}
#[derive(Debug, Clone)]
struct ProductResult {
recipe: String,
renderer: String,
passed: bool,
#[allow(dead_code)]
agent_model: String,
}
impl Comparable for ProductResult {
fn recipe_name(&self) -> &str {
&self.recipe
}
fn renderer(&self) -> &str {
&self.renderer
}
fn passed(&self) -> bool {
self.passed
}
}
fn product(name: &str, passed: bool) -> ProductResult {
ProductResult {
recipe: name.to_string(),
renderer: "ink".to_string(),
passed,
agent_model: "gpt-5.5".to_string(),
}
}
#[test]
fn a_foreign_result_type_can_be_compared() {
let r = build_before_after(
vec![product("login", true), product("search", true)],
vec![product("login", false), product("search", true)],
);
assert_eq!(r.deltas.len(), 1);
assert_eq!(r.deltas[0].explanation(), "login [ink]: PASS -> FAIL");
assert_eq!("gpt-5.5", r.after[0].agent_model);
}
#[test]
fn skips_work_for_a_foreign_type_too() {
let deltas = compute_deltas(&[product("login", true)], &[]);
assert_eq!(deltas[0].explanation(), "login [ink]: PASS -> SKIP");
}
}