skilltest_core/

report.rs

//! Run results and the JSON report. The serialized shape here is the **stable
//! contract** the language SDKs parse. These types are the source of truth:
//! their JSON Schemas (via `skilltest schema`, goldens in `schemas/`) are what
//! the SDK contract tests compare their Pydantic/Zod models against.

use schemars::JsonSchema;
use serde::{Deserialize, Serialize};

use crate::conversation::Transcript;
use crate::error::{Error, ProviderErrorKind};
use crate::eval::EvalOutcome;
use crate::mock::MockCall;
use crate::provider::Usage;
use crate::skill::Finding;

/// The result of running one test case on one (platform, model) pair.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct CaseRun {
    /// The test case name.
    pub case: String,
    /// Absolute-ish path to the skill that was exercised.
    pub skill: String,
    /// The harness platform this run used.
    pub platform: String,
    /// The model this run used.
    pub model: String,
    /// True iff every eval in this run passed.
    pub passed: bool,
    /// Number of assistant turns produced.
    pub turns: usize,
    /// Per-eval outcomes, in declaration order.
    pub evals: Vec<EvalOutcome>,
    /// The full conversation, for debugging and deterministic mix-in checks.
    pub transcript: Transcript,
    /// Aggregated token/cost usage across every provider call in this run
    /// (skill turns + simulated-user turns + judge calls). Omitted when no
    /// usage was reported (e.g. the fake provider or a harness that doesn't
    /// surface usage).
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub usage: Option<Usage>,
    /// Every tool call the mock/spy channel observed, in order, with the
    /// original (pre-rewrite) input and the verdict applied. `null` when the
    /// channel was off for this run (no `mocks`, no `spy`); an empty array
    /// means the channel was on and the skill made no tool calls — SDKs use
    /// that distinction so a spy on a channel-less run errs instead of reading
    /// as "zero calls".
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub mock_calls: Option<Vec<MockCall>>,
}

/// Aggregate pass/fail counts for a report.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct Summary {
    /// Distinct test cases represented.
    pub cases: usize,
    /// Total (case × platform × model) runs.
    pub runs: usize,
    /// Runs that passed.
    pub passed: usize,
    /// Runs that failed.
    pub failed: usize,
    /// Aggregated token/cost usage across every run in the report. Omitted
    /// when no run reported usage.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub usage: Option<Usage>,
}

/// The top-level report for a `skilltest run` invocation.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct Report {
    /// True iff every run passed.
    pub passed: bool,
    /// Aggregate counts.
    pub summary: Summary,
    /// Every individual run.
    pub runs: Vec<CaseRun>,
}

impl Report {
    /// Build a report from runs, computing the summary and overall pass.
    #[must_use]
    pub fn new(runs: Vec<CaseRun>) -> Self {
        let mut case_names: Vec<&str> = runs.iter().map(|r| r.case.as_str()).collect();
        case_names.sort_unstable();
        case_names.dedup();
        let passed_runs = runs.iter().filter(|r| r.passed).count();
        let mut total_usage = Usage::default();
        for run in &runs {
            if let Some(u) = &run.usage {
                total_usage.add(u);
            }
        }
        let usage = (!total_usage.is_empty()).then_some(total_usage);
        let summary = Summary {
            cases: case_names.len(),
            runs: runs.len(),
            passed: passed_runs,
            failed: runs.len() - passed_runs,
            usage,
        };
        Report {
            passed: summary.failed == 0 && !runs.is_empty(),
            summary,
            runs,
        }
    }

    /// Serialize to pretty JSON (the `--format json` output).
    ///
    /// # Errors
    /// [`serde_json::Error`] only if a contained value cannot serialize, which
    /// should not happen for these types.
    pub fn to_json(&self) -> Result<String, serde_json::Error> {
        serde_json::to_string_pretty(self)
    }

    /// A compact, human-readable summary line per run plus a total. Quiet by
    /// design: this is context the next reader has to parse.
    #[must_use]
    pub fn to_human(&self) -> String {
        let mut out = String::new();
        for run in &self.runs {
            let mark = if run.passed { "PASS" } else { "FAIL" };
            out.push_str(&format!(
                "{mark}  {} [{}/{}]\n",
                run.case, run.platform, run.model
            ));
            for eval in &run.evals {
                if !eval.passed {
                    out.push_str(&format!(
                        "      - {}: {} ({})\n",
                        eval.label,
                        eval.detail.summary(),
                        eval.reason
                    ));
                }
            }
        }
        out.push_str(&format!(
            "{}/{} runs passed\n",
            self.summary.passed, self.summary.runs
        ));
        if let Some(usage) = &self.summary.usage {
            let mut parts = Vec::new();
            if let Some(cost) = usage.cost_usd {
                parts.push(format!("${cost:.4}"));
            }
            if let (Some(i), Some(o)) = (usage.input_tokens, usage.output_tokens) {
                parts.push(format!("{} in / {} out tokens", i, o));
            } else {
                if let Some(i) = usage.input_tokens {
                    parts.push(format!("{i} input tokens"));
                }
                if let Some(o) = usage.output_tokens {
                    parts.push(format!("{o} output tokens"));
                }
            }
            if !parts.is_empty() {
                out.push_str(&format!("usage: {}\n", parts.join(", ")));
            }
        }
        out
    }
}

/// One problem found while validating a skill, as serialized in the
/// `skilltest validate --format json` output.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct ValidationFinding {
    /// The skill directory the finding is about.
    pub skill: String,
    /// What is wrong and how to fix it.
    pub message: String,
}

/// The top-level report for a `skilltest validate` invocation.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct ValidationReport {
    /// True iff no findings were produced.
    pub valid: bool,
    /// Every finding, in discovery order.
    pub findings: Vec<ValidationFinding>,
}

impl ValidationReport {
    /// Build a validation report from raw findings.
    #[must_use]
    pub fn new(findings: &[Finding]) -> Self {
        ValidationReport {
            valid: findings.is_empty(),
            findings: findings
                .iter()
                .map(|f| ValidationFinding {
                    skill: f.skill.to_string_lossy().into_owned(),
                    message: f.message.clone(),
                })
                .collect(),
        }
    }

    /// Serialize to pretty JSON (the `--format json` output).
    ///
    /// # Errors
    /// [`serde_json::Error`] only if a contained value cannot serialize, which
    /// should not happen for these types.
    pub fn to_json(&self) -> Result<String, serde_json::Error> {
        serde_json::to_string_pretty(self)
    }
}

/// Which class of failure a [`ReportError`] describes. Mirrors the process exit
/// code so a JSON consumer gets the same coarse classification as a shell script
/// branching on `$?`.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum ErrorCode {
    /// Bad usage or input (exit 2): malformed config/case YAML, a missing file,
    /// a semantically invalid test definition.
    Usage,
    /// A provider/environment failure (exit 3): the harness or judge could not
    /// be reached or misbehaved.
    Provider,
}

/// A structured error, emitted as the `--format json` / `json-stream` output
/// when a `skilltest run` cannot produce a [`Report`].
///
/// This is the machine-readable counterpart to the human hint the CLI prints on
/// stderr: it rides on stdout so SDK/plugin consumers get the [`ProviderErrorKind`]
/// (and the `code`/`context`) for targeted handling — retry on
/// [`ProviderErrorKind::Timeout`], fail fast on [`ProviderErrorKind::Auth`] —
/// instead of matching substrings in the message. For `json` the object is
/// emitted bare; for `json-stream` it is the terminal
/// `{"type":"error","error":{…}}` line.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct ReportError {
    /// The coarse failure class, matching the process exit code.
    pub code: ErrorCode,
    /// The structured provider-failure category, when skilltest could classify
    /// it. Absent for usage errors and for unclassified provider failures.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub kind: Option<ProviderErrorKind>,
    /// The provider context the failure came from (e.g. `oneharness:claude-code`,
    /// `api-judge`). Absent for usage errors.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub context: Option<String>,
    /// A human-readable description of what went wrong (the same text printed on
    /// stderr, minus the suggested-action hint).
    pub message: String,
}

impl ReportError {
    /// Build the structured error for a core [`Error`]. The mapping mirrors the
    /// CLI's error→exit-code mapping (`report_error`): [`Error::Provider`] is a
    /// provider failure carrying its `kind`/`context`; everything else is a
    /// usage error.
    #[must_use]
    pub fn from_error(err: &Error) -> Self {
        match err {
            Error::Provider {
                context,
                message,
                kind,
            } => ReportError {
                code: ErrorCode::Provider,
                kind: *kind,
                context: Some(context.clone()),
                message: message.clone(),
            },
            other => ReportError {
                code: ErrorCode::Usage,
                kind: None,
                context: None,
                message: other.to_string(),
            },
        }
    }

    /// Serialize to pretty JSON (the bare `--format json` error output).
    ///
    /// # Errors
    /// [`serde_json::Error`] only if a contained value cannot serialize, which
    /// should not happen for these types.
    pub fn to_json(&self) -> Result<String, serde_json::Error> {
        serde_json::to_string_pretty(self)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::conversation::Transcript;
    use crate::eval::{Comparator, EvalDetail, EvalOutcome};

    fn run(case: &str, passed: bool, evals: Vec<EvalOutcome>, usage: Option<Usage>) -> CaseRun {
        CaseRun {
            case: case.to_string(),
            skill: "/tmp/skill".to_string(),
            platform: "claude-code".to_string(),
            model: "sonnet".to_string(),
            passed,
            turns: 1,
            evals,
            transcript: Transcript::from_input("hi"),
            usage,
            mock_calls: None,
        }
    }

    fn bool_eval(label: &str, passed: bool) -> EvalOutcome {
        EvalOutcome {
            label: label.to_string(),
            passed,
            detail: EvalDetail::Boolean {
                value: passed,
                expected: true,
            },
            reason: "because".to_string(),
        }
    }

    #[test]
    fn new_computes_summary_and_dedups_cases() {
        let report = Report::new(vec![
            run("a", true, vec![bool_eval("x", true)], None),
            run("a", false, vec![bool_eval("y", false)], None),
            run("b", true, vec![bool_eval("z", true)], None),
        ]);
        // Two distinct cases, three runs, one failure -> overall fail.
        assert_eq!(report.summary.cases, 2);
        assert_eq!(report.summary.runs, 3);
        assert_eq!(report.summary.passed, 2);
        assert_eq!(report.summary.failed, 1);
        assert!(!report.passed);
        // No run reported usage, so the summary omits it.
        assert!(report.summary.usage.is_none());
    }

    #[test]
    fn empty_report_is_not_passed() {
        let report = Report::new(vec![]);
        assert!(!report.passed, "an empty run set is not a pass");
        assert_eq!(report.summary.runs, 0);
    }

    #[test]
    fn new_aggregates_usage_across_runs() {
        let report = Report::new(vec![
            run(
                "a",
                true,
                vec![bool_eval("x", true)],
                Some(Usage {
                    input_tokens: Some(10),
                    output_tokens: Some(2),
                    cost_usd: Some(0.01),
                }),
            ),
            run(
                "b",
                true,
                vec![bool_eval("y", true)],
                Some(Usage {
                    input_tokens: Some(5),
                    output_tokens: None,
                    cost_usd: Some(0.02),
                }),
            ),
        ]);
        let usage = report.summary.usage.unwrap();
        assert_eq!(usage.input_tokens, Some(15));
        assert_eq!(usage.output_tokens, Some(2));
        assert!((usage.cost_usd.unwrap() - 0.03).abs() < 1e-9);
    }

    #[test]
    fn to_json_round_trips() {
        let report = Report::new(vec![run("a", true, vec![bool_eval("x", true)], None)]);
        let json = report.to_json().unwrap();
        let parsed: Report = serde_json::from_str(&json).unwrap();
        assert_eq!(parsed, report);
    }

    #[test]
    fn to_human_lists_runs_and_failed_evals() {
        let numeric = EvalOutcome {
            label: "warmth".to_string(),
            passed: false,
            detail: EvalDetail::Numeric {
                value: 4.0,
                threshold: 7.0,
                comparator: Comparator::Gte,
            },
            reason: "too cold".to_string(),
        };
        let report = Report::new(vec![
            run("greets", true, vec![bool_eval("names", true)], None),
            run("warm", false, vec![numeric], None),
        ]);
        let human = report.to_human();
        assert!(human.contains("PASS  greets [claude-code/sonnet]"));
        assert!(human.contains("FAIL  warm"));
        // Only the failing eval is itemized, with its summary and reason.
        assert!(human.contains("warmth: 4 >= 7 (too cold)"), "got:\n{human}");
        assert!(human.contains("1/2 runs passed"));
    }

    #[test]
    fn to_human_renders_usage_line_variants() {
        // Cost + both token counts.
        let full = Report::new(vec![run(
            "a",
            true,
            vec![bool_eval("x", true)],
            Some(Usage {
                input_tokens: Some(100),
                output_tokens: Some(50),
                cost_usd: Some(0.1234),
            }),
        )]);
        let human = full.to_human();
        assert!(
            human.contains("usage: $0.1234, 100 in / 50 out tokens"),
            "got:\n{human}"
        );

        // Only an input-token count (no cost, no output) hits the singular branch.
        let partial = Report::new(vec![run(
            "a",
            true,
            vec![bool_eval("x", true)],
            Some(Usage {
                input_tokens: Some(7),
                output_tokens: None,
                cost_usd: None,
            }),
        )]);
        assert!(partial.to_human().contains("usage: 7 input tokens"));

        // Only an output-token count.
        let out_only = Report::new(vec![run(
            "a",
            true,
            vec![bool_eval("x", true)],
            Some(Usage {
                input_tokens: None,
                output_tokens: Some(9),
                cost_usd: None,
            }),
        )]);
        assert!(out_only.to_human().contains("usage: 9 output tokens"));
    }

    #[test]
    fn to_human_without_usage_has_no_usage_line() {
        let report = Report::new(vec![run("a", true, vec![bool_eval("x", true)], None)]);
        assert!(!report.to_human().contains("usage:"));
    }

    #[test]
    fn validation_report_new_and_json() {
        use crate::skill::Finding;
        let empty = ValidationReport::new(&[]);
        assert!(empty.valid);
        assert!(empty.findings.is_empty());

        let findings = vec![
            Finding {
                skill: std::path::PathBuf::from("/tmp/a"),
                message: "missing name".to_string(),
            },
            Finding {
                skill: std::path::PathBuf::from("/tmp/b"),
                message: "no body".to_string(),
            },
        ];
        let report = ValidationReport::new(&findings);
        assert!(!report.valid);
        assert_eq!(report.findings.len(), 2);
        assert_eq!(report.findings[0].skill, "/tmp/a");
        let json = report.to_json().unwrap();
        let parsed: ValidationReport = serde_json::from_str(&json).unwrap();
        assert_eq!(parsed, report);
    }

    #[test]
    fn report_error_from_classified_provider_error() {
        let err = Error::provider_classified(
            "oneharness:claude-code",
            "harness run failed: deadline",
            ProviderErrorKind::Timeout,
        );
        let structured = ReportError::from_error(&err);
        assert_eq!(structured.code, ErrorCode::Provider);
        assert_eq!(structured.kind, Some(ProviderErrorKind::Timeout));
        assert_eq!(
            structured.context.as_deref(),
            Some("oneharness:claude-code")
        );
        assert!(structured.message.contains("deadline"));
        // Round-trips through the JSON contract, kind as a snake_case string.
        let json = structured.to_json().unwrap();
        assert!(json.contains("\"kind\": \"timeout\""));
        let parsed: ReportError = serde_json::from_str(&json).unwrap();
        assert_eq!(parsed, structured);
    }

    #[test]
    fn report_error_from_unclassified_provider_error() {
        let err = Error::provider("oneharness", "boom");
        let structured = ReportError::from_error(&err);
        assert_eq!(structured.code, ErrorCode::Provider);
        assert_eq!(structured.kind, None);
        assert_eq!(structured.context.as_deref(), Some("oneharness"));
    }

    #[test]
    fn report_error_from_usage_error() {
        let structured = ReportError::from_error(&Error::Invalid("bad case".into()));
        assert_eq!(structured.code, ErrorCode::Usage);
        assert_eq!(structured.kind, None);
        assert!(structured.context.is_none());
        assert!(structured.message.contains("bad case"));
    }
}