assay-core 5.2.0

use super::super::Runner;
use crate::cache::key::cache_key;
use crate::model::{EvalConfig, LlmResponse, TestCase, TestResultRow, TestStatus};
use tracing::{info_span, Instrument};

pub(crate) async fn run_test_once_impl(
    runner: &Runner,
    cfg: &EvalConfig,
    tc: &TestCase,
) -> anyhow::Result<(TestResultRow, LlmResponse)> {
    // Keep direct internal callers fail-closed and bind external bytes to the
    // same snapshot used by fingerprinting and metric evaluation. `run_suite_impl`
    // already does this before spawning tasks, so this is a no-op on that path.
    let mut bound_tc = tc.clone();
    crate::model::bind_external_expected_inputs(&mut bound_tc.expected)?;
    crate::model::validate_test_case_for_execution(&bound_tc)?;
    let tc = &bound_tc;

    let expected_json = serde_json::to_string(&tc.expected).unwrap_or_default();
    let metric_versions = [("assay", env!("CARGO_PKG_VERSION"))];

    let policy_hash = if let Some(path) = tc.expected.get_policy_path() {
        match std::fs::read_to_string(path) {
            Ok(content) => Some(crate::fingerprint::sha256_hex(&content)),
            Err(_) => None,
        }
    } else {
        None
    };

    let fp = crate::fingerprint::compute(crate::fingerprint::Context {
        suite: &cfg.suite,
        model: &cfg.model,
        test_id: &tc.id,
        prompt: &tc.input.prompt,
        context: tc.input.context.as_deref(),
        expected_canonical: &expected_json,
        policy_hash: policy_hash.as_deref(),
        metric_versions: &metric_versions,
    });

    if runner.incremental && !runner.refresh_cache {
        if let Some(prev) = runner.store.get_last_passing_by_fingerprint(&fp.hex)? {
            let row = TestResultRow {
                test_id: tc.id.clone(),
                status: TestStatus::Skipped,
                score: prev.score,
                cached: true,
                message: "skipped: fingerprint match".into(),
                details: serde_json::json!({
                    "skip": {
                         "reason": "fingerprint_match",
                         "fingerprint": fp.hex,
                         "previous_run_id": prev.details.get("skip").and_then(|s: &serde_json::Value| s.get("previous_run_id")).and_then(|v: &serde_json::Value| v.as_i64()),
                         "previous_at": prev.details.get("skip").and_then(|s: &serde_json::Value| s.get("previous_at")).and_then(|v: &serde_json::Value| v.as_str()),
                         "origin_run_id": prev.details.get("skip").and_then(|s: &serde_json::Value| s.get("origin_run_id")).and_then(|v: &serde_json::Value| v.as_i64()),
                         "previous_score": prev.score
                    }
                }),
                duration_ms: Some(0),
                fingerprint: Some(fp.hex.clone()),
                skip_reason: Some("fingerprint_match".into()),
                attempts: None,
                error_policy_applied: None,
            };

            let resp = LlmResponse {
                text: "".into(),
                provider: "skipped".into(),
                model: cfg.model.clone(),
                cached: true,
                meta: serde_json::json!({}),
            };
            return Ok((row, resp));
        }
    }

    let key = cache_key(
        &cfg.model,
        &tc.input.prompt,
        &fp.hex,
        runner.client.fingerprint().as_deref(),
    );

    let start = std::time::Instant::now();
    let mut cached = false;

    let mut resp: LlmResponse = if cfg.settings.cache.unwrap_or(true) && !runner.refresh_cache {
        if let Some(r) = runner.cache.get(&key)? {
            cached = true;
            eprintln!(
                "  [CACHE HIT] key={} prompt_len={}",
                key,
                tc.input.prompt.len()
            );
            r
        } else {
            let r = runner.call_llm(cfg, tc).await?;
            runner.cache.put(&key, &r)?;
            r
        }
    } else {
        runner.call_llm(cfg, tc).await?
    };
    resp.cached = resp.cached || cached;

    runner.enrich_semantic(cfg, tc, &mut resp).await?;
    runner.enrich_judge(cfg, tc, &mut resp).await?;

    let mut final_status = TestStatus::Pass;
    let mut final_score: Option<f64> = None;
    let mut msg = String::new();
    let mut details = serde_json::json!({ "metrics": {} });

    for m in &runner.metrics {
        let metric_name = m.name();
        let metric_span = info_span!(
            "assay.eval.metric",
            "assay.eval.test_id" = tc.id.as_str(),
            "assay.eval.metric.name" = metric_name,
            "assay.eval.response.cached" = resp.cached,
            "assay.eval.metric.score" = tracing::field::Empty,
            "assay.eval.metric.passed" = tracing::field::Empty,
            "assay.eval.metric.unstable" = tracing::field::Empty,
            "assay.eval.metric.duration_ms" = tracing::field::Empty,
            "error" = tracing::field::Empty,
            "error.message" = tracing::field::Empty
        );
        let metric_start = std::time::Instant::now();
        let metric_result = async { m.evaluate(tc, &tc.expected, &resp).await }
            .instrument(metric_span.clone())
            .await;
        let metric_duration_ms = metric_start.elapsed().as_millis() as u64;
        metric_span.record("assay.eval.metric.duration_ms", metric_duration_ms);

        let r = match metric_result {
            Ok(result) => {
                metric_span.record("assay.eval.metric.score", result.score);
                metric_span.record("assay.eval.metric.passed", result.passed);
                metric_span.record("assay.eval.metric.unstable", result.unstable);
                result
            }
            Err(err) => {
                let error_message = err.to_string();
                metric_span.record("error", true);
                metric_span.record("error.message", error_message.as_str());
                return Err(err);
            }
        };

        details["metrics"][metric_name] = serde_json::json!({
            "score": r.score,
            "passed": r.passed,
            "unstable": r.unstable,
            "exercised": exercised_label(r.exercised),
            "details": r.details
        });
        // Only a metric that actually evaluated something may set the test's score.
        //
        // This used to be unconditional, so the score belonged to whichever metric ran last. All
        // thirteen registered metrics run against every test and eleven of them return a
        // not-applicable 1.0, so a semantic test that scored 0.87 reported 1.0 -- the number came
        // from a metric that was never asked to run. `pass_rate_masking_is_reported` pins it.
        final_score = score_after(final_score, &r);

        if r.unstable {
            final_status = TestStatus::Warn;
            msg = format!("unstable metric: {}", metric_name);
            break;
        }
        if !r.passed {
            final_status = TestStatus::Fail;
            msg = format!("failed: {}", metric_name);
            break;
        }
    }

    if let Some(baseline) = &runner.baseline {
        if let Some((new_status, new_msg)) =
            runner.check_baseline_regressions(tc, cfg, &details, &runner.metrics, baseline)
        {
            if matches!(new_status, TestStatus::Fail | TestStatus::Warn) {
                final_status = new_status;
                msg = new_msg;
            }
        }
    }

    let duration_ms = start.elapsed().as_millis() as u64;
    let mut row = TestResultRow {
        test_id: tc.id.clone(),
        status: final_status,
        score: final_score,
        cached: resp.cached,
        message: if msg.is_empty() { "ok".into() } else { msg },
        details,
        duration_ms: Some(duration_ms),
        fingerprint: Some(fp.hex),
        skip_reason: None,
        attempts: None,
        error_policy_applied: None,
    };

    if runner.client.provider_name() == "trace" {
        row.details["assay.replay"] = serde_json::json!(true);
    }

    row.details["prompt"] = serde_json::Value::String(tc.input.prompt.clone());

    Ok((row, resp))
}

/// The test's score after folding in one metric result.
///
/// A metric that evaluated nothing does not get to set the score. Extracted rather than written
/// inline so the loop above and the tests below apply the same rule -- two implementations of one
/// rule drift, and this one decides a number that reaches `run.json` and SARIF.
fn score_after(current: Option<f64>, r: &crate::metrics_api::MetricResult) -> Option<f64> {
    if r.is_exercised() {
        Some(r.score)
    } else {
        current
    }
}

/// The stable string for an `Exercised` value in `details["metrics"][…]`.
///
/// Delegates to [`crate::metrics_api::Exercised::label`], which is where the vocabulary moved once
/// it acquired a reader (`report::exercised`). Kept as a named function because the tests below
/// pin the mapping at the point the runner uses it.
fn exercised_label(e: crate::metrics_api::Exercised) -> &'static str {
    e.label()
}

#[cfg(test)]
mod exercised_tests {
    use super::*;
    use crate::metrics_api::{Exercised, MetricResult};

    /// Folds with `score_after`, the same function the runner's loop calls. A test that reimplemented
    /// the rule would assert its own copy and stay green while the loop drifted away from it.
    fn fold_score(results: &[MetricResult]) -> Option<f64> {
        results.iter().fold(None, score_after)
    }

    #[test]
    fn a_not_applicable_metric_does_not_set_the_score() {
        // The shape that made this issue: one metric evaluates and scores 0.87, then eight metrics
        // that do not handle the variant return a not-applicable 1.0 after it.
        let mut results = vec![MetricResult::pass(0.87)];
        results.extend((0..8).map(|_| MetricResult::not_applicable()));
        assert_eq!(
            fold_score(&results),
            Some(0.87),
            "the score came from a metric that was never asked to run"
        );
    }

    #[test]
    fn a_not_exercised_metric_does_not_set_the_score() {
        let results = vec![
            MetricResult::pass(0.62),
            MetricResult::not_exercised("no tool definitions in the trace"),
        ];
        assert_eq!(fold_score(&results), Some(0.62));
    }

    #[test]
    fn a_test_with_no_exercised_metric_has_no_score() {
        // Not 1.0. A test nothing evaluated has no score, and saying so is the point of the
        // dimension -- a 1.0 here is the vacuous pass the issue is about.
        let results = vec![
            MetricResult::not_applicable(),
            MetricResult::not_exercised("no output schemas configured"),
        ];
        assert_eq!(fold_score(&results), None);
    }

    #[test]
    fn a_real_evaluation_still_sets_the_score() {
        assert_eq!(fold_score(&[MetricResult::pass(0.5)]), Some(0.5));
        assert_eq!(fold_score(&[MetricResult::fail(0.1, "nope")]), Some(0.1));
    }

    #[test]
    fn the_label_vocabulary_is_stable() {
        // These strings land in run.json, so they are an interface rather than a rendering.
        assert_eq!(exercised_label(Exercised::Exercised), "exercised");
        assert_eq!(exercised_label(Exercised::NotApplicable), "not_applicable");
        assert_eq!(exercised_label(Exercised::NotExercised), "not_exercised");
    }

    #[test]
    fn not_applicable_and_not_exercised_never_fail_a_test() {
        // Over-eager vacuity detection earns a suppression and takes real findings with it, so
        // these report and never decide.
        for r in [
            MetricResult::not_applicable(),
            MetricResult::not_exercised("nothing to check"),
        ] {
            assert!(r.passed, "a non-exercised metric must not fail the test");
            assert!(!r.unstable);
            assert!(!r.is_exercised());
        }
    }
}