assay-cli 3.31.0

use crate::exit_codes::{ReasonCode, RunOutcome};
use std::path::{Path, PathBuf};

fn reason_code_from_run_error_kind(kind: assay_core::errors::RunErrorKind) -> Option<ReasonCode> {
    use assay_core::errors::RunErrorKind;

    match kind {
        RunErrorKind::TraceNotFound => Some(ReasonCode::ETraceNotFound),
        RunErrorKind::MissingConfig => Some(ReasonCode::EMissingConfig),
        RunErrorKind::ConfigParse => Some(ReasonCode::ECfgParse),
        RunErrorKind::InvalidArgs => Some(ReasonCode::EInvalidArgs),
        RunErrorKind::ProviderRateLimit => Some(ReasonCode::ERateLimit),
        RunErrorKind::ProviderTimeout => Some(ReasonCode::ETimeout),
        RunErrorKind::ProviderServer => Some(ReasonCode::EProvider5xx),
        RunErrorKind::Network => Some(ReasonCode::ENetworkError),
        RunErrorKind::JudgeUnavailable => Some(ReasonCode::EJudgeUnavailable),
        RunErrorKind::Other => None,
    }
}

pub(crate) fn reason_code_from_run_error(err: &assay_core::errors::RunError) -> Option<ReasonCode> {
    reason_code_from_run_error_kind(err.kind.clone())
}

fn run_error_kind_from_details(
    details: &serde_json::Value,
) -> Option<assay_core::errors::RunErrorKind> {
    use assay_core::errors::RunErrorKind;

    let kind = details.get("run_error_kind")?.as_str()?;
    match kind {
        "trace_not_found" => Some(RunErrorKind::TraceNotFound),
        "missing_config" => Some(RunErrorKind::MissingConfig),
        "config_parse" => Some(RunErrorKind::ConfigParse),
        "invalid_args" => Some(RunErrorKind::InvalidArgs),
        "provider_rate_limit" => Some(RunErrorKind::ProviderRateLimit),
        "provider_timeout" => Some(RunErrorKind::ProviderTimeout),
        "provider_server" => Some(RunErrorKind::ProviderServer),
        "network" => Some(RunErrorKind::Network),
        "judge_unavailable" => Some(RunErrorKind::JudgeUnavailable),
        "other" => Some(RunErrorKind::Other),
        _ => None,
    }
}

fn reason_code_from_result_row(row: &assay_core::model::TestResultRow) -> Option<ReasonCode> {
    // Typed-first mapping in hot path; message classification remains explicit legacy fallback.
    if let Some(kind) = run_error_kind_from_details(&row.details) {
        return reason_code_from_run_error_kind(kind);
    }
    reason_code_from_error_message(&row.message)
}

pub(crate) fn reason_code_from_error_message(message: &str) -> Option<ReasonCode> {
    use assay_core::errors::RunError;

    // Legacy-only fallback for untyped paths.
    let classified = RunError::legacy_classify_message(message.to_string());
    reason_code_from_run_error(&classified)
}

pub(crate) fn decide_run_outcome(
    results: &[assay_core::model::TestResultRow],
    strict: bool,
    version: crate::exit_codes::ExitCodeVersion,
) -> crate::exit_codes::RunOutcome {
    use assay_core::model::TestStatus;

    // Helper to ensure exit code matches requested version
    let make_outcome = |reason: ReasonCode, msg: Option<String>, context: Option<&str>| {
        let mut o = RunOutcome::from_reason(reason, msg, context);
        o.exit_code = reason.exit_code_for(version);
        o
    };

    // Priority 1: Config/Argument Errors (Exit 2)
    for r in results {
        if let Some(reason) = reason_code_from_result_row(r) {
            if matches!(
                reason,
                ReasonCode::ETraceNotFound
                    | ReasonCode::EMissingConfig
                    | ReasonCode::ECfgParse
                    | ReasonCode::EInvalidArgs
            ) {
                return make_outcome(reason, Some(r.message.clone()), None);
            }
        }
    }

    // Priority 2: Infrastructure Failures (Refined Heuristics)
    let infra_errors: Vec<&assay_core::model::TestResultRow> = results
        .iter()
        .filter(|r| matches!(r.status, TestStatus::Error))
        .collect();

    if !infra_errors.is_empty() {
        let reason = pick_infra_reason(&infra_errors);
        return make_outcome(
            reason,
            Some("Infrastructure failures detected".into()),
            None,
        );
    }

    // Priority 3: Judge uncertain (abstain) — exit 1, E_JUDGE_UNCERTAIN
    let abstain_count = results
        .iter()
        .filter(|r| has_judge_verdict_abstain(&r.details))
        .count();
    if abstain_count > 0 {
        let mut o = RunOutcome::judge_uncertain(abstain_count);
        o.exit_code = ReasonCode::EJudgeUncertain.exit_code_for(version);
        return o;
    }

    // Priority 4: Test Failures
    let fails = results
        .iter()
        .filter(|r| matches!(r.status, TestStatus::Fail))
        .count();
    if fails > 0 {
        let mut o = RunOutcome::test_failure(fails);
        o.exit_code = ReasonCode::ETestFailed.exit_code_for(version);
        return o;
    }

    // Priority 5: Strict Mode Violations
    if strict {
        let violations = results
            .iter()
            .filter(|r| {
                matches!(
                    r.status,
                    TestStatus::Warn | TestStatus::Flaky | TestStatus::Unstable
                )
            })
            .count();
        if violations > 0 {
            return make_outcome(
                ReasonCode::EPolicyViolation,
                Some(format!("Strict mode: {} policy violations", violations)),
                None,
            );
        }
    }

    // Priority 6: Success (ensure version compliance though Success is usually 0 in all versions)
    let mut o = RunOutcome::success();
    o.exit_code = ReasonCode::Success.exit_code_for(version);
    o
}

/// True if this result row has any judge metric with verdict "Abstain" (E7.5).
pub(crate) fn has_judge_verdict_abstain(details: &serde_json::Value) -> bool {
    let Some(metrics) = details.get("metrics").and_then(|m| m.as_object()) else {
        return false;
    };
    for (_name, metric_val) in metrics {
        if let Some(inner) = metric_val.get("details").and_then(|d| d.get("verdict")) {
            if inner.as_str() == Some("Abstain") {
                return true;
            }
        }
    }
    false
}

fn pick_infra_reason(
    errors: &[&assay_core::model::TestResultRow],
) -> crate::exit_codes::ReasonCode {
    for r in errors {
        if let Some(reason) = reason_code_from_result_row(r) {
            if matches!(
                reason,
                ReasonCode::ERateLimit
                    | ReasonCode::ETimeout
                    | ReasonCode::EProvider5xx
                    | ReasonCode::ENetworkError
                    | ReasonCode::EJudgeUnavailable
            ) {
                return reason;
            }
        }
    }
    ReasonCode::EJudgeUnavailable
}

/// Build a Summary from RunOutcome for writing summary.json (same dir as run.json).
pub(crate) fn summary_from_outcome(
    outcome: &crate::exit_codes::RunOutcome,
    verify_enabled: bool,
) -> assay_core::report::summary::Summary {
    let assay_version = env!("CARGO_PKG_VERSION");
    if outcome.exit_code == 0 {
        assay_core::report::summary::Summary::success(assay_version, verify_enabled)
    } else {
        assay_core::report::summary::Summary::failure(
            outcome.exit_code,
            &outcome.reason_code,
            outcome.message.as_deref().unwrap_or(""),
            outcome.next_step.as_deref().unwrap_or(""),
            assay_version,
            verify_enabled,
        )
    }
}

pub(crate) fn write_extended_run_json(
    artifacts: &assay_core::report::RunArtifacts,
    outcome: &crate::exit_codes::RunOutcome,
    path: &Path,
    sarif_omitted: Option<u64>,
) -> anyhow::Result<()> {
    // Manually construct the JSON to inject outcome fields
    let mut v = serde_json::to_value(artifacts)?;
    if let Some(obj) = v.as_object_mut() {
        // Inject top-level outcome fields for machine-readability (Canonical Contract)
        obj.insert(
            "exit_code".to_string(),
            serde_json::json!(outcome.exit_code),
        );
        obj.insert(
            "reason_code".to_string(),
            serde_json::json!(outcome.reason_code),
        );
        obj.insert(
            "reason_code_version".to_string(),
            serde_json::json!(assay_core::report::summary::REASON_CODE_VERSION),
        );

        // E7.2: seeds always present; order_seed/judge_seed as string or null (SPEC: avoid JSON number precision loss)
        obj.insert(
            "seed_version".to_string(),
            serde_json::json!(assay_core::report::summary::SEED_VERSION),
        );
        let order_seed_json = match artifacts.order_seed {
            Some(n) => serde_json::Value::String(n.to_string()),
            None => serde_json::Value::Null,
        };
        obj.insert("order_seed".to_string(), order_seed_json);
        obj.insert("judge_seed".to_string(), serde_json::Value::Null);

        // E7.3: judge metrics when present
        if let Some(metrics) =
            assay_core::report::summary::judge_metrics_from_results(&artifacts.results)
        {
            obj.insert("judge_metrics".to_string(), serde_json::to_value(metrics)?);
        }

        // E2.3: SARIF truncation metadata when SARIF was truncated
        if let Some(n) = sarif_omitted {
            if n > 0 {
                obj.insert("sarif".to_string(), serde_json::json!({ "omitted": n }));
            }
        }

        // Conflict avoidance: Move full details to 'resolution' object
        // Do NOT inject 'message' or 'next_step' top-level to avoid collisions with artifact fields.
        obj.insert("resolution".to_string(), serde_json::to_value(outcome)?);

        if !outcome.warnings.is_empty() {
            obj.insert("warnings".into(), serde_json::json!(outcome.warnings));
        }
    }

    write_sanitized_run_json(path, v)
}

/// Sanitize a run.json document and write it. Render-safety (MCP01a-5): EVERY run.json writer routes
/// its assembled value through the same `render_details_safe` walk as the `--format json` stdout
/// renderer (`assay_core::report::json::render_json`), so the untrusted result `message` /
/// `details.*` / `resolution.message` content is redacted on disk too — the file-writers used to
/// serialize raw artifacts and bypass redaction. As a record sink it redacts and control-strips but
/// does not truncate (`usize::MAX`); assay-owned keys (ids, status, reason_code, exit_code, digests,
/// seeds) stay byte-stable. Shared by both the extended and minimal (early-error) writers so neither
/// path can drift back to raw.
fn write_sanitized_run_json(path: &Path, v: serde_json::Value) -> anyhow::Result<()> {
    let v = assay_core::render_safety::render_details_safe(
        assay_core::render_safety::Sink::Json,
        &v,
        usize::MAX,
    );
    if let Some(parent) = path.parent() {
        std::fs::create_dir_all(parent)?;
    }
    std::fs::write(path, serde_json::to_string_pretty(&v)?)?;
    Ok(())
}

pub(crate) fn write_run_json_minimal(
    outcome: &crate::exit_codes::RunOutcome,
    path: &Path,
) -> anyhow::Result<()> {
    // Minimal JSON for early exits (no artifacts available). E7.2: seed fields present for schema stability (null when unknown).
    // `resolution` carries the outcome (incl. its `message`, which on config/arg errors echoes
    // external path/parse text), so this goes through the shared sanitizer like the extended writer.
    let v = serde_json::json!({
        "exit_code": outcome.exit_code,
        "reason_code": outcome.reason_code,
        "reason_code_version": assay_core::report::summary::REASON_CODE_VERSION,
        "seed_version": assay_core::report::summary::SEED_VERSION,
        "order_seed": null,
        "judge_seed": null,
        "resolution": outcome
    });
    write_sanitized_run_json(path, v)
}

pub(crate) fn export_baseline(
    path: &PathBuf,
    config_path: &Path,
    cfg: &assay_core::model::EvalConfig,
    results: &[assay_core::model::TestResultRow],
) -> anyhow::Result<()> {
    let mut entries = Vec::new();

    for r in results {
        if let Some(metrics) = r.details.get("metrics").and_then(|v| v.as_object()) {
            for (metric_name, m_val) in metrics {
                if let Some(score) = m_val.get("score").and_then(|s| s.as_f64()) {
                    entries.push(assay_core::baseline::BaselineEntry {
                        test_id: r.test_id.clone(),
                        metric: metric_name.clone(),
                        score,
                        meta: None,
                    });
                }
            }
        }
    }

    let b = assay_core::baseline::Baseline {
        schema_version: 1,
        suite: cfg.suite.clone(),
        assay_version: env!("CARGO_PKG_VERSION").to_string(),
        created_at: chrono::Utc::now().to_rfc3339(),
        config_fingerprint: assay_core::baseline::compute_config_fingerprint(config_path),
        git_info: None,
        entries,
    };

    b.save(path)?;
    eprintln!("exported baseline to {}", path.display());
    Ok(())
}

#[cfg(test)]
mod run_json_render_safety_tests {
    use super::write_extended_run_json;
    use crate::exit_codes::{ReasonCode, RunOutcome};
    use assay_core::model::{TestResultRow, TestStatus};
    use assay_core::report::RunArtifacts;

    /// MCP01a-5 regression: the on-disk run.json file writer must redact untrusted result
    /// `message` / `details.*` content, matching the `--format json` stdout renderer. This guards the
    /// gap where `write_extended_run_json` serialized raw artifacts and leaked secrets to the file.
    #[test]
    fn on_disk_run_json_is_render_safe() {
        let secret = format!("ghp_{}", "A".repeat(36));
        let artifacts = RunArtifacts {
            run_id: 7,
            suite: "owned-suite".into(),
            results: vec![TestResultRow {
                test_id: "t_owned".into(),
                status: TestStatus::Fail,
                score: Some(0.0),
                cached: false,
                message: format!("failed: leaked {secret}"),
                details: serde_json::json!({
                    "prompt": format!("ask {secret} alice@example.com"),
                    "metrics": { "must_contain": { "details": { "message": format!("missing {secret}") } } },
                }),
                duration_ms: Some(3),
                fingerprint: Some("fp_owned".into()),
                skip_reason: None,
                attempts: None,
                error_policy_applied: None,
            }],
            order_seed: None,
            runner_clone_ms: None,
        };
        let outcome = RunOutcome::from_reason(ReasonCode::ETestFailed, None, None);
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("run.json");
        write_extended_run_json(&artifacts, &outcome, &path, None).unwrap();
        let out = std::fs::read_to_string(&path).unwrap();

        // Structure preserved + hostile values gone, markers present.
        let parsed: serde_json::Value = serde_json::from_str(&out).unwrap();
        assert!(!out.contains(&secret), "on-disk run.json leaked secret");
        assert!(!out.contains("ghp_"), "raw token prefix leaked to run.json");
        assert!(
            !out.contains("alice@example.com"),
            "on-disk run.json leaked pii"
        );
        assert!(
            out.contains("<redacted:"),
            "on-disk run.json fired no redaction"
        );
        // Assay-owned fields stay byte-stable.
        assert_eq!(parsed["suite"], "owned-suite");
        assert_eq!(parsed["results"][0]["test_id"], "t_owned");
        assert_eq!(parsed["results"][0]["fingerprint"], "fp_owned");
        assert_eq!(parsed["reason_code"], "E_TEST_FAILED");
    }

    /// MCP01a-5 regression: the early-error / replay-failure minimal writer shares the same sanitizer,
    /// so a config/arg-error `resolution.message` (which can echo external path/parse text) is redacted
    /// on disk too — not just the artifact-bearing extended writer.
    #[test]
    fn minimal_run_json_resolution_message_is_render_safe() {
        let secret = format!("ghp_{}", "Z".repeat(36));
        let outcome = RunOutcome::from_reason(
            ReasonCode::ETraceNotFound,
            Some(format!("trace not found: {secret}")),
            None,
        );
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("run.json");
        super::write_run_json_minimal(&outcome, &path).unwrap();
        let out = std::fs::read_to_string(&path).unwrap();
        let parsed: serde_json::Value = serde_json::from_str(&out).unwrap();
        assert!(
            !out.contains(&secret),
            "minimal run.json leaked secret in resolution.message"
        );
        assert!(!out.contains("ghp_"), "raw token prefix leaked to run.json");
        assert!(
            out.contains("<redacted:"),
            "minimal run.json fired no redaction"
        );
        // Assay-owned reason_code stays byte-stable.
        assert_eq!(parsed["reason_code"], "E_TRACE_NOT_FOUND");
    }
}

#[cfg(test)]
mod run_outcome_tests {
    use super::{
        decide_run_outcome, has_judge_verdict_abstain, reason_code_from_error_message,
        reason_code_from_run_error,
    };
    use crate::exit_codes::{ExitCodeVersion, ReasonCode, EXIT_INFRA_ERROR};
    use assay_core::errors::RunError;
    use assay_core::model::{TestResultRow, TestStatus};

    #[test]
    fn test_infra_beats_abstain_precedence() {
        // Precedence: infra (exit 3) must win over abstain (exit 1). One Error + one Abstain → exit 3.
        let results = vec![
            TestResultRow {
                test_id: "infra".into(),
                status: TestStatus::Error,
                score: None,
                cached: false,
                message: "Request timeout".into(),
                details: serde_json::json!({}),
                duration_ms: None,
                fingerprint: None,
                skip_reason: None,
                attempts: None,
                error_policy_applied: None,
            },
            TestResultRow {
                test_id: "abstain".into(),
                status: TestStatus::Pass,
                score: Some(0.5),
                cached: false,
                message: String::new(),
                details: serde_json::json!({
                    "metrics": {
                        "faithfulness": {
                            "details": { "verdict": "Abstain", "score": 0.5 }
                        }
                    }
                }),
                duration_ms: None,
                fingerprint: None,
                skip_reason: None,
                attempts: None,
                error_policy_applied: None,
            },
        ];
        let outcome = decide_run_outcome(&results, false, ExitCodeVersion::V2);
        assert_eq!(
            outcome.exit_code, EXIT_INFRA_ERROR,
            "infra must beat abstain: expected exit 3"
        );
        assert!(
            outcome.reason_code == ReasonCode::ETimeout.as_str()
                || outcome.reason_code == ReasonCode::EJudgeUnavailable.as_str(),
            "reason should be infra (E_TIMEOUT or E_JUDGE_UNAVAILABLE), got {}",
            outcome.reason_code
        );
    }

    #[test]
    fn test_has_judge_verdict_abstain_detects_abstain() {
        let details = serde_json::json!({
            "metrics": {
                "faithfulness": {
                    "score": 0.5,
                    "passed": false,
                    "unstable": true,
                    "details": { "verdict": "Abstain", "score": 0.5 }
                }
            }
        });
        assert!(has_judge_verdict_abstain(&details));
    }

    #[test]
    fn test_has_judge_verdict_abstain_ignores_pass() {
        let details = serde_json::json!({
            "metrics": {
                "faithfulness": {
                    "score": 1.0,
                    "passed": true,
                    "unstable": false,
                    "details": { "verdict": "Pass", "score": 1.0 }
                }
            }
        });
        assert!(!has_judge_verdict_abstain(&details));
    }

    #[test]
    fn test_has_judge_verdict_abstain_no_metrics() {
        let details = serde_json::json!({});
        assert!(!has_judge_verdict_abstain(&details));
    }

    #[test]
    fn test_reason_code_from_error_message_maps_config_family() {
        assert_eq!(
            reason_code_from_error_message("trace not found: traces/missing.jsonl"),
            Some(ReasonCode::ETraceNotFound)
        );
        assert_eq!(
            reason_code_from_error_message("Config file not found: eval.yaml"),
            Some(ReasonCode::EMissingConfig)
        );
        assert_eq!(
            reason_code_from_error_message("config error: unknown field `foo`"),
            Some(ReasonCode::ECfgParse)
        );
    }

    #[test]
    fn test_reason_code_from_error_message_maps_infra_family() {
        assert_eq!(
            reason_code_from_error_message("provider returned 429 rate limit"),
            Some(ReasonCode::ERateLimit)
        );
        assert_eq!(
            reason_code_from_error_message("request timeout while calling judge"),
            Some(ReasonCode::ETimeout)
        );
        assert_eq!(
            reason_code_from_error_message("provider error: 503"),
            Some(ReasonCode::EProvider5xx)
        );
        assert_eq!(
            reason_code_from_error_message("network connection reset by peer"),
            Some(ReasonCode::ENetworkError)
        );
    }

    #[test]
    fn test_reason_code_from_run_error_uses_typed_kind() {
        let typed = RunError::missing_config("eval.yaml", "missing");
        assert_eq!(
            reason_code_from_run_error(&typed),
            Some(ReasonCode::EMissingConfig)
        );
        assert!(!typed.legacy_classified);
    }

    #[test]
    fn test_decide_outcome_uses_typed_details_before_legacy_message_fallback() {
        let row = TestResultRow {
            test_id: "typed".into(),
            status: TestStatus::Error,
            score: None,
            cached: false,
            message: "untyped error text".into(),
            details: serde_json::json!({
                "run_error_kind": "invalid_args"
            }),
            duration_ms: None,
            fingerprint: None,
            skip_reason: None,
            attempts: None,
            error_policy_applied: None,
        };

        let outcome = decide_run_outcome(&[row], true, ExitCodeVersion::V2);
        assert_eq!(outcome.reason_code, ReasonCode::EInvalidArgs.as_str());
        assert_eq!(
            outcome.exit_code,
            ReasonCode::EInvalidArgs.exit_code_for(ExitCodeVersion::V2)
        );
    }
}