remem-ai 0.6.93

//! GH969 executable ship matrix and outcome scorecard.
//!
//! Existing eval and benchmark evidence is composed here without converting
//! missing official runs into zeroes or successful claims.

use std::collections::BTreeMap;
use std::fs;
use std::path::{Path, PathBuf};

use serde::Serialize;
use serde_json::Value;
use sha2::{Digest, Sha256};

use crate::eval::bench_artifact::{
    AuthorityVerdict, PublicBaselineReport, SecurityReportAuthorityVerdict,
};
use crate::eval::gates::EvalGateDelta;

mod rows;
mod scorecard;

pub const DEFAULT_PUBLIC_ROOT: &str = "eval/public";
pub const DEFAULT_SECURITY_REPORT: &str = "eval/public/memory/reports/adversarial-policy-v2.json";
pub const LINUX_X86_64_SECURITY_REPORT: &str =
    "eval/public/memory/reports/adversarial-policy-v2-linux-x86_64.json";
pub const DEFAULT_CROSS_HOST_CHARTER: &str = "eval/cross-host/benchmark-charter.json";
pub const DEFAULT_CLAIM_REGISTRY: &str = "eval/claims/registry.json";

#[derive(Debug, Clone)]
pub struct ShipMatrixOptions {
    pub baseline_path: String,
    pub thresholds_path: String,
    pub golden_dataset_path: String,
    pub public_root: PathBuf,
    pub security_report_path: PathBuf,
    pub cross_host_charter_path: PathBuf,
    pub claim_registry_path: PathBuf,
    pub input_artifact_sha256: BTreeMap<String, String>,
}

impl Default for ShipMatrixOptions {
    fn default() -> Self {
        Self {
            baseline_path: crate::eval::gates::DEFAULT_BASELINE_PATH.to_string(),
            thresholds_path: crate::eval::gates::DEFAULT_THRESHOLDS_PATH.to_string(),
            golden_dataset_path: crate::eval::gates::DEFAULT_GOLDEN_DATASET_PATH.to_string(),
            public_root: PathBuf::from(DEFAULT_PUBLIC_ROOT),
            security_report_path: default_security_report_path(),
            cross_host_charter_path: PathBuf::from(DEFAULT_CROSS_HOST_CHARTER),
            claim_registry_path: PathBuf::from(DEFAULT_CLAIM_REGISTRY),
            input_artifact_sha256: BTreeMap::new(),
        }
    }
}

fn default_security_report_path() -> PathBuf {
    security_report_for_platform(std::env::consts::OS, std::env::consts::ARCH)
}

fn security_report_for_platform(os: &str, arch: &str) -> PathBuf {
    match (os, arch) {
        ("macos", "aarch64") => PathBuf::from(DEFAULT_SECURITY_REPORT),
        ("linux", "x86_64") => PathBuf::from(LINUX_X86_64_SECURITY_REPORT),
        ("macos", "x86_64") => PathBuf::from(
            "eval/public/memory/reports/adversarial-policy-v2-x86_64-apple-darwin.json",
        ),
        ("linux", "aarch64") => PathBuf::from(
            "eval/public/memory/reports/adversarial-policy-v2-aarch64-unknown-linux-gnu.json",
        ),
        _ => PathBuf::from(format!(
            "eval/public/memory/reports/adversarial-policy-v2-{os}-{arch}.json"
        )),
    }
}

#[derive(Debug, Clone, Serialize)]
pub struct ShipEvidence {
    pub ship_matrix: ShipMatrixReport,
    pub outcome_scorecard: OutcomeScorecard,
}

#[derive(Debug, Clone, Serialize)]
pub struct ShipMatrixReport {
    pub schema_version: u32,
    pub implementation: ImplementationIdentity,
    pub summary: ShipMatrixSummary,
    pub gates: Vec<ShipGateRow>,
}

#[derive(Debug, Clone, Serialize)]
pub struct ImplementationIdentity {
    pub git_sha: Option<String>,
    pub checkout_git_sha: Option<String>,
    pub build_source_dirty: Option<bool>,
    pub source_dirty: Option<bool>,
    pub production_input_tree_sha256: Option<String>,
    pub checkout_production_input_tree_sha256: Option<String>,
    pub production_pathspec_sha256: Option<String>,
    pub executable_source_equivalent: bool,
    pub package_version: &'static str,
    pub os: &'static str,
    pub arch: &'static str,
}

#[derive(Debug, Clone, Serialize)]
pub struct ShipMatrixSummary {
    pub command_passed: bool,
    pub merge_ready: bool,
    pub release_ready: bool,
    pub implementation_identified: bool,
    pub source_clean: bool,
    pub default_on_ready: bool,
    pub cross_host_claim_ready: bool,
    pub coding_outcome_claim_ready: bool,
    pub public_claim_ready: bool,
}

#[derive(Debug, Clone, Serialize)]
pub struct ShipGateRow {
    pub id: &'static str,
    pub owner: &'static str,
    pub status: ShipGateStatus,
    pub blocks: Vec<&'static str>,
    pub required_for_command_success: bool,
    pub claim_level: String,
    pub condition_completeness: String,
    pub config_identity: String,
    pub model_identity: String,
    pub platform_identity: String,
    pub metric_deltas: BTreeMap<String, f64>,
    pub stop_loss_verdict: String,
    pub exclusions: Vec<String>,
    pub evidence: Vec<ArtifactEvidence>,
    pub diagnostics: Vec<String>,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum ShipGateStatus {
    Pass,
    Fail,
    Incomplete,
    Unavailable,
    #[allow(dead_code)]
    NotApplicable,
}

#[derive(Debug, Clone, Serialize)]
pub struct ArtifactEvidence {
    pub path: String,
    pub state: ArtifactState,
    pub sha256: Option<String>,
    pub detail: String,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum ArtifactState {
    Verified,
    Present,
    Missing,
    Invalid,
}

#[derive(Debug, Clone, Serialize)]
pub struct OutcomeScorecard {
    pub schema_version: u32,
    pub measurement_states: [MeasurementState; 3],
    pub fields: Vec<ScorecardField>,
}

#[derive(Debug, Clone, Serialize)]
pub struct ScorecardField {
    pub id: &'static str,
    pub measurement_state: MeasurementState,
    pub eligible_population: String,
    pub numerator: ScorecardComponent,
    pub denominator: ScorecardComponent,
    pub values: BTreeMap<String, f64>,
    pub threshold: String,
    pub source: Option<String>,
    pub claim_level: String,
    pub note: String,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum MeasurementState {
    Measured,
    Unavailable,
    NotApplicable,
}

#[derive(Debug, Clone, Serialize)]
pub struct ScorecardComponent {
    pub definition: String,
    pub value: Option<f64>,
}

pub(super) struct PublicEvidence {
    pub(super) report: Option<PublicBaselineReport>,
    pub(super) report_error: Option<String>,
    pub(super) security_claim_level: Option<String>,
    pub(super) security_source: Option<String>,
    pub(super) security_error: Option<String>,
    pub(super) security_authority: Option<SecurityReportAuthorityVerdict>,
}

impl PublicEvidence {
    pub(super) fn authority_verdict(&self) -> Option<&AuthorityVerdict> {
        self.report
            .as_ref()
            .map(|report| &report.artifact_verifier.authority_verdict)
    }
}

pub fn build_ship_evidence(
    deltas: &[EvalGateDelta],
    capacity_applicable: bool,
    legacy_gates_passed: bool,
    options: ShipMatrixOptions,
) -> ShipEvidence {
    let public = load_public_evidence(&options);
    let implementation = implementation_identity(&public);
    let gates = rows::build_gate_rows(deltas, capacity_applicable, &options, &public);
    let command_passed = legacy_gates_passed
        && gates
            .iter()
            .filter(|gate| gate.required_for_command_success)
            .all(pass_or_not_applicable);
    let implementation_identified = implementation.git_sha.is_some()
        && implementation.checkout_git_sha.is_some()
        && implementation.executable_source_equivalent;
    let source_clean = source_is_clean(
        implementation.build_source_dirty,
        implementation.source_dirty,
    );
    let summary = ShipMatrixSummary {
        command_passed,
        merge_ready: merge_is_ready(&gates, legacy_gates_passed),
        release_ready: release_is_ready(
            &gates,
            legacy_gates_passed,
            public
                .authority_verdict()
                .is_some_and(|verdict| verdict.release.ready),
        ),
        implementation_identified,
        source_clean,
        default_on_ready: required_rows_pass(&gates, "retrieval_default")
            && required_rows_pass(&gates, "context_default"),
        cross_host_claim_ready: gate_passes(&gates, "cross_host"),
        coding_outcome_claim_ready: gate_passes(&gates, "coding_outcome"),
        public_claim_ready: gate_passes(&gates, "public_claim"),
    };
    ShipEvidence {
        ship_matrix: ShipMatrixReport {
            schema_version: 1,
            implementation,
            summary,
            gates,
        },
        outcome_scorecard: scorecard::build_scorecard(&public, &options.security_report_path),
    }
}

fn load_public_evidence(options: &ShipMatrixOptions) -> PublicEvidence {
    let (report, report_error) = match crate::eval::bench_artifact::generate_public_baseline_report(
        &options.public_root,
        &options.claim_registry_path,
    ) {
        Ok(report) => (Some(report), None),
        Err(error) => (
            None,
            Some(format!("public artifact verification failed: {error:#}")),
        ),
    };
    let selected_relative = options
        .security_report_path
        .strip_prefix(&options.public_root)
        .ok()
        .map(|path| path.to_string_lossy().replace('\\', "/"));
    let verified = report
        .as_ref()
        .map(|baseline| &baseline.artifact_verifier.verified_artifacts);
    let selected = selected_relative.as_deref().and_then(|relative| {
        verified?
            .reports
            .iter()
            .find(|artifact| artifact.path == relative)
    });
    let security_claim_level = selected.map(|artifact| artifact.value.claim_level.clone());
    let security_authority = selected.and_then(|artifact| {
        report
            .as_ref()?
            .artifact_verifier
            .authority_verdict
            .security
            .reports
            .iter()
            .find(|authority| {
                authority.report_path == artifact.path && authority.report_sha256 == artifact.sha256
            })
            .cloned()
    });
    let security_source = security_authority.as_ref().map(|authority| {
        format!(
            "{}#sha256={}",
            options
                .public_root
                .join(&authority.report_path)
                .to_string_lossy(),
            authority.report_sha256
        )
    });
    let security_error = security_authority.is_none().then(|| {
        format!(
            "runtime authority verdict omitted an exact binding for selected security report {}",
            options.security_report_path.display()
        )
    });
    PublicEvidence {
        report,
        report_error,
        security_claim_level,
        security_source,
        security_error,
        security_authority,
    }
}

pub(super) fn read_json_value(path: &Path) -> Result<Value, String> {
    let bytes = fs::read(path).map_err(|error| format!("read {}: {error}", path.display()))?;
    serde_json::from_slice(&bytes).map_err(|error| format!("parse {}: {error}", path.display()))
}

pub(super) fn evidence_for_path(path: &Path, desired_state: ArtifactState) -> ArtifactEvidence {
    match fs::read(path) {
        Ok(bytes) => ArtifactEvidence {
            path: path.to_string_lossy().to_string(),
            state: desired_state,
            sha256: Some(format!("{:x}", Sha256::digest(bytes))),
            detail: "exact file content hash".to_string(),
        },
        Err(error) => ArtifactEvidence {
            path: path.to_string_lossy().to_string(),
            state: ArtifactState::Missing,
            sha256: None,
            detail: error.to_string(),
        },
    }
}

pub(super) fn evidence_for_evaluated_path(
    path: &Path,
    expected_sha256: Option<&str>,
) -> ArtifactEvidence {
    match (fs::read(path), expected_sha256) {
        (Ok(bytes), Some(expected)) if format!("{:x}", Sha256::digest(&bytes)) == expected => {
            ArtifactEvidence {
                path: path.to_string_lossy().to_string(),
                state: ArtifactState::Verified,
                sha256: Some(expected.to_string()),
                detail: "exact bytes consumed by eval-gates".to_string(),
            }
        }
        (Ok(_), Some(expected)) => ArtifactEvidence {
            path: path.to_string_lossy().to_string(),
            state: ArtifactState::Invalid,
            sha256: Some(expected.to_string()),
            detail: "file changed after eval-gates loaded it".to_string(),
        },
        (Ok(_), None) => ArtifactEvidence {
            path: path.to_string_lossy().to_string(),
            state: ArtifactState::Invalid,
            sha256: None,
            detail: "no consumed-byte identity was supplied".to_string(),
        },
        (Err(error), _) => ArtifactEvidence {
            path: path.to_string_lossy().to_string(),
            state: ArtifactState::Missing,
            sha256: expected_sha256.map(str::to_string),
            detail: error.to_string(),
        },
    }
}

fn implementation_identity(public: &PublicEvidence) -> ImplementationIdentity {
    let binding = public
        .authority_verdict()
        .map(|verdict| &verdict.implementation);
    ImplementationIdentity {
        git_sha: binding.and_then(|binding| binding.build_git_sha.clone()),
        checkout_git_sha: binding.and_then(|binding| binding.checkout_git_sha.clone()),
        build_source_dirty: binding.and_then(|binding| binding.build_source_dirty),
        source_dirty: binding.and_then(|binding| binding.checkout_source_dirty),
        production_input_tree_sha256: binding
            .and_then(|binding| binding.build_production_input_tree_sha256.clone()),
        checkout_production_input_tree_sha256: binding
            .and_then(|binding| binding.checkout_production_input_tree_sha256.clone()),
        production_pathspec_sha256: binding
            .and_then(|binding| binding.production_pathspec_sha256.clone()),
        executable_source_equivalent: binding
            .is_some_and(|binding| binding.executable_source_equivalent),
        package_version: env!("CARGO_PKG_VERSION"),
        os: std::env::consts::OS,
        arch: std::env::consts::ARCH,
    }
}

fn source_is_clean(build_source_dirty: Option<bool>, checkout_source_dirty: Option<bool>) -> bool {
    build_source_dirty == Some(false) && checkout_source_dirty == Some(false)
}

fn required_rows_pass(gates: &[ShipGateRow], scope: &str) -> bool {
    gates
        .iter()
        .filter(|gate| gate.blocks.contains(&scope))
        .all(pass_or_not_applicable)
}

fn merge_is_ready(gates: &[ShipGateRow], legacy_gates_passed: bool) -> bool {
    legacy_gates_passed && required_rows_pass(gates, "merge")
}

fn release_is_ready(
    gates: &[ShipGateRow],
    legacy_gates_passed: bool,
    verifier_release_ready: bool,
) -> bool {
    legacy_gates_passed && verifier_release_ready && required_rows_pass(gates, "release")
}

fn gate_passes(gates: &[ShipGateRow], scope: &str) -> bool {
    gates
        .iter()
        .filter(|gate| gate.blocks.contains(&scope))
        .all(|gate| gate.status == ShipGateStatus::Pass)
}

fn pass_or_not_applicable(gate: &ShipGateRow) -> bool {
    matches!(
        gate.status,
        ShipGateStatus::Pass | ShipGateStatus::NotApplicable
    )
}

#[cfg(test)]
mod tests;