sharpebench_core/

forecast.rs

//! Strict ingestion and independent analysis of prospective forecast evidence.
//!
//! SharpeArena owns the commit-time ledger. SharpeBench accepts only the closed
//! `sharpe.forecast-evidence.v1` and `.v2` file contracts, reconstructs every score locally,
//! and compares two agents only on the exact contracts both resolved, settled to the
//! exact same outcome. A pair receives an interval, a p-value and a Holm verdict only
//! when each agent resolved every contract the other did, so one agent's gaps can
//! neither shrink another pair's comparison nor choose its own. Bench takes documents
//! from any producer, so contract identity alone does not license differencing two
//! losses; the realized outcome and its availability time are retained past scoring
//! and must agree. This module does not import [`crate::composite`] and its report is
//! not an input to trading-rank eligibility.

use std::collections::{BTreeMap, BTreeSet};
use std::error::Error;
use std::fmt::{Display, Formatter};

use serde::{Deserialize, Serialize};
use serde_json::Value;
use sha2::{Digest, Sha256};
use sharpebench_protocol::canonical::{versioned_preimage, CANONICAL_JSON_VERSION};

use crate::stats::norm_cdf;

pub const FORECAST_EVIDENCE_SCHEMA: &str = "sharpe.forecast-evidence.v1";
/// The v1 envelope plus `contract_digest_encoding` on every revision. A v2
/// revision is verified under the encoding it declares and under nothing else.
pub const FORECAST_EVIDENCE_SCHEMA_V2: &str = "sharpe.forecast-evidence.v2";
/// Version 2 replaces the field-wide support intersection with exact pairwise
/// support: `common_support` names the digests any agent resolved and charges each
/// agent's gaps to that agent, and a pair with unequal resolved support carries
/// `support_gap` and a withheld inference.
pub const FORECAST_QUALITY_SCHEMA: &str = "sharpebench.forecast-quality.v2";
/// The report written against a declared [`ForecastContractPlan`]: v2 with the
/// plan as the field support and `common_support.outside_plan_by_agent` listing
/// the resolved digests the plan does not name, which are not scored. A report
/// without a plan stays v2.
pub const FORECAST_QUALITY_PLAN_SCHEMA: &str = "sharpebench.forecast-quality.v3";
/// Schema of the declared contract universe file.
pub const FORECAST_CONTRACT_PLAN_SCHEMA: &str = "sharpebench.forecast-contract-plan.v1";
const SUPPORT_RULE: &str = "a pair is differenced on the contract digests both agents resolved \
     and receives inference only when each agent resolved every digest the other did";
const PLAN_SUPPORT_RULE: &str = "a pair is differenced on the declared plan's contract digests \
     both agents resolved and receives inference only when each agent resolved every plan digest \
     the other did; resolved digests outside the plan are listed and not scored";
const CONTRACT_SCHEMA: &str = "sharpearena.forecast-contract.v1";

/// The encoding a revision's `contract_sha256` was verified under.
///
/// A contract has two accepted digests. The current one hashes the
/// [`versioned_preimage`] of `sharpebench/canonical-json/v1`, so it carries its
/// version and cannot collide with a digest taken under another form. The legacy
/// one hashes the unframed pre-migration text (`serde_json` number rendering with
/// Python exponent padding) and is accepted only because published evidence pins
/// it: `paper/evidence/prospective-forecast-field/` carries 24 such digests. A
/// revision digest that recomputes under neither is refused, and the version each
/// scored contract verified under is reported, so a legacy field is never mistaken
/// for a current one.
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub enum ContractDigestVersion {
    /// SHA-256 over `versioned_preimage` under `sharpebench/canonical-json/v1`.
    CanonicalJsonV1,
    /// SHA-256 over the unframed pre-migration canonical text.
    Legacy,
}

impl ContractDigestVersion {
    /// The label written into the report: the canonical-form version string for a
    /// current digest, `legacy` for a pre-migration one.
    pub const fn as_str(self) -> &'static str {
        match self {
            Self::CanonicalJsonV1 => CANONICAL_JSON_VERSION,
            Self::Legacy => "legacy",
        }
    }

    /// The inverse of [`Self::as_str`]: the encoding a v2 revision declares in
    /// `contract_digest_encoding`. Any other label is not an encoding.
    pub fn from_label(label: &str) -> Option<Self> {
        [Self::CanonicalJsonV1, Self::Legacy]
            .into_iter()
            .find(|version| version.as_str() == label)
    }
}

/// A v2 revision whose `contract_sha256` does not recompute under the encoding
/// it declares. Carries the digest, the declared encoding and the encoding the
/// digest does recompute under, or `None` when it matches no contract at all.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ContractDigestEncodingMismatch {
    pub revision_id: String,
    pub contract_sha256: String,
    pub declared: ContractDigestVersion,
    pub recomputed: Option<ContractDigestVersion>,
}

impl Display for ContractDigestEncodingMismatch {
    fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result {
        write!(
            formatter,
            "revision {} declares contract digest {} under {}, but it recomputes under ",
            self.revision_id,
            self.contract_sha256,
            self.declared.as_str()
        )?;
        match self.recomputed {
            Some(recomputed) => write!(formatter, "{}", recomputed.as_str()),
            None => write!(
                formatter,
                "neither {CANONICAL_JSON_VERSION} nor the legacy encoding of any contract"
            ),
        }
    }
}

impl Error for ContractDigestEncodingMismatch {}

impl From<ContractDigestEncodingMismatch> for ForecastError {
    fn from(mismatch: ContractDigestEncodingMismatch) -> Self {
        ForecastError(mismatch.to_string())
    }
}

impl Serialize for ContractDigestVersion {
    fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
        serializer.serialize_str(self.as_str())
    }
}

/// Both digests one contract answers to.
struct ContractDigests {
    canonical_json_v1: String,
    legacy: String,
}

/// Contracts keyed by every digest they answer to, with the version each key is.
type ContractIndex<'a> = BTreeMap<String, (&'a ForecastContract, ContractDigestVersion)>;

fn index_contracts<'a>(
    contracts: impl IntoIterator<Item = (ContractDigests, &'a ForecastContract)>,
) -> ContractIndex<'a> {
    let mut index = BTreeMap::new();
    for (digests, contract) in contracts {
        index.insert(
            digests.canonical_json_v1,
            (contract, ContractDigestVersion::CanonicalJsonV1),
        );
        // The v1 digest is over the versioned frame and the legacy digest over
        // the bare text, so the two keys never coincide and this never
        // displaces the entry above.
        index
            .entry(digests.legacy)
            .or_insert((contract, ContractDigestVersion::Legacy));
    }
    index
}

fn unknown_contract_digest() -> ForecastError {
    reject(format!(
        "revision names an unknown contract digest: it recomputes under neither \
         {CANONICAL_JSON_VERSION} nor the legacy encoding of any contract"
    ))
}

#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ForecastError(pub String);

impl Display for ForecastError {
    fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result {
        formatter.write_str(&self.0)
    }
}

impl Error for ForecastError {}

fn reject(message: impl Into<String>) -> ForecastError {
    ForecastError(message.into())
}

#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ForecastProducer {
    pub name: String,
    pub contract: String,
}

#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ForecastRunIdentity {
    pub agent_id: String,
    pub model_id: String,
    pub model_sha256: String,
    pub scaffold_id: String,
    pub scaffold_sha256: String,
    pub prompt_sha256: String,
    pub operator_id: String,
    pub config_sha256: String,
}

#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ForecastContract {
    pub schema_version: String,
    pub contract_id: String,
    pub question: String,
    pub instrument: String,
    pub target: String,
    pub kind: String,
    pub opens_at: u64,
    pub deadline: u64,
    pub resolves_at: u64,
    pub observation_source: String,
    pub open_definition: String,
    pub close_definition: String,
    pub unit: String,
    pub scoring_rule: String,
    pub neutral_threshold: f64,
    pub boundary_ownership: String,
    pub missing_data_policy: String,
    pub fallback_policy: String,
    pub categories: Vec<String>,
    pub interval_alpha: Option<f64>,
}

#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub struct InformationExposure {
    pub observed_at: u64,
    pub market_snapshot_sha256: Option<String>,
    pub consensus_visible: bool,
    pub consensus_snapshot_sha256: Option<String>,
    pub source_ids: Vec<String>,
}

#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ForecastRevision {
    pub revision_id: String,
    pub claim_id: String,
    pub ordinal: u64,
    pub supersedes: Option<String>,
    pub contract_sha256: String,
    /// Present on every revision of a `sharpe.forecast-evidence.v2` document and
    /// on none of a v1 one; the parser refuses either envelope with the other
    /// shape. Absent rather than `null` on the wire, so a v1 document round-trips
    /// byte for byte.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub contract_digest_encoding: Option<String>,
    pub prediction: Vec<f64>,
    pub confidence: f64,
    pub rationale: String,
    pub submitted_at: u64,
    pub status: String,
    pub reason: Option<String>,
    pub trigger_event_id: Option<String>,
    pub revision_reason: Option<String>,
    pub exposure: InformationExposure,
    pub idempotency_key: String,
}

#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ForecastResolution {
    pub claim_id: String,
    pub status: String,
    pub outcome: Option<Value>,
    pub available_at: Option<u64>,
    pub recorded_at: u64,
    pub reason: Option<String>,
}

#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ForecastEvidence {
    pub schema_version: String,
    pub producer: ForecastProducer,
    pub generated_at: u64,
    pub identity: ForecastRunIdentity,
    pub contracts: Vec<ForecastContract>,
    pub revisions: Vec<ForecastRevision>,
    pub resolutions: Vec<ForecastResolution>,
}

/// Parse the closed evidence envelope and validate all semantic links.
pub fn parse_forecast_evidence(payload: &str) -> Result<ForecastEvidence, ForecastError> {
    let evidence: ForecastEvidence = serde_json::from_str(payload)
        .map_err(|error| reject(format!("invalid forecast evidence JSON: {error}")))?;
    validate_evidence(&evidence)?;
    Ok(evidence)
}

/// The contract digests every agent in a field is asked to answer, declared
/// before resolution.
///
/// Without a plan the field support is the union of the digests the documents
/// resolved, and a document chooses its part of that union: resolving one
/// contract nobody else resolved withholds inference on every pair it is in and
/// charges that contract to every other agent. A plan fixes the support
/// instead. A resolved digest the plan does not name is listed and not scored,
/// and an agent's gaps are the planned digests it did not resolve.
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct ForecastContractPlan {
    pub schema_version: String,
    pub contract_sha256: Vec<String>,
}

/// Parse and validate a [`ForecastContractPlan`]: the exact schema, at least one
/// digest, every digest a lowercase SHA-256, and no digest twice.
pub fn parse_forecast_contract_plan(payload: &str) -> Result<ForecastContractPlan, ForecastError> {
    let plan: ForecastContractPlan = serde_json::from_str(payload)
        .map_err(|error| reject(format!("invalid forecast contract plan: {error}")))?;
    if plan.schema_version != FORECAST_CONTRACT_PLAN_SCHEMA {
        return Err(reject("unsupported forecast contract plan schema_version"));
    }
    if plan.contract_sha256.is_empty() {
        return Err(reject(
            "a forecast contract plan names at least one contract",
        ));
    }
    let mut seen = BTreeSet::new();
    for value in &plan.contract_sha256 {
        digest(value, "plan contract_sha256")?;
        if !seen.insert(value.as_str()) {
            return Err(reject(format!(
                "forecast contract plan names contract digest {value} twice"
            )));
        }
    }
    Ok(plan)
}

fn nonempty(value: &str, field: &str) -> Result<(), ForecastError> {
    if value.trim().is_empty() {
        Err(reject(format!("{field} must be non-empty")))
    } else {
        Ok(())
    }
}

fn digest(value: &str, field: &str) -> Result<(), ForecastError> {
    if value.len() != 64
        || !value
            .bytes()
            .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
    {
        return Err(reject(format!(
            "{field} must be a lowercase SHA-256 digest"
        )));
    }
    Ok(())
}

fn finite(value: f64, field: &str) -> Result<(), ForecastError> {
    if value.is_finite() {
        Ok(())
    } else {
        Err(reject(format!("{field} must be finite")))
    }
}

fn validate_contract(contract: &ForecastContract) -> Result<ContractDigests, ForecastError> {
    if contract.schema_version != CONTRACT_SCHEMA {
        return Err(reject(format!(
            "contract {} has unsupported schema_version",
            contract.contract_id
        )));
    }
    for (field, value) in [
        ("contract_id", contract.contract_id.as_str()),
        ("question", contract.question.as_str()),
        ("instrument", contract.instrument.as_str()),
        ("target", contract.target.as_str()),
        ("observation_source", contract.observation_source.as_str()),
        ("open_definition", contract.open_definition.as_str()),
        ("close_definition", contract.close_definition.as_str()),
        ("unit", contract.unit.as_str()),
        ("boundary_ownership", contract.boundary_ownership.as_str()),
        ("missing_data_policy", contract.missing_data_policy.as_str()),
        ("fallback_policy", contract.fallback_policy.as_str()),
    ] {
        nonempty(value, field)?;
    }
    if contract.opens_at > contract.deadline || contract.deadline >= contract.resolves_at {
        return Err(reject(format!(
            "contract {} must satisfy opens_at <= deadline < resolves_at",
            contract.contract_id
        )));
    }
    finite(contract.neutral_threshold, "neutral_threshold")?;
    if contract.neutral_threshold < 0.0 {
        return Err(reject("neutral_threshold must be non-negative"));
    }
    let allowed = match contract.kind.as_str() {
        "point" => &["point_errors"][..],
        "probability" => &["binary_brier", "binary_log"][..],
        "categorical" => &["categorical_brier", "categorical_log"][..],
        "normal" => &["normal_crps"][..],
        "direction" => &["direction_accuracy"][..],
        "interval" => &["interval_score"][..],
        other => return Err(reject(format!("unknown forecast kind {other:?}"))),
    };
    if !allowed.contains(&contract.scoring_rule.as_str()) {
        return Err(reject(format!(
            "scoring rule {:?} is invalid for kind {:?}",
            contract.scoring_rule, contract.kind
        )));
    }
    let category_set: BTreeSet<&str> = contract.categories.iter().map(String::as_str).collect();
    if contract.kind == "categorical" {
        if contract.categories.len() < 2 || category_set.len() != contract.categories.len() {
            return Err(reject(
                "categorical contracts need at least two unique categories",
            ));
        }
        for category in &contract.categories {
            nonempty(category, "categories[]")?;
        }
    } else if !contract.categories.is_empty() {
        return Err(reject(
            "categories are valid only for categorical contracts",
        ));
    }
    if contract.kind == "interval" {
        match contract.interval_alpha {
            Some(alpha) if alpha.is_finite() && alpha > 0.0 && alpha < 1.0 => {}
            _ => return Err(reject("interval contracts need interval_alpha in (0, 1)")),
        }
    } else if contract.interval_alpha.is_some() {
        return Err(reject(
            "interval_alpha is valid only for interval contracts",
        ));
    }
    contract_digests(contract)
}

fn validate_prediction(
    contract: &ForecastContract,
    prediction: &[f64],
) -> Result<(), ForecastError> {
    if prediction.iter().any(|value| !value.is_finite()) {
        return Err(reject("prediction values must be finite"));
    }
    match contract.kind.as_str() {
        "point" => {
            if prediction.len() != 1 {
                return Err(reject("point prediction needs exactly one value"));
            }
        }
        "probability" => {
            if prediction.len() != 1 || !(0.0..=1.0).contains(&prediction[0]) {
                return Err(reject("probability prediction must be one value in [0, 1]"));
            }
            if contract.scoring_rule == "binary_log"
                && (prediction[0] <= 0.0 || prediction[0] >= 1.0)
            {
                return Err(reject("binary_log probability must lie inside (0, 1)"));
            }
        }
        "categorical" => {
            let sum: f64 = prediction.iter().sum();
            if prediction.len() != contract.categories.len()
                || prediction.iter().any(|value| !(0.0..=1.0).contains(value))
                || (sum - 1.0).abs() > 1e-12
            {
                return Err(reject(
                    "categorical prediction must be a simplex matching categories",
                ));
            }
            if contract.scoring_rule == "categorical_log"
                && prediction.iter().any(|value| *value <= 0.0)
            {
                return Err(reject("categorical_log probabilities must be positive"));
            }
        }
        "normal" => {
            if prediction.len() != 2 || prediction[1] <= 0.0 {
                return Err(reject("normal prediction must be [mean, positive sigma]"));
            }
        }
        "direction" => {
            if prediction.len() != 1 || !matches!(prediction[0], -1.0 | 1.0) {
                return Err(reject("direction prediction must be -1 or 1"));
            }
        }
        "interval" => {
            if prediction.len() != 2 || prediction[0] > prediction[1] {
                return Err(reject("interval prediction must be [lo, hi] with lo <= hi"));
            }
        }
        _ => return Err(reject("unknown forecast kind")),
    }
    Ok(())
}

/// Resolve one revision's contract under the rules of its envelope.
///
/// A v1 revision names a digest and Bench infers the encoding by recomputation
/// under both. A v2 revision declares the encoding beside the digest and is
/// verified under that encoding only: a digest that recomputes under the other
/// encoding, or under neither, is refused with the declared and the recomputed
/// encoding named, so a producer that mislabels its field learns which label
/// its digests actually answer to.
fn resolve_revision_contract<'a>(
    declares_encoding: bool,
    revision: &ForecastRevision,
    contracts: &ContractIndex<'a>,
) -> Result<&'a ForecastContract, ForecastError> {
    let recomputed = contracts.get(&revision.contract_sha256).copied();
    match (declares_encoding, &revision.contract_digest_encoding) {
        (false, None) => recomputed
            .map(|(contract, _)| contract)
            .ok_or_else(unknown_contract_digest),
        (false, Some(_)) => Err(reject(format!(
            "contract_digest_encoding is not a field of {FORECAST_EVIDENCE_SCHEMA}; \
             a document that declares it must be {FORECAST_EVIDENCE_SCHEMA_V2}"
        ))),
        (true, None) => Err(reject(format!(
            "{FORECAST_EVIDENCE_SCHEMA_V2} requires contract_digest_encoding on every revision"
        ))),
        (true, Some(label)) => {
            let declared = ContractDigestVersion::from_label(label).ok_or_else(|| {
                reject(format!(
                    "unknown contract_digest_encoding {label:?}; expected \
                     {CANONICAL_JSON_VERSION} or legacy"
                ))
            })?;
            match recomputed {
                Some((contract, version)) if version == declared => Ok(contract),
                other => Err(ContractDigestEncodingMismatch {
                    revision_id: revision.revision_id.clone(),
                    contract_sha256: revision.contract_sha256.clone(),
                    declared,
                    recomputed: other.map(|(_, version)| version),
                }
                .into()),
            }
        }
    }
}

fn validate_evidence(evidence: &ForecastEvidence) -> Result<(), ForecastError> {
    let declares_encoding = match evidence.schema_version.as_str() {
        FORECAST_EVIDENCE_SCHEMA => false,
        FORECAST_EVIDENCE_SCHEMA_V2 => true,
        _ => return Err(reject("unsupported forecast evidence schema_version")),
    };
    if evidence.producer.name != "sharpearena" || evidence.producer.contract != "native" {
        return Err(reject("producer must be the native SharpeArena contract"));
    }
    for (field, value) in [
        ("agent_id", evidence.identity.agent_id.as_str()),
        ("model_id", evidence.identity.model_id.as_str()),
        ("scaffold_id", evidence.identity.scaffold_id.as_str()),
        ("operator_id", evidence.identity.operator_id.as_str()),
    ] {
        nonempty(value, field)?;
    }
    for (field, value) in [
        ("model_sha256", evidence.identity.model_sha256.as_str()),
        (
            "scaffold_sha256",
            evidence.identity.scaffold_sha256.as_str(),
        ),
        ("prompt_sha256", evidence.identity.prompt_sha256.as_str()),
        ("config_sha256", evidence.identity.config_sha256.as_str()),
    ] {
        digest(value, field)?;
    }
    if evidence.contracts.is_empty()
        || evidence.revisions.is_empty()
        || evidence.resolutions.is_empty()
    {
        return Err(reject(
            "contracts, revisions, and resolutions must all be non-empty",
        ));
    }

    let mut validated = Vec::with_capacity(evidence.contracts.len());
    let mut contract_ids = BTreeSet::new();
    for contract in &evidence.contracts {
        if !contract_ids.insert(contract.contract_id.as_str()) {
            return Err(reject(format!(
                "duplicate contract_id {:?}",
                contract.contract_id
            )));
        }
        validated.push((validate_contract(contract)?, contract));
    }
    let contracts = index_contracts(validated);

    let mut revision_ids = BTreeSet::new();
    let mut idempotency_keys = BTreeSet::new();
    let mut by_claim: BTreeMap<&str, Vec<&ForecastRevision>> = BTreeMap::new();
    let mut claim_contracts: BTreeMap<&str, &str> = BTreeMap::new();
    for revision in &evidence.revisions {
        nonempty(&revision.revision_id, "revision_id")?;
        nonempty(&revision.claim_id, "claim_id")?;
        nonempty(&revision.idempotency_key, "idempotency_key")?;
        if !revision_ids.insert(revision.revision_id.as_str()) {
            return Err(reject(format!(
                "duplicate revision_id {:?}",
                revision.revision_id
            )));
        }
        if !idempotency_keys.insert(revision.idempotency_key.as_str()) {
            return Err(reject(format!(
                "duplicate idempotency_key {:?}",
                revision.idempotency_key
            )));
        }
        digest(&revision.contract_sha256, "contract_sha256")?;
        let contract = resolve_revision_contract(declares_encoding, revision, &contracts)?;
        match claim_contracts.insert(&revision.claim_id, &revision.contract_sha256) {
            Some(prior) if prior != revision.contract_sha256 => {
                return Err(reject("a claim changes contract across revisions"));
            }
            _ => {}
        }
        validate_prediction(contract, &revision.prediction)?;
        finite(revision.confidence, "confidence")?;
        if !(0.0..=1.0).contains(&revision.confidence) {
            return Err(reject("confidence must lie in [0, 1]"));
        }
        if revision.exposure.observed_at != revision.submitted_at {
            return Err(reject("exposure.observed_at must equal submitted_at"));
        }
        if let Some(hash) = &revision.exposure.market_snapshot_sha256 {
            digest(hash, "market_snapshot_sha256")?;
        }
        match (
            revision.exposure.consensus_visible,
            &revision.exposure.consensus_snapshot_sha256,
        ) {
            (true, Some(hash)) => digest(hash, "consensus_snapshot_sha256")?,
            (true, None) => return Err(reject("visible consensus requires its snapshot digest")),
            (false, Some(_)) => {
                return Err(reject("hidden consensus cannot carry a snapshot digest"));
            }
            (false, None) => {}
        }
        let sources: BTreeSet<&str> = revision
            .exposure
            .source_ids
            .iter()
            .map(String::as_str)
            .collect();
        if sources.len() != revision.exposure.source_ids.len()
            || sources.iter().any(|source| source.trim().is_empty())
        {
            return Err(reject("source_ids must be unique and non-empty"));
        }
        match revision.status.as_str() {
            "eligible"
                if revision.submitted_at >= contract.opens_at
                    && revision.submitted_at <= contract.deadline
                    && revision.reason.is_none() => {}
            "late"
                if revision.submitted_at > contract.deadline
                    && revision
                        .reason
                        .as_deref()
                        .is_some_and(|reason| !reason.trim().is_empty()) => {}
            "rejected"
                if revision.submitted_at < contract.opens_at
                    && revision
                        .reason
                        .as_deref()
                        .is_some_and(|reason| !reason.trim().is_empty()) => {}
            _ => {
                return Err(reject(
                    "revision status, clock, and reason are inconsistent",
                ))
            }
        }
        if revision
            .trigger_event_id
            .as_deref()
            .is_some_and(|value| value.trim().is_empty())
        {
            return Err(reject("trigger_event_id must be non-empty when present"));
        }
        let chain = by_claim.entry(&revision.claim_id).or_default();
        if revision.ordinal as usize != chain.len() {
            return Err(reject("claim revisions are not in append order"));
        }
        let expected = chain.last().map(|prior| prior.revision_id.as_str());
        if revision.supersedes.as_deref() != expected {
            return Err(reject("revision has the wrong supersedes link"));
        }
        if revision.ordinal == 0 && revision.revision_reason.is_some() {
            return Err(reject("initial forecast cannot have revision_reason"));
        }
        if revision.ordinal > 0
            && revision
                .revision_reason
                .as_deref()
                .is_none_or(|reason| reason.trim().is_empty())
        {
            return Err(reject("a revision requires revision_reason"));
        }
        chain.push(revision);
    }
    // Count contracts, not digests: a contract answers to two digests, and every
    // referenced digest was resolved to its contract above.
    let referenced: BTreeSet<&str> = claim_contracts
        .values()
        .map(|digest| contracts[*digest].0.contract_id.as_str())
        .collect();
    if referenced.len() != evidence.contracts.len() {
        return Err(reject(
            "every exported contract must be referenced by a claim",
        ));
    }

    let mut resolutions = BTreeMap::new();
    for resolution in &evidence.resolutions {
        nonempty(&resolution.claim_id, "resolution.claim_id")?;
        let revisions = by_claim
            .get(resolution.claim_id.as_str())
            .ok_or_else(|| reject("resolution names an unknown claim"))?;
        if resolutions
            .insert(resolution.claim_id.as_str(), resolution)
            .is_some()
        {
            return Err(reject("duplicate resolution for a claim"));
        }
        let eligible = revisions
            .iter()
            .rev()
            .find(|revision| revision.status == "eligible")
            .copied();
        match resolution.status.as_str() {
            "resolved" => {
                let revision =
                    eligible.ok_or_else(|| reject("resolved claim has no eligible revision"))?;
                let available_at = resolution
                    .available_at
                    .ok_or_else(|| reject("resolved record needs available_at"))?;
                let outcome = resolution
                    .outcome
                    .as_ref()
                    .ok_or_else(|| reject("resolved record needs an outcome"))?;
                if resolution.reason.is_some() {
                    return Err(reject("resolved record cannot carry a reason"));
                }
                let (contract, _) = contracts
                    .get(&revision.contract_sha256)
                    .ok_or_else(|| reject("resolved revision lost its contract"))?;
                if available_at <= revision.submitted_at || available_at < contract.resolves_at {
                    return Err(reject(
                        "resolution predates the frozen information boundary",
                    ));
                }
                score_prediction(contract, &revision.prediction, outcome)?;
            }
            "pending" => {
                if eligible.is_none()
                    || resolution.outcome.is_some()
                    || resolution.available_at.is_some()
                    || resolution.reason.is_some()
                {
                    return Err(reject("pending resolution fields are inconsistent"));
                }
            }
            "cancelled" => {
                if resolution.outcome.is_some()
                    || resolution.available_at.is_some()
                    || resolution
                        .reason
                        .as_deref()
                        .is_none_or(|reason| reason.trim().is_empty())
                {
                    return Err(reject("cancelled resolution fields are inconsistent"));
                }
            }
            "rejected" => {
                if eligible.is_some()
                    || resolution.outcome.is_some()
                    || resolution
                        .reason
                        .as_deref()
                        .is_none_or(|reason| reason.trim().is_empty())
                {
                    return Err(reject("rejected resolution fields are inconsistent"));
                }
            }
            _ => return Err(reject("unknown resolution status")),
        }
    }
    if resolutions.len() != by_claim.len() {
        return Err(reject("resolutions must cover every claim exactly once"));
    }
    Ok(())
}

/// The pre-migration canonical text: sorted keys, no whitespace, `serde_json`
/// number rendering with the exponent padded to two digits the way Python's
/// `repr` pads it. Kept verbatim because the digests it produced are pinned in
/// published evidence; new digests come from [`versioned_preimage`].
fn legacy_canonical_json(value: &Value, output: &mut String) -> Result<(), ForecastError> {
    match value {
        Value::Null => output.push_str("null"),
        Value::Bool(value) => output.push_str(if *value { "true" } else { "false" }),
        Value::Number(number) => output.push_str(&python_number(number)),
        Value::String(value) => output.push_str(
            &serde_json::to_string(value)
                .map_err(|error| reject(format!("cannot encode contract string: {error}")))?,
        ),
        Value::Array(values) => {
            output.push('[');
            for (index, value) in values.iter().enumerate() {
                if index > 0 {
                    output.push(',');
                }
                legacy_canonical_json(value, output)?;
            }
            output.push(']');
        }
        Value::Object(values) => {
            output.push('{');
            let mut fields: Vec<_> = values.iter().collect();
            fields.sort_by(|left, right| left.0.cmp(right.0));
            for (index, (key, value)) in fields.into_iter().enumerate() {
                if index > 0 {
                    output.push(',');
                }
                output.push_str(
                    &serde_json::to_string(key)
                        .map_err(|error| reject(format!("cannot encode contract key: {error}")))?,
                );
                output.push(':');
                legacy_canonical_json(value, output)?;
            }
            output.push('}');
        }
    }
    Ok(())
}

fn python_number(number: &serde_json::Number) -> String {
    let rendered = number.to_string();
    let Some((mantissa, exponent)) = rendered.split_once('e') else {
        return rendered;
    };
    let (sign, digits) = if let Some(rest) = exponent.strip_prefix('-') {
        ("-", rest)
    } else if let Some(rest) = exponent.strip_prefix('+') {
        ("+", rest)
    } else {
        ("+", exponent)
    };
    format!("{mantissa}e{sign}{digits:0>2}")
}

fn contract_digests(contract: &ForecastContract) -> Result<ContractDigests, ForecastError> {
    let value = serde_json::to_value(contract)
        .map_err(|error| reject(format!("cannot serialize forecast contract: {error}")))?;
    let framed = versioned_preimage(&value)
        .map_err(|error| reject(format!("cannot encode forecast contract: {error}")))?;
    let mut legacy = String::new();
    legacy_canonical_json(&value, &mut legacy)?;
    Ok(ContractDigests {
        canonical_json_v1: crate::lower_hex(&Sha256::digest(&framed)),
        legacy: crate::lower_hex(&Sha256::digest(legacy.as_bytes())),
    })
}

/// Content address of a realized outcome, over the same canonical encoding as the
/// contract digest.
///
/// The contract digest fixes the question. It says nothing about how the question
/// was settled, so two producers can present the identical contract with opposite
/// realized outcomes. Comparing agents requires both, and the outcome must survive
/// scoring rather than be discarded once a loss has been computed from it.
fn outcome_sha256(outcome: &Value) -> Result<String, ForecastError> {
    let preimage = versioned_preimage(outcome)
        .map_err(|error| reject(format!("cannot encode forecast outcome: {error}")))?;
    Ok(crate::lower_hex(&Sha256::digest(&preimage)))
}

fn number_outcome(outcome: &Value) -> Result<f64, ForecastError> {
    let value = outcome
        .as_f64()
        .ok_or_else(|| reject("forecast outcome must be numeric"))?;
    finite(value, "outcome")?;
    Ok(value)
}

#[derive(Clone, Debug)]
struct ScoredForecast {
    contract_sha256: String,
    contract_digest_version: ContractDigestVersion,
    instrument: String,
    resolves_at: u64,
    scoring_rule: String,
    /// Retained settlement identity: which outcome this loss was computed against.
    outcome_sha256: String,
    /// Retained settlement availability, when the resolution record carries one.
    outcome_available_at: Option<u64>,
    loss: f64,
    probability: Option<(f64, bool)>,
    categorical_confidence: Option<(f64, bool)>,
    normal_pit: Option<f64>,
    consensus_visible: bool,
}

type ScoreParts = (f64, Option<(f64, bool)>, Option<(f64, bool)>, Option<f64>);

fn score_prediction(
    contract: &ForecastContract,
    prediction: &[f64],
    outcome: &Value,
) -> Result<ScoreParts, ForecastError> {
    validate_prediction(contract, prediction)?;
    match contract.kind.as_str() {
        "point" => {
            let error = number_outcome(outcome)? - prediction[0];
            Ok((error * error, None, None, None))
        }
        "probability" => {
            let realized = number_outcome(outcome)?;
            if realized != 0.0 && realized != 1.0 {
                return Err(reject("binary outcome must be 0 or 1"));
            }
            let event = realized == 1.0;
            let probability = prediction[0];
            let loss = if contract.scoring_rule == "binary_brier" {
                (probability - realized).powi(2)
            } else if event {
                -probability.ln()
            } else {
                -(1.0 - probability).ln()
            };
            Ok((loss, Some((probability, event)), None, None))
        }
        "categorical" => {
            let realized = outcome
                .as_str()
                .ok_or_else(|| reject("categorical outcome must be a string"))?;
            let index = contract
                .categories
                .iter()
                .position(|category| category == realized)
                .ok_or_else(|| reject("categorical outcome is not in the frozen categories"))?;
            let loss = if contract.scoring_rule == "categorical_brier" {
                prediction
                    .iter()
                    .enumerate()
                    .map(|(at, probability)| {
                        let target = if at == index { 1.0 } else { 0.0 };
                        (probability - target).powi(2)
                    })
                    .sum()
            } else {
                -prediction[index].ln()
            };
            let (chosen, confidence) = prediction
                .iter()
                .copied()
                .enumerate()
                .max_by(|left, right| left.1.total_cmp(&right.1))
                .ok_or_else(|| reject("categorical prediction is empty"))?;
            Ok((loss, None, Some((confidence, chosen == index)), None))
        }
        "normal" => {
            let realized = number_outcome(outcome)?;
            let mean = prediction[0];
            let sigma = prediction[1];
            let z = (realized - mean) / sigma;
            let phi = (-0.5 * z * z).exp() / (2.0 * std::f64::consts::PI).sqrt();
            let cdf = norm_cdf(z);
            let crps =
                sigma * (z * (2.0 * cdf - 1.0) + 2.0 * phi - 1.0 / std::f64::consts::PI.sqrt());
            Ok((crps, None, None, Some(cdf)))
        }
        "direction" => {
            let realized = number_outcome(outcome)?;
            let direction = if realized.abs() <= contract.neutral_threshold {
                0.0
            } else {
                realized.signum()
            };
            let correct = direction != 0.0 && direction == prediction[0];
            Ok((if correct { 0.0 } else { 1.0 }, None, None, None))
        }
        "interval" => {
            let realized = number_outcome(outcome)?;
            let alpha = contract
                .interval_alpha
                .ok_or_else(|| reject("interval_alpha is missing"))?;
            let (lower, upper) = (prediction[0], prediction[1]);
            let penalty = if realized < lower {
                2.0 * (lower - realized) / alpha
            } else if realized > upper {
                2.0 * (realized - upper) / alpha
            } else {
                0.0
            };
            Ok((upper - lower + penalty, None, None, None))
        }
        _ => Err(reject("unknown forecast kind")),
    }
}

fn scored_forecasts(evidence: &ForecastEvidence) -> Result<Vec<ScoredForecast>, ForecastError> {
    let mut digests = Vec::with_capacity(evidence.contracts.len());
    for contract in &evidence.contracts {
        digests.push((contract_digests(contract)?, contract));
    }
    let contracts = index_contracts(digests);
    let mut effective: BTreeMap<&str, &ForecastRevision> = BTreeMap::new();
    for revision in &evidence.revisions {
        if revision.status == "eligible" {
            effective.insert(&revision.claim_id, revision);
        }
    }
    let resolutions: BTreeMap<&str, &ForecastResolution> = evidence
        .resolutions
        .iter()
        .map(|resolution| (resolution.claim_id.as_str(), resolution))
        .collect();
    let mut used_contracts = BTreeSet::new();
    let mut scored = Vec::new();
    for (claim_id, revision) in effective {
        let Some(resolution) = resolutions.get(claim_id) else {
            continue;
        };
        if resolution.status != "resolved" {
            continue;
        }
        let (contract, contract_digest_version) = *contracts
            .get(&revision.contract_sha256)
            .ok_or_else(|| reject("effective revision names an unknown contract"))?;
        // By contract, not by digest: the same contract under its v1 and its
        // legacy digest is still one contract.
        if !used_contracts.insert(contract.contract_id.as_str()) {
            return Err(reject(
                "one agent has multiple effective claims for the same contract",
            ));
        }
        let outcome = resolution
            .outcome
            .as_ref()
            .ok_or_else(|| reject("resolved record has no outcome"))?;
        let (loss, probability, categorical_confidence, normal_pit) =
            score_prediction(contract, &revision.prediction, outcome)?;
        scored.push(ScoredForecast {
            contract_sha256: revision.contract_sha256.clone(),
            contract_digest_version,
            instrument: contract.instrument.clone(),
            resolves_at: contract.resolves_at,
            scoring_rule: contract.scoring_rule.clone(),
            outcome_sha256: outcome_sha256(outcome)?,
            outcome_available_at: resolution.available_at,
            loss,
            probability,
            categorical_confidence,
            normal_pit,
            consensus_visible: revision.exposure.consensus_visible,
        });
    }
    Ok(scored)
}

#[derive(Clone, Copy, Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ForecastAnalysisConfig {
    pub bootstrap_seed: u64,
    pub bootstrap_samples: usize,
    pub confidence: f64,
    pub familywise_alpha: f64,
    pub calibration_bins: usize,
}

impl Default for ForecastAnalysisConfig {
    fn default() -> Self {
        Self {
            bootstrap_seed: 0xF0EC_A57A_2026,
            bootstrap_samples: 2_000,
            confidence: 0.95,
            familywise_alpha: 0.05,
            calibration_bins: 10,
        }
    }
}

#[derive(Clone, Debug, Serialize)]
pub struct MetricMean {
    pub scoring_rule: String,
    pub n: usize,
    pub mean_loss: f64,
}

#[derive(Clone, Debug, Serialize)]
pub struct CalibrationBin {
    pub lower: f64,
    pub upper: f64,
    pub n: usize,
    pub mean_forecast: Option<f64>,
    pub event_rate: Option<f64>,
}

#[derive(Clone, Debug, Serialize)]
pub struct BinaryCalibration {
    pub n: usize,
    pub brier: f64,
    pub base_rate: f64,
    pub reliability: f64,
    pub resolution: f64,
    pub uncertainty: f64,
    pub brier_skill: Option<f64>,
    pub bins: Vec<CalibrationBin>,
}

#[derive(Clone, Debug, Serialize)]
pub struct ConfidenceCalibration {
    pub n: usize,
    pub mean_confidence: f64,
    pub accuracy: f64,
    pub bins: Vec<CalibrationBin>,
}

#[derive(Clone, Debug, Serialize)]
pub struct DistributionCalibration {
    pub n: usize,
    pub pit_mean: f64,
    pub pit_variance: f64,
    pub uniform_reference_mean: f64,
    pub uniform_reference_variance: f64,
    pub bin_counts: Vec<usize>,
}

#[derive(Clone, Debug, Serialize)]
pub struct AgentForecastSummary {
    pub agent_id: String,
    pub model_id: String,
    pub n_claims: usize,
    pub n_revisions: usize,
    pub n_eligible_claims: usize,
    pub n_resolved: usize,
    pub n_pending: usize,
    pub n_cancelled: usize,
    pub n_rejected: usize,
    pub resolution_rate: f64,
    pub blind_resolved: usize,
    pub consensus_exposed_resolved: usize,
    pub metrics: Vec<MetricMean>,
    pub binary_calibration: Option<BinaryCalibration>,
    pub categorical_confidence_calibration: Option<ConfidenceCalibration>,
    pub normal_distribution_calibration: Option<DistributionCalibration>,
}

/// The field contracts one agent did not resolve, each listed once under that agent's
/// own status for it.
///
/// A digest named by a pending claim is listed as pending even if another claim on it
/// was cancelled or rejected, and a cancelled one outranks a rejected one: the list
/// records the most recoverable reason the agent gave.
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
pub struct AgentUnresolvedSupport {
    pub n_unresolved: usize,
    /// Named by no claim in this agent's document. Support is exact by digest, so a
    /// contract the agent claimed under its other encoding is listed here.
    pub not_claimed: Vec<String>,
    pub pending: Vec<String>,
    pub cancelled: Vec<String>,
    /// Claimed, but the claim had no eligible revision to settle.
    pub rejected: Vec<String>,
}

/// A contract some agent resolved while another agent's document leaves it pending
/// or cancelled. Recorded, never differenced: the non-resolving agent is charged the
/// gap and its pairs are withheld.
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub struct SettlementStatusDisagreement {
    pub contract_sha256: String,
    pub resolved_by: Vec<String>,
    pub pending_by: Vec<String>,
    pub cancelled_by: Vec<String>,
}

#[derive(Clone, Debug, Serialize)]
pub struct CommonSupport {
    pub rule: &'static str,
    /// The field support each agent's gaps are charged against: every contract
    /// digest at least one agent resolved, or, against a plan, the planned
    /// digests. It equals every pair's support when the whole field resolved the
    /// same digests.
    pub n_contracts: usize,
    pub contract_sha256: Vec<String>,
    pub unresolved_by_agent: BTreeMap<String, AgentUnresolvedSupport>,
    pub settlement_status_disagreements: Vec<SettlementStatusDisagreement>,
    /// Against a plan only: per agent, the resolved digests the plan does not
    /// name, sorted. They enter no score, calibration or comparison.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub outside_plan_by_agent: Option<BTreeMap<String, Vec<String>>>,
}

/// How far apart two agents' resolved supports are. Each count is charged to the
/// agent that did not resolve those digests.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize)]
pub struct SupportGap {
    /// Digests agent B resolved and agent A did not.
    pub agent_a_unresolved: usize,
    /// Digests agent A resolved and agent B did not.
    pub agent_b_unresolved: usize,
}

#[derive(Clone, Debug, Serialize)]
pub struct PairwiseForecastComparison {
    pub agent_a: String,
    pub agent_b: String,
    /// Contracts both agents resolved, which are the contracts differenced.
    pub n_contracts: usize,
    pub n_settlement_blocks: usize,
    /// Mean loss(A) minus mean loss(B); negative favors A. Descriptive only when
    /// inference is withheld: under a `support_gap` it covers the contracts the
    /// incomplete agent chose to resolve.
    pub mean_loss_difference: f64,
    /// Absent, together with the other three inference fields, when
    /// `inference_error` says the block resampling law cannot support the claim.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub confidence_lower: Option<f64>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub confidence_upper: Option<f64>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub raw_p_value: Option<f64>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub holm_adjusted_p_value: Option<f64>,
    pub familywise_significant: bool,
    /// The measured rejection rate of this comparison's test under the null, at
    /// this comparison's block count, against the nominal 5%.
    ///
    /// It travels with the flag because the flag is not delivered at the block
    /// counts the protocol admits: at four blocks, the first count the
    /// pair-support gate passes, the test rejects a true null 18.8% of the
    /// time. Read `familywise_significant` against this, not against 0.05.
    /// `None` when the comparison carries no p-value.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub measured_size_at_nominal_5pct: Option<f64>,
    /// Present only when the two agents did not resolve the same digests.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub support_gap: Option<SupportGap>,
    /// Present only when inference was withheld. Omitted for a supported comparison.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub inference_error: Option<String>,
}

/// Why a paired comparison carries no interval, no p-value and no significance.
#[derive(Clone, Copy, Debug, PartialEq)]
enum ForecastInferenceError {
    UnequalResolvedSupport(SupportGap),
    NoSettlementBlock,
    LevelBelowResamplingResolution {
        blocks: usize,
        degenerate_mass: f64,
        familywise_alpha: f64,
    },
}

impl Display for ForecastInferenceError {
    fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result {
        match self {
            Self::UnequalResolvedSupport(gap) => write!(
                formatter,
                "unequal resolved support: agent_a did not resolve {} contract(s) that agent_b \
                 resolved and agent_b did not resolve {} that agent_a resolved; a pair receives \
                 inference only when both agents resolved the same contracts",
                gap.agent_a_unresolved, gap.agent_b_unresolved
            ),
            Self::NoSettlementBlock => formatter.write_str(
                "no settlement block on exact common support: there is nothing to resample",
            ),
            Self::LevelBelowResamplingResolution {
                blocks,
                degenerate_mass,
                familywise_alpha,
            } => write!(
                formatter,
                "familywise level {familywise_alpha} is finer than a {blocks}-block resampling \
                 law can resolve: single-block resamples hold {degenerate_mass} of its mass, and \
                 more bootstrap replications estimate that same law without adding blocks"
            ),
        }
    }
}

impl Error for ForecastInferenceError {}

/// Mass the block resampling law places on resamples that are one block repeated.
///
/// The comparison resamples whole resolution-clock blocks with replacement. A draw
/// that takes the same block every time reproduces one block's mean and carries no
/// information about variation between blocks. Drawing `blocks` times from `blocks`
/// blocks lands on such a draw with probability `blocks * blocks^-blocks`, that is
/// `blocks^(1 - blocks)`: 1 at one block, 1/2 at two, 1/9 at three, 1/64 at four.
/// What `familywise_significant` actually delivers, as written into the report.
pub const FAMILYWISE_SIZE_RULE: &str =
    "the paired test resamples whole blocks and centres the raw \
     mean, not a studentised statistic, so it rejects a true null more often than its nominal 5%: \
     measured 0.188 at 4 blocks, 0.162 at 5, 0.138 at 6, 0.096 at 10, 0.072 at 20 and 0.060 at 40 \
     over 6000 i.i.d. normal draws of 5 contracts a block, and 0.237 at 4 blocks under lognormal \
     contract differences; read familywise_significant against measured_size_at_nominal_5pct";

/// The measured size of the block bootstrap at a block count, against a nominal
/// 5%, under i.i.d. normal per-contract differences with five contracts a block
/// and the shipped 2000 replications. Entries are `(blocks, size)` ascending.
///
/// These are measurements of this implementation, not properties of a block
/// bootstrap in general, and the data-generating process is named because the
/// number moves with it: the same table under lognormal differences reads 0.237
/// at four blocks rather than 0.188. The table is reported rather than corrected
/// because no available replacement holds its size across the admitted counts.
/// Studentising the statistic swaps the error for the opposite one, measuring
/// 0.013 at four blocks and rejecting a half-sigma effect 12.1% of the time
/// against this test's 74.5%; a Student-t on the block means is exact under
/// normal blocks (0.050) but reads 0.094 under skewed ones, and it tests the
/// unweighted block mean rather than the contract mean this report publishes as
/// `mean_loss_difference`.
const MEASURED_SIZE: &[(usize, f64)] = &[
    (4, 0.188),
    (5, 0.162),
    (6, 0.138),
    (10, 0.096),
    (20, 0.072),
    (40, 0.060),
];

/// The measured size at or below `blocks`, so a reader is never handed a
/// friendlier number than the block count earns. Above the largest measured
/// count the last entry is carried forward, which is the closest to nominal the
/// table states.
fn measured_size(blocks: usize) -> f64 {
    MEASURED_SIZE
        .iter()
        .rev()
        .find(|(at, _)| *at <= blocks)
        .map_or(MEASURED_SIZE[0].1, |(_, size)| *size)
}

fn degenerate_resample_mass(blocks: usize) -> f64 {
    if blocks == 0 {
        return 1.0;
    }
    let blocks = blocks as f64;
    blocks.powf(1.0 - blocks)
}

/// The independent-block requirement this protocol enforces before it reports
/// significance.
///
/// A claim at familywise level `familywise_alpha` is withheld unless the block
/// resampling law places at most that much mass on its own degenerate draws. At one
/// block every resample is the observed sample: the percentile interval has zero
/// width by construction and the exceedance count is zero, so the reported p-value
/// is the `(0 + 1) / (samples + 1)` smoothing floor and shrinks as replications are
/// added. At two blocks half the law's mass is degenerate, at three blocks a ninth.
/// The bar is passed at four blocks for the default alpha of 0.05.
///
/// This is a necessary condition on the reported level, not a certificate of
/// calibration. Clearing it does not establish that blocks are independent, that the
/// resolution clock is the right dependence unit, or that the percentile interval has
/// its nominal coverage in finite samples. Those remain assumptions of the protocol.
fn inference_support(blocks: usize, familywise_alpha: f64) -> Result<(), ForecastInferenceError> {
    if blocks == 0 {
        return Err(ForecastInferenceError::NoSettlementBlock);
    }
    let degenerate_mass = degenerate_resample_mass(blocks);
    if degenerate_mass > familywise_alpha {
        return Err(ForecastInferenceError::LevelBelowResamplingResolution {
            blocks,
            degenerate_mass,
            familywise_alpha,
        });
    }
    Ok(())
}

#[derive(Clone, Debug, Serialize)]
pub struct ForecastQualityReport {
    pub schema_version: &'static str,
    pub rank_effect: &'static str,
    pub dependence_unit: &'static str,
    /// What `familywise_significant` delivers at the admitted block counts.
    pub familywise_size: &'static str,
    pub config: ForecastAnalysisConfig,
    pub common_support: CommonSupport,
    /// The encoding each scored contract digest verified under, for every digest
    /// an effective resolved revision in the field names. Support is exact by
    /// digest, so a contract presented under `legacy` by one agent and under
    /// `sharpebench/canonical-json/v1` by another is two digests and not common.
    pub contract_digest_versions: BTreeMap<String, ContractDigestVersion>,
    pub agents: Vec<AgentForecastSummary>,
    pub comparisons: Vec<PairwiseForecastComparison>,
}

/// Recompute forecast quality without changing or consulting trading-rank eligibility.
///
/// The field support is the union of the digests the documents resolved, so a
/// document that resolves a contract nobody else did withholds inference on its
/// own pairs and charges that contract to every other agent. Use
/// [`analyze_forecast_quality_against_plan`] when the question set is declared.
pub fn analyze_forecast_quality(
    evidence: &[ForecastEvidence],
    config: ForecastAnalysisConfig,
) -> Result<ForecastQualityReport, ForecastError> {
    analyze(evidence, config, None)
}

/// [`analyze_forecast_quality`] against a declared contract universe. The plan
/// is the field support: every agent's gaps are the planned digests it did not
/// resolve, and a resolved digest outside the plan is listed in
/// `outside_plan_by_agent` and enters no score, calibration or comparison. Its
/// settlement is still checked against every other document that resolved it.
/// The report is [`FORECAST_QUALITY_PLAN_SCHEMA`].
pub fn analyze_forecast_quality_against_plan(
    evidence: &[ForecastEvidence],
    config: ForecastAnalysisConfig,
    plan: &ForecastContractPlan,
) -> Result<ForecastQualityReport, ForecastError> {
    analyze(evidence, config, Some(plan))
}

/// Every unordered pair of documents in a field, by index, in report order. One
/// statement of the pairing, so the settlement check and the comparisons cover
/// the same pairs and neither compares a document with itself.
fn document_pairs(documents: usize) -> impl Iterator<Item = (usize, usize)> {
    (0..documents).flat_map(move |left| ((left + 1)..documents).map(move |right| (left, right)))
}

fn analyze(
    evidence: &[ForecastEvidence],
    config: ForecastAnalysisConfig,
    plan: Option<&ForecastContractPlan>,
) -> Result<ForecastQualityReport, ForecastError> {
    if evidence.is_empty() {
        return Err(reject(
            "at least one forecast evidence document is required",
        ));
    }
    if config.bootstrap_samples == 0
        || config.calibration_bins == 0
        || !config.confidence.is_finite()
        || !(0.0..1.0).contains(&config.confidence)
        || !config.familywise_alpha.is_finite()
        || !(0.0..1.0).contains(&config.familywise_alpha)
    {
        return Err(reject(
            "forecast analysis configuration is outside its valid range",
        ));
    }
    let mut agent_ids = BTreeSet::new();
    let mut rows = Vec::new();
    for document in evidence {
        validate_evidence(document)?;
        if !agent_ids.insert(document.identity.agent_id.as_str()) {
            return Err(reject("forecast field contains duplicate agent_id values"));
        }
        rows.push(scored_forecasts(document)?);
    }
    let mut outside_plan_by_agent = None;
    let mut planned = None;
    if let Some(plan) = plan {
        // Every settlement is checked before the plan drops anything, so an
        // unscored contract still has one outcome across the field. The pairs
        // are the ones compared below.
        for (left, right) in document_pairs(rows.len()) {
            check_shared_settlements(&rows[left], &rows[right])?;
        }
        let digests: BTreeSet<&str> = plan.contract_sha256.iter().map(String::as_str).collect();
        let mut outside = BTreeMap::new();
        for (document, rows) in evidence.iter().zip(&mut rows) {
            let unplanned: BTreeSet<String> = rows
                .iter()
                .filter(|row| !digests.contains(row.contract_sha256.as_str()))
                .map(|row| row.contract_sha256.clone())
                .collect();
            rows.retain(|row| digests.contains(row.contract_sha256.as_str()));
            outside.insert(
                document.identity.agent_id.clone(),
                unplanned.into_iter().collect::<Vec<_>>(),
            );
        }
        outside_plan_by_agent = Some(outside);
        planned = Some(digests);
    }
    let mut common_support = field_support(evidence, &rows, planned.as_ref());
    common_support.outside_plan_by_agent = outside_plan_by_agent;
    let agents = evidence
        .iter()
        .zip(&rows)
        .map(|(document, rows)| summarize_agent(document, rows, config.calibration_bins))
        .collect();
    let mut comparisons = Vec::new();
    for (left, right) in document_pairs(evidence.len()) {
        comparisons.push(compare_agents(
            &evidence[left].identity.agent_id,
            &rows[left],
            &evidence[right].identity.agent_id,
            &rows[right],
            config,
        )?);
    }
    holm_adjust(&mut comparisons, config.familywise_alpha);
    let contract_digest_versions = rows
        .iter()
        .flatten()
        .map(|row| (row.contract_sha256.clone(), row.contract_digest_version))
        .collect();
    Ok(ForecastQualityReport {
        schema_version: if plan.is_some() {
            FORECAST_QUALITY_PLAN_SCHEMA
        } else {
            FORECAST_QUALITY_SCHEMA
        },
        rank_effect: "reported_only_never_trading_rank",
        dependence_unit: "whole resolution-clock block across assets and questions",
        familywise_size: FAMILYWISE_SIZE_RULE,
        config,
        common_support,
        contract_digest_versions,
        agents,
        comparisons,
    })
}

/// Charge every field contract an agent did not resolve to that agent, and record
/// each contract one agent resolved while another left it pending or cancelled.
///
/// Without a plan the field support is the union of resolved digests, so an agent's
/// gaps are measured against what the rest of the field settled. With one it is the
/// planned digests, and `rows` hold only those. It is disclosure only: no comparison
/// reads it, so a document with gaps or extra contracts cannot change the support of
/// a pair it is not part of.
fn field_support(
    evidence: &[ForecastEvidence],
    rows: &[Vec<ScoredForecast>],
    plan: Option<&BTreeSet<&str>>,
) -> CommonSupport {
    let resolved: Vec<BTreeSet<&str>> = rows
        .iter()
        .map(|rows| {
            rows.iter()
                .map(|row| row.contract_sha256.as_str())
                .collect()
        })
        .collect();
    let field: BTreeSet<&str> = match plan {
        Some(plan) => plan.clone(),
        None => resolved.iter().flatten().copied().collect(),
    };
    let mut unresolved_by_agent = BTreeMap::new();
    let mut disputed: BTreeMap<&str, SettlementStatusDisagreement> = BTreeMap::new();
    for (document, resolved) in evidence.iter().zip(&resolved) {
        let agent_id = &document.identity.agent_id;
        let statuses = claim_statuses_by_digest(document);
        let mut gap = AgentUnresolvedSupport::default();
        for digest in field.difference(resolved) {
            let record = || SettlementStatusDisagreement {
                contract_sha256: digest.to_string(),
                resolved_by: Vec::new(),
                pending_by: Vec::new(),
                cancelled_by: Vec::new(),
            };
            match statuses.get(digest) {
                None => gap.not_claimed.push(digest.to_string()),
                Some(status) if status.contains("pending") => {
                    gap.pending.push(digest.to_string());
                    disputed
                        .entry(*digest)
                        .or_insert_with(record)
                        .pending_by
                        .push(agent_id.clone());
                }
                Some(status) if status.contains("cancelled") => {
                    gap.cancelled.push(digest.to_string());
                    disputed
                        .entry(*digest)
                        .or_insert_with(record)
                        .cancelled_by
                        .push(agent_id.clone());
                }
                Some(_) => gap.rejected.push(digest.to_string()),
            }
        }
        gap.n_unresolved = field.len() - resolved.len();
        unresolved_by_agent.insert(agent_id.clone(), gap);
    }
    for (document, resolved) in evidence.iter().zip(&resolved) {
        for (digest, record) in &mut disputed {
            if resolved.contains(digest) {
                record.resolved_by.push(document.identity.agent_id.clone());
            }
        }
    }
    let settlement_status_disagreements = disputed
        .into_values()
        .map(|mut record| {
            record.resolved_by.sort();
            record.pending_by.sort();
            record.cancelled_by.sort();
            record
        })
        .collect();
    CommonSupport {
        rule: if plan.is_some() {
            PLAN_SUPPORT_RULE
        } else {
            SUPPORT_RULE
        },
        n_contracts: field.len(),
        contract_sha256: field.into_iter().map(str::to_string).collect(),
        unresolved_by_agent,
        settlement_status_disagreements,
        outside_plan_by_agent: None,
    }
}

/// Every resolution status this document gives a contract digest, over all claims
/// naming it. Validation holds a claim to one digest across its revisions, so any
/// revision identifies it.
fn claim_statuses_by_digest(document: &ForecastEvidence) -> BTreeMap<&str, BTreeSet<&str>> {
    let digest_by_claim: BTreeMap<&str, &str> = document
        .revisions
        .iter()
        .map(|revision| {
            (
                revision.claim_id.as_str(),
                revision.contract_sha256.as_str(),
            )
        })
        .collect();
    let mut statuses: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new();
    for resolution in &document.resolutions {
        statuses
            .entry(digest_by_claim[resolution.claim_id.as_str()])
            .or_default()
            .insert(resolution.status.as_str());
    }
    statuses
}

fn summarize_agent(
    document: &ForecastEvidence,
    rows: &[ScoredForecast],
    calibration_bins: usize,
) -> AgentForecastSummary {
    let claim_ids: BTreeSet<&str> = document
        .revisions
        .iter()
        .map(|revision| revision.claim_id.as_str())
        .collect();
    let eligible_ids: BTreeSet<&str> = document
        .revisions
        .iter()
        .filter(|revision| revision.status == "eligible")
        .map(|revision| revision.claim_id.as_str())
        .collect();
    let status_count = |status: &str| {
        document
            .resolutions
            .iter()
            .filter(|resolution| resolution.status == status)
            .count()
    };
    let mut metrics: BTreeMap<&str, Vec<f64>> = BTreeMap::new();
    for row in rows {
        metrics
            .entry(row.scoring_rule.as_str())
            .or_default()
            .push(row.loss);
    }
    let metrics = metrics
        .into_iter()
        .map(|(rule, values)| MetricMean {
            scoring_rule: rule.to_string(),
            n: values.len(),
            mean_loss: mean(&values),
        })
        .collect();
    let binary: Vec<_> = rows.iter().filter_map(|row| row.probability).collect();
    let categorical: Vec<_> = rows
        .iter()
        .filter_map(|row| row.categorical_confidence)
        .collect();
    let pits: Vec<_> = rows.iter().filter_map(|row| row.normal_pit).collect();
    AgentForecastSummary {
        agent_id: document.identity.agent_id.clone(),
        model_id: document.identity.model_id.clone(),
        n_claims: claim_ids.len(),
        n_revisions: document.revisions.len(),
        n_eligible_claims: eligible_ids.len(),
        n_resolved: status_count("resolved"),
        n_pending: status_count("pending"),
        n_cancelled: status_count("cancelled"),
        n_rejected: status_count("rejected"),
        resolution_rate: status_count("resolved") as f64 / claim_ids.len() as f64,
        blind_resolved: rows.iter().filter(|row| !row.consensus_visible).count(),
        consensus_exposed_resolved: rows.iter().filter(|row| row.consensus_visible).count(),
        metrics,
        binary_calibration: binary_calibration(&binary, calibration_bins),
        categorical_confidence_calibration: confidence_calibration(&categorical, calibration_bins),
        normal_distribution_calibration: distribution_calibration(&pits, calibration_bins),
    }
}

fn calibration_bins(values: &[(f64, bool)], bins: usize) -> Vec<CalibrationBin> {
    (0..bins)
        .map(|index| {
            let selected: Vec<_> = values
                .iter()
                .filter(|(probability, _)| {
                    ((*probability * bins as f64).floor() as usize).min(bins - 1) == index
                })
                .collect();
            CalibrationBin {
                lower: index as f64 / bins as f64,
                upper: (index + 1) as f64 / bins as f64,
                n: selected.len(),
                mean_forecast: (!selected.is_empty()).then(|| {
                    selected
                        .iter()
                        .map(|(probability, _)| probability)
                        .sum::<f64>()
                        / selected.len() as f64
                }),
                event_rate: (!selected.is_empty()).then(|| {
                    selected.iter().filter(|(_, event)| *event).count() as f64
                        / selected.len() as f64
                }),
            }
        })
        .collect()
}

fn binary_calibration(values: &[(f64, bool)], bins: usize) -> Option<BinaryCalibration> {
    if values.is_empty() {
        return None;
    }
    let bins = calibration_bins(values, bins);
    let n = values.len() as f64;
    let base_rate = values.iter().filter(|(_, event)| *event).count() as f64 / n;
    let brier = values
        .iter()
        .map(|(probability, event)| (probability - if *event { 1.0 } else { 0.0 }).powi(2))
        .sum::<f64>()
        / n;
    let reliability = bins
        .iter()
        .filter_map(|bin| Some((bin.n as f64 / n, bin.mean_forecast?, bin.event_rate?)))
        .map(|(weight, forecast, observed)| weight * (forecast - observed).powi(2))
        .sum();
    let resolution = bins
        .iter()
        .filter_map(|bin| Some((bin.n as f64 / n, bin.event_rate?)))
        .map(|(weight, observed)| weight * (observed - base_rate).powi(2))
        .sum();
    let uncertainty = base_rate * (1.0 - base_rate);
    Some(BinaryCalibration {
        n: values.len(),
        brier,
        base_rate,
        reliability,
        resolution,
        uncertainty,
        brier_skill: (uncertainty > 0.0).then_some(1.0 - brier / uncertainty),
        bins,
    })
}

fn confidence_calibration(values: &[(f64, bool)], bins: usize) -> Option<ConfidenceCalibration> {
    if values.is_empty() {
        return None;
    }
    Some(ConfidenceCalibration {
        n: values.len(),
        mean_confidence: values.iter().map(|(confidence, _)| confidence).sum::<f64>()
            / values.len() as f64,
        accuracy: values.iter().filter(|(_, correct)| *correct).count() as f64
            / values.len() as f64,
        bins: calibration_bins(values, bins),
    })
}

fn distribution_calibration(values: &[f64], bins: usize) -> Option<DistributionCalibration> {
    if values.is_empty() {
        return None;
    }
    let average = mean(values);
    let variance = values
        .iter()
        .map(|value| (value - average).powi(2))
        .sum::<f64>()
        / values.len() as f64;
    let mut bin_counts = vec![0; bins];
    for value in values {
        let index = ((*value * bins as f64).floor() as usize).min(bins - 1);
        bin_counts[index] += 1;
    }
    Some(DistributionCalibration {
        n: values.len(),
        pit_mean: average,
        pit_variance: variance,
        uniform_reference_mean: 0.5,
        uniform_reference_variance: 1.0 / 12.0,
        bin_counts,
    })
}

fn mean(values: &[f64]) -> f64 {
    values.iter().sum::<f64>() / values.len() as f64
}

#[derive(Clone, Copy)]
struct SplitMix64(u64);

impl SplitMix64 {
    fn next(&mut self) -> u64 {
        self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15);
        let mut value = self.0;
        value = (value ^ (value >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
        value = (value ^ (value >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
        value ^ (value >> 31)
    }

    fn below(&mut self, upper: usize) -> usize {
        ((self.next() as u128 * upper as u128) >> 64) as usize
    }
}

fn percentile(sorted: &[f64], probability: f64) -> f64 {
    let index = ((sorted.len() - 1) as f64 * probability).round() as usize;
    sorted[index]
}

/// Refuse two documents that settle a contract digest they both resolved
/// differently: other contract semantics, another realized outcome, or another
/// outcome availability time.
fn check_shared_settlements(
    rows_a: &[ScoredForecast],
    rows_b: &[ScoredForecast],
) -> Result<(), ForecastError> {
    let by_hash_a: BTreeMap<_, _> = rows_a
        .iter()
        .map(|row| (row.contract_sha256.as_str(), row))
        .collect();
    let by_hash_b: BTreeMap<_, _> = rows_b
        .iter()
        .map(|row| (row.contract_sha256.as_str(), row))
        .collect();
    for (contract_hash, left) in &by_hash_a {
        let Some(right) = by_hash_b.get(contract_hash) else {
            continue;
        };
        if left.resolves_at != right.resolves_at
            || left.instrument != right.instrument
            || left.scoring_rule != right.scoring_rule
        {
            return Err(reject(
                "equal contract digest resolved to unequal contract semantics",
            ));
        }
        // Contract equality is not outcome equality. Bench takes documents from any
        // producer, so the settlement each loss was computed against has to agree
        // before the two losses may be differenced.
        if left.outcome_sha256 != right.outcome_sha256 {
            return Err(reject(
                "equal contract digest settled to unequal realized outcomes",
            ));
        }
        if left.outcome_available_at != right.outcome_available_at {
            return Err(reject(
                "equal contract digest settled at unequal outcome availability times",
            ));
        }
    }
    Ok(())
}

fn compare_agents(
    agent_a: &str,
    rows_a: &[ScoredForecast],
    agent_b: &str,
    rows_b: &[ScoredForecast],
    config: ForecastAnalysisConfig,
) -> Result<PairwiseForecastComparison, ForecastError> {
    // Settlement agreement is checked on every contract both resolved, including a
    // pair whose inference is withheld below, so a disputed settlement is refused
    // whichever other documents the field holds.
    check_shared_settlements(rows_a, rows_b)?;
    let by_hash_a: BTreeMap<_, _> = rows_a
        .iter()
        .map(|row| (row.contract_sha256.as_str(), row))
        .collect();
    let by_hash_b: BTreeMap<_, _> = rows_b
        .iter()
        .map(|row| (row.contract_sha256.as_str(), row))
        .collect();
    let mut blocks: BTreeMap<u64, Vec<f64>> = BTreeMap::new();
    for (contract_hash, left) in &by_hash_a {
        let Some(right) = by_hash_b.get(contract_hash) else {
            continue;
        };
        blocks
            .entry(left.resolves_at)
            .or_default()
            .push(left.loss - right.loss);
    }
    let observed_values: Vec<f64> = blocks.values().flatten().copied().collect();
    let block_values: Vec<&Vec<f64>> = blocks.values().collect();
    let gap = SupportGap {
        agent_a_unresolved: by_hash_b.len() - observed_values.len(),
        agent_b_unresolved: by_hash_a.len() - observed_values.len(),
    };
    let supported = if gap.agent_a_unresolved > 0 || gap.agent_b_unresolved > 0 {
        Err(ForecastInferenceError::UnequalResolvedSupport(gap))
    } else {
        inference_support(block_values.len(), config.familywise_alpha)
    };
    if let Err(unsupported) = supported {
        return Ok(PairwiseForecastComparison {
            agent_a: agent_a.to_string(),
            agent_b: agent_b.to_string(),
            n_contracts: observed_values.len(),
            n_settlement_blocks: block_values.len(),
            mean_loss_difference: if observed_values.is_empty() {
                0.0
            } else {
                mean(&observed_values)
            },
            confidence_lower: None,
            confidence_upper: None,
            raw_p_value: None,
            holm_adjusted_p_value: None,
            familywise_significant: false,
            measured_size_at_nominal_5pct: None,
            support_gap: match unsupported {
                ForecastInferenceError::UnequalResolvedSupport(gap) => Some(gap),
                _ => None,
            },
            inference_error: Some(unsupported.to_string()),
        });
    }
    let observed = mean(&observed_values);
    let mut rng = SplitMix64(config.bootstrap_seed ^ stable_pair_seed(agent_a, agent_b));
    let mut samples = Vec::with_capacity(config.bootstrap_samples);
    let mut null_extreme = 0usize;
    for _ in 0..config.bootstrap_samples {
        let mut sample = Vec::new();
        for _ in 0..block_values.len() {
            sample.extend_from_slice(block_values[rng.below(block_values.len())]);
        }
        let sample_mean = mean(&sample);
        samples.push(sample_mean);
        let centered =
            sample.iter().map(|value| value - observed).sum::<f64>() / sample.len() as f64;
        if centered.abs() >= observed.abs() {
            null_extreme += 1;
        }
    }
    samples.sort_by(f64::total_cmp);
    let tail = (1.0 - config.confidence) / 2.0;
    Ok(PairwiseForecastComparison {
        agent_a: agent_a.to_string(),
        agent_b: agent_b.to_string(),
        n_contracts: observed_values.len(),
        n_settlement_blocks: block_values.len(),
        mean_loss_difference: observed,
        confidence_lower: Some(percentile(&samples, tail)),
        confidence_upper: Some(percentile(&samples, 1.0 - tail)),
        raw_p_value: Some((null_extreme as f64 + 1.0) / (config.bootstrap_samples as f64 + 1.0)),
        holm_adjusted_p_value: None,
        familywise_significant: false,
        measured_size_at_nominal_5pct: Some(measured_size(block_values.len())),
        support_gap: None,
        inference_error: None,
    })
}

fn stable_pair_seed(left: &str, right: &str) -> u64 {
    let (first, second) = if left <= right {
        (left, right)
    } else {
        (right, left)
    };
    let digest = Sha256::digest(format!("{first}\0{second}").as_bytes());
    u64::from_be_bytes(digest[..8].try_into().unwrap_or([0; 8]))
}

/// Holm over the comparisons that produced a p-value.
///
/// A withheld comparison is not a test and receives no adjusted p-value, but it stays
/// in the family size: dropping it would shrink the multiplier and make the surviving
/// comparisons look more significant because a neighbour could not be evaluated.
fn holm_adjust(comparisons: &mut [PairwiseForecastComparison], alpha: f64) {
    let mut order: Vec<usize> = (0..comparisons.len())
        .filter(|index| comparisons[*index].raw_p_value.is_some())
        .collect();
    order.sort_by(|left, right| {
        comparisons[*left]
            .raw_p_value
            .unwrap_or(1.0)
            .total_cmp(&comparisons[*right].raw_p_value.unwrap_or(1.0))
    });
    let mut prior = 0.0_f64;
    let family = comparisons.len();
    for (rank, index) in order.into_iter().enumerate() {
        let adjusted = ((family - rank) as f64 * comparisons[index].raw_p_value.unwrap_or(1.0))
            .max(prior)
            .min(1.0);
        comparisons[index].holm_adjusted_p_value = Some(adjusted);
        comparisons[index].familywise_significant = adjusted <= alpha;
        prior = adjusted;
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn fixture(agent: &str, probabilities: &[f64], outcomes: &[f64]) -> String {
        assert_eq!(probabilities.len(), outcomes.len());
        let contracts: Vec<_> = probabilities
            .iter()
            .enumerate()
            .map(|(index, _)| ForecastContract {
                schema_version: CONTRACT_SCHEMA.to_string(),
                contract_id: format!("c{index}"),
                question: format!("question {index}"),
                instrument: if index % 2 == 0 { "ES" } else { "NQ" }.to_string(),
                target: "close_up".to_string(),
                kind: "probability".to_string(),
                opens_at: index as u64 * 2,
                deadline: index as u64 * 2 + 1,
                resolves_at: (index as u64 / 2 + 1) * 10,
                observation_source: "fixture:v1".to_string(),
                open_definition: "close at opens_at".to_string(),
                close_definition: "close at resolves_at".to_string(),
                unit: "binary".to_string(),
                scoring_rule: "binary_brier".to_string(),
                neutral_threshold: 0.001,
                boundary_ownership: "threshold is false".to_string(),
                missing_data_policy: "cancel".to_string(),
                fallback_policy: "cancel".to_string(),
                categories: vec![],
                interval_alpha: None,
            })
            .collect();
        let hashes: Vec<_> = contracts
            .iter()
            .map(|contract| contract_digests(contract).unwrap().canonical_json_v1)
            .collect();
        let evidence = ForecastEvidence {
            schema_version: FORECAST_EVIDENCE_SCHEMA.to_string(),
            producer: ForecastProducer {
                name: "sharpearena".to_string(),
                contract: "native".to_string(),
            },
            generated_at: 100,
            identity: ForecastRunIdentity {
                agent_id: agent.to_string(),
                model_id: format!("model-{agent}"),
                model_sha256: "a".repeat(64),
                scaffold_id: "scaffold".to_string(),
                scaffold_sha256: "b".repeat(64),
                prompt_sha256: "c".repeat(64),
                operator_id: "operator".to_string(),
                config_sha256: "d".repeat(64),
            },
            revisions: probabilities
                .iter()
                .enumerate()
                .map(|(index, probability)| ForecastRevision {
                    revision_id: format!("claim-{index}:r0"),
                    claim_id: format!("claim-{index}"),
                    ordinal: 0,
                    supersedes: None,
                    contract_sha256: hashes[index].clone(),
                    contract_digest_encoding: None,
                    prediction: vec![*probability],
                    confidence: *probability,
                    rationale: "evidence".to_string(),
                    submitted_at: index as u64 * 2,
                    status: "eligible".to_string(),
                    reason: None,
                    trigger_event_id: None,
                    revision_reason: None,
                    exposure: InformationExposure {
                        observed_at: index as u64 * 2,
                        market_snapshot_sha256: Some("e".repeat(64)),
                        consensus_visible: false,
                        consensus_snapshot_sha256: None,
                        source_ids: vec!["market".to_string()],
                    },
                    idempotency_key: format!("request-{index}"),
                })
                .collect(),
            resolutions: outcomes
                .iter()
                .enumerate()
                .map(|(index, outcome)| ForecastResolution {
                    claim_id: format!("claim-{index}"),
                    status: "resolved".to_string(),
                    outcome: Some(Value::from(*outcome)),
                    available_at: Some(contracts[index].resolves_at),
                    recorded_at: 100,
                    reason: None,
                })
                .collect(),
            contracts,
        };
        serde_json::to_string(&evidence).unwrap()
    }

    #[test]
    fn strict_ingestion_rejects_unknown_fields_and_contract_drift() {
        let payload = fixture("a", &[0.8], &[1.0]);
        let mut value: Value = serde_json::from_str(&payload).unwrap();
        value["revisions"][0]["unknown"] = Value::Bool(true);
        assert!(parse_forecast_evidence(&value.to_string()).is_err());

        let mut value: Value = serde_json::from_str(&payload).unwrap();
        value["contracts"][0]["question"] = Value::String("changed".to_string());
        let error = parse_forecast_evidence(&value.to_string()).unwrap_err();
        assert!(error.0.contains("unknown contract digest"));
    }

    #[test]
    fn sharpearena_golden_file_crosses_the_artifact_boundary_exactly() {
        let payload = include_str!("../tests/fixtures/sharpearena-forecast-evidence-v1.json");
        let document = parse_forecast_evidence(payload).unwrap();
        let report =
            analyze_forecast_quality(&[document], ForecastAnalysisConfig::default()).unwrap();
        assert_eq!(report.agents[0].agent_id, "arena-reference");
        assert_eq!(report.agents[0].metrics[0].scoring_rule, "binary_brier");
        assert!((report.agents[0].metrics[0].mean_loss - 0.09).abs() < 1e-12);
    }

    /// The v2 golden is the v1 golden with the envelope bumped and the encoding
    /// its digest recomputes under declared; the report is the same report plus the
    /// declared label under `contract_digest_versions`.
    #[test]
    fn sharpearena_v2_golden_file_crosses_the_artifact_boundary_exactly() {
        let v1 = include_str!("../tests/fixtures/sharpearena-forecast-evidence-v1.json");
        let v2 = include_str!("../tests/fixtures/sharpearena-forecast-evidence-v2.json");
        let config = ForecastAnalysisConfig::default();
        let baseline =
            analyze_forecast_quality(&[parse_forecast_evidence(v1).unwrap()], config).unwrap();

        let document = parse_forecast_evidence(v2).unwrap();
        assert_eq!(document.schema_version, FORECAST_EVIDENCE_SCHEMA_V2);
        assert_eq!(
            document.revisions[0].contract_digest_encoding.as_deref(),
            Some("legacy")
        );
        let report = analyze_forecast_quality(&[document], config).unwrap();
        assert_eq!(report.agents[0].agent_id, "arena-reference");
        assert_eq!(report.agents[0].metrics[0].scoring_rule, "binary_brier");
        assert!((report.agents[0].metrics[0].mean_loss - 0.09).abs() < 1e-12);
        assert_eq!(
            report.contract_digest_versions,
            BTreeMap::from([(
                "0eb3250fcb87a5e225f476e8a6e18ccdf65ac458919ad0b150594eb5761ecd2d".to_string(),
                ContractDigestVersion::Legacy
            )])
        );
        assert_eq!(
            serde_json::to_string_pretty(&report).unwrap(),
            serde_json::to_string_pretty(&baseline).unwrap()
        );
    }

    /// Verify one tutorial field against its manifest and recompute its frozen report
    /// byte for byte with the tutorial's analysis settings.
    fn recompute_tutorial_field(
        alpha: &str,
        beta: &str,
        manifest: &str,
        expected_report: &str,
    ) -> ForecastQualityReport {
        let manifest: Value = serde_json::from_str(manifest).unwrap();
        for (name, payload) in [("agent-alpha.json", alpha), ("agent-beta.json", beta)] {
            let digest = crate::lower_hex(&Sha256::digest(payload.as_bytes()));
            assert_eq!(manifest["files"][name].as_str(), Some(digest.as_str()));
        }

        let documents = [
            parse_forecast_evidence(alpha).unwrap(),
            parse_forecast_evidence(beta).unwrap(),
        ];
        let report = analyze_forecast_quality(
            &documents,
            ForecastAnalysisConfig {
                bootstrap_seed: 23,
                bootstrap_samples: 400,
                confidence: 0.9,
                familywise_alpha: 0.05,
                calibration_bins: 5,
            },
        )
        .unwrap();
        let actual = serde_json::to_string_pretty(&report).unwrap() + "\n";
        assert_eq!(actual, expected_report);
        assert_eq!(report.rank_effect, "reported_only_never_trading_rank");
        report
    }

    #[test]
    fn cross_product_tutorial_is_content_addressed_and_recomputes_exactly() {
        let report = recompute_tutorial_field(
            include_str!("../../../examples/forecast-quality/fixtures/agent-alpha.json"),
            include_str!("../../../examples/forecast-quality/fixtures/agent-beta.json"),
            include_str!("../../../examples/forecast-quality/fixtures/manifest.json"),
            include_str!("../../../examples/forecast-quality/fixtures/report.json"),
        );
        assert_eq!(report.common_support.n_contracts, 12);
        let comparison = &report.comparisons[0];
        assert_eq!(comparison.n_contracts, 12);
        assert_eq!(comparison.support_gap, None);
        assert_eq!(comparison.n_settlement_blocks, 6);
        assert_eq!(comparison.inference_error, None);
        assert!(comparison.confidence_lower.unwrap() < comparison.confidence_upper.unwrap());
        assert!(comparison.familywise_significant);
    }

    #[test]
    fn cross_product_tutorial_withheld_field_records_the_reason_and_recomputes_exactly() {
        let report = recompute_tutorial_field(
            include_str!("../../../examples/forecast-quality/fixtures/withheld/agent-alpha.json"),
            include_str!("../../../examples/forecast-quality/fixtures/withheld/agent-beta.json"),
            include_str!("../../../examples/forecast-quality/fixtures/withheld/manifest.json"),
            include_str!("../../../examples/forecast-quality/fixtures/withheld/report.json"),
        );
        assert_eq!(report.common_support.n_contracts, 8);
        let comparison = &report.comparisons[0];
        assert_eq!(comparison.n_contracts, 8);
        assert_eq!(comparison.support_gap, None);
        assert_eq!(comparison.n_settlement_blocks, 2);
        assert!(comparison.inference_error.is_some());
        assert_eq!(comparison.raw_p_value, None);
        assert!(!comparison.familywise_significant);
    }

    #[test]
    fn scoring_recomputes_brier_and_calibration_from_raw_evidence() {
        let document =
            parse_forecast_evidence(&fixture("a", &[0.8, 0.2, 0.7, 0.1], &[1.0, 0.0, 1.0, 0.0]))
                .unwrap();
        let report =
            analyze_forecast_quality(&[document], ForecastAnalysisConfig::default()).unwrap();
        let summary = &report.agents[0];
        assert!((summary.metrics[0].mean_loss - 0.045).abs() < 1e-12);
        let calibration = summary.binary_calibration.as_ref().unwrap();
        assert!((calibration.brier - 0.045).abs() < 1e-12);
        assert!((calibration.brier_skill.unwrap() - 0.82).abs() < 1e-12);
        assert_eq!(summary.blind_resolved, 4);
        assert_eq!(report.rank_effect, "reported_only_never_trading_rank");
    }

    #[test]
    fn pairwise_bootstrap_is_invariant_to_input_file_order() {
        let left = parse_forecast_evidence(&fixture(
            "left",
            &[0.81, 0.22, 0.73, 0.34, 0.66, 0.15, 0.59, 0.27],
            &[1.0, 0.0, 1.0, 0.0, 0.0, 1.0, 1.0, 0.0],
        ))
        .unwrap();
        let right = parse_forecast_evidence(&fixture(
            "right",
            &[0.61, 0.42, 0.53, 0.54, 0.46, 0.35, 0.39, 0.47],
            &[1.0, 0.0, 1.0, 0.0, 0.0, 1.0, 1.0, 0.0],
        ))
        .unwrap();
        let config = ForecastAnalysisConfig {
            bootstrap_seed: 260_904,
            bootstrap_samples: 500,
            confidence: 0.95,
            familywise_alpha: 0.05,
            calibration_bins: 5,
        };

        let forward = analyze_forecast_quality(&[left.clone(), right.clone()], config)
            .unwrap()
            .comparisons
            .remove(0);
        let reverse = analyze_forecast_quality(&[right, left], config)
            .unwrap()
            .comparisons
            .remove(0);

        assert_eq!(forward.raw_p_value, reverse.raw_p_value);
        assert_eq!(forward.holm_adjusted_p_value, reverse.holm_adjusted_p_value);
        assert_eq!(
            forward.familywise_significant,
            reverse.familywise_significant
        );
        assert_eq!(forward.mean_loss_difference, -reverse.mean_loss_difference);
        let negated = |value: Option<f64>| value.map(|value| -value);
        assert_eq!(forward.confidence_lower, negated(reverse.confidence_upper));
        assert_eq!(forward.confidence_upper, negated(reverse.confidence_lower));
        assert!(forward.inference_error.is_none());
    }

    #[test]
    fn fixed_point_brier_model_matches_the_executable_float_rule() {
        let base = serde_json::from_str::<ForecastEvidence>(&fixture("formal", &[0.5], &[1.0]))
            .unwrap()
            .contracts
            .remove(0);
        let scale = 1_000_i64;
        for probability in [0_i64, 100, 500, 900, 1_000] {
            for truth in [0_i64, 200, 500, 800, 1_000] {
                let p = probability as f64 / scale as f64;
                let q = truth as f64 / scale as f64;
                let loss_zero = score_prediction(&base, &[p], &Value::from(0.0)).unwrap().0;
                let loss_one = score_prediction(&base, &[p], &Value::from(1.0)).unwrap().0;
                let expected = q * loss_one + (1.0 - q) * loss_zero;
                let numerator =
                    truth * (scale - probability).pow(2) + (scale - truth) * probability.pow(2);
                let decomposed =
                    truth * (scale - truth) * scale + scale * (probability - truth).pow(2);
                assert_eq!(numerator, decomposed);
                assert!((expected - numerator as f64 / (scale.pow(3) as f64)).abs() < 1e-15);
            }
        }
    }

    #[test]
    fn late_revision_is_audited_but_never_becomes_the_scored_forecast() {
        let mut document =
            serde_json::from_str::<ForecastEvidence>(&fixture("a", &[0.8], &[1.0])).unwrap();
        let mut late = document.revisions[0].clone();
        late.revision_id = "claim-0:r1".to_string();
        late.ordinal = 1;
        late.supersedes = Some("claim-0:r0".to_string());
        late.prediction = vec![0.01];
        late.submitted_at = document.contracts[0].deadline + 1;
        late.status = "late".to_string();
        late.reason = Some("submitted after the contract deadline".to_string());
        late.revision_reason = Some("answer became easier".to_string());
        late.exposure.observed_at = late.submitted_at;
        late.idempotency_key = "late-request".to_string();
        document.revisions.push(late);
        validate_evidence(&document).unwrap();

        let report =
            analyze_forecast_quality(&[document], ForecastAnalysisConfig::default()).unwrap();
        assert_eq!(report.agents[0].n_revisions, 2);
        assert!((report.agents[0].metrics[0].mean_loss - 0.04).abs() < 1e-12);
    }

    #[test]
    fn comparison_uses_exact_pair_support_and_whole_time_blocks() {
        let a =
            parse_forecast_evidence(&fixture("a", &[0.9, 0.1, 0.8, 0.2], &[1.0, 0.0, 1.0, 0.0]))
                .unwrap();
        let b = parse_forecast_evidence(&fixture(
            "b",
            &[0.6, 0.4, 0.55, 0.45],
            &[1.0, 0.0, 1.0, 0.0],
        ))
        .unwrap();
        let report = analyze_forecast_quality(&[a, b], ForecastAnalysisConfig::default()).unwrap();
        assert_eq!(report.common_support.n_contracts, 4);
        assert_eq!(report.comparisons[0].n_settlement_blocks, 2);
        assert!(report.comparisons[0].mean_loss_difference < 0.0);
    }

    #[test]
    fn equivalent_numeric_settlements_compare_but_distinct_outcomes_refuse() {
        for (left, right) in [
            (serde_json::json!(1), serde_json::json!(1.0)),
            (serde_json::json!(0.0), serde_json::json!(-0.0)),
        ] {
            let a = parse_forecast_evidence(&fixture("a", &[0.8], &[1.0])).unwrap();
            let b = parse_forecast_evidence(&fixture("b", &[0.6], &[1.0])).unwrap();
            let mut docs = [a, b];
            docs[0].resolutions[0].outcome = Some(left);
            docs[1].resolutions[0].outcome = Some(right);
            let report =
                analyze_forecast_quality(&docs, ForecastAnalysisConfig::default()).unwrap();
            assert_eq!(report.comparisons[0].n_contracts, 1);
            docs[1].resolutions[0].outcome = Some(serde_json::json!(
                1.0 - number_outcome(docs[0].resolutions[0].outcome.as_ref().unwrap()).unwrap()
            ));
            assert!(
                analyze_forecast_quality(&docs, ForecastAnalysisConfig::default())
                    .unwrap_err()
                    .to_string()
                    .contains("unequal realized outcomes")
            );
        }
    }

    /// Replaces the pre-v2 pin that charged `b`'s pending settlement to `a` as one
    /// excluded contract. The gap is `b`'s, and the pair receives no inference.
    #[test]
    fn unmatched_questions_are_charged_to_the_agent_missing_them_and_withhold_the_pair() {
        let a =
            parse_forecast_evidence(&fixture("a", &[0.9, 0.1, 0.8, 0.2], &[1.0, 0.0, 1.0, 0.0]))
                .unwrap();
        let mut b = parse_forecast_evidence(&fixture(
            "b",
            &[0.6, 0.4, 0.55, 0.45],
            &[1.0, 0.0, 1.0, 0.0],
        ))
        .unwrap();
        b.resolutions[3].status = "pending".to_string();
        b.resolutions[3].outcome = None;
        b.resolutions[3].available_at = None;
        let pending = b.revisions[3].contract_sha256.clone();
        let report = analyze_forecast_quality(&[a, b], ForecastAnalysisConfig::default()).unwrap();
        let support = &report.common_support;
        assert_eq!(report.schema_version, "sharpebench.forecast-quality.v2");
        assert_eq!(support.n_contracts, 4);
        assert_eq!(
            support.unresolved_by_agent["a"],
            AgentUnresolvedSupport::default()
        );
        assert_eq!(
            support.unresolved_by_agent["b"],
            AgentUnresolvedSupport {
                n_unresolved: 1,
                pending: vec![pending.clone()],
                ..AgentUnresolvedSupport::default()
            }
        );
        assert_eq!(
            support.settlement_status_disagreements,
            vec![SettlementStatusDisagreement {
                contract_sha256: pending,
                resolved_by: vec!["a".to_string()],
                pending_by: vec!["b".to_string()],
                cancelled_by: vec![],
            }]
        );
        let comparison = &report.comparisons[0];
        assert_eq!(comparison.n_contracts, 3);
        assert_eq!(
            comparison.support_gap,
            Some(SupportGap {
                agent_a_unresolved: 0,
                agent_b_unresolved: 1,
            })
        );
        assert_eq!(comparison.raw_p_value, None);
        assert_eq!(
            comparison.inference_error.as_deref(),
            Some(
                "unequal resolved support: agent_a did not resolve 0 contract(s) that agent_b \
                 resolved and agent_b did not resolve 1 that agent_a resolved; a pair receives \
                 inference only when both agents resolved the same contracts"
            )
        );
    }

    #[test]
    fn holm_adjustment_is_monotone_and_controls_the_family() {
        let mut comparisons = vec![0.01, 0.03, 0.04]
            .into_iter()
            .enumerate()
            .map(|(index, p)| PairwiseForecastComparison {
                agent_a: format!("a{index}"),
                agent_b: "b".to_string(),
                n_contracts: 10,
                n_settlement_blocks: 5,
                mean_loss_difference: -0.1,
                confidence_lower: Some(-0.2),
                confidence_upper: Some(-0.01),
                raw_p_value: Some(p),
                holm_adjusted_p_value: None,
                familywise_significant: false,
                measured_size_at_nominal_5pct: Some(measured_size(5)),
                support_gap: None,
                inference_error: None,
            })
            .collect::<Vec<_>>();
        holm_adjust(&mut comparisons, 0.05);
        assert_eq!(comparisons[0].holm_adjusted_p_value, Some(0.03));
        assert_eq!(comparisons[1].holm_adjusted_p_value, Some(0.06));
        assert_eq!(comparisons[2].holm_adjusted_p_value, Some(0.06));
        assert!(comparisons[0].familywise_significant);
        assert!(!comparisons[1].familywise_significant);
    }

    /// `familywise_significant` carries the size it is actually run at.
    ///
    /// The paired test resamples blocks and centres the raw mean rather than a
    /// studentised statistic, so its rejection rate under the null is well above
    /// its nominal 5% at every block count `inference_support` admits. The flag
    /// is reported with that number beside it rather than silently, and the
    /// table is monotone toward nominal so a larger block count never reads
    /// worse than a smaller one.
    #[test]
    fn the_measured_size_travels_with_the_familywise_flag() {
        // The table is the one the book prints and the report states.
        assert_eq!(measured_size(4), 0.188);
        assert_eq!(measured_size(5), 0.162);
        assert_eq!(measured_size(6), 0.138);
        assert_eq!(measured_size(10), 0.096);
        assert_eq!(measured_size(20), 0.072);
        assert_eq!(measured_size(40), 0.060);
        // Every entry overstates the error that a nominal 5% claims.
        for (blocks, size) in MEASURED_SIZE {
            assert!(*size > 0.05, "{blocks} blocks reads {size}");
        }
        // A count between entries reads the lower one, never the friendlier.
        assert_eq!(measured_size(7), measured_size(6));
        assert_eq!(measured_size(19), measured_size(10));
        assert_eq!(measured_size(1000), measured_size(40));
        // Monotone toward nominal.
        let sizes: Vec<f64> = (4..=60).map(measured_size).collect();
        assert!(sizes.windows(2).all(|w| w[1] <= w[0]), "{sizes:?}");
        // Four blocks is the first count the resampling gate admits, and it is
        // where the flag is furthest from what it claims.
        assert!(inference_support(4, 0.05).is_ok());
        assert!(inference_support(3, 0.05).is_err());
        assert!(measured_size(4) > 3.0 * 0.05);

        // The statement the report carries names the same numbers.
        for number in [
            "0.188", "0.162", "0.138", "0.096", "0.072", "0.060", "0.237",
        ] {
            assert!(
                FAMILYWISE_SIZE_RULE.contains(number),
                "the report statement must name {number}"
            );
        }
    }

    #[test]
    fn normal_crps_and_interval_score_match_closed_form_values() {
        let mut normal =
            serde_json::from_str::<ForecastEvidence>(&fixture("n", &[0.5], &[1.0])).unwrap();
        normal.contracts[0].kind = "normal".to_string();
        normal.contracts[0].scoring_rule = "normal_crps".to_string();
        normal.contracts[0].unit = "USD".to_string();
        normal.revisions[0].prediction = vec![1.0, 2.0];
        normal.revisions[0].contract_sha256 = contract_digests(&normal.contracts[0])
            .unwrap()
            .canonical_json_v1;
        let score = score_prediction(
            &normal.contracts[0],
            &normal.revisions[0].prediction,
            normal.resolutions[0].outcome.as_ref().unwrap(),
        )
        .unwrap()
        .0;
        let expected =
            2.0 * ((2.0 / std::f64::consts::PI).sqrt() - 1.0 / std::f64::consts::PI.sqrt());
        assert!((score - expected).abs() < 1e-7);
    }

    #[test]
    fn every_declared_scoring_rule_is_recomputed_from_raw_values() {
        let base = serde_json::from_str::<ForecastEvidence>(&fixture("rules", &[0.5], &[1.0]))
            .unwrap()
            .contracts
            .remove(0);

        let mut binary_log = base.clone();
        binary_log.scoring_rule = "binary_log".to_string();
        let score = score_prediction(&binary_log, &[0.8], &Value::from(1.0))
            .unwrap()
            .0;
        assert!((score - -0.8_f64.ln()).abs() < 1e-12);

        let mut categorical = base.clone();
        categorical.kind = "categorical".to_string();
        categorical.scoring_rule = "categorical_brier".to_string();
        categorical.categories = vec!["up".into(), "flat".into(), "down".into()];
        let score = score_prediction(
            &categorical,
            &[0.2, 0.3, 0.5],
            &Value::String("down".to_string()),
        )
        .unwrap()
        .0;
        assert!((score - 0.38).abs() < 1e-12);
        categorical.scoring_rule = "categorical_log".to_string();
        let score = score_prediction(
            &categorical,
            &[0.2, 0.3, 0.5],
            &Value::String("flat".to_string()),
        )
        .unwrap()
        .0;
        assert!((score - -0.3_f64.ln()).abs() < 1e-12);

        let mut direction = base.clone();
        direction.kind = "direction".to_string();
        direction.scoring_rule = "direction_accuracy".to_string();
        direction.neutral_threshold = 0.01;
        assert_eq!(
            score_prediction(&direction, &[1.0], &Value::from(0.009))
                .unwrap()
                .0,
            1.0
        );

        let mut interval = base;
        interval.kind = "interval".to_string();
        interval.scoring_rule = "interval_score".to_string();
        interval.interval_alpha = Some(0.1);
        let score = score_prediction(&interval, &[8.0, 12.0], &Value::from(14.0))
            .unwrap()
            .0;
        assert!((score - 44.0).abs() < 1e-12);
    }

    #[test]
    fn forecast_analysis_cannot_change_the_trading_board() {
        use crate::composite::{rank, AgentSubmission, Run, ScoreConfig};

        let submission = AgentSubmission {
            agent_id: "trader".to_string(),
            runs: vec![Run {
                returns: (0..80)
                    .map(|index| 0.002 + 0.0005 * (index as f64 * 0.7).sin())
                    .collect(),
                ..Run::default()
            }],
            ..AgentSubmission::default()
        };
        let before = rank(std::slice::from_ref(&submission), &ScoreConfig::default());
        let evidence = parse_forecast_evidence(&fixture("forecast-only", &[0.9], &[0.0])).unwrap();
        let report =
            analyze_forecast_quality(&[evidence], ForecastAnalysisConfig::default()).unwrap();
        let after = rank(&[submission], &ScoreConfig::default());
        assert_eq!(before, after);
        assert_eq!(report.rank_effect, "reported_only_never_trading_rank");
    }
    /// Audit finding R08 (deferred, not repaired): the pairwise comparison pools
    /// every common contract into one `mean_loss_difference` regardless of
    /// scoring rule or target unit. A binary Brier loss is dimensionless and a
    /// point squared error carries the target unit squared, so rescaling the
    /// point contract from dollars to cents changes the sign of the pooled
    /// difference while the forecasts are unchanged. This test pins that the
    /// analysis averages silently today. Every committed field is one stratum,
    /// so no reported verdict depends on it; the deferral reopens when a
    /// committed comparison mixes strata and stratifying changes its verdict.
    #[test]
    fn mixed_scoring_rules_are_silently_pooled_into_one_mean_loss_difference_r08() {
        fn field(agent: &str, probability: f64, point: f64, unit: &str) -> ForecastEvidence {
            let mut evidence = serde_json::from_str::<ForecastEvidence>(&fixture(
                agent,
                &[probability, 0.5],
                &[1.0, 0.0],
            ))
            .unwrap();
            let contract = &mut evidence.contracts[1];
            contract.kind = "point".to_string();
            contract.scoring_rule = "point_errors".to_string();
            contract.target = "price".to_string();
            contract.unit = unit.to_string();
            let digest = contract_digests(contract).unwrap().canonical_json_v1;
            evidence.revisions[1].prediction = vec![point];
            evidence.revisions[1].contract_sha256 = digest;
            evidence
        }
        let mut pooled = Vec::new();
        for (scale, unit) in [(1.0, "USD"), (100.0, "US_cents")] {
            let a = field("a", 0.9, 0.2 * scale, unit);
            let b = field("b", 0.1, 0.1 * scale, unit);
            let report =
                analyze_forecast_quality(&[a, b], ForecastAnalysisConfig::default()).unwrap();
            // Per-agent means are grouped by rule; the pairwise comparison is not.
            assert_eq!(report.agents[0].metrics.len(), 2);
            let comparison = &report.comparisons[0];
            assert_eq!(comparison.n_contracts, 2);
            let brier_difference = (0.9f64 - 1.0).powi(2) - (0.1f64 - 1.0).powi(2);
            let point_difference = (0.2 * scale).powi(2) - (0.1 * scale).powi(2);
            let expected = (brier_difference + point_difference) / 2.0;
            assert!((comparison.mean_loss_difference - expected).abs() < 1e-9);
            pooled.push(comparison.mean_loss_difference);
        }
        assert!(
            pooled[0] < 0.0,
            "in USD agent a looks better: {}",
            pooled[0]
        );
        assert!(
            pooled[1] > 0.0,
            "in cents agent b looks better: {}",
            pooled[1]
        );
    }
}