chappi_core/

measure.rs

use std::collections::{BTreeMap, BTreeSet};
use std::fs;
use std::path::{Component, Path};

use chappi_syntax::parse_document;
use serde::{Deserialize, Serialize};

use crate::patch::AddValue;
use crate::{
    BudgetUnit, CognitiveState, ConstraintStrength, Diagnostic, FocusCue, GoalStatus,
    LexicalTokenEstimator, Node, NodeKind, Patch, PatchOp, ProjectionFormat, QuestionStatus,
    ReentryCapsule, Source, TensionStatus, TokenEstimator, apply_patch, project, snapshot_source,
    state_hash, validate, validate_patch,
};

const CORPUS_SCHEMA: &str = "chappi.corpus";
const EPISODE_SCHEMA: &str = "chappi.episode";
const MEASUREMENT_SCHEMA: &str = "chappi.measurement";
const VERSION: u32 = 1;
const FACT_ONTOLOGY_VERSION: u32 = 1;
const CONTROLLED_ENGLISH_VERSION: u32 = 1;
const EPISODE_COUNT: usize = 24;
const KNOWN_DIAGNOSTIC_CODES: &[&str] = &[
    "cyclic-supersession",
    "duplicate-focus-reference",
    "duplicate-focus-stream",
    "duplicate-header",
    "duplicate-node-id",
    "duplicate-patch-header",
    "duplicate-relation",
    "duplicate-revision",
    "empty-focus-stream",
    "expected-symbol",
    "focus-stream-mismatch",
    "invalid-add-arity",
    "invalid-add-payload",
    "invalid-add-revision",
    "invalid-affect-intensity",
    "invalid-append-focus-arity",
    "invalid-append-focus-payload",
    "invalid-attention-mode",
    "invalid-character",
    "invalid-confidence",
    "invalid-constraint-strength",
    "invalid-desire-force",
    "invalid-entity-target-kind",
    "invalid-escape",
    "invalid-evidence-source",
    "invalid-expected-revision",
    "invalid-focus-append",
    "invalid-focus-arity",
    "invalid-focus-bucket",
    "invalid-focus-owner-kind",
    "invalid-focus-sequence",
    "invalid-focus-sequence-number",
    "invalid-focus-start",
    "invalid-focus-transition",
    "invalid-goal-status",
    "invalid-header",
    "invalid-lifetime",
    "invalid-metacognition-status",
    "invalid-node-arity",
    "invalid-owner-kind",
    "invalid-patch-base-hash",
    "invalid-patch-header",
    "invalid-patch-top-level-form",
    "invalid-patch-version",
    "invalid-prediction-status",
    "invalid-question-status",
    "invalid-relation-arity",
    "invalid-relation-endpoints",
    "invalid-relation-kind",
    "invalid-retire-arity",
    "invalid-retire-node-id",
    "invalid-revise-arity",
    "invalid-revise-payload",
    "invalid-revision",
    "invalid-revision-node",
    "invalid-salience",
    "invalid-tension-status",
    "invalid-top-level-form",
    "invalid-unicode-escape",
    "invalid-version",
    "missing-header",
    "missing-patch-header",
    "missing-perspective",
    "nesting-depth-exceeded",
    "node-still-referenced",
    "noncanonical-decimal",
    "noncanonical-integer",
    "noncanonical-symbol",
    "noncanonical-text",
    "overlapping-focus-reference",
    "revision-conflict",
    "revision-overflow",
    "stale-patch-base",
    "unclosed-list",
    "unclosed-string",
    "undefined-entity-target",
    "undefined-focus-owner",
    "undefined-focus-reference",
    "undefined-owner",
    "undefined-relation-source",
    "undefined-relation-target",
    "unexpected-close-paren",
    "unexpected-eof",
    "unknown-node-id",
    "unknown-patch-operation",
    "unknown-revision-node",
    "unknown-top-level-form",
];

#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct Corpus {
    pub schema: String,
    pub version: u32,
    pub episodes: Vec<Episode>,
}

#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub struct Episode {
    pub schema: String,
    pub version: u32,
    pub id: String,
    pub category: EpisodeCategory,
    pub snapshot_source: String,
    pub patch_sources: Vec<String>,
    pub natural_language_baseline: String,
    pub capsule_budget_bytes: usize,
    #[serde(default)]
    pub comparison: ComparisonFixture,
    pub golden: GoldenAnswers,
    pub recovery_questions: Vec<RecoveryQuestion>,
    pub seeded_invalid_sources: Vec<SeededInvalidSource>,
}

#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ComparisonFixture {
    pub fact_ontology_version: u32,
    pub controlled_english_version: u32,
    pub selected_node_ids: Vec<String>,
    pub atomic_facts: Vec<AtomicFact>,
}

#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)]
#[serde(rename_all = "kebab-case")]
pub enum FactFamily {
    PayloadKind,
    IdentityLifecycle,
    Relation,
    Focus,
}

impl FactFamily {
    const ALL: [Self; 4] = [
        Self::PayloadKind,
        Self::IdentityLifecycle,
        Self::Relation,
        Self::Focus,
    ];
}

#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)]
#[serde(rename_all = "kebab-case")]
pub enum NodeFactField {
    Id,
    Revision,
    Owner,
    Lifetime,
    Salience,
    Kind,
    EntityKind,
    ClaimConfidence,
    EvidenceKind,
    EvidenceSourceType,
    EvidenceSourceValue,
    GoalStatus,
    ConstraintStrength,
    QuestionStatus,
    RelationshipTarget,
    RelationshipStatus,
    DesireForce,
    AttentionMode,
    TensionStatus,
    PredictionConfidence,
    PredictionStatus,
    PerspectiveTarget,
    PerspectiveConfidence,
    AffectTarget,
    AffectKind,
    AffectIntensity,
    MetacognitionStatus,
    Payload,
}

impl NodeFactField {
    fn family(self) -> FactFamily {
        match self {
            Self::Id | Self::Revision | Self::Owner | Self::Lifetime | Self::Salience => {
                FactFamily::IdentityLifecycle
            }
            _ => FactFamily::PayloadKind,
        }
    }

    fn key(self) -> &'static str {
        match self {
            Self::Id => "id",
            Self::Revision => "revision",
            Self::Owner => "owner",
            Self::Lifetime => "lifetime",
            Self::Salience => "salience",
            Self::Kind => "kind",
            Self::EntityKind => "entity-kind",
            Self::ClaimConfidence => "claim-confidence",
            Self::EvidenceKind => "evidence-kind",
            Self::EvidenceSourceType => "evidence-source-type",
            Self::EvidenceSourceValue => "evidence-source-value",
            Self::GoalStatus => "goal-status",
            Self::ConstraintStrength => "constraint-strength",
            Self::QuestionStatus => "question-status",
            Self::RelationshipTarget => "relationship-target",
            Self::RelationshipStatus => "relationship-status",
            Self::DesireForce => "desire-force",
            Self::AttentionMode => "attention-mode",
            Self::TensionStatus => "tension-status",
            Self::PredictionConfidence => "prediction-confidence",
            Self::PredictionStatus => "prediction-status",
            Self::PerspectiveTarget => "perspective-target",
            Self::PerspectiveConfidence => "perspective-confidence",
            Self::AffectTarget => "affect-target",
            Self::AffectKind => "affect-kind",
            Self::AffectIntensity => "affect-intensity",
            Self::MetacognitionStatus => "metacognition-status",
            Self::Payload => "payload",
        }
    }
}

#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)]
#[serde(rename_all = "kebab-case")]
pub enum FocusFactField {
    StreamId,
    Sequence,
    Owner,
    Transition,
    Trigger,
}

impl FocusFactField {
    fn key(self) -> &'static str {
        match self {
            Self::StreamId => "stream",
            Self::Sequence => "sequence",
            Self::Owner => "owner",
            Self::Transition => "transition",
            Self::Trigger => "trigger",
        }
    }
}

#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)]
#[serde(rename_all = "kebab-case")]
pub enum FocusBucket {
    Foreground,
    Background,
    Residue,
}

impl FocusBucket {
    fn key(self) -> &'static str {
        match self {
            Self::Foreground => "foreground",
            Self::Background => "background",
            Self::Residue => "residue",
        }
    }
}

#[derive(Clone, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)]
#[serde(tag = "fact", rename_all = "kebab-case", deny_unknown_fields)]
pub enum AtomicFact {
    Node {
        node_id: String,
        field: NodeFactField,
        value: String,
    },
    Relation {
        kind: String,
        source: String,
        target: String,
    },
    Focus {
        stream_id: String,
        sequence: u64,
        field: FocusFactField,
        value: String,
    },
    FocusBucketMember {
        stream_id: String,
        sequence: u64,
        bucket: FocusBucket,
        node_id: String,
    },
    FocusBucketEmpty {
        stream_id: String,
        sequence: u64,
        bucket: FocusBucket,
    },
}

impl AtomicFact {
    pub fn family(&self) -> FactFamily {
        match self {
            Self::Node { field, .. } => field.family(),
            Self::Relation { .. } => FactFamily::Relation,
            Self::Focus { .. } | Self::FocusBucketMember { .. } | Self::FocusBucketEmpty { .. } => {
                FactFamily::Focus
            }
        }
    }
}

#[derive(Clone, Debug)]
pub struct DerivedComparison {
    pub fixture: ComparisonFixture,
    pub canonical_capsule: String,
    pub natural_language_baseline: String,
    pub capsule_budget_bytes: usize,
}

#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)]
#[serde(rename_all = "kebab-case")]
pub enum EpisodeCategory {
    Technical,
    Interpersonal,
    AffectiveMemory,
    Tension,
    PredictionSurprise,
    InterruptedResumption,
}

impl EpisodeCategory {
    const ALL: [Self; 6] = [
        Self::Technical,
        Self::Interpersonal,
        Self::AffectiveMemory,
        Self::Tension,
        Self::PredictionSurprise,
        Self::InterruptedResumption,
    ];
}

#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub struct GoldenAnswers {
    pub required_constraint_ids: Vec<String>,
    pub committed_goal_ids: Vec<String>,
    pub active_question_ids: Vec<String>,
    pub active_tension_ids: Vec<String>,
    pub owner_target_expectations: Vec<OwnerTargetExpectation>,
}

#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "kebab-case")]
pub enum AnswerPolarity {
    Affirmed,
    Negated,
}

#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub struct SemanticAnswer {
    pub value: String,
    #[serde(default)]
    pub aliases: Vec<String>,
    pub qualifiers: Vec<Vec<String>>,
    pub polarity: AnswerPolarity,
}

#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)]
#[serde(rename_all = "kebab-case")]
pub enum RecoveryFactKind {
    OwnerContext,
    RequiredConstraint,
    CommittedGoal,
    ActiveQuestion,
    ActiveTension,
}

#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub struct RecoveryQuestion {
    pub id: String,
    pub prompt: String,
    pub fact_kind: RecoveryFactKind,
    pub covered_node_id: String,
    pub expected: SemanticAnswer,
}

#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub struct OwnerTargetExpectation {
    pub node_id: String,
    pub owner: String,
    pub target: Option<String>,
}

#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub struct SeededInvalidSource {
    pub source: String,
    pub expected_diagnostic_codes: Vec<String>,
}

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum CorpusErrorKind {
    Io,
    Invalid,
}

#[derive(Clone, Debug)]
pub struct CorpusError {
    pub kind: CorpusErrorKind,
    pub diagnostics: Vec<CorpusDiagnostic>,
}

#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub struct CorpusDiagnostic {
    pub code: String,
    pub path: String,
    pub message: String,
}

#[derive(Clone, Debug, Serialize)]
pub struct MeasurementReport {
    pub schema: &'static str,
    pub version: u32,
    pub episodes: usize,
    pub semantic: SemanticMetrics,
    pub continuity: ContinuityMetrics,
    pub precision: PrecisionMetrics,
    pub compression: CompressionMetrics,
    pub gates: Vec<GateResult>,
    pub passed: bool,
}

#[derive(Clone, Debug, Serialize)]
pub struct SemanticMetrics {
    pub loaded: usize,
    pub replayed: usize,
    pub replay_ratio: f64,
}

#[derive(Clone, Debug, Serialize)]
pub struct ContinuityMetrics {
    pub constraints_and_committed_goals: f64,
    pub active_questions_and_tensions: f64,
    pub owner_target_expectations: f64,
}

#[derive(Clone, Debug, Serialize)]
pub struct PrecisionMetrics {
    pub seeded_diagnostics: f64,
    pub chappi_task_accuracy: f64,
    pub baseline_task_accuracy: f64,
    pub task_accuracy_advantage: f64,
    pub fact_families: Vec<FactFamilyMetrics>,
}

#[derive(Clone, Debug, Serialize)]
pub struct FactFamilyMetrics {
    pub family: FactFamily,
    pub total: usize,
    pub chappi_hits: usize,
    pub baseline_hits: usize,
    pub chappi_accuracy: Option<f64>,
    pub baseline_accuracy: Option<f64>,
    pub included_in_macro: bool,
}

#[derive(Clone, Debug)]
pub struct FactRecoveryScore {
    pub families: Vec<FactFamilyMetrics>,
    pub chappi_macro_accuracy: f64,
    pub baseline_macro_accuracy: f64,
}

pub fn score_fact_recovery(
    expected: &[AtomicFact],
    chappi: &[AtomicFact],
    baseline: &[AtomicFact],
) -> FactRecoveryScore {
    let expected = expected.iter().cloned().collect::<BTreeSet<_>>();
    let chappi = chappi.iter().cloned().collect::<BTreeSet<_>>();
    let baseline = baseline.iter().cloned().collect::<BTreeSet<_>>();
    let mut families = Vec::new();
    let mut chappi_sum = 0.0;
    let mut baseline_sum = 0.0;
    let mut included = 0usize;
    for family in FactFamily::ALL {
        let family_expected = expected
            .iter()
            .filter(|fact| fact.family() == family)
            .collect::<BTreeSet<_>>();
        let total = family_expected.len();
        let chappi_hits = family_expected
            .iter()
            .filter(|fact| chappi.contains(**fact))
            .count();
        let baseline_hits = family_expected
            .iter()
            .filter(|fact| baseline.contains(**fact))
            .count();
        let chappi_accuracy = (total > 0).then(|| ratio(chappi_hits, total));
        let baseline_accuracy = (total > 0).then(|| ratio(baseline_hits, total));
        if let (Some(chappi_accuracy), Some(baseline_accuracy)) =
            (chappi_accuracy, baseline_accuracy)
        {
            chappi_sum += chappi_accuracy;
            baseline_sum += baseline_accuracy;
            included += 1;
        }
        families.push(FactFamilyMetrics {
            family,
            total,
            chappi_hits,
            baseline_hits,
            chappi_accuracy,
            baseline_accuracy,
            included_in_macro: total > 0,
        });
    }
    FactRecoveryScore {
        families,
        chappi_macro_accuracy: if included == 0 {
            0.0
        } else {
            chappi_sum / included as f64
        },
        baseline_macro_accuracy: if included == 0 {
            0.0
        } else {
            baseline_sum / included as f64
        },
    }
}

#[derive(Clone, Debug, Serialize)]
pub struct CompressionMetrics {
    pub snapshot_to_json: f64,
    pub capsule_to_baseline: f64,
    pub canonical_snapshot_bytes: usize,
    pub pretty_json_bytes: usize,
    pub capsule_bytes: usize,
    pub baseline_bytes: usize,
    pub canonical_snapshot_tokens: usize,
    pub pretty_json_tokens: usize,
    pub capsule_tokens: usize,
    pub baseline_tokens: usize,
}

#[derive(Clone, Debug, Serialize)]
pub struct GateResult {
    pub name: &'static str,
    pub passed: bool,
    pub actual: f64,
    pub required: f64,
}

#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct Manifest {
    schema: String,
    version: u32,
    episodes: Vec<String>,
}

/// Derives the complete semantic comparison for an already fixed node selection.
///
/// The selection is never changed to satisfy a byte limit. The returned byte budget is the exact
/// size of the resulting canonical content and is therefore a consequence of semantics.
pub fn derive_semantic_comparison(
    state: &CognitiveState,
    selected_node_ids: &[String],
) -> Result<DerivedComparison, String> {
    let selected = selected_node_ids.iter().cloned().collect::<BTreeSet<_>>();
    if selected.len() != selected_node_ids.len() {
        return Err("selected node IDs must be unique".into());
    }
    let known = state
        .nodes
        .iter()
        .map(|node| node.id.as_str())
        .collect::<BTreeSet<_>>();
    if let Some(id) = selected.iter().find(|id| !known.contains(id.as_str())) {
        return Err(format!("selected node `{id}` does not exist"));
    }
    for node in state
        .nodes
        .iter()
        .filter(|node| selected.contains(&node.id))
    {
        if !selected.contains(&node.owner) {
            return Err(format!(
                "selection omits owner `{}` for `{}`",
                node.owner, node.id
            ));
        }
        if let Some(target) = node_target(node) {
            if !selected.contains(target) {
                return Err(format!(
                    "selection omits target `{target}` for `{}`",
                    node.id
                ));
            }
        }
    }

    let capsule = exact_capsule(state, &selected);
    let canonical_capsule = capsule.canonical_content();
    let atomic_facts = facts_from_capsule(&capsule);
    let natural_language_baseline = render_controlled_english(&atomic_facts)?;
    Ok(DerivedComparison {
        fixture: ComparisonFixture {
            fact_ontology_version: FACT_ONTOLOGY_VERSION,
            controlled_english_version: CONTROLLED_ENGLISH_VERSION,
            selected_node_ids: selected.into_iter().collect(),
            atomic_facts,
        },
        capsule_budget_bytes: canonical_capsule.len(),
        canonical_capsule,
        natural_language_baseline,
    })
}

fn exact_capsule(state: &CognitiveState, selected: &BTreeSet<String>) -> ReentryCapsule {
    let mut nodes = state
        .nodes
        .iter()
        .filter(|node| selected.contains(&node.id))
        .cloned()
        .collect::<Vec<_>>();
    nodes.sort_by(|left, right| left.id.cmp(&right.id));

    let mut relations = state
        .relations
        .iter()
        .filter(|relation| {
            selected.contains(&relation.source) && selected.contains(&relation.target)
        })
        .cloned()
        .collect::<Vec<_>>();
    relations.sort_by(|left, right| {
        left.kind
            .as_str()
            .cmp(right.kind.as_str())
            .then_with(|| left.source.cmp(&right.source))
            .then_with(|| left.target.cmp(&right.target))
    });

    let mut focus_cues = state
        .focus_streams
        .iter()
        .filter_map(|stream| stream.frames.iter().max_by_key(|frame| frame.sequence))
        .filter(|frame| frame.transition != crate::TransitionKind::Resolved)
        .filter(|frame| {
            selected.contains(&frame.owner)
                && frame
                    .foreground
                    .iter()
                    .chain(&frame.background)
                    .chain(&frame.residue)
                    .all(|id| selected.contains(id))
        })
        .map(|frame| FocusCue {
            stream_id: frame.stream_id.clone(),
            sequence: frame.sequence,
            owner: frame.owner.clone(),
            transition: frame.transition,
            foreground: sorted_owned(&frame.foreground),
            background: sorted_owned(&frame.background),
            residue: sorted_owned(&frame.residue),
            trigger: crate::resume::canonical_form(&frame.trigger),
        })
        .collect::<Vec<_>>();
    focus_cues.sort_by(|left, right| {
        left.stream_id
            .cmp(&right.stream_id)
            .then_with(|| left.sequence.cmp(&right.sequence))
    });

    let mut omitted_node_ids = state
        .nodes
        .iter()
        .filter(|node| !selected.contains(&node.id))
        .map(|node| node.id.clone())
        .collect::<Vec<_>>();
    omitted_node_ids.sort();
    let mut capsule = ReentryCapsule {
        schema_version: VERSION,
        source_state_hash: state_hash(state),
        nodes,
        relations,
        focus_cues,
        omitted_node_ids,
        used: 0,
        requested_limit: 0,
        requested_unit: BudgetUnit::Bytes,
        over_budget: false,
    };
    let used = capsule.canonical_content().len();
    capsule.used = used;
    capsule.requested_limit = used;
    capsule
}

fn sorted_owned(values: &[String]) -> Vec<String> {
    let mut values = values.to_vec();
    values.sort();
    values
}

fn facts_from_capsule(capsule: &ReentryCapsule) -> Vec<AtomicFact> {
    let mut facts = Vec::new();
    for node in &capsule.nodes {
        push_node_facts(&mut facts, node);
    }
    for relation in &capsule.relations {
        facts.push(AtomicFact::Relation {
            kind: relation.kind.as_str().into(),
            source: relation.source.clone(),
            target: relation.target.clone(),
        });
    }
    for cue in &capsule.focus_cues {
        facts.push(focus_scalar(cue, FocusFactField::StreamId, &cue.stream_id));
        facts.push(focus_scalar(
            cue,
            FocusFactField::Sequence,
            &cue.sequence.to_string(),
        ));
        facts.push(focus_scalar(cue, FocusFactField::Owner, &cue.owner));
        facts.push(focus_scalar(
            cue,
            FocusFactField::Transition,
            cue.transition.as_str(),
        ));
        push_bucket_facts(&mut facts, cue, FocusBucket::Foreground, &cue.foreground);
        push_bucket_facts(&mut facts, cue, FocusBucket::Background, &cue.background);
        push_bucket_facts(&mut facts, cue, FocusBucket::Residue, &cue.residue);
        facts.push(focus_scalar(cue, FocusFactField::Trigger, &cue.trigger));
    }
    facts
}

fn push_node_facts(facts: &mut Vec<AtomicFact>, node: &Node) {
    let mut push = |field, value: String| {
        facts.push(AtomicFact::Node {
            node_id: node.id.clone(),
            field,
            value,
        });
    };
    push(NodeFactField::Id, node.id.clone());
    push(NodeFactField::Revision, node.revision.to_string());
    push(NodeFactField::Owner, node.owner.clone());
    push(
        NodeFactField::Lifetime,
        crate::resume::lifetime_text(node.lifetime).into(),
    );
    push(NodeFactField::Salience, node.salience.to_string());
    push(NodeFactField::Kind, node_kind_name(&node.kind).into());
    match &node.kind {
        NodeKind::Entity { entity_kind } => push(NodeFactField::EntityKind, entity_kind.clone()),
        NodeKind::Claim { confidence } => push(
            NodeFactField::ClaimConfidence,
            crate::resume::confidence_text(*confidence).into(),
        ),
        NodeKind::Evidence {
            evidence_kind,
            source,
        } => {
            push(NodeFactField::EvidenceKind, evidence_kind.clone());
            push(
                NodeFactField::EvidenceSourceType,
                match source {
                    Source::Symbol(_) => "symbol",
                    Source::String(_) => "string",
                }
                .into(),
            );
            push(
                NodeFactField::EvidenceSourceValue,
                match source {
                    Source::Symbol(value) | Source::String(value) => value.clone(),
                },
            );
        }
        NodeKind::Goal { status } => push(
            NodeFactField::GoalStatus,
            crate::resume::goal_status(*status).into(),
        ),
        NodeKind::Constraint { strength } => push(
            NodeFactField::ConstraintStrength,
            crate::resume::constraint_text(*strength).into(),
        ),
        NodeKind::Question { status } => push(
            NodeFactField::QuestionStatus,
            crate::resume::question_text(*status).into(),
        ),
        NodeKind::Relationship { target, status } => {
            push(NodeFactField::RelationshipTarget, target.clone());
            push(NodeFactField::RelationshipStatus, status.clone());
        }
        NodeKind::Desire { force } => push(
            NodeFactField::DesireForce,
            crate::resume::desire_text(*force).into(),
        ),
        NodeKind::Attention { mode } => push(
            NodeFactField::AttentionMode,
            crate::resume::attention_text(*mode).into(),
        ),
        NodeKind::Association => {}
        NodeKind::Tension { status } => push(
            NodeFactField::TensionStatus,
            crate::resume::tension_text(*status).into(),
        ),
        NodeKind::Prediction { confidence, status } => {
            push(
                NodeFactField::PredictionConfidence,
                crate::resume::confidence_text(*confidence).into(),
            );
            push(
                NodeFactField::PredictionStatus,
                crate::resume::prediction_text(*status).into(),
            );
        }
        NodeKind::Perspective(value) => {
            push(NodeFactField::PerspectiveTarget, value.target.clone());
            push(
                NodeFactField::PerspectiveConfidence,
                crate::resume::confidence_text(value.confidence).into(),
            );
        }
        NodeKind::Affect {
            target,
            affect_kind,
            intensity,
        } => {
            push(NodeFactField::AffectTarget, target.clone());
            push(NodeFactField::AffectKind, affect_kind.clone());
            push(
                NodeFactField::AffectIntensity,
                crate::resume::affect_text(*intensity).into(),
            );
        }
        NodeKind::Metacognition { status } => push(
            NodeFactField::MetacognitionStatus,
            crate::resume::metacognition_text(*status).into(),
        ),
    }
    push(
        NodeFactField::Payload,
        crate::resume::canonical_form(&node.payload),
    );
}

fn focus_scalar(cue: &FocusCue, field: FocusFactField, value: &str) -> AtomicFact {
    AtomicFact::Focus {
        stream_id: cue.stream_id.clone(),
        sequence: cue.sequence,
        field,
        value: value.into(),
    }
}

fn push_bucket_facts(
    facts: &mut Vec<AtomicFact>,
    cue: &FocusCue,
    bucket: FocusBucket,
    ids: &[String],
) {
    if ids.is_empty() {
        facts.push(AtomicFact::FocusBucketEmpty {
            stream_id: cue.stream_id.clone(),
            sequence: cue.sequence,
            bucket,
        });
    } else {
        facts.extend(ids.iter().map(|id| AtomicFact::FocusBucketMember {
            stream_id: cue.stream_id.clone(),
            sequence: cue.sequence,
            bucket,
            node_id: id.clone(),
        }));
    }
}

fn render_controlled_english(facts: &[AtomicFact]) -> Result<String, String> {
    let mut records = Vec::new();
    let mut index = 0;
    while index < facts.len() {
        match &facts[index] {
            AtomicFact::Node { node_id, .. } => {
                let start = index;
                while index < facts.len()
                    && matches!(&facts[index], AtomicFact::Node { node_id: id, .. } if id == node_id)
                {
                    index += 1;
                }
                records.push(render_node_record(&facts[start..index])?);
            }
            AtomicFact::Relation {
                kind,
                source,
                target,
            } => {
                records.push(format!(
                    "Relation kind={kind} source={} target={}.",
                    quote(source),
                    quote(target)
                ));
                index += 1;
            }
            AtomicFact::Focus {
                stream_id,
                sequence,
                ..
            }
            | AtomicFact::FocusBucketMember {
                stream_id,
                sequence,
                ..
            }
            | AtomicFact::FocusBucketEmpty {
                stream_id,
                sequence,
                ..
            } => {
                let start = index;
                while index < facts.len() && same_focus_record(&facts[index], stream_id, *sequence)
                {
                    index += 1;
                }
                records.push(render_focus_record(&facts[start..index])?);
            }
        }
    }
    Ok(records.join(" "))
}

fn render_node_record(facts: &[AtomicFact]) -> Result<String, String> {
    let mut output = String::from("Node");
    let mut seen = BTreeSet::new();
    let mut node_id = None;
    for fact in facts {
        let AtomicFact::Node {
            node_id: id,
            field,
            value,
        } = fact
        else {
            return Err("node record contains a non-node fact".into());
        };
        if node_id.get_or_insert(id) != &id {
            return Err("node record mixes node IDs".into());
        }
        if !seen.insert(*field) {
            return Err(format!("node `{id}` duplicates `{}`", field.key()));
        }
        output.push(' ');
        output.push_str(field.key());
        output.push('=');
        if node_field_is_quoted(*field) {
            output.push_str(&quote(value));
        } else {
            output.push_str(value);
        }
    }
    output.push('.');
    Ok(output)
}

fn render_focus_record(facts: &[AtomicFact]) -> Result<String, String> {
    let (stream_id, sequence) = focus_key(
        facts
            .first()
            .ok_or_else(|| "focus record must not be empty".to_string())?,
    )
    .ok_or_else(|| "focus record must start with a focus fact".to_string())?;
    let mut scalars = BTreeMap::new();
    let mut buckets = BTreeMap::<FocusBucket, Vec<String>>::new();
    let mut empty = BTreeSet::new();
    for fact in facts {
        if focus_key(fact) != Some((stream_id, sequence)) {
            return Err("focus record mixes keys".into());
        }
        match fact {
            AtomicFact::Focus { field, value, .. } => {
                if scalars.insert(*field, value.as_str()).is_some() {
                    return Err(format!("focus record duplicates `{}`", field.key()));
                }
            }
            AtomicFact::FocusBucketMember {
                bucket, node_id, ..
            } => buckets.entry(*bucket).or_default().push(node_id.clone()),
            AtomicFact::FocusBucketEmpty { bucket, .. } => {
                empty.insert(*bucket);
            }
            AtomicFact::Node { .. } | AtomicFact::Relation { .. } => unreachable!(),
        }
    }
    for bucket in FocusBucket::ALL {
        let members = buckets.entry(bucket).or_default();
        members.sort();
        if members.is_empty() != empty.contains(&bucket) {
            return Err(format!(
                "focus bucket `{}` must have members or one emptiness fact",
                bucket.key()
            ));
        }
    }
    let scalar = |field| {
        scalars
            .get(&field)
            .copied()
            .ok_or_else(|| format!("focus record omits `{}`", field.key()))
    };
    if scalar(FocusFactField::StreamId)? != stream_id
        || scalar(FocusFactField::Sequence)? != sequence.to_string()
    {
        return Err("focus key facts do not match their record key".into());
    }
    Ok(format!(
        "Focus stream={} sequence={} owner={} transition={} foreground={} background={} residue={} trigger={}.",
        quote(stream_id),
        sequence,
        quote(scalar(FocusFactField::Owner)?),
        scalar(FocusFactField::Transition)?,
        serde_json::to_string(&buckets[&FocusBucket::Foreground]).expect("strings serialize"),
        serde_json::to_string(&buckets[&FocusBucket::Background]).expect("strings serialize"),
        serde_json::to_string(&buckets[&FocusBucket::Residue]).expect("strings serialize"),
        quote(scalar(FocusFactField::Trigger)?),
    ))
}

impl FocusBucket {
    const ALL: [Self; 3] = [Self::Foreground, Self::Background, Self::Residue];
}

fn same_focus_record(fact: &AtomicFact, stream_id: &str, sequence: u64) -> bool {
    focus_key(fact) == Some((stream_id, sequence))
}

fn focus_key(fact: &AtomicFact) -> Option<(&str, u64)> {
    match fact {
        AtomicFact::Focus {
            stream_id,
            sequence,
            ..
        }
        | AtomicFact::FocusBucketMember {
            stream_id,
            sequence,
            ..
        }
        | AtomicFact::FocusBucketEmpty {
            stream_id,
            sequence,
            ..
        } => Some((stream_id, *sequence)),
        AtomicFact::Node { .. } | AtomicFact::Relation { .. } => None,
    }
}

fn node_field_is_quoted(field: NodeFactField) -> bool {
    matches!(
        field,
        NodeFactField::Id
            | NodeFactField::Owner
            | NodeFactField::EntityKind
            | NodeFactField::EvidenceKind
            | NodeFactField::EvidenceSourceValue
            | NodeFactField::RelationshipTarget
            | NodeFactField::RelationshipStatus
            | NodeFactField::PerspectiveTarget
            | NodeFactField::AffectTarget
            | NodeFactField::AffectKind
            | NodeFactField::Payload
    )
}

fn quote(value: &str) -> String {
    serde_json::to_string(value).expect("strings serialize")
}

/// Decodes only complete records from the v1 controlled-English grammar. A truncated or malformed
/// sentence contributes no facts for that record.
pub fn decode_controlled_english_facts(input: &str) -> Vec<AtomicFact> {
    complete_sentences(input)
        .into_iter()
        .filter_map(decode_controlled_record)
        .flatten()
        .collect()
}

fn complete_sentences(input: &str) -> Vec<&str> {
    let mut records = Vec::new();
    let mut start = 0;
    let mut quoted = false;
    let mut escaped = false;
    for (index, character) in input.char_indices() {
        if quoted {
            if escaped {
                escaped = false;
            } else if character == '\\' {
                escaped = true;
            } else if character == '"' {
                quoted = false;
            }
        } else if character == '"' {
            quoted = true;
        } else if character == '.' {
            records.push(input[start..=index].trim());
            start = index + 1;
        }
    }
    records
}

fn decode_controlled_record(record: &str) -> Option<Vec<AtomicFact>> {
    let body = record.strip_suffix('.')?;
    let tokens = split_controlled_tokens(body)?;
    let (head, fields) = tokens.split_first()?;
    match head.as_str() {
        "Node" => decode_controlled_node(fields),
        "Relation" => decode_controlled_relation(fields),
        "Focus" => decode_controlled_focus(fields),
        _ => None,
    }
}

fn split_controlled_tokens(input: &str) -> Option<Vec<String>> {
    let mut tokens = Vec::new();
    let mut start = None;
    let mut quoted = false;
    let mut escaped = false;
    let mut brackets = 0usize;
    for (index, character) in input.char_indices() {
        if quoted {
            if escaped {
                escaped = false;
            } else if character == '\\' {
                escaped = true;
            } else if character == '"' {
                quoted = false;
            }
        } else {
            match character {
                '"' => quoted = true,
                '[' => brackets += 1,
                ']' => brackets = brackets.checked_sub(1)?,
                ' ' if brackets == 0 => {
                    if let Some(begin) = start.take() {
                        tokens.push(input[begin..index].to_string());
                    }
                    continue;
                }
                _ => {}
            }
        }
        start.get_or_insert(index);
    }
    if quoted || brackets != 0 {
        return None;
    }
    if let Some(begin) = start {
        tokens.push(input[begin..].to_string());
    }
    Some(tokens)
}

fn key_values(fields: &[String]) -> Option<Vec<(&str, &str)>> {
    fields.iter().map(|field| field.split_once('=')).collect()
}

fn decode_controlled_node(fields: &[String]) -> Option<Vec<AtomicFact>> {
    let pairs = key_values(fields)?;
    let id = decode_quoted(pairs.first().filter(|(key, _)| *key == "id")?.1)?;
    let mut facts = Vec::new();
    for (key, raw) in pairs {
        let field = node_field_from_key(key)?;
        let value = if node_field_is_quoted(field) {
            decode_quoted(raw)?
        } else {
            if raw.is_empty()
                || !raw
                    .chars()
                    .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit())
            {
                return None;
            }
            raw.to_string()
        };
        facts.push(AtomicFact::Node {
            node_id: id.clone(),
            field,
            value,
        });
    }
    valid_node_fact_record(&facts).then_some(facts)
}

fn decode_controlled_relation(fields: &[String]) -> Option<Vec<AtomicFact>> {
    let pairs = key_values(fields)?;
    if pairs.len() != 3 || pairs[0].0 != "kind" || pairs[1].0 != "source" || pairs[2].0 != "target"
    {
        return None;
    }
    Some(vec![AtomicFact::Relation {
        kind: pairs[0].1.to_string(),
        source: decode_quoted(pairs[1].1)?,
        target: decode_quoted(pairs[2].1)?,
    }])
}

fn decode_controlled_focus(fields: &[String]) -> Option<Vec<AtomicFact>> {
    let pairs = key_values(fields)?;
    let expected = [
        "stream",
        "sequence",
        "owner",
        "transition",
        "foreground",
        "background",
        "residue",
        "trigger",
    ];
    if pairs.len() != expected.len() || pairs.iter().map(|(key, _)| *key).ne(expected) {
        return None;
    }
    let stream_id = decode_quoted(pairs[0].1)?;
    let sequence = pairs[1].1.parse::<u64>().ok()?;
    if sequence.to_string() != pairs[1].1 {
        return None;
    }
    let owner = decode_quoted(pairs[2].1)?;
    let transition = pairs[3].1.to_string();
    let mut facts = vec![
        AtomicFact::Focus {
            stream_id: stream_id.clone(),
            sequence,
            field: FocusFactField::StreamId,
            value: stream_id.clone(),
        },
        AtomicFact::Focus {
            stream_id: stream_id.clone(),
            sequence,
            field: FocusFactField::Sequence,
            value: sequence.to_string(),
        },
        AtomicFact::Focus {
            stream_id: stream_id.clone(),
            sequence,
            field: FocusFactField::Owner,
            value: owner,
        },
        AtomicFact::Focus {
            stream_id: stream_id.clone(),
            sequence,
            field: FocusFactField::Transition,
            value: transition,
        },
    ];
    for (bucket, (_, raw)) in FocusBucket::ALL.into_iter().zip(&pairs[4..7]) {
        let members = serde_json::from_str::<Vec<String>>(raw).ok()?;
        if members.is_empty() {
            facts.push(AtomicFact::FocusBucketEmpty {
                stream_id: stream_id.clone(),
                sequence,
                bucket,
            });
        } else {
            facts.extend(
                members
                    .into_iter()
                    .map(|node_id| AtomicFact::FocusBucketMember {
                        stream_id: stream_id.clone(),
                        sequence,
                        bucket,
                        node_id,
                    }),
            );
        }
    }
    facts.push(AtomicFact::Focus {
        stream_id,
        sequence,
        field: FocusFactField::Trigger,
        value: decode_quoted(pairs[7].1)?,
    });
    Some(facts)
}

fn decode_quoted(raw: &str) -> Option<String> {
    serde_json::from_str::<String>(raw).ok()
}

fn node_field_from_key(key: &str) -> Option<NodeFactField> {
    Some(match key {
        "id" => NodeFactField::Id,
        "revision" => NodeFactField::Revision,
        "owner" => NodeFactField::Owner,
        "lifetime" => NodeFactField::Lifetime,
        "salience" => NodeFactField::Salience,
        "kind" => NodeFactField::Kind,
        "entity-kind" => NodeFactField::EntityKind,
        "claim-confidence" => NodeFactField::ClaimConfidence,
        "evidence-kind" => NodeFactField::EvidenceKind,
        "evidence-source-type" => NodeFactField::EvidenceSourceType,
        "evidence-source-value" => NodeFactField::EvidenceSourceValue,
        "goal-status" => NodeFactField::GoalStatus,
        "constraint-strength" => NodeFactField::ConstraintStrength,
        "question-status" => NodeFactField::QuestionStatus,
        "relationship-target" => NodeFactField::RelationshipTarget,
        "relationship-status" => NodeFactField::RelationshipStatus,
        "desire-force" => NodeFactField::DesireForce,
        "attention-mode" => NodeFactField::AttentionMode,
        "tension-status" => NodeFactField::TensionStatus,
        "prediction-confidence" => NodeFactField::PredictionConfidence,
        "prediction-status" => NodeFactField::PredictionStatus,
        "perspective-target" => NodeFactField::PerspectiveTarget,
        "perspective-confidence" => NodeFactField::PerspectiveConfidence,
        "affect-target" => NodeFactField::AffectTarget,
        "affect-kind" => NodeFactField::AffectKind,
        "affect-intensity" => NodeFactField::AffectIntensity,
        "metacognition-status" => NodeFactField::MetacognitionStatus,
        "payload" => NodeFactField::Payload,
        _ => return None,
    })
}

fn valid_node_fact_record(facts: &[AtomicFact]) -> bool {
    let fields = facts
        .iter()
        .filter_map(|fact| match fact {
            AtomicFact::Node { field, .. } => Some(*field),
            _ => None,
        })
        .collect::<Vec<_>>();
    if fields.len() != facts.len() || fields.len() < 7 {
        return false;
    }
    let base = [
        NodeFactField::Id,
        NodeFactField::Revision,
        NodeFactField::Owner,
        NodeFactField::Lifetime,
        NodeFactField::Salience,
        NodeFactField::Kind,
    ];
    if !fields.starts_with(&base) || fields.last() != Some(&NodeFactField::Payload) {
        return false;
    }
    let kind = facts.iter().find_map(|fact| match fact {
        AtomicFact::Node {
            field: NodeFactField::Kind,
            value,
            ..
        } => Some(value.as_str()),
        _ => None,
    });
    let expected_middle: &[NodeFactField] = match kind {
        Some("entity") => &[NodeFactField::EntityKind],
        Some("claim") => &[NodeFactField::ClaimConfidence],
        Some("evidence") => &[
            NodeFactField::EvidenceKind,
            NodeFactField::EvidenceSourceType,
            NodeFactField::EvidenceSourceValue,
        ],
        Some("goal") => &[NodeFactField::GoalStatus],
        Some("constraint") => &[NodeFactField::ConstraintStrength],
        Some("question") => &[NodeFactField::QuestionStatus],
        Some("relationship") => &[
            NodeFactField::RelationshipTarget,
            NodeFactField::RelationshipStatus,
        ],
        Some("desire") => &[NodeFactField::DesireForce],
        Some("attention") => &[NodeFactField::AttentionMode],
        Some("association") => &[],
        Some("tension") => &[NodeFactField::TensionStatus],
        Some("prediction") => &[
            NodeFactField::PredictionConfidence,
            NodeFactField::PredictionStatus,
        ],
        Some("perspective") => &[
            NodeFactField::PerspectiveTarget,
            NodeFactField::PerspectiveConfidence,
        ],
        Some("affect") => &[
            NodeFactField::AffectTarget,
            NodeFactField::AffectKind,
            NodeFactField::AffectIntensity,
        ],
        Some("metacognition") => &[NodeFactField::MetacognitionStatus],
        _ => return false,
    };
    fields[base.len()..fields.len() - 1] == *expected_middle
}

/// Decodes only complete canonical compact records. Operational wrapper fields on
/// [`ReentryCapsule`] are outside this comparison scope; `canonical_content()` and the
/// controlled-English records both cover nodes, enclosed relations, and closed latest focus cues.
pub fn decode_compact_semantic_facts(input: &str) -> Vec<AtomicFact> {
    complete_compact_records(input)
        .into_iter()
        .filter_map(decode_compact_record)
        .flatten()
        .collect()
}

fn complete_compact_records(input: &str) -> Vec<&str> {
    let mut records = Vec::new();
    let mut start = None;
    let mut depth = 0usize;
    let mut quoted = false;
    let mut escaped = false;
    for (index, character) in input.char_indices() {
        if quoted {
            if escaped {
                escaped = false;
            } else if character == '\\' {
                escaped = true;
            } else if character == '"' {
                quoted = false;
            }
            continue;
        }
        match character {
            '"' if depth > 0 => quoted = true,
            '(' => {
                if depth == 0 {
                    start = Some(index);
                }
                depth += 1;
            }
            ')' if depth > 0 => {
                depth -= 1;
                if depth == 0 {
                    if let Some(begin) = start.take() {
                        records.push(&input[begin..=index]);
                    }
                }
            }
            _ => {}
        }
    }
    records
}

fn decode_compact_record(record: &str) -> Option<Vec<AtomicFact>> {
    let inner = record.strip_prefix('(')?.strip_suffix(')')?;
    if inner.starts_with('R') {
        decode_compact_relation(inner)
    } else if inner.starts_with('!') {
        decode_compact_focus(inner)
    } else {
        decode_compact_node(inner)
    }
}

fn decode_compact_node(inner: &str) -> Option<Vec<AtomicFact>> {
    let (header, payload) = split_once_outside_quotes(inner, ' ')?;
    if parse_document(payload).ok()?.forms.len() != 1 {
        return None;
    }
    let (id_raw, rest) = split_once_outside_quotes(header, '~')?;
    let (revision_raw, rest) = split_once_outside_quotes(rest, '@')?;
    let (owner_raw, rest) = split_once_outside_quotes(rest, '/')?;
    let (life_salience, kind_raw) = split_once_outside_quotes(rest, ':')?;
    let mut life_chars = life_salience.chars();
    let lifetime = match life_chars.next()? {
        'p' => "persistent",
        'w' => "working",
        't' => "transient",
        _ => return None,
    };
    let salience = decode_base36(life_chars.as_str())?;
    if salience > u8::MAX as u64 {
        return None;
    }
    let id = decode_compact_atom(id_raw)?;
    let owner = decode_compact_atom(owner_raw)?;
    let revision = decode_base36(revision_raw)?;
    let (kind, attributes) = decode_compact_kind(kind_raw)?;
    let mut facts = vec![
        node_fact(&id, NodeFactField::Id, &id),
        node_fact(&id, NodeFactField::Revision, &revision.to_string()),
        node_fact(&id, NodeFactField::Owner, &owner),
        node_fact(&id, NodeFactField::Lifetime, lifetime),
        node_fact(&id, NodeFactField::Salience, &salience.to_string()),
        node_fact(&id, NodeFactField::Kind, kind),
    ];
    facts.extend(
        attributes
            .into_iter()
            .map(|(field, value)| node_fact(&id, field, &value)),
    );
    facts.push(node_fact(&id, NodeFactField::Payload, payload));
    valid_node_fact_record(&facts).then_some(facts)
}

fn node_fact(node_id: &str, field: NodeFactField, value: &str) -> AtomicFact {
    AtomicFact::Node {
        node_id: node_id.into(),
        field,
        value: value.into(),
    }
}

fn decode_compact_kind(raw: &str) -> Option<(&'static str, Vec<(NodeFactField, String)>)> {
    let parts = split_outside_quotes(raw, '/')?;
    let head = parts.first()?.as_str();
    let result = match head {
        "ep" if parts.len() == 1 => ("entity", vec![(NodeFactField::EntityKind, "person".into())]),
        "e" if parts.len() == 2 => (
            "entity",
            vec![(NodeFactField::EntityKind, decode_compact_atom(&parts[1])?)],
        ),
        value if value.len() == 2 && value.starts_with('c') && parts.len() == 1 => (
            "claim",
            vec![(
                NodeFactField::ClaimConfidence,
                decode_confidence(value.chars().nth(1)?)?.into(),
            )],
        ),
        "v" if parts.len() == 4 => (
            "evidence",
            vec![
                (NodeFactField::EvidenceKind, decode_compact_atom(&parts[1])?),
                (
                    NodeFactField::EvidenceSourceType,
                    match parts[2].as_str() {
                        "s" => "symbol",
                        "q" => "string",
                        _ => return None,
                    }
                    .into(),
                ),
                (
                    NodeFactField::EvidenceSourceValue,
                    decode_compact_atom(&parts[3])?,
                ),
            ],
        ),
        value if value.len() == 2 && value.starts_with('g') && parts.len() == 1 => (
            "goal",
            vec![(
                NodeFactField::GoalStatus,
                match value.chars().nth(1)? {
                    'n' => "considering",
                    'a' => "active",
                    'c' => "committed",
                    'd' => "done",
                    'x' => "dropped",
                    _ => return None,
                }
                .into(),
            )],
        ),
        value if value.len() == 2 && value.starts_with('k') && parts.len() == 1 => (
            "constraint",
            vec![(
                NodeFactField::ConstraintStrength,
                match value.chars().nth(1)? {
                    's' => "soft",
                    'h' => "hard",
                    _ => return None,
                }
                .into(),
            )],
        ),
        value if value.len() == 2 && value.starts_with('q') && parts.len() == 1 => (
            "question",
            vec![(
                NodeFactField::QuestionStatus,
                match value.chars().nth(1)? {
                    'o' => "open",
                    'a' => "answered",
                    'd' => "deferred",
                    _ => return None,
                }
                .into(),
            )],
        ),
        "r" if parts.len() == 3 => (
            "relationship",
            vec![
                (
                    NodeFactField::RelationshipTarget,
                    decode_compact_atom(&parts[1])?,
                ),
                (
                    NodeFactField::RelationshipStatus,
                    decode_compact_atom(&parts[2])?,
                ),
            ],
        ),
        value if value.len() == 2 && value.starts_with('d') && parts.len() == 1 => (
            "desire",
            vec![(
                NodeFactField::DesireForce,
                match value.chars().nth(1)? {
                    'w' => "weak",
                    'm' => "moderate",
                    's' => "strong",
                    _ => return None,
                }
                .into(),
            )],
        ),
        value if value.len() == 2 && value.starts_with('a') && parts.len() == 1 => (
            "attention",
            vec![(
                NodeFactField::AttentionMode,
                match value.chars().nth(1)? {
                    'f' => "foreground",
                    'b' => "background",
                    _ => return None,
                }
                .into(),
            )],
        ),
        "s" if parts.len() == 1 => ("association", vec![]),
        value if value.len() == 2 && value.starts_with('t') && parts.len() == 1 => (
            "tension",
            vec![(
                NodeFactField::TensionStatus,
                match value.chars().nth(1)? {
                    'o' => "open",
                    'r' => "resolved",
                    _ => return None,
                }
                .into(),
            )],
        ),
        value if value.len() == 3 && value.starts_with('p') && parts.len() == 1 => (
            "prediction",
            vec![
                (
                    NodeFactField::PredictionConfidence,
                    decode_confidence(value.chars().nth(1)?)?.into(),
                ),
                (
                    NodeFactField::PredictionStatus,
                    match value.chars().nth(2)? {
                        'p' => "pending",
                        'm' => "matched",
                        's' => "surprised",
                        _ => return None,
                    }
                    .into(),
                ),
            ],
        ),
        "w" if parts.len() == 3 && parts[2].len() == 1 => (
            "perspective",
            vec![
                (
                    NodeFactField::PerspectiveTarget,
                    decode_compact_atom(&parts[1])?,
                ),
                (
                    NodeFactField::PerspectiveConfidence,
                    decode_confidence(parts[2].chars().next()?)?.into(),
                ),
            ],
        ),
        "f" if parts.len() == 4 => (
            "affect",
            vec![
                (NodeFactField::AffectTarget, decode_compact_atom(&parts[1])?),
                (NodeFactField::AffectKind, decode_compact_atom(&parts[2])?),
                (
                    NodeFactField::AffectIntensity,
                    match parts[3].as_str() {
                        "w" => "weak",
                        "m" => "moderate",
                        "s" => "strong",
                        _ => return None,
                    }
                    .into(),
                ),
            ],
        ),
        value if value.len() == 2 && value.starts_with('m') && parts.len() == 1 => (
            "metacognition",
            vec![(
                NodeFactField::MetacognitionStatus,
                match value.chars().nth(1)? {
                    'u' => "uncertain",
                    'm' => "monitoring",
                    'r' => "resolved",
                    _ => return None,
                }
                .into(),
            )],
        ),
        _ => return None,
    };
    Some(result)
}

fn decode_confidence(code: char) -> Option<&'static str> {
    Some(match code {
        't' => "tentative",
        'p' => "plausible",
        'l' => "likely",
        's' => "settled",
        _ => return None,
    })
}

fn decode_compact_relation(inner: &str) -> Option<Vec<AtomicFact>> {
    let (kind_raw, endpoints) = split_once_outside_quotes(inner.strip_prefix('R')?, ':')?;
    let (source_raw, target_raw) = split_once_outside_quotes(endpoints, '>')?;
    let kind = match kind_raw {
        "u" => "supports",
        "x" => "contradicts",
        "r" => "refines",
        "c" => "causes",
        "s" => "supersedes",
        "a" => "about",
        "b" => "blocks",
        "o" => "associates",
        _ => return None,
    };
    Some(vec![AtomicFact::Relation {
        kind: kind.into(),
        source: decode_compact_atom(source_raw)?,
        target: decode_compact_atom(target_raw)?,
    }])
}

fn decode_compact_focus(inner: &str) -> Option<Vec<AtomicFact>> {
    let (stream_raw, rest) = split_once_outside_quotes(inner.strip_prefix('!')?, '~')?;
    let (sequence_raw, rest) = split_once_outside_quotes(rest, '@')?;
    let (owner_raw, rest) = split_once_outside_quotes(rest, ':')?;
    let (transition_raw, buckets_trigger) = split_once_outside_quotes(rest, '/')?;
    let (buckets, trigger) = split_once_outside_quotes(buckets_trigger, '#')?;
    let parts = split_outside_quotes(buckets, '/')?;
    if parts.len() != 3 {
        return None;
    }
    let residue_raw = &parts[2];
    let stream_id = decode_compact_atom(stream_raw)?;
    let sequence = decode_base36(sequence_raw)?;
    let owner = decode_compact_atom(owner_raw)?;
    let transition = match transition_raw {
        "e" => "evoked",
        "a" => "associated",
        "i" => "interrupted",
        "r" => "returned",
        "f" => "reframed",
        "s" => "resolved",
        _ => return None,
    };
    if parse_document(trigger).ok()?.forms.len() != 1 {
        return None;
    }
    let mut facts = vec![
        AtomicFact::Focus {
            stream_id: stream_id.clone(),
            sequence,
            field: FocusFactField::StreamId,
            value: stream_id.clone(),
        },
        AtomicFact::Focus {
            stream_id: stream_id.clone(),
            sequence,
            field: FocusFactField::Sequence,
            value: sequence.to_string(),
        },
        AtomicFact::Focus {
            stream_id: stream_id.clone(),
            sequence,
            field: FocusFactField::Owner,
            value: owner,
        },
        AtomicFact::Focus {
            stream_id: stream_id.clone(),
            sequence,
            field: FocusFactField::Transition,
            value: transition.into(),
        },
    ];
    for (bucket, raw) in [
        (FocusBucket::Foreground, parts[0].as_str()),
        (FocusBucket::Background, parts[1].as_str()),
        (FocusBucket::Residue, residue_raw),
    ] {
        let members = decode_compact_bucket(raw, bucket)?;
        if members.is_empty() {
            facts.push(AtomicFact::FocusBucketEmpty {
                stream_id: stream_id.clone(),
                sequence,
                bucket,
            });
        } else {
            facts.extend(
                members
                    .into_iter()
                    .map(|node_id| AtomicFact::FocusBucketMember {
                        stream_id: stream_id.clone(),
                        sequence,
                        bucket,
                        node_id,
                    }),
            );
        }
    }
    facts.push(AtomicFact::Focus {
        stream_id,
        sequence,
        field: FocusFactField::Trigger,
        value: trigger.into(),
    });
    Some(facts)
}

fn decode_compact_bucket(raw: &str, bucket: FocusBucket) -> Option<Vec<String>> {
    let prefix = match bucket {
        FocusBucket::Foreground => "f[",
        FocusBucket::Background => "b[",
        FocusBucket::Residue => "r[",
    };
    let values = raw.strip_prefix(prefix)?.strip_suffix(']')?;
    if values.is_empty() {
        return Some(Vec::new());
    }
    split_outside_quotes(values, ',')?
        .into_iter()
        .map(|value| decode_compact_atom(&value))
        .collect()
}

fn decode_compact_atom(raw: &str) -> Option<String> {
    let document = parse_document(raw).ok()?;
    let form = document
        .forms
        .first()
        .filter(|_| document.forms.len() == 1)?;
    match &form.kind {
        chappi_syntax::FormKind::Atom(chappi_syntax::Atom::Symbol(value))
        | chappi_syntax::FormKind::Atom(chappi_syntax::Atom::Integer(value))
        | chappi_syntax::FormKind::Atom(chappi_syntax::Atom::Decimal(value))
        | chappi_syntax::FormKind::Atom(chappi_syntax::Atom::String(value)) => Some(value.clone()),
        chappi_syntax::FormKind::List(_) => None,
    }
}

fn decode_base36(raw: &str) -> Option<u64> {
    if raw.is_empty()
        || (raw.len() > 1 && raw.starts_with('0'))
        || !raw
            .chars()
            .all(|character| character.is_ascii_digit() || character.is_ascii_lowercase())
    {
        return None;
    }
    u64::from_str_radix(raw, 36).ok()
}

fn split_once_outside_quotes(input: &str, delimiter: char) -> Option<(&str, &str)> {
    let mut quoted = false;
    let mut escaped = false;
    for (index, character) in input.char_indices() {
        if quoted {
            if escaped {
                escaped = false;
            } else if character == '\\' {
                escaped = true;
            } else if character == '"' {
                quoted = false;
            }
        } else if character == '"' {
            quoted = true;
        } else if character == delimiter {
            let after = index + character.len_utf8();
            return Some((&input[..index], &input[after..]));
        }
    }
    None
}

fn split_outside_quotes(input: &str, delimiter: char) -> Option<Vec<String>> {
    let mut output = Vec::new();
    let mut rest = input;
    while let Some((left, right)) = split_once_outside_quotes(rest, delimiter) {
        output.push(left.to_string());
        rest = right;
    }
    output.push(rest.to_string());
    Some(output)
}

pub fn load_corpus(root: &Path) -> Result<Corpus, CorpusError> {
    let manifest_path = root.join("manifest.json");
    let manifest_bytes = read(&manifest_path)?;
    let manifest: Manifest = decode_json(&manifest_bytes, "manifest.json")?;
    let mut diagnostics = Vec::new();
    if manifest.schema != CORPUS_SCHEMA {
        push(
            &mut diagnostics,
            "invalid-corpus-schema",
            "manifest.json.schema",
            format!("corpus schema must be `{CORPUS_SCHEMA}`"),
        );
    }
    if manifest.version != VERSION {
        push(
            &mut diagnostics,
            "invalid-corpus-version",
            "manifest.json.version",
            "corpus version must be 1",
        );
    }
    if manifest.episodes.len() != EPISODE_COUNT {
        push(
            &mut diagnostics,
            "invalid-episode-count",
            "manifest.json.episodes",
            "corpus must contain exactly 24 episodes",
        );
    }
    let mut manifest_paths = BTreeSet::new();
    for (index, path) in manifest.episodes.iter().enumerate() {
        if !manifest_paths.insert(path.as_str()) {
            push(
                &mut diagnostics,
                "duplicate-episode-path",
                format!("manifest.json.episodes.{index}"),
                "episode paths must be unique",
            );
        }
    }
    for pair in manifest.episodes.windows(2) {
        if pair[0] > pair[1] {
            push(
                &mut diagnostics,
                "unsorted-episode-paths",
                "manifest.json.episodes",
                "episode paths must be sorted",
            );
            break;
        }
    }
    for (index, path) in manifest.episodes.iter().enumerate() {
        if !safe_episode_path(path) {
            push(
                &mut diagnostics,
                "unsafe-episode-path",
                format!("manifest.json.episodes.{index}"),
                "episode path must be a relative episodes/<id>.json path",
            );
        }
    }
    if !diagnostics.is_empty() {
        return Err(CorpusError {
            kind: CorpusErrorKind::Invalid,
            diagnostics,
        });
    }
    let episode_directory = root.join("episodes");
    let entries = fs::read_dir(&episode_directory).map_err(|error| CorpusError {
        kind: CorpusErrorKind::Io,
        diagnostics: vec![CorpusDiagnostic {
            code: "corpus-io-error".into(),
            path: episode_directory.display().to_string(),
            message: error.to_string(),
        }],
    })?;
    let mut disk_paths = Vec::new();
    for entry in entries {
        let entry = entry.map_err(|error| CorpusError {
            kind: CorpusErrorKind::Io,
            diagnostics: vec![CorpusDiagnostic {
                code: "corpus-io-error".into(),
                path: episode_directory.display().to_string(),
                message: error.to_string(),
            }],
        })?;
        if entry.path().extension().and_then(|value| value.to_str()) == Some("json") {
            disk_paths.push(format!("episodes/{}", entry.file_name().to_string_lossy()));
        }
    }
    disk_paths.sort();
    for path in disk_paths {
        if !manifest_paths.contains(path.as_str()) {
            push(
                &mut diagnostics,
                "unlisted-episode-file",
                path,
                "every episode JSON file must be listed by the manifest",
            );
        }
    }
    let canonical_root = fs::canonicalize(root).map_err(|error| CorpusError {
        kind: CorpusErrorKind::Io,
        diagnostics: vec![CorpusDiagnostic {
            code: "corpus-io-error".into(),
            path: root.display().to_string(),
            message: error.to_string(),
        }],
    })?;

    let mut episodes = Vec::new();
    let mut ids = BTreeSet::new();
    for relative in &manifest.episodes {
        let path = root.join(relative);
        let resolved = fs::canonicalize(&path).map_err(|error| CorpusError {
            kind: CorpusErrorKind::Io,
            diagnostics: vec![CorpusDiagnostic {
                code: "corpus-io-error".into(),
                path: path.display().to_string(),
                message: error.to_string(),
            }],
        })?;
        if !resolved.starts_with(&canonical_root) {
            push(
                &mut diagnostics,
                "unsafe-episode-path",
                relative,
                "resolved episode path must remain below the corpus root",
            );
            continue;
        }
        let bytes = read(&resolved)?;
        let episode: Episode = match decode_json(&bytes, relative) {
            Ok(episode) => episode,
            Err(error) => {
                diagnostics.extend(error.diagnostics);
                continue;
            }
        };
        validate_episode_metadata(&episode, relative, &mut diagnostics);
        let expected_id = Path::new(relative)
            .file_stem()
            .and_then(|value| value.to_str())
            .unwrap_or_default();
        if episode.id != expected_id {
            push(
                &mut diagnostics,
                "mismatched-episode-id",
                format!("{relative}.id"),
                format!("episode ID must match `{expected_id}`"),
            );
        }
        if !ids.insert(episode.id.clone()) {
            push(
                &mut diagnostics,
                "duplicate-episode-id",
                format!("{relative}.id"),
                "episode IDs must be unique",
            );
        }
        validate_episode_content(&episode, relative, &mut diagnostics);
        episodes.push(episode);
    }

    let mut counts = BTreeMap::new();
    for episode in &episodes {
        *counts.entry(episode.category).or_insert(0usize) += 1;
    }
    for category in EpisodeCategory::ALL {
        if counts.get(&category).copied().unwrap_or_default() != 4 {
            push(
                &mut diagnostics,
                "invalid-category-count",
                "manifest.json.episodes",
                format!("category {category:?} must contain exactly four episodes"),
            );
        }
    }
    validate_corpus_diversity(&episodes, &mut diagnostics);
    if diagnostics.is_empty() {
        Ok(Corpus {
            schema: manifest.schema,
            version: manifest.version,
            episodes,
        })
    } else {
        Err(CorpusError {
            kind: CorpusErrorKind::Invalid,
            diagnostics,
        })
    }
}

pub fn measure_corpus(corpus: &Corpus) -> MeasurementReport {
    let mut replayed = 0usize;
    let mut mandatory_hit = 0usize;
    let mut mandatory_total = 0usize;
    let mut active_hit = 0usize;
    let mut active_total = 0usize;
    let mut owner_target_hit = 0usize;
    let mut owner_target_total = 0usize;
    let mut diagnostic_hit = 0usize;
    let mut diagnostic_total = 0usize;
    let mut fact_counts = FactFamily::ALL
        .into_iter()
        .map(|family| (family, (0usize, 0usize, 0usize)))
        .collect::<BTreeMap<_, _>>();
    let mut snapshot_bytes = 0usize;
    let mut json_bytes = 0usize;
    let mut capsule_bytes = 0usize;
    let mut baseline_bytes = 0usize;
    let mut snapshot_tokens = 0usize;
    let mut json_tokens = 0usize;
    let mut capsule_tokens = 0usize;
    let mut baseline_tokens = 0usize;
    let estimator = LexicalTokenEstimator;

    for episode in &corpus.episodes {
        let Ok(state) = replay_episode(episode) else {
            continue;
        };
        replayed += 1;
        let fixed_selection = derive_required_selection(episode, &state)
            .into_iter()
            .collect::<Vec<_>>();
        let Ok(derived) = derive_semantic_comparison(&state, &fixed_selection) else {
            continue;
        };
        let compact_prefix = utf8_prefix(&derived.canonical_capsule, episode.capsule_budget_bytes);
        let baseline_prefix = utf8_prefix(
            &episode.natural_language_baseline,
            episode.capsule_budget_bytes,
        );
        let compact_facts = decode_compact_semantic_facts(compact_prefix);
        let baseline_facts = decode_controlled_english_facts(baseline_prefix);
        let selected = compact_facts
            .iter()
            .filter_map(|fact| match fact {
                AtomicFact::Node {
                    node_id,
                    field: NodeFactField::Id,
                    value,
                } if node_id == value => Some(node_id.as_str()),
                _ => None,
            })
            .collect::<BTreeSet<_>>();
        for id in episode
            .golden
            .required_constraint_ids
            .iter()
            .chain(&episode.golden.committed_goal_ids)
        {
            mandatory_total += 1;
            mandatory_hit += usize::from(selected.contains(id.as_str()));
        }
        for id in episode
            .golden
            .active_question_ids
            .iter()
            .chain(&episode.golden.active_tension_ids)
        {
            active_total += 1;
            active_hit += usize::from(selected.contains(id.as_str()));
        }
        let recovery_ids = episode
            .golden
            .required_constraint_ids
            .iter()
            .chain(&episode.golden.committed_goal_ids)
            .chain(&episode.golden.active_question_ids)
            .chain(&episode.golden.active_tension_ids)
            .collect::<BTreeSet<_>>();
        for expectation in &episode.golden.owner_target_expectations {
            if !recovery_ids.contains(&expectation.node_id) && expectation.target.is_none() {
                continue;
            }
            owner_target_total += 1;
            owner_target_hit += usize::from(
                derived
                    .fixture
                    .selected_node_ids
                    .iter()
                    .filter(|id| selected.contains(id.as_str()))
                    .filter_map(|id| state.nodes.iter().find(|node| node.id == *id))
                    .any(|node| expectation_matches(node, expectation)),
            );
        }
        let score = score_fact_recovery(
            &episode.comparison.atomic_facts,
            &compact_facts,
            &baseline_facts,
        );
        for family in score.families {
            let counts = fact_counts
                .get_mut(&family.family)
                .expect("all families initialized");
            counts.0 += family.total;
            counts.1 += family.chappi_hits;
            counts.2 += family.baseline_hits;
        }
        for seeded in &episode.seeded_invalid_sources {
            let actual = diagnostic_codes(&seeded.source);
            let expected = seeded
                .expected_diagnostic_codes
                .iter()
                .cloned()
                .collect::<BTreeSet<_>>();
            diagnostic_total += 1;
            diagnostic_hit += usize::from(actual == expected);
        }
        let canonical_snapshot = snapshot_source(&state);
        let compact_json = project(&state, ProjectionFormat::Json);
        let json: serde_json::Value =
            serde_json::from_str(&compact_json).expect("projection is JSON");
        let pretty_json = serde_json::to_string_pretty(&json).expect("projection serializes");
        let canonical_capsule = derived.canonical_capsule;
        snapshot_bytes += canonical_snapshot.len();
        json_bytes += pretty_json.len();
        capsule_bytes += canonical_capsule.len();
        baseline_bytes += episode.natural_language_baseline.len();
        snapshot_tokens += estimator
            .estimate(std::str::from_utf8(&canonical_snapshot).expect("snapshot is UTF-8"));
        json_tokens += estimator.estimate(&pretty_json);
        capsule_tokens += estimator.estimate(&canonical_capsule);
        baseline_tokens += estimator.estimate(&episode.natural_language_baseline);
    }

    let semantic = SemanticMetrics {
        loaded: corpus.episodes.len(),
        replayed,
        replay_ratio: ratio(replayed.min(EPISODE_COUNT), EPISODE_COUNT),
    };
    let continuity = ContinuityMetrics {
        constraints_and_committed_goals: ratio(mandatory_hit, mandatory_total),
        active_questions_and_tensions: ratio(active_hit, active_total),
        owner_target_expectations: ratio(owner_target_hit, owner_target_total),
    };
    let fact_families = FactFamily::ALL
        .into_iter()
        .map(|family| {
            let (total, chappi_hits, baseline_hits) = fact_counts[&family];
            FactFamilyMetrics {
                family,
                total,
                chappi_hits,
                baseline_hits,
                chappi_accuracy: (total > 0).then(|| ratio(chappi_hits, total)),
                baseline_accuracy: (total > 0).then(|| ratio(baseline_hits, total)),
                included_in_macro: total > 0,
            }
        })
        .collect::<Vec<_>>();
    let included = fact_families
        .iter()
        .filter(|family| family.included_in_macro)
        .count();
    let chappi_accuracy = if included == 0 {
        0.0
    } else {
        fact_families
            .iter()
            .filter_map(|family| family.chappi_accuracy)
            .sum::<f64>()
            / included as f64
    };
    let baseline_accuracy = if included == 0 {
        0.0
    } else {
        fact_families
            .iter()
            .filter_map(|family| family.baseline_accuracy)
            .sum::<f64>()
            / included as f64
    };
    let precision = PrecisionMetrics {
        seeded_diagnostics: ratio(diagnostic_hit, diagnostic_total),
        chappi_task_accuracy: chappi_accuracy,
        baseline_task_accuracy: baseline_accuracy,
        task_accuracy_advantage: chappi_accuracy - baseline_accuracy,
        fact_families,
    };
    let compression = CompressionMetrics {
        snapshot_to_json: ratio_usize(snapshot_bytes, json_bytes),
        capsule_to_baseline: ratio_usize(capsule_bytes, baseline_bytes),
        canonical_snapshot_bytes: snapshot_bytes,
        pretty_json_bytes: json_bytes,
        capsule_bytes,
        baseline_bytes,
        canonical_snapshot_tokens: snapshot_tokens,
        pretty_json_tokens: json_tokens,
        capsule_tokens,
        baseline_tokens,
    };
    let gates = vec![
        GateResult {
            name: "semantic-replay",
            passed: semantic.loaded == EPISODE_COUNT && semantic.replayed == EPISODE_COUNT,
            actual: semantic.replay_ratio,
            required: 1.0,
        },
        at_least(
            "constraints-and-committed-goals",
            continuity.constraints_and_committed_goals,
            1.0,
        ),
        at_least(
            "active-questions-and-tensions",
            continuity.active_questions_and_tensions,
            0.90,
        ),
        at_least(
            "owner-target-expectations",
            continuity.owner_target_expectations,
            1.0,
        ),
        at_least("seeded-diagnostics", precision.seeded_diagnostics, 1.0),
        at_least(
            "task-accuracy-advantage",
            precision.task_accuracy_advantage,
            0.10,
        ),
        at_most(
            "snapshot-to-json-compression",
            compression.snapshot_to_json,
            0.70,
        ),
        at_most(
            "capsule-to-baseline-compression",
            compression.capsule_to_baseline,
            0.60,
        ),
    ];
    let passed = gates.iter().all(|gate| gate.passed);
    MeasurementReport {
        schema: MEASUREMENT_SCHEMA,
        version: VERSION,
        episodes: corpus.episodes.len(),
        semantic,
        continuity,
        precision,
        compression,
        gates,
        passed,
    }
}

fn validate_episode_metadata(
    episode: &Episode,
    path: &str,
    diagnostics: &mut Vec<CorpusDiagnostic>,
) {
    if episode.schema != EPISODE_SCHEMA {
        push(
            diagnostics,
            "invalid-episode-schema",
            format!("{path}.schema"),
            format!("episode schema must be `{EPISODE_SCHEMA}`"),
        );
    }
    if episode.version != VERSION {
        push(
            diagnostics,
            "invalid-episode-version",
            format!("{path}.version"),
            "episode version must be 1",
        );
    }
}

fn validate_episode_content(
    episode: &Episode,
    path: &str,
    diagnostics: &mut Vec<CorpusDiagnostic>,
) {
    check_nonempty_unique(
        &episode.golden.required_constraint_ids,
        &format!("{path}.golden.required_constraint_ids"),
        "required_constraint_ids",
        diagnostics,
    );
    check_nonempty_unique(
        &episode.golden.committed_goal_ids,
        &format!("{path}.golden.committed_goal_ids"),
        "committed_goal_ids",
        diagnostics,
    );
    check_nonempty_unique(
        &episode.golden.active_question_ids,
        &format!("{path}.golden.active_question_ids"),
        "active_question_ids",
        diagnostics,
    );
    check_nonempty_unique(
        &episode.golden.active_tension_ids,
        &format!("{path}.golden.active_tension_ids"),
        "active_tension_ids",
        diagnostics,
    );
    if episode.golden.owner_target_expectations.is_empty() {
        push(
            diagnostics,
            "empty-golden-list",
            format!("{path}.golden.owner_target_expectations"),
            "owner/target expectations must not be empty",
        );
    }
    let mut expectation_ids = BTreeSet::new();
    for expectation in &episode.golden.owner_target_expectations {
        if !expectation_ids.insert(expectation.node_id.as_str()) {
            push(
                diagnostics,
                "duplicate-golden-id",
                format!("{path}.golden.owner_target_expectations"),
                format!("duplicate expectation for `{}`", expectation.node_id),
            );
        }
    }
    let state = match replay_episode(episode) {
        Ok(state) => state,
        Err(items) => {
            for item in items {
                push(
                    diagnostics,
                    &item.code,
                    format!("{path}.sources"),
                    item.message,
                );
            }
            return;
        }
    };
    check_golden_ids(episode, &state, path, diagnostics);
    validate_recovery_questions(episode, &state, path, diagnostics);
    validate_comparison_fixture(episode, &state, path, diagnostics);
    if episode.seeded_invalid_sources.is_empty() {
        push(
            diagnostics,
            "missing-seeded-invalid-source",
            format!("{path}.seeded_invalid_sources"),
            "episode must seed at least one invalid source",
        );
    }
    for (index, seeded) in episode.seeded_invalid_sources.iter().enumerate() {
        check_nonempty_unique(
            &seeded.expected_diagnostic_codes,
            &format!("{path}.seeded_invalid_sources.{index}.expected_diagnostic_codes"),
            "expected_diagnostic_codes",
            diagnostics,
        );
        if seeded.source.trim().is_empty() {
            push(
                diagnostics,
                "empty-seeded-invalid-source",
                format!("{path}.seeded_invalid_sources.{index}.source"),
                "seeded source must not be empty",
            );
        }
        for code in &seeded.expected_diagnostic_codes {
            if !KNOWN_DIAGNOSTIC_CODES.contains(&code.as_str()) {
                push(
                    diagnostics,
                    "unknown-expected-diagnostic-code",
                    format!("{path}.seeded_invalid_sources.{index}.expected_diagnostic_codes"),
                    format!("`{code}` is not in the v1 diagnostic vocabulary"),
                );
            }
        }
    }
}

fn validate_comparison_fixture(
    episode: &Episode,
    state: &CognitiveState,
    path: &str,
    diagnostics: &mut Vec<CorpusDiagnostic>,
) {
    if episode.comparison.fact_ontology_version != FACT_ONTOLOGY_VERSION {
        push(
            diagnostics,
            "invalid-fact-ontology-version",
            format!("{path}.comparison.fact_ontology_version"),
            "fact ontology version must be 1",
        );
    }
    if episode.comparison.controlled_english_version != CONTROLLED_ENGLISH_VERSION {
        push(
            diagnostics,
            "invalid-controlled-english-version",
            format!("{path}.comparison.controlled_english_version"),
            "controlled-English grammar version must be 1",
        );
    }
    let selected = derive_required_selection(episode, state);
    let selected_ids = selected.into_iter().collect::<Vec<_>>();
    if episode.comparison.selected_node_ids != selected_ids {
        push(
            diagnostics,
            "selected-node-set-mismatch",
            format!("{path}.comparison.selected_node_ids"),
            format!(
                "declared {:?}, derived {selected_ids:?}",
                episode.comparison.selected_node_ids
            ),
        );
    }
    let Ok(derived) = derive_semantic_comparison(state, &selected_ids) else {
        push(
            diagnostics,
            "invalid-semantic-comparison",
            format!("{path}.comparison"),
            "derived selected node set is not closed",
        );
        return;
    };
    if episode.comparison.atomic_facts != derived.fixture.atomic_facts {
        push(
            diagnostics,
            "atomic-facts-mismatch",
            format!("{path}.comparison.atomic_facts"),
            "literal facts must exactly match the typed target capsule",
        );
    }
    if episode.capsule_budget_bytes != derived.capsule_budget_bytes {
        push(
            diagnostics,
            "capsule-budget-mismatch",
            format!("{path}.capsule_budget_bytes"),
            format!(
                "budget {} must equal exact canonical size {}",
                episode.capsule_budget_bytes, derived.capsule_budget_bytes
            ),
        );
    }
    let rendered = render_controlled_english(&episode.comparison.atomic_facts);
    match rendered {
        Ok(rendered) if rendered == episode.natural_language_baseline => {}
        Ok(_) => push(
            diagnostics,
            "controlled-english-baseline-mismatch",
            format!("{path}.natural_language_baseline"),
            "stored baseline must equal the v1 renderer byte-for-byte",
        ),
        Err(message) => push(
            diagnostics,
            "invalid-atomic-fact-records",
            format!("{path}.comparison.atomic_facts"),
            message,
        ),
    }
    if decode_compact_semantic_facts(&derived.canonical_capsule) != episode.comparison.atomic_facts
    {
        push(
            diagnostics,
            "compact-fact-decode-mismatch",
            format!("{path}.comparison.atomic_facts"),
            "independent compact decoding must reproduce the literal facts",
        );
    }
    if decode_controlled_english_facts(&episode.natural_language_baseline)
        != episode.comparison.atomic_facts
    {
        push(
            diagnostics,
            "controlled-english-fact-decode-mismatch",
            format!("{path}.natural_language_baseline"),
            "independent controlled-English decoding must reproduce the literal facts",
        );
    }
}

fn derive_required_selection(episode: &Episode, state: &CognitiveState) -> BTreeSet<String> {
    let mut selected = episode
        .golden
        .required_constraint_ids
        .iter()
        .chain(&episode.golden.committed_goal_ids)
        .chain(&episode.golden.active_question_ids)
        .chain(&episode.golden.active_tension_ids)
        .cloned()
        .collect::<BTreeSet<_>>();
    selected.extend(
        episode
            .golden
            .owner_target_expectations
            .iter()
            .map(|expectation| expectation.node_id.clone()),
    );
    loop {
        let before = selected.len();
        let dependencies = state
            .nodes
            .iter()
            .filter(|node| selected.contains(&node.id))
            .flat_map(|node| {
                std::iter::once(node.owner.clone()).chain(node_target(node).map(ToOwned::to_owned))
            })
            .collect::<Vec<_>>();
        selected.extend(dependencies);
        if selected.len() == before {
            return selected;
        }
    }
}

fn replay_episode(episode: &Episode) -> Result<CognitiveState, Vec<Diagnostic>> {
    let mut state = decode_state(&episode.snapshot_source)?;
    if episode.patch_sources.is_empty() {
        return Err(vec![synthetic_diagnostic(
            "missing-patch-source",
            "episode must apply at least one patch",
        )]);
    }
    for source in &episode.patch_sources {
        let document = parse_document(source)?;
        let patch = Patch::try_from(document)?;
        if patch.ops.is_empty() {
            return Err(vec![synthetic_diagnostic(
                "empty-patch-operations",
                "every corpus patch must contain at least one operation",
            )]);
        }
        let patch_diagnostics = validate_patch(&patch);
        if !patch_diagnostics.is_empty() {
            return Err(patch_diagnostics);
        }
        state = apply_patch(&state, &patch)?;
    }
    Ok(state)
}

fn decode_state(source: &str) -> Result<CognitiveState, Vec<Diagnostic>> {
    let document = parse_document(source)?;
    let state = CognitiveState::try_from(document)?;
    let diagnostics = validate(&state);
    if diagnostics.is_empty() {
        Ok(state)
    } else {
        Err(diagnostics)
    }
}

fn diagnostic_codes(source: &str) -> BTreeSet<String> {
    let diagnostics = match parse_document(source) {
        Err(items) => items,
        Ok(document) => match CognitiveState::try_from(document) {
            Err(items) => items,
            Ok(state) => validate(&state),
        },
    };
    diagnostics.into_iter().map(|item| item.code).collect()
}

fn check_golden_ids(
    episode: &Episode,
    state: &CognitiveState,
    path: &str,
    diagnostics: &mut Vec<CorpusDiagnostic>,
) {
    for id in &episode.golden.required_constraint_ids {
        check_id(
            state,
            id,
            path,
            diagnostics,
            "required_constraint_ids",
            |kind| {
                matches!(
                    kind,
                    NodeKind::Constraint {
                        strength: ConstraintStrength::Hard
                    }
                )
            },
        );
    }
    for id in &episode.golden.committed_goal_ids {
        check_id(state, id, path, diagnostics, "committed_goal_ids", |kind| {
            matches!(
                kind,
                NodeKind::Goal {
                    status: GoalStatus::Committed
                }
            )
        });
    }
    for id in &episode.golden.active_question_ids {
        check_id(
            state,
            id,
            path,
            diagnostics,
            "active_question_ids",
            |kind| {
                matches!(
                    kind,
                    NodeKind::Question {
                        status: QuestionStatus::Open
                    }
                )
            },
        );
    }
    for id in &episode.golden.active_tension_ids {
        check_id(state, id, path, diagnostics, "active_tension_ids", |kind| {
            matches!(
                kind,
                NodeKind::Tension {
                    status: TensionStatus::Open
                }
            )
        });
    }
    for expectation in &episode.golden.owner_target_expectations {
        if !state
            .nodes
            .iter()
            .any(|node| expectation_matches(node, expectation))
        {
            push(
                diagnostics,
                "invalid-owner-target-expectation",
                format!("{path}.golden.owner_target_expectations"),
                format!(
                    "expectation for `{}` does not match final state",
                    expectation.node_id
                ),
            );
        }
    }
    check_complete_golden_sets(episode, state, path, diagnostics);
}

fn check_complete_golden_sets(
    episode: &Episode,
    state: &CognitiveState,
    path: &str,
    diagnostics: &mut Vec<CorpusDiagnostic>,
) {
    let exact = [
        (
            "required_constraint_ids",
            episode
                .golden
                .required_constraint_ids
                .iter()
                .cloned()
                .collect::<BTreeSet<_>>(),
            state
                .nodes
                .iter()
                .filter(|node| {
                    matches!(
                        node.kind,
                        NodeKind::Constraint {
                            strength: ConstraintStrength::Hard
                        }
                    )
                })
                .map(|node| node.id.clone())
                .collect::<BTreeSet<_>>(),
        ),
        (
            "committed_goal_ids",
            episode
                .golden
                .committed_goal_ids
                .iter()
                .cloned()
                .collect::<BTreeSet<_>>(),
            state
                .nodes
                .iter()
                .filter(|node| {
                    matches!(
                        node.kind,
                        NodeKind::Goal {
                            status: GoalStatus::Committed
                        }
                    )
                })
                .map(|node| node.id.clone())
                .collect(),
        ),
        (
            "active_question_ids",
            episode
                .golden
                .active_question_ids
                .iter()
                .cloned()
                .collect::<BTreeSet<_>>(),
            state
                .nodes
                .iter()
                .filter(|node| {
                    matches!(
                        node.kind,
                        NodeKind::Question {
                            status: QuestionStatus::Open
                        }
                    )
                })
                .map(|node| node.id.clone())
                .collect(),
        ),
        (
            "active_tension_ids",
            episode
                .golden
                .active_tension_ids
                .iter()
                .cloned()
                .collect::<BTreeSet<_>>(),
            state
                .nodes
                .iter()
                .filter(|node| {
                    matches!(
                        node.kind,
                        NodeKind::Tension {
                            status: TensionStatus::Open
                        }
                    )
                })
                .map(|node| node.id.clone())
                .collect(),
        ),
    ];
    for (field, declared, derived) in exact {
        if declared != derived {
            push(
                diagnostics,
                "incomplete-golden-semantic-set",
                format!("{path}.golden.{field}"),
                format!("declared {declared:?}, derived {derived:?}"),
            );
        }
    }

    let declared = episode
        .golden
        .owner_target_expectations
        .iter()
        .map(|item| {
            (
                item.node_id.clone(),
                item.owner.clone(),
                item.target.clone(),
            )
        })
        .collect::<BTreeSet<_>>();
    let derived = state
        .nodes
        .iter()
        .map(|node| {
            (
                node.id.clone(),
                node.owner.clone(),
                node_target(node).map(ToOwned::to_owned),
            )
        })
        .collect::<BTreeSet<_>>();
    if declared != derived {
        push(
            diagnostics,
            "incomplete-owner-target-golden",
            format!("{path}.golden.owner_target_expectations"),
            format!("declared {declared:?}, derived {derived:?}"),
        );
    }
}

fn validate_recovery_questions(
    episode: &Episode,
    state: &CognitiveState,
    path: &str,
    diagnostics: &mut Vec<CorpusDiagnostic>,
) {
    let mut required = [
        (
            RecoveryFactKind::RequiredConstraint,
            &episode.golden.required_constraint_ids,
        ),
        (
            RecoveryFactKind::CommittedGoal,
            &episode.golden.committed_goal_ids,
        ),
        (
            RecoveryFactKind::ActiveQuestion,
            &episode.golden.active_question_ids,
        ),
        (
            RecoveryFactKind::ActiveTension,
            &episode.golden.active_tension_ids,
        ),
    ]
    .into_iter()
    .flat_map(|(kind, ids)| ids.iter().cloned().map(move |id| (kind, id)))
    .collect::<BTreeSet<_>>();
    let recovery_owners = required
        .iter()
        .filter_map(|(_, id)| state.nodes.iter().find(|node| node.id == *id))
        .map(|node| node.owner.clone())
        .collect::<BTreeSet<_>>();
    required.extend(
        recovery_owners
            .into_iter()
            .map(|owner| (RecoveryFactKind::OwnerContext, owner)),
    );
    let declared = episode
        .recovery_questions
        .iter()
        .map(|question| (question.fact_kind, question.covered_node_id.clone()))
        .collect::<BTreeSet<_>>();
    let unique_ids = episode
        .recovery_questions
        .iter()
        .map(|question| question.id.as_str())
        .collect::<BTreeSet<_>>();
    if required != declared
        || declared.len() != episode.recovery_questions.len()
        || unique_ids.len() != episode.recovery_questions.len()
    {
        push(
            diagnostics,
            "incomplete-recovery-question-set",
            format!("{path}.recovery_questions"),
            "questions must uniquely cover every required semantic fact exactly once",
        );
    }

    for (index, question) in episode.recovery_questions.iter().enumerate() {
        if question.prompt.trim().is_empty()
            || question.expected.value.trim().is_empty()
            || question.expected.qualifiers.is_empty()
            || question.expected.qualifiers.iter().any(|alternatives| {
                alternatives.is_empty() || alternatives.iter().any(|v| v.trim().is_empty())
            })
        {
            push(
                diagnostics,
                "invalid-semantic-question",
                format!("{path}.recovery_questions.{index}"),
                "question, answer value, and non-empty qualifier alternatives are required",
            );
            continue;
        }
        let Some(node) = state
            .nodes
            .iter()
            .find(|node| node.id == question.covered_node_id)
        else {
            continue;
        };
        let payload = crate::resume::canonical_form(&node.payload);
        if !exact_lexical_containment(&payload, &question.expected.value) {
            push(
                diagnostics,
                "semantic-value-not-payload-backed",
                format!("{path}.recovery_questions.{index}.expected.value"),
                "the canonical answer value must be a contiguous phrase in the covered node payload",
            );
        }
        if !semantic_answer_matches(&semantic_evidence_for_node(node, state), &question.expected) {
            push(
                diagnostics,
                "semantic-answer-mismatches-state",
                format!("{path}.recovery_questions.{index}.expected"),
                "expected answer is not recoverable from the covered node's semantic content",
            );
        }
    }
}

fn semantic_evidence_for_node(node: &Node, state: &CognitiveState) -> String {
    let mut parts = vec![match &node.kind {
        NodeKind::Entity { .. } => "entity owner owns responsible".to_string(),
        NodeKind::Constraint { strength } => format!(
            "constraint {}",
            match strength {
                ConstraintStrength::Hard => "hard required must",
                ConstraintStrength::Soft => "soft preferred",
            }
        ),
        NodeKind::Goal { status } => format!(
            "goal {}",
            match status {
                GoalStatus::Committed => "committed will",
                GoalStatus::Active => "active",
                GoalStatus::Considering => "considering",
                GoalStatus::Done => "done",
                GoalStatus::Dropped => "dropped",
            }
        ),
        NodeKind::Question { status } => format!(
            "question {}",
            match status {
                QuestionStatus::Open => "open unresolved",
                QuestionStatus::Answered => "answered",
                QuestionStatus::Deferred => "deferred",
            }
        ),
        NodeKind::Tension { status } => format!(
            "tension tradeoff {}",
            match status {
                TensionStatus::Open => "open unresolved",
                TensionStatus::Resolved => "resolved",
            }
        ),
        _ => "context".to_string(),
    }];
    parts.push(crate::resume::canonical_form(&node.payload));
    if let Some(owner) = state
        .nodes
        .iter()
        .find(|candidate| candidate.id == node.owner)
    {
        parts.push(crate::resume::canonical_form(&owner.payload));
    }
    if let Some(target) =
        node_target(node).and_then(|id| state.nodes.iter().find(|candidate| candidate.id == id))
    {
        parts.push(crate::resume::canonical_form(&target.payload));
    }
    parts.join(" ")
}

// This is a deliberately controlled evaluator, not general-purpose NLP. It recognizes complete
// value/alias phrases and qualifier groups inside one clause, then assigns value polarity from
// the parity of the finite predicate-negator vocabulary before that value. Sentence punctuation,
// commas, and coordinating/subordinate boundaries split clauses. `not only` is the one explicit
// non-negating construction. `without` binds to the following matched predicate across only the
// small adverb whitelist in `without_reaches`; a comma keeps an introductory phrase such as
// `Without hesitation, ...` from negating a later clause.
pub fn semantic_answer_matches(text: &str, answer: &SemanticAnswer) -> bool {
    semantic_clauses(text)
        .iter()
        .any(|clause| semantic_answer_matches_clause(clause, answer))
}

fn semantic_answer_matches_clause(text: &str, answer: &SemanticAnswer) -> bool {
    let tokens = normalized_tokens(text);
    if !answer.qualifiers.iter().all(|alternatives| {
        alternatives.iter().any(|value| {
            phrase_positions(&tokens, &normalized_tokens(value))
                .next()
                .is_some()
        })
    }) {
        return false;
    }
    let mut values = Vec::with_capacity(answer.aliases.len() + 1);
    values.push(answer.value.as_str());
    values.extend(answer.aliases.iter().map(String::as_str));
    values.into_iter().any(|value| {
        phrase_positions(&tokens, &normalized_tokens(value)).any(|position| {
            is_negated(&tokens, position) == (answer.polarity == AnswerPolarity::Negated)
        })
    })
}

fn normalized_tokens(text: &str) -> Vec<String> {
    lexical_tokens(text)
        .into_iter()
        .map(|token| semantic_stem(&token))
        .collect()
}

fn exact_lexical_containment(text: &str, value: &str) -> bool {
    let text = text.to_lowercase();
    let value = value.to_lowercase();
    if value.is_empty() {
        return false;
    }
    text.match_indices(&value).any(|(start, matched)| {
        let end = start + matched.len();
        let before = text[..start].chars().next_back();
        let after = text[end..].chars().next();
        before.is_none_or(|character| !character.is_alphanumeric())
            && after.is_none_or(|character| !character.is_alphanumeric())
    })
}

fn lexical_tokens(text: &str) -> Vec<String> {
    let mut expanded = text.to_lowercase().replace('’', "'");
    for (contraction, replacement) in [
        ("can't", "cannot"),
        ("won't", "will not"),
        ("wouldn't", "would not"),
        ("shouldn't", "should not"),
        ("couldn't", "could not"),
        ("mustn't", "must not"),
        ("isn't", "is not"),
        ("aren't", "are not"),
        ("doesn't", "does not"),
        ("don't", "do not"),
        ("didn't", "did not"),
    ] {
        expanded = expanded.replace(contraction, replacement);
    }
    let mut prepared = String::new();
    for character in expanded.chars() {
        if character.is_alphanumeric() {
            prepared.push(character);
        } else {
            prepared.push(' ');
        }
    }
    prepared
        .split_whitespace()
        .filter(|token| !matches!(*token, "a" | "an" | "the" | "to" | "is" | "are"))
        .map(ToOwned::to_owned)
        .collect()
}

fn semantic_stem(token: &str) -> String {
    let mut value = token.to_string();
    if value.len() >= 6 && value.ends_with("ing") {
        value.truncate(value.len() - 3);
    } else if value.len() > 5 && value.ends_with("ed") {
        value.truncate(value.len() - 2);
    } else if value.len() > 5 && value.ends_with('s') {
        value.truncate(value.len() - 1);
    }
    if value.len() > 4 && value.ends_with('e') {
        value.pop();
    }
    if value.len() > 3 {
        let mut chars = value.chars().rev();
        if chars.next() == chars.next() {
            value.pop();
        }
    }
    value
}

fn phrase_positions<'a>(
    tokens: &'a [String],
    phrase: &'a [String],
) -> impl Iterator<Item = usize> + 'a {
    tokens
        .windows(phrase.len().max(1))
        .enumerate()
        .filter(move |(_, window)| !phrase.is_empty() && *window == phrase)
        .map(|(index, _)| index)
}

fn is_negated(tokens: &[String], position: usize) -> bool {
    tokens[..position]
        .iter()
        .enumerate()
        .filter(|(index, token)| {
            if token.as_str() == "not" && tokens.get(index + 1).is_some_and(|next| next == "only") {
                return false;
            }
            if token.as_str() == "without" {
                return without_reaches(tokens, *index, position);
            }
            matches!(
                token.as_str(),
                "not"
                    | "cannot"
                    | "cant"
                    | "no"
                    | "never"
                    | "unabl"
                    | "avoid"
                    | "forbid"
                    | "forbidden"
                    | "prohibit"
            )
        })
        .count()
        % 2
        == 1
}

fn without_reaches(tokens: &[String], index: usize, position: usize) -> bool {
    index < position
        && tokens[index + 1..position].iter().all(|token| {
            matches!(
                token.as_str(),
                "ever" | "fully" | "adequately" | "properly" | "reliably" | "safely"
            )
        })
}

fn semantic_clauses(text: &str) -> Vec<String> {
    let expanded = text.to_lowercase().replace('’', "'");
    let mut clauses = Vec::new();
    let mut current = Vec::new();
    for token in expanded.split_whitespace() {
        let trimmed = token
            .trim_matches(|character: char| matches!(character, ',' | '.' | '!' | '?' | ';' | ':'));
        let begins_new_clause =
            matches!(trimmed, "but" | "however" | "whereas" | "although" | "yet");
        if begins_new_clause && !current.is_empty() {
            clauses.push(current.join(" "));
            current.clear();
        }
        if !trimmed.is_empty() && !begins_new_clause {
            current.push(trimmed.to_string());
        }
        if token.ends_with(['.', '!', '?', ';', ':', ',']) && !current.is_empty() {
            clauses.push(current.join(" "));
            current.clear();
        }
    }
    if !current.is_empty() {
        clauses.push(current.join(" "));
    }
    clauses
}

fn check_id(
    state: &CognitiveState,
    id: &str,
    path: &str,
    diagnostics: &mut Vec<CorpusDiagnostic>,
    field: &str,
    predicate: impl Fn(&NodeKind) -> bool,
) {
    match state.nodes.iter().find(|node| node.id == id) {
        Some(node) if predicate(&node.kind) => {}
        Some(_) => push(
            diagnostics,
            "golden-id-wrong-kind-or-status",
            format!("{path}.golden.{field}"),
            format!("golden ID `{id}` has the wrong kind or status"),
        ),
        None => push(
            diagnostics,
            "golden-id-absent",
            format!("{path}.golden.{field}"),
            format!("golden ID `{id}` is absent"),
        ),
    }
}

fn expectation_matches(node: &Node, expectation: &OwnerTargetExpectation) -> bool {
    node.id == expectation.node_id
        && node.owner == expectation.owner
        && expectation.target.as_deref() == node_target(node)
}

fn check_nonempty_unique(
    values: &[String],
    field_path: &str,
    field: &str,
    diagnostics: &mut Vec<CorpusDiagnostic>,
) {
    if values.is_empty() {
        push(
            diagnostics,
            "empty-golden-list",
            field_path,
            format!("`{field}` must not be empty"),
        );
        return;
    }
    let mut unique = BTreeSet::new();
    for value in values {
        if !unique.insert(value) {
            push(
                diagnostics,
                "duplicate-golden-id",
                field_path,
                format!("duplicate value `{value}`"),
            );
        }
    }
}

fn validate_corpus_diversity(episodes: &[Episode], diagnostics: &mut Vec<CorpusDiagnostic>) {
    let unique_snapshots = episodes
        .iter()
        .map(|episode| episode.snapshot_source.as_str())
        .collect::<BTreeSet<_>>()
        .len();
    let mut initial_cardinalities = BTreeSet::new();
    let mut final_cardinalities = BTreeSet::new();
    let mut patch_counts = BTreeSet::new();
    let mut operation_kinds = BTreeSet::new();
    let mut revision_episodes = 0usize;
    let mut retirement_episodes = 0usize;
    let mut transitions = BTreeSet::new();
    let mut category_shapes = BTreeMap::<EpisodeCategory, BTreeSet<&'static str>>::new();
    let mut causal_signatures = BTreeSet::new();
    let mut revised_semantic_kinds = BTreeSet::new();
    let mut retired_semantic_kinds = BTreeSet::new();
    let mut initial_recovery_episodes = 0usize;
    let mut owner_handoff_episodes = 0usize;
    let mut relation_history_episodes = 0usize;
    let mut residue_history_episodes = 0usize;

    for episode in episodes {
        let all_sources = std::iter::once(episode.snapshot_source.as_str())
            .chain(episode.patch_sources.iter().map(String::as_str))
            .collect::<Vec<_>>()
            .join("\n");
        if [
            "(working note ",
            "(reviewed question ",
            "(next action ",
            "(remaining risk ",
            "(evidence needed ",
        ]
        .iter()
        .any(|scaffold| all_sources.contains(scaffold))
        {
            push(
                diagnostics,
                "formulaic-history-scaffold",
                format!("episodes/{}.json.sources", episode.id),
                "history contains the rejected d/x0/x1/x2 benchmark scaffold",
            );
        }
        let Ok(initial) = decode_state(&episode.snapshot_source) else {
            continue;
        };
        let Ok(final_state) = replay_episode(episode) else {
            continue;
        };
        initial_cardinalities.insert(initial.nodes.len());
        final_cardinalities.insert(final_state.nodes.len());
        patch_counts.insert(episode.patch_sources.len());
        let mut revised = false;
        let mut retired = false;
        let mut owner_handoff = false;
        let mut relation_history = false;
        let mut residue_history = false;
        let mut current = initial.clone();
        let mut signature = initial
            .nodes
            .iter()
            .map(|node| format!("initial:{}", history_node_label(&node.kind)))
            .collect::<Vec<_>>();
        signature.sort();
        initial_recovery_episodes += usize::from(initial.nodes.iter().any(|node| {
            matches!(
                node.kind,
                NodeKind::Constraint { .. }
                    | NodeKind::Goal { .. }
                    | NodeKind::Question { .. }
                    | NodeKind::Tension { .. }
            )
        }));
        for source in &episode.patch_sources {
            let Ok(document) = parse_document(source) else {
                continue;
            };
            let Ok(patch) = Patch::try_from(document) else {
                continue;
            };
            for operation in &patch.ops {
                match operation {
                    PatchOp::Add(add) => {
                        operation_kinds.insert("add");
                        match &add.value {
                            AddValue::Node(node) => {
                                signature.push(format!("add:{}", history_node_label(&node.kind)))
                            }
                            AddValue::Relation(relation) => {
                                relation_history = true;
                                signature.push(format!("relation:{}", relation.kind.as_str()));
                            }
                        }
                    }
                    PatchOp::Revise(revision) => {
                        operation_kinds.insert("revise");
                        revised = true;
                        let before = current
                            .nodes
                            .iter()
                            .find(|node| node.id == revision.node.id);
                        if let Some(before) = before {
                            let before_kind = history_node_label(&before.kind);
                            let after_kind = history_node_label(&revision.node.kind);
                            signature.push(format!("revise:{before_kind}->{after_kind}"));
                            if semantic_history_kind(&before.kind) {
                                revised_semantic_kinds.insert(node_kind_name(&before.kind));
                            }
                            if before.owner != revision.node.owner {
                                owner_handoff = true;
                                signature.push("owner-handoff".into());
                            }
                        }
                    }
                    PatchOp::Retire(retirement) => {
                        operation_kinds.insert("retire");
                        retired = true;
                        if let Some(before) = current
                            .nodes
                            .iter()
                            .find(|node| node.id == retirement.node_id)
                        {
                            signature.push(format!("retire:{}", history_node_label(&before.kind)));
                            if semantic_history_kind(&before.kind) {
                                retired_semantic_kinds.insert(node_kind_name(&before.kind));
                            }
                        }
                    }
                    PatchOp::AppendFocus(append) => {
                        operation_kinds.insert("append-focus");
                        residue_history |= !append.frame.residue.is_empty();
                        signature.push(format!(
                            "focus:{}:{}",
                            append.frame.transition.as_str(),
                            if append.frame.residue.is_empty() {
                                "clear"
                            } else {
                                "residue"
                            }
                        ));
                    }
                }
            }
            if let Ok(next) = apply_patch(&current, &patch) {
                current = next;
            }
        }
        causal_signatures.insert(signature);
        revision_episodes += usize::from(revised);
        retirement_episodes += usize::from(retired);
        owner_handoff_episodes += usize::from(owner_handoff);
        relation_history_episodes += usize::from(relation_history);
        residue_history_episodes += usize::from(residue_history);
        for frame in final_state
            .focus_streams
            .iter()
            .flat_map(|stream| &stream.frames)
        {
            transitions.insert(frame.transition.as_str());
        }
        let shape = category_shapes.entry(episode.category).or_default();
        for node in &final_state.nodes {
            shape.insert(node_kind_name(&node.kind));
        }
    }
    let distinct_category_shapes = category_shapes
        .values()
        .map(|shape| shape.iter().copied().collect::<Vec<_>>())
        .collect::<BTreeSet<_>>()
        .len();
    let requirements = [
        (
            unique_snapshots == EPISODE_COUNT,
            format!("{unique_snapshots} unique starting snapshots"),
        ),
        (
            initial_cardinalities.len() >= 3,
            format!("{} starting cardinalities", initial_cardinalities.len()),
        ),
        (
            final_cardinalities.len() >= 4,
            format!("{} final cardinalities", final_cardinalities.len()),
        ),
        (
            patch_counts.len() >= 3,
            format!("{} patch-count shapes", patch_counts.len()),
        ),
        (
            operation_kinds.len() == 4,
            format!("operations {operation_kinds:?}"),
        ),
        (
            revision_episodes >= 6,
            format!("{revision_episodes} episodes revise nodes"),
        ),
        (
            retirement_episodes >= 6,
            format!("{retirement_episodes} episodes retire nodes"),
        ),
        (
            transitions.len() >= 5,
            format!("transitions {transitions:?}"),
        ),
        (
            distinct_category_shapes == 6,
            format!("{distinct_category_shapes} category shapes"),
        ),
    ];
    for (passed, detail) in requirements {
        if !passed {
            push(
                diagnostics,
                "insufficient-corpus-diversity",
                "manifest.json.episodes",
                detail,
            );
        }
    }
    let causal_requirements = [
        (
            causal_signatures.len() >= 8,
            format!("{} structural history signatures", causal_signatures.len()),
        ),
        (
            initial_recovery_episodes >= 12,
            format!("{initial_recovery_episodes} histories begin with a recovery fact"),
        ),
        (
            revised_semantic_kinds.len() >= 3,
            format!("revised semantic kinds {revised_semantic_kinds:?}"),
        ),
        (
            retired_semantic_kinds.len() >= 3,
            format!("retired semantic kinds {retired_semantic_kinds:?}"),
        ),
        (
            owner_handoff_episodes >= 3,
            format!("{owner_handoff_episodes} owner handoff histories"),
        ),
        (
            relation_history_episodes >= 3,
            format!("{relation_history_episodes} relation-bearing histories"),
        ),
        (
            residue_history_episodes >= 3,
            format!("{residue_history_episodes} histories preserve focus residue"),
        ),
    ];
    for (passed, detail) in causal_requirements {
        if !passed {
            push(
                diagnostics,
                "insufficient-causal-history-diversity",
                "manifest.json.episodes",
                detail,
            );
        }
    }
}

fn semantic_history_kind(kind: &NodeKind) -> bool {
    matches!(
        kind,
        NodeKind::Constraint { .. }
            | NodeKind::Goal { .. }
            | NodeKind::Question { .. }
            | NodeKind::Tension { .. }
    )
}

fn history_node_label(kind: &NodeKind) -> String {
    match kind {
        NodeKind::Constraint { strength } => format!("constraint:{strength:?}"),
        NodeKind::Goal { status } => format!("goal:{status:?}"),
        NodeKind::Question { status } => format!("question:{status:?}"),
        NodeKind::Tension { status } => format!("tension:{status:?}"),
        _ => node_kind_name(kind).to_string(),
    }
}

fn node_kind_name(kind: &NodeKind) -> &'static str {
    match kind {
        NodeKind::Entity { .. } => "entity",
        NodeKind::Claim { .. } => "claim",
        NodeKind::Evidence { .. } => "evidence",
        NodeKind::Goal { .. } => "goal",
        NodeKind::Constraint { .. } => "constraint",
        NodeKind::Question { .. } => "question",
        NodeKind::Relationship { .. } => "relationship",
        NodeKind::Desire { .. } => "desire",
        NodeKind::Attention { .. } => "attention",
        NodeKind::Association => "association",
        NodeKind::Tension { .. } => "tension",
        NodeKind::Prediction { .. } => "prediction",
        NodeKind::Perspective(_) => "perspective",
        NodeKind::Affect { .. } => "affect",
        NodeKind::Metacognition { .. } => "metacognition",
    }
}

fn node_target(node: &Node) -> Option<&str> {
    match &node.kind {
        NodeKind::Relationship { target, .. } | NodeKind::Affect { target, .. } => Some(target),
        NodeKind::Perspective(value) => Some(&value.target),
        _ => None,
    }
}

fn safe_episode_path(path: &str) -> bool {
    let path = Path::new(path);
    !path.is_absolute()
        && path.extension().and_then(|value| value.to_str()) == Some("json")
        && matches!(path.components().next(), Some(Component::Normal(value)) if value == "episodes")
        && path
            .components()
            .all(|component| matches!(component, Component::Normal(_)))
        && path.components().count() == 2
}

fn utf8_prefix(value: &str, limit: usize) -> &str {
    let mut end = value.len().min(limit);
    while !value.is_char_boundary(end) {
        end -= 1;
    }
    &value[..end]
}

fn ratio(numerator: usize, denominator: usize) -> f64 {
    if denominator == 0 {
        0.0
    } else {
        numerator as f64 / denominator as f64
    }
}

fn ratio_usize(numerator: usize, denominator: usize) -> f64 {
    if denominator == 0 {
        if numerator == 0 { 0.0 } else { 1.0 }
    } else {
        numerator as f64 / denominator as f64
    }
}

fn at_least(name: &'static str, actual: f64, required: f64) -> GateResult {
    GateResult {
        name,
        passed: actual >= required,
        actual,
        required,
    }
}

fn at_most(name: &'static str, actual: f64, required: f64) -> GateResult {
    GateResult {
        name,
        passed: actual <= required,
        actual,
        required,
    }
}

fn read(path: &Path) -> Result<Vec<u8>, CorpusError> {
    fs::read(path).map_err(|error| CorpusError {
        kind: CorpusErrorKind::Io,
        diagnostics: vec![CorpusDiagnostic {
            code: "corpus-io-error".into(),
            path: path.display().to_string(),
            message: error.to_string(),
        }],
    })
}

fn decode_json<T: for<'de> Deserialize<'de>>(bytes: &[u8], path: &str) -> Result<T, CorpusError> {
    serde_json::from_slice(bytes).map_err(|error| CorpusError {
        kind: CorpusErrorKind::Invalid,
        diagnostics: vec![CorpusDiagnostic {
            code: "invalid-corpus-json".into(),
            path: path.into(),
            message: error.to_string(),
        }],
    })
}

fn push(
    diagnostics: &mut Vec<CorpusDiagnostic>,
    code: &str,
    path: impl Into<String>,
    message: impl Into<String>,
) {
    diagnostics.push(CorpusDiagnostic {
        code: code.into(),
        path: path.into(),
        message: message.into(),
    });
}

fn synthetic_diagnostic(code: &str, message: &str) -> Diagnostic {
    Diagnostic {
        code: code.into(),
        severity: crate::Severity::Error,
        span: crate::Span {
            start: 0,
            end: 0,
            start_line: 1,
            start_column: 1,
            end_line: 1,
            end_column: 1,
        },
        semantic_path: Vec::new(),
        message: message.into(),
        hint: None,
    }
}