car-memgine 0.48.0

//! Episode/intent credit model — turns per-turn outcomes into routing-grade
//! credit for the model that produced each turn.
//!
//! Per-turn outcomes (see [`crate::outcome_signal`]) are the raw material; this
//! groups them into *episodes* (a contiguous run of attempts at one intent) and
//! assigns one credit per assistant turn, modulated by how the episode ended:
//!
//! - **Resolved** (user advanced / cleanly exited after the intent): the turn
//!   that resolved it is credited a success; earlier circled-on attempts are
//!   debited but *discounted* — the system recovered, so each failure cost less.
//! - **Abandoned** (explicit user abandonment): every attempt in the loop is
//!   debited, *boosted* — the whole episode failed the user.
//! - **Ambiguous** (repair-then-silence): the circled-on attempts are known
//!   failures and are debited; the final unknowable turn is **deferred** (no
//!   credit emitted) until the conversation resolves it.
//! - **Ongoing** (conversation ends mid-loop on a circle): the circled-on
//!   attempts are known failures; there's no resolver yet.
//!
//! This is *intended* as the single credit source for routing — emit one credit
//! per turn, never count a turn twice. Deferring ambiguous terminal turns is
//! exactly the "only feed newly-resolved" property the tracker wire-up needs.
//!
//! ## Requirements on the Task-3 wire-up (this module alone does not enforce them)
//!
//! - **Route every credit through the one reversing door.** The tracker books a
//!   *mechanical* success when a turn completes with output; only
//!   `record_inferred_outcome` (which pops `pending`) knows how to *reverse* it.
//!   A `Failure` credit fed any other way leaves a phantom success — defeating
//!   intent-aware credit. Map `Success→Accepted`, `Failure→Rejected`, and skip
//!   `trace_id == None` (observer turns never bind a decision).
//! - **Be idempotent on `(turn_ref, episode_id)`.** Deferral means the emitted
//!   credit set *grows* as the conversation grows; re-running must not re-credit
//!   already-resolved turns.
//! - **Replace, don't run alongside, the existing action-sequence resolution.**
//!   Two sources resolving the same `pending` traces double-counts.
//! - **`confidence` is an ordinal EMA weight, not a count.** Today the routing
//!   arm (Thompson) reads integer success/fail counts; the discount/boost only
//!   move the quality EMA. Task 3 decides whether a discounted failure becomes a
//!   *fractional* count (giving the modulation teeth) or stays EMA-only.
//!
//! ## Known limitation: topic-drift-after-defeat
//!
//! If a user abandons intent A by pivoting to an unrelated intent B *without an
//! explicit marker*, the substrate reads the pivot as `Advance` (it can't yet
//! tell "advanced the same intent" from "gave up and changed subject"). This
//! module then labels A's circles as Resolved-recovered and credits the pivot
//! turn for "resolving" A — wrong, and biased toward rewarding whatever model
//! preceded a topic change. Fixing it needs the intent-continuity discriminator
//! noted in [`crate::outcome_signal`] (the embedding upgrade). Frequency is low
//! (explicit abandonment fires `FrustratedExit`, which is handled correctly);
//! named here so it isn't mistaken for recovery in the data.

use crate::outcome_signal::{classify_turns, AssistantTurnOutcome, ConversationTurn, TurnOutcome};

/// Whether a turn helped or hurt the user getting their outcome.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Credit {
    Success,
    Failure,
}

/// How the episode a turn belonged to ended.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum EpisodeResolution {
    /// User got their outcome and moved on / ended cleanly.
    Resolved,
    /// User explicitly abandoned the intent.
    Abandoned,
    /// Ended on a repair-then-silence — the final turn's result is unknowable.
    Ambiguous,
    /// Conversation ended mid-loop (last attempt was circled-on, no resolver).
    Ongoing,
}

/// One routing-grade credit decision for an assistant turn, attributed to the
/// model/trace that produced it.
#[derive(Debug, Clone)]
pub struct TurnCredit {
    pub turn_ref: String,
    pub model_id: Option<String>,
    pub trace_id: Option<String>,
    pub credit: Credit,
    /// 0.0–1.0 ordinal weight (episode-modulated), for EMA aggregation.
    pub confidence: f64,
    /// Groups turns of one intent thread (the "episode_ref" for routing learning).
    pub episode_id: usize,
    pub resolution: EpisodeResolution,
}

/// Failures in an episode that *eventually resolved* cost less — the system
/// recovered.
const RECOVERED_DISCOUNT: f64 = 0.7;
/// Failures in an *abandoned* episode are worse — the loop failed the user.
const ABANDONED_BOOST: f64 = 1.15;

/// Convenience: classify typed turns and assign episode credit in one call.
pub fn assign_credit(turns: &[ConversationTurn]) -> Vec<TurnCredit> {
    assign_credit_from_outcomes(&classify_turns(turns))
}

/// Group an ordered sequence of per-turn outcomes into episodes and emit one
/// episode-modulated credit per turn. Ambiguous terminal turns are deferred
/// (not emitted). Input must be in chronological order (as [`classify_turns`]
/// returns it).
pub fn assign_credit_from_outcomes(outcomes: &[AssistantTurnOutcome]) -> Vec<TurnCredit> {
    let mut credits = Vec::new();
    let mut episode_id = 0usize;
    let mut start = 0usize; // index of the current episode's first outcome

    for (i, o) in outcomes.iter().enumerate() {
        if o.outcome != TurnOutcome::Circle {
            // Non-circle = terminator: close the episode [start..=i].
            close_episode(&outcomes[start..=i], episode_id, &mut credits);
            episode_id += 1;
            start = i + 1;
        }
    }
    // Trailing episode with no terminator (ends on a circle) → Ongoing.
    if start < outcomes.len() {
        close_episode(&outcomes[start..], episode_id, &mut credits);
    }

    credits
}

/// Emit episode-modulated credit for one episode `ep` (its terminator, if any,
/// is the last element). Deferred (ambiguous-terminal) turns emit nothing.
fn close_episode(ep: &[AssistantTurnOutcome], episode_id: usize, credits: &mut Vec<TurnCredit>) {
    let resolution = match ep.last().unwrap().outcome {
        TurnOutcome::Advance | TurnOutcome::CleanExit => EpisodeResolution::Resolved,
        TurnOutcome::FrustratedExit => EpisodeResolution::Abandoned,
        TurnOutcome::AmbiguousExit => EpisodeResolution::Ambiguous,
        // Only reachable via the trailing (no-terminator) path, where the last
        // element is itself a Circle. Exhaustive match (no wildcard) by design.
        TurnOutcome::Circle => EpisodeResolution::Ongoing,
    };
    let last = ep.len() - 1;
    for (i, o) in ep.iter().enumerate() {
        let is_terminator = i == last;
        let (credit, confidence) = match resolution {
            EpisodeResolution::Resolved => {
                if is_terminator {
                    (Credit::Success, o.confidence)
                } else {
                    (Credit::Failure, o.confidence * RECOVERED_DISCOUNT)
                }
            }
            EpisodeResolution::Abandoned => {
                (Credit::Failure, (o.confidence * ABANDONED_BOOST).min(1.0))
            }
            EpisodeResolution::Ambiguous => {
                if is_terminator {
                    continue; // defer — unknowable until resolved
                }
                (Credit::Failure, o.confidence)
            }
            EpisodeResolution::Ongoing => (Credit::Failure, o.confidence),
        };
        credits.push(TurnCredit {
            turn_ref: o.turn_ref.clone(),
            model_id: o.model_id.clone(),
            trace_id: o.trace_id.clone(),
            credit,
            confidence,
            episode_id,
            resolution,
        });
    }
}

/// Outcome-denominated conversation metrics — the "minimize turns-to-outcome"
/// scoreboard, the inverse of an engagement metric. Computed over a batch of
/// episode credits (one or more conversations).
///
/// LIMITATION (neo): fully-deferred episodes (ambiguous terminal with no
/// preceding circles) emit no credits, so they appear in NEITHER the numerator
/// nor the denominator here — `churn_rate` and `episodes` *undercount* unknown
/// intents, and a model that ends more turns ambiguously looks artificially
/// clean. The deferral judgment is honest (don't fabricate an outcome on an
/// unknowable turn), but a true scoreboard must add a `pending`/`unknown` count
/// — which `TurnCredit` alone can't supply (deferred turns emit none). That
/// count has to come from the ledger or raw conversation, i.e. the persistent
/// aggregator, not this per-batch primitive.
#[derive(Debug, Clone, Default, PartialEq)]
pub struct ConversationMetrics {
    pub episodes: usize,
    pub resolved: usize,
    pub abandoned: usize,
    pub ambiguous: usize,
    pub ongoing: usize,
    /// Episodes the user got on the FIRST attempt (one turn, no circling).
    pub first_turn_resolutions: usize,
    /// `first_turn_resolutions / resolved` — the north star (1.0 = every
    /// resolved intent was nailed in one turn). Higher is better.
    pub first_turn_resolution_rate: f64,
    /// Mean attempts per resolved episode (≥ 1.0). LOWER is better — the goal is
    /// to minimize turns to a correct outcome, never to maximize interaction.
    pub avg_turns_to_resolution: f64,
    /// `abandoned / episodes` — churn. Lower is better.
    pub churn_rate: f64,
}

/// Aggregate episode credits into outcome-denominated metrics. All credits of an
/// episode share an `episode_id` and `resolution`; turn count per episode is the
/// number of attempts.
pub fn conversation_metrics(credits: &[TurnCredit]) -> ConversationMetrics {
    use std::collections::HashMap;
    let mut episodes: HashMap<usize, (usize, EpisodeResolution)> = HashMap::new();
    for c in credits {
        // All credits of an episode share `resolution` (stamped by
        // close_episode), so the first insert fixes it; only the count grows.
        let e = episodes.entry(c.episode_id).or_insert((0, c.resolution));
        e.0 += 1;
    }

    let mut m = ConversationMetrics::default();
    let mut resolved_turns = 0usize;
    for (count, resolution) in episodes.values() {
        m.episodes += 1;
        match resolution {
            EpisodeResolution::Resolved => {
                m.resolved += 1;
                resolved_turns += count;
                if *count == 1 {
                    m.first_turn_resolutions += 1;
                }
            }
            EpisodeResolution::Abandoned => m.abandoned += 1,
            EpisodeResolution::Ambiguous => m.ambiguous += 1,
            EpisodeResolution::Ongoing => m.ongoing += 1,
        }
    }
    if m.resolved > 0 {
        m.avg_turns_to_resolution = resolved_turns as f64 / m.resolved as f64;
        m.first_turn_resolution_rate = m.first_turn_resolutions as f64 / m.resolved as f64;
    }
    if m.episodes > 0 {
        m.churn_rate = m.abandoned as f64 / m.episodes as f64;
    }
    m
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::outcome_signal::ConversationTurn;

    fn u(text: &str, r: &str) -> ConversationTurn {
        ConversationTurn::user(text, r)
    }
    fn a(text: &str, r: &str, model: &str) -> ConversationTurn {
        ConversationTurn::assistant(text, r, Some(model.into()), Some(format!("trace-{r}")))
    }

    #[test]
    fn clean_single_answer_is_resolved_success() {
        let turns = [u("what's 2+2", "u1"), a("4", "t1", "m1")];
        let c = assign_credit(&turns);
        assert_eq!(c.len(), 1);
        assert_eq!(c[0].credit, Credit::Success);
        assert_eq!(c[0].resolution, EpisodeResolution::Resolved);
        assert_eq!(c[0].episode_id, 0);
    }

    #[test]
    fn recovery_debits_circle_credits_resolver_across_models() {
        // m1 fails (user circles), m2 resolves (user advances to new intent).
        let turns = [
            u("convert to async", "u1"),
            a("threaded version", "t1", "m1"),
            u("no, that's not what i asked, i said async", "u2"),
            a("async version", "t2", "m2"),
            u("thanks, now add tests", "u3"),
            a("tests added", "t3", "m2"),
        ];
        let c = assign_credit(&turns);
        // t1: failed attempt in a resolved episode → debited, discounted.
        let t1 = c.iter().find(|x| x.turn_ref == "t1").unwrap();
        assert_eq!(t1.credit, Credit::Failure);
        assert_eq!(t1.model_id.as_deref(), Some("m1"));
        // marker-circle confidence is 0.85; discounted (×0.7) is below it.
        assert!(t1.confidence < 0.85, "recovered failure is discounted");
        assert_eq!(t1.resolution, EpisodeResolution::Resolved);
        // t2: resolved the intent → success.
        let t2 = c.iter().find(|x| x.turn_ref == "t2").unwrap();
        assert_eq!(t2.credit, Credit::Success);
        assert_eq!(t2.model_id.as_deref(), Some("m2"));
        // t1 and t2 share an episode; t3 is a new episode.
        assert_eq!(t1.episode_id, t2.episode_id);
        let t3 = c.iter().find(|x| x.turn_ref == "t3").unwrap();
        assert!(t3.episode_id > t2.episode_id);
        assert_eq!(t3.credit, Credit::Success);
    }

    #[test]
    fn abandoned_episode_debits_all_attempts_boosted() {
        let turns = [
            u("fix the deploy", "u1"),
            a("attempt 1", "t1", "m1"),
            u("no that's wrong", "u2"),
            a("attempt 2", "t2", "m1"),
            u("forget it, this is useless", "u3"),
        ];
        let c = assign_credit(&turns);
        assert!(c.iter().all(|x| x.credit == Credit::Failure));
        assert!(c
            .iter()
            .all(|x| x.resolution == EpisodeResolution::Abandoned));
        // boosted confidence (>= the raw circle/abandon weights, capped at 1.0).
        assert!(c.iter().all(|x| x.confidence <= 1.0));
    }

    #[test]
    fn ambiguous_terminal_turn_is_deferred() {
        // circle then a repair attempt then silence: the circle is a known
        // failure; the final repair attempt is deferred (unknowable).
        let turns = [
            u("fix the test", "u1"),
            a("attempt 1", "t1", "m1"),
            u("that didn't work, still failing", "u2"),
            a("attempt 2 (maybe fixed)", "t2", "m1"),
        ];
        let c = assign_credit(&turns);
        // t1 circled-on → debited; t2 ambiguous → NOT emitted.
        assert!(c
            .iter()
            .any(|x| x.turn_ref == "t1" && x.credit == Credit::Failure));
        assert!(
            !c.iter().any(|x| x.turn_ref == "t2"),
            "ambiguous terminal turn must be deferred"
        );
    }

    #[test]
    fn ongoing_loop_debits_known_circles() {
        // ends with the user circling and no further assistant reply.
        let turns = [
            u("do X", "u1"),
            a("wrong 1", "t1", "m1"),
            u("no, try again", "u2"),
            a("wrong 2", "t2", "m1"),
            u("still wrong, that failed", "u3"),
        ];
        let c = assign_credit(&turns);
        // both attempts were circled-on → known failures.
        assert_eq!(c.len(), 2);
        assert!(c.iter().all(|x| x.credit == Credit::Failure));
        assert!(c.iter().all(|x| x.resolution == EpisodeResolution::Ongoing));
    }

    #[test]
    fn distinct_intents_are_separate_episodes() {
        let turns = [
            u("question one", "u1"),
            a("answer one", "t1", "m1"),
            u("unrelated question two", "u2"),
            a("answer two", "t2", "m1"),
        ];
        let c = assign_credit(&turns);
        assert_eq!(c.len(), 2);
        assert_ne!(c[0].episode_id, c[1].episode_id);
        assert!(c.iter().all(|x| x.credit == Credit::Success));
    }

    #[test]
    fn empty_is_safe() {
        assert!(assign_credit(&[]).is_empty());
        assert_eq!(conversation_metrics(&[]), ConversationMetrics::default());
    }

    #[test]
    fn metrics_one_shot_resolution() {
        let turns = [u("what's 2+2", "u1"), a("4", "t1", "m1")];
        let m = conversation_metrics(&assign_credit(&turns));
        assert_eq!(m.episodes, 1);
        assert_eq!(m.resolved, 1);
        assert_eq!(m.first_turn_resolutions, 1);
        assert!((m.first_turn_resolution_rate - 1.0).abs() < 1e-9);
        assert!((m.avg_turns_to_resolution - 1.0).abs() < 1e-9);
        assert!((m.churn_rate - 0.0).abs() < 1e-9);
    }

    #[test]
    fn metrics_count_turns_to_resolution_across_episodes() {
        // ep0 takes 2 turns (circle then resolve); ep1 is one-shot.
        let turns = [
            u("convert to async", "u1"),
            a("threaded", "t1", "m1"),
            u("no, that's not what i asked", "u2"),
            a("async", "t2", "m2"),
            u("thanks, now a new question", "u3"),
            a("answer", "t3", "m2"),
        ];
        let m = conversation_metrics(&assign_credit(&turns));
        assert_eq!(m.resolved, 2);
        assert_eq!(m.first_turn_resolutions, 1, "only ep1 was one-shot");
        // avg = (2 attempts + 1 attempt) / 2 resolved = 1.5
        assert!((m.avg_turns_to_resolution - 1.5).abs() < 1e-9);
        assert!((m.first_turn_resolution_rate - 0.5).abs() < 1e-9);
    }

    #[test]
    fn metrics_ongoing_loop_is_not_churn() {
        // Ends mid-loop on a circle (no resolver, no abandonment) → Ongoing.
        // Pins the rates: not resolved, not churn.
        let turns = [
            u("do X", "u1"),
            a("wrong", "t1", "m1"),
            u("no, try again", "u2"),
            a("wrong2", "t2", "m1"),
            u("still wrong, that failed", "u3"),
        ];
        let m = conversation_metrics(&assign_credit(&turns));
        assert_eq!(m.episodes, 1);
        assert_eq!(m.ongoing, 1);
        assert_eq!(m.resolved, 0);
        assert_eq!(m.abandoned, 0);
        assert!((m.churn_rate - 0.0).abs() < 1e-9);
    }

    #[test]
    fn metrics_churn_on_abandonment() {
        let turns = [
            u("fix it", "u1"),
            a("attempt", "t1", "m1"),
            u("no that's wrong", "u2"),
            a("attempt2", "t2", "m1"),
            u("forget it, useless", "u3"),
        ];
        let m = conversation_metrics(&assign_credit(&turns));
        assert_eq!(m.episodes, 1);
        assert_eq!(m.abandoned, 1);
        assert_eq!(m.resolved, 0);
        assert!((m.churn_rate - 1.0).abs() < 1e-9);
    }

    #[test]
    fn mixed_resolution_episodes_coexist() {
        // ep0 resolves; ep1 is abandoned — both in one conversation.
        let turns = [
            u("q1", "u1"),
            a("ans1", "t1", "m1"),
            u("thanks, now fix the deploy", "u2"),
            a("attempt", "t2", "m1"),
            u("no that's wrong", "u3"),
            a("attempt2", "t3", "m1"),
            u("forget it, useless", "u4"),
        ];
        let c = assign_credit(&turns);
        let t1 = c.iter().find(|x| x.turn_ref == "t1").unwrap();
        assert_eq!(t1.resolution, EpisodeResolution::Resolved);
        assert_eq!(t1.credit, Credit::Success);
        let t2 = c.iter().find(|x| x.turn_ref == "t2").unwrap();
        let t3 = c.iter().find(|x| x.turn_ref == "t3").unwrap();
        assert_eq!(t2.resolution, EpisodeResolution::Abandoned);
        assert_eq!(t3.resolution, EpisodeResolution::Abandoned);
        assert_ne!(t1.episode_id, t2.episode_id);
        assert_eq!(t2.episode_id, t3.episode_id);
    }

    #[test]
    fn multi_circle_then_ambiguous_emits_all_circles() {
        let turns = [
            u("fix the test", "u1"),
            a("attempt 1", "t1", "m1"),
            u("no, that's wrong", "u2"),
            a("attempt 2", "t2", "m1"),
            u("still failing, didn't work", "u3"),
            a("attempt 3 (maybe)", "t3", "m1"),
        ];
        let c = assign_credit(&turns);
        // both circled attempts emitted as Failure; the ambiguous terminal deferred.
        assert!(c
            .iter()
            .any(|x| x.turn_ref == "t1" && x.credit == Credit::Failure));
        assert!(c
            .iter()
            .any(|x| x.turn_ref == "t2" && x.credit == Credit::Failure));
        assert!(!c.iter().any(|x| x.turn_ref == "t3"));
        assert_eq!(c.len(), 2);
    }

    #[test]
    fn confidence_discount_and_boost_are_exact() {
        // Recovered circle: marker-circle 0.85 × 0.7 = 0.595.
        let resolved = [
            u("convert to async", "u1"),
            a("threaded", "t1", "m1"),
            u("no, that's not what i asked", "u2"),
            a("async", "t2", "m2"),
            u("thanks, new question", "u3"),
            a("answer", "t3", "m2"),
        ];
        let rc = assign_credit(&resolved);
        let t1 = rc.iter().find(|x| x.turn_ref == "t1").unwrap();
        assert!((t1.confidence - 0.85 * 0.7).abs() < 1e-9);

        // Abandoned circle: 0.85 × 1.15 = 0.9775 (under the 1.0 cap).
        let abandoned = [
            u("fix it", "u1"),
            a("attempt", "t1", "m1"),
            u("no that's wrong", "u2"),
            a("attempt2", "t2", "m1"),
            u("forget it, useless", "u3"),
        ];
        let ac = assign_credit(&abandoned);
        let at1 = ac.iter().find(|x| x.turn_ref == "t1").unwrap();
        assert!((at1.confidence - (0.85 * 1.15)).abs() < 1e-9);
        assert!(ac.iter().all(|x| x.confidence <= 1.0));
    }

    #[test]
    fn episode_id_advances_even_when_terminal_deferred() {
        // ep0 resolved (1 credit); ep1 = circle + ambiguous-terminal (1 credit
        // emitted, terminal deferred). episode_id must still advance to 1.
        let turns = [
            u("q1", "u1"),
            a("a1", "t1", "m1"),
            u("now fix the bug", "u2"),
            a("fix1", "t2", "m1"),
            u("still broken, didn't work", "u3"),
            a("fix2", "t3", "m1"),
        ];
        let c = assign_credit(&turns);
        let t1 = c.iter().find(|x| x.turn_ref == "t1").unwrap();
        let t2 = c.iter().find(|x| x.turn_ref == "t2").unwrap();
        assert_eq!(t1.episode_id, 0);
        assert_eq!(
            t2.episode_id, 1,
            "episode_id advances despite deferred terminal"
        );
        assert!(
            !c.iter().any(|x| x.turn_ref == "t3"),
            "ambiguous terminal deferred"
        );
    }
}