polyc-agent 2026.9.0

#![allow(clippy::unwrap_used)] // test: panics are acceptable
//! Compaction recall eval — summarizer-side tiers (#1134, INV-C2/CONF-2).
//!
//! Golden transcripts seeded with identifiers of every extractable class
//! (opaque ids, amounts, URLs, person names, quoted open items) live as
//! portable data files owned by `polyc-conformance-vectors`
//! (`crates/conformance-vectors/vectors/compaction/`). The seeded
//! material is scenario prose (freight, event planning, inventory
//! monitoring) written independently of the summarizer prompt's own
//! vocabulary, so surviving identifiers prove retention, not keyword echo.
//! The identifier definition is the normative extractor in
//! [`polyc_agent::identifiers`] — shared with the mint-time retention gate
//! (#1135) so the eval measures exactly the property the gate enforces.
//!
//! Two tiers run here:
//!
//! - **Rendering mechanics (CI-gated, deterministic):** the
//!   [`LlmSummarizer`] transcript rendering must deliver every required
//!   identifier into the prompt the provider receives — including
//!   identifiers inside tool args/results up to the render clips (1024-byte
//!   args, 4096-byte results). Identifiers seeded BEYOND the clips are the
//!   documented standing miss: the clip provably eats them before any model
//!   sees them, so no prompt improvement can recover them. Those are pinned
//!   by `clip_shadowed` assertions — if the clip logic ever changes to keep
//!   them, the absence assertions fail and the identifiers must be promoted
//!   into `required`.
//! - **Model-backed recall (`#[ignore]`d by default — NOT run in CI):** the
//!   same vectors driven through the real [`LlmSummarizer`] against a live
//!   chat-completions-compatible endpoint (the `ollama_e2e` convention:
//!   `POLYCHROME_OLLAMA_BASE_URL` / `POLYCHROME_OLLAMA_MODEL`), K = 3
//!   successive anchored compactions, with measured per-family floors.
//!
//! **What CI actually asserts:** only the rendering-mechanics tier above. The
//! model-backed recall floors are never asserted in CI — the test is
//! `#[ignore]`d, so it runs only when invoked explicitly (see the command
//! below), against whatever the operator has pointed
//! `POLYCHROME_OLLAMA_BASE_URL` / `POLYCHROME_OLLAMA_MODEL` at. A green CI run
//! says nothing about a real model's recall; it says only that every
//! identifier reaches the prompt.
//!
//! The pipeline tier (event-log fold, turn-boundary cuts, anchor
//! persistence) lives with the compaction pipeline in the control plane:
//! `crates/control-plane/src/grpc/recall_eval.rs`.
//!
//! Run the model-backed tier locally with (plain `cargo test`, not nextest —
//! a local model needs minutes per family and nextest's per-test terminate
//! would kill it), against whatever `POLYCHROME_OLLAMA_BASE_URL` /
//! `POLYCHROME_OLLAMA_MODEL` point at:
//!   cargo test -p polyc-agent --test compaction_recall -- --ignored --nocapture

#![allow(clippy::pedantic, clippy::nursery, missing_docs)]

use std::sync::{Arc, Mutex};

use async_trait::async_trait;
use futures::stream::{self, StreamExt};
use polyc_agent::golden_vectors::{Vector, VectorMessage};
use polyc_agent::identifiers::{IdentifierClass, extract_identifiers, is_retained};
use polyc_agent::{LlmSummarizer, Summarizer};
use polyc_llm::error::DummyError;
use polyc_llm::{
    Chunk, CompletionRequest, Content as LlmContent, LlmProvider, Message as LlmMessage, Role,
    StopReason,
};

fn vectors() -> Vec<Vector> {
    polyc_conformance_vectors::compaction_vectors()
        .into_iter()
        .map(|s| serde_json::from_str(s).expect("valid vector JSON"))
        .collect()
}

fn to_llm(msg: &VectorMessage) -> LlmMessage {
    match msg {
        VectorMessage::User { text } => LlmMessage::user(text.clone()),
        VectorMessage::Assistant { text } => LlmMessage::assistant(text.clone()),
        VectorMessage::ToolCall {
            id,
            name,
            args_json,
        } => LlmMessage {
            role: Role::Assistant,
            content: vec![LlmContent::tool_use(
                id.clone(),
                name.clone(),
                args_json.clone(),
            )],
        },
        VectorMessage::ToolResult { id, result_json } => LlmMessage {
            role: Role::Tool,
            content: vec![LlmContent::tool_result(
                id.clone(),
                result_json.clone(),
                false,
                true,
            )],
        },
    }
}

/// All messages of one round, in order.
fn round_messages(vector: &Vector, round: usize) -> Vec<LlmMessage> {
    vector.rounds[round]
        .turns
        .iter()
        .flat_map(|t| t.messages.iter().map(to_llm))
        .collect()
}

/// Every seeded byte of the vector, flattened (prose + tool payloads).
fn seeded_text(vector: &Vector) -> String {
    let mut s = String::new();
    for round in &vector.rounds {
        for turn in &round.turns {
            for msg in &turn.messages {
                match msg {
                    VectorMessage::User { text } | VectorMessage::Assistant { text } => {
                        s.push_str(text);
                    }
                    VectorMessage::ToolCall { args_json, .. } => s.push_str(args_json),
                    VectorMessage::ToolResult { result_json, .. } => s.push_str(result_json),
                }
                s.push('\n');
            }
        }
    }
    s
}

fn class_from_str(s: &str) -> IdentifierClass {
    match s {
        "url" => IdentifierClass::Url,
        "amount" => IdentifierClass::Amount,
        "opaque_id" => IdentifierClass::OpaqueId,
        "proper_noun" => IdentifierClass::ProperNoun,
        "quoted" => IdentifierClass::Quoted,
        other => panic!("unknown identifier class in vector: {other}"),
    }
}

// ---------------------------------------------------------------------------
// Vector sanity: the eval and the gate must agree on what an identifier is.
// ---------------------------------------------------------------------------

/// Every `required` identifier must be recognized — with its declared class —
/// by the normative extractor over the seeded material. This pins the vectors
/// to the shared definition in `polyc_agent::identifiers`: a vector seeding an
/// identifier the gate cannot extract would measure recall the retention gate
/// (#1135) could never enforce.
#[test]
fn required_identifiers_are_extractable_by_the_normative_definition() {
    for vector in vectors() {
        let extracted = extract_identifiers(&seeded_text(&vector));
        for req in &vector.required {
            let class = class_from_str(&req.class);
            assert!(
                extracted
                    .iter()
                    .any(|id| id.class == class && id.text == req.text),
                "[{}] required identifier {:?} (class {:?}) is not extractable \
                 from the seeded transcript by the normative extractor",
                vector.family,
                req.text,
                class,
            );
        }
        // Each vector drives K >= 3 successive compactions and seeds every
        // round, so no round's material rides for free in the recent tail.
        assert!(vector.rounds.len() >= 3, "[{}] needs K >= 3", vector.family);
        for round in 0..vector.rounds.len() {
            assert!(
                vector.required.iter().any(|r| r.round == round),
                "[{}] round {round} seeds no required identifier",
                vector.family,
            );
        }
    }
}

// ---------------------------------------------------------------------------
// Tier: rendering mechanics (CI-gated, deterministic)
// ---------------------------------------------------------------------------

/// Provider double that records the rendered request and returns a canned
/// completion, so `LlmSummarizer`'s transcript rendering (including the
/// arg/result clips) is observable at the provider boundary.
struct CapturingProvider {
    seen_user_text: Mutex<Vec<String>>,
}

impl CapturingProvider {
    fn new() -> Self {
        Self {
            seen_user_text: Mutex::new(Vec::new()),
        }
    }
}

#[async_trait]
impl LlmProvider for CapturingProvider {
    type Error = DummyError;

    async fn complete(
        &self,
        req: CompletionRequest,
    ) -> Result<futures::stream::BoxStream<'static, Result<Chunk, Self::Error>>, Self::Error> {
        let user_text = req
            .messages
            .iter()
            .flat_map(|m| m.content.iter())
            .filter_map(|c| match c {
                LlmContent::Text(t) => Some(t.as_str()),
                _ => None,
            })
            .collect::<Vec<_>>()
            .join("\n");
        self.seen_user_text.lock().unwrap().push(user_text);
        let chunks = vec![
            Ok(Chunk::text_delta("ok".to_owned())),
            Ok(Chunk::Stop(StopReason::EndTurn)),
        ];
        Ok(stream::iter(chunks).boxed())
    }
}

/// The transcript rendering must deliver every required identifier to the
/// model's input — prose, tool args within the 1024-byte clip, and tool
/// results within the 4096-byte clip. A structural drop here is invisible to
/// any downstream retention gate (the model can't keep what it never saw), so
/// the floor for this tier is 100%: any miss is a bug, not a tuning target.
#[tokio::test]
async fn rendering_delivers_required_identifiers_to_the_model_input() {
    for vector in vectors() {
        let provider = Arc::new(CapturingProvider::new());
        let summarizer = LlmSummarizer::new(provider.clone(), "eval-model", 2048);
        let messages: Vec<LlmMessage> = (0..vector.rounds.len())
            .flat_map(|r| round_messages(&vector, r))
            .collect();
        let _ = summarizer.summarize("", &messages).await;
        let rendered = provider.seen_user_text.lock().unwrap().join("\n");
        for req in &vector.required {
            assert!(
                is_retained(&rendered, &req.text),
                "[{}] required identifier {:?} (round {}) never reached the \
                 summarizer's rendered input — the pipeline structurally \
                 dropped it before any model could retain it",
                vector.family,
                req.text,
                req.round,
            );
        }
    }
}

/// STANDING MISS — RATIFIED AS PERMANENT (#1297): an identifier placed
/// beyond the render clips — past 1024 bytes into a tool call's args or past
/// 4096 bytes into a tool result — is eaten by `clip()` before the summary
/// model sees it, so it can never survive into the anchor. This is a real
/// retention boundary of the shipped clip constants, not an eval artifact,
/// and the #1297 spike closed on accepting it permanently rather than
/// raising the clips: the identifiers stranded here are inside raw machine
/// payloads — the hardest-to-read material a summarizer prompt could spend
/// its token budget on — and the recovery path already ships:
/// `conversation_read_tool_result` (#1148, INV-C4) reads the recorded bytes back
/// verbatim on request, so nothing is unrecoverable, only absent from the
/// compacted summary. No follow-up implementation is planned. The absence is
/// asserted so the boundary stays visible: if the clip logic ever changes to
/// keep these bytes, this test fails and the `clip_shadowed` identifiers
/// must be promoted into `required` (tightening the eval, never silently
/// loosening it).
#[tokio::test]
async fn clip_shadowed_identifiers_are_a_documented_standing_miss() {
    let mut saw_shadowed_family = false;
    for vector in vectors() {
        if vector.clip_shadowed.is_empty() {
            continue;
        }
        saw_shadowed_family = true;
        let provider = Arc::new(CapturingProvider::new());
        let summarizer = LlmSummarizer::new(provider.clone(), "eval-model", 2048);
        let messages: Vec<LlmMessage> = (0..vector.rounds.len())
            .flat_map(|r| round_messages(&vector, r))
            .collect();
        let _ = summarizer.summarize("", &messages).await;
        let rendered = provider.seen_user_text.lock().unwrap().join("\n");
        for shadowed in &vector.clip_shadowed {
            // The identifier IS in the seeded transcript…
            assert!(
                is_retained(&seeded_text(&vector), &shadowed.text),
                "[{}] clip_shadowed {:?} missing from the seeded transcript",
                vector.family,
                shadowed.text,
            );
            // …but the clip eats it before the model's input.
            assert!(
                !is_retained(&rendered, &shadowed.text),
                "[{}] {:?} ({}) survived the render clip — the standing miss \
                 has closed; promote it into the vector's `required` set",
                vector.family,
                shadowed.text,
                shadowed.location,
            );
        }
    }
    assert!(
        saw_shadowed_family,
        "the tool-noise vector pins the clip boundary"
    );
}

// ---------------------------------------------------------------------------
// Tier: model-backed recall (env-gated; skipped in CI)
// ---------------------------------------------------------------------------

/// Floors for the model-backed tier, per family: the minimum number of
/// `required` identifiers that must appear verbatim in the final anchor
/// after K = 3 successive anchored compactions.
///
/// Reference model: the local reference model configured via
/// `POLYCHROME_OLLAMA_BASE_URL` / `POLYCHROME_OLLAMA_MODEL` (a 12B-class
/// open-weight model, temperature 0.2 as shipped) — deliberately a SMALL
/// summarization-grade model, matching the production note that
/// summarization may run on a cheaper model than turns. A larger hosted
/// model should clear these floors with room; a regression below them means
/// the prompt or the pipeline got worse, not the model.
///
/// STANDING NOTE (explicit, not a silent cap): only the identifier-dense
/// family has a measured baseline so far — 12/12 survived on the reference
/// model (single run, 2026-07-16); its floor leaves one grace slot for
/// sampling jitter. The contradiction-heavy and tool-noise-heavy floors are
/// PROVISIONAL pending a first measured run on the reference model; they
/// are set conservatively (two grace slots) and must be tightened to the
/// measurement once one lands, per the retrieval-eval practice of measured
/// floors, never aspirations.
fn family_floor(family: &str, required: usize) -> usize {
    match family {
        // Measured: 12/12 on the reference model.
        "identifier-dense" => required - 1,
        // PROVISIONAL — supersession costs attention; unmeasured.
        "contradiction-heavy" => required - 2,
        // PROVISIONAL — identifiers inside clipped machine payloads are the
        // hardest read; unmeasured.
        "tool-noise-heavy" => required - 2,
        other => panic!("no floor recorded for family {other}"),
    }
}

/// The same golden vectors, driven through the REAL `LlmSummarizer` against a
/// live chat-completions-compatible endpoint, K = 3 successive anchored
/// compactions (each round's summary is the next round's prior anchor).
/// Prints the full per-identifier report; asserts the per-family floors
/// above. NOT run in CI — see the module docs for exactly what CI does
/// assert instead.
#[tokio::test]
#[ignore = "requires a live chat-completions-compatible endpoint (set \
            POLYCHROME_OLLAMA_BASE_URL / POLYCHROME_OLLAMA_MODEL; run with \
            cargo test -- --ignored)"]
async fn model_backed_recall_meets_family_floors() {
    use polyc_llm_openai::{OpenAiConfig, OpenAiProvider};

    let base_url = std::env::var("POLYCHROME_OLLAMA_BASE_URL")
        .unwrap_or_else(|_| "http://localhost:11434/v1".to_owned());
    let model =
        std::env::var("POLYCHROME_OLLAMA_MODEL").unwrap_or_else(|_| "gemma4:12b-mlx".to_owned());
    let provider = Arc::new(OpenAiProvider::new(OpenAiConfig {
        base_url,
        api_key: None,
        default_model: model.clone(),
        web_search_forced: false,
    }));
    let summarizer = LlmSummarizer::new(provider, model.clone(), 2048);

    let mut failures = Vec::new();
    for vector in vectors() {
        let mut anchor = String::new();
        for round in 0..vector.rounds.len() {
            let messages = round_messages(&vector, round);
            let next = summarizer.summarize(&anchor, &messages).await;
            assert!(
                next != anchor || anchor.is_empty(),
                "[{}] round {round}: summarizer failed soft (provider error?) — \
                 anchor unchanged",
                vector.family,
            );
            anchor = next;
            println!(
                "[{}] round {round}: anchor {} bytes",
                vector.family,
                anchor.len()
            );
        }
        let mut survived = 0usize;
        println!("\n== {} (model {model}) ==", vector.family);
        for req in &vector.required {
            let ok = is_retained(&anchor, &req.text);
            if ok {
                survived += 1;
            }
            println!(
                "  [{}] {:?} (class {}, round {})",
                if ok { "KEPT" } else { "LOST" },
                req.text,
                req.class,
                req.round,
            );
        }
        // Supersession report (not gated): a good summary demotes or drops
        // replaced values; keeping them verbatim is not wrong, only noisy.
        for old in &vector.superseded {
            println!(
                "  [superseded {}] {:?}",
                if is_retained(&anchor, old) {
                    "kept"
                } else {
                    "dropped"
                },
                old,
            );
        }
        let floor = family_floor(&vector.family, vector.required.len());
        println!(
            "  {}: {survived}/{} survived (floor {floor})",
            vector.family,
            vector.required.len(),
        );
        if survived < floor {
            failures.push(format!(
                "{}: {survived}/{} survived, floor {floor}",
                vector.family,
                vector.required.len(),
            ));
        }
    }
    assert!(
        failures.is_empty(),
        "model-backed recall regressed below its measured floors:\n  {}",
        failures.join("\n  "),
    );
}