car-memgine 0.48.0

//! Conversation compaction — embedding-aware, graph-informed progressive summarization.
//!
//! Instead of dumb fixed-size batches, compaction uses:
//! 1. **Embeddings** — cluster turns by semantic similarity (same-topic turns summarized together)
//! 2. **Importance scoring** — turns with decisions, graph edges, or high relevance stay verbatim longer
//! 3. **Redundancy detection** — turns whose content is already captured in fact nodes can be dropped
//! 4. **Inference summarization** — when available, uses the model; falls back to graph-informed heuristics

use crate::graph::MemNode;
use car_ir::json_extract::extract_json_object;
use car_ir::linalg::cosine_similarity;
use petgraph::stable_graph::NodeIndex;
use serde::{Deserialize, Serialize};

/// Result of compacting a batch of conversation turns.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CompactionResult {
    pub summary: String,
    pub key_facts: Vec<String>,
}

/// Report from a conversation compaction pass.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct ConversationCompactionReport {
    pub turns_summarized: usize,
    pub summaries_created: usize,
    pub facts_extracted: usize,
    pub summaries_promoted: usize,
    pub redundant_dropped: usize,
    /// Extracted facts refused because the text they were derived from does not
    /// support them (car#703). A non-zero value means the summarizer invented
    /// content that would otherwise have become durable memory.
    #[serde(default)]
    pub facts_rejected_ungrounded: usize,
    pub tokens_before: usize,
    pub tokens_after: usize,
}

impl ConversationCompactionReport {
    /// Emit a structured telemetry event for this compaction pass.
    /// Only emits if compaction actually did something (turns_summarized > 0 or redundant_dropped > 0).
    pub fn emit_telemetry(&self, layer: &str) {
        if self.turns_summarized == 0
            && self.redundant_dropped == 0
            && self.facts_rejected_ungrounded == 0
        {
            return;
        }
        let compression_ratio = if self.tokens_before > 0 {
            self.tokens_after as f64 / self.tokens_before as f64
        } else {
            1.0
        };
        tracing::info!(
            layer = %layer,
            tokens_before = self.tokens_before,
            tokens_after = self.tokens_after,
            turns_summarized = self.turns_summarized,
            summaries_created = self.summaries_created,
            facts_extracted = self.facts_extracted,
            summaries_promoted = self.summaries_promoted,
            redundant_dropped = self.redundant_dropped,
            facts_rejected_ungrounded = self.facts_rejected_ungrounded,
            compression_ratio = format_args!("{:.2}", compression_ratio),
            "conversation compaction completed"
        );
    }
}

/// Importance score for a conversation turn (higher = keep longer).
#[derive(Debug, Clone)]
pub struct TurnImportance {
    pub nix: NodeIndex,
    pub score: f32,
    pub redundant: bool,
}

// --- Importance scoring ---

const DECISION_KEYWORDS: &[&str] = &[
    "decided",
    "agreed",
    "will",
    "should",
    "must",
    "need to",
    "budget",
    "deadline",
    "plan",
    "commit",
    "approve",
    "reject",
    "choose",
    "selected",
    "confirmed",
    "assigned",
    "action item",
];

const STRONG_DECISION_KEYWORDS: &[&str] = &[
    "decided",
    "agreed",
    "confirmed",
    "approved",
    "selected",
    "assigned",
];

/// Score a conversation turn's importance based on content signals.
/// Returns 0.0–1.0 (higher = more important to keep).
pub fn content_importance(text: &str) -> f32 {
    let lower = text.to_lowercase();
    let mut score: f32 = 0.0;

    // Decision language
    let decision_hits = DECISION_KEYWORDS
        .iter()
        .filter(|kw| lower.contains(*kw))
        .count();
    score += (decision_hits as f32 * 0.15).min(0.5);

    // Questions (may contain unanswered queries)
    if lower.contains('?') {
        score += 0.1;
    }

    // Numbers, dates, proper nouns (factual content)
    let has_numbers = text.chars().any(|c| c.is_ascii_digit());
    if has_numbers {
        score += 0.1;
    }

    // Length bonus (longer turns tend to carry more info)
    let word_count = text.split_whitespace().count();
    if word_count > 20 {
        score += 0.1;
    }

    score.min(1.0)
}

/// Score importance using graph signals: how connected is this turn?
pub fn graph_importance(nix: NodeIndex, graph: &crate::graph::MemoryGraph) -> f32 {
    let edges = graph.inner.edges(nix).count()
        + graph
            .inner
            .edges_directed(nix, petgraph::Direction::Incoming)
            .count();
    // Subtract the 1-2 TemporalNext edges that every conv node has
    let non_temporal = edges.saturating_sub(2);
    (non_temporal as f32 * 0.2).min(0.5)
}

/// Check if a turn's content is already captured in existing fact nodes.
/// Uses embedding similarity: if a turn is >0.85 similar to any fact, it's redundant.
/// Contrast, negation and decision markers. A turn carrying one of these is
/// almost always saying something a flat fact does not: what was rejected, what
/// does *not* hold, or why one option beat another.
const CONTRAST_MARKERS: &[&str] = &[
    "not", "no", "never", "except", "instead", "rather", "but", "however", "unless", "without",
    "chose", "chosen", "choosing", "decided", "rejected", "declined", "because", "since", "over",
    "vs", "versus", "although", "though", "despite", "avoid", "avoided", "stopped", "reverted",
];

/// Capitalized words that carry no information — sentence-initial articles and
/// pronouns. Without this list every sentence's first word reads as a proper
/// noun, so "The project uses Postgres" and "project uses Postgres" would
/// disagree on `the` and nothing would ever be droppable or grounded.
const CAPITALIZED_STOPWORDS: &[&str] = &[
    "the", "a", "an", "we", "i", "it", "this", "that", "these", "those", "there", "they", "he",
    "she", "you", "our", "my", "its", "their", "in", "on", "at", "to", "for", "of", "and", "or",
    "is", "was", "were", "are", "be", "been", "has", "have", "had", "will", "would", "can",
    "could", "should", "if", "when", "while", "as", "so", "then", "also", "with", "by", "from",
];

/// Entity-like terms: numbers and proper-noun/identifier tokens.
///
/// Deliberately narrow. It is *not* every content word — requiring full lexical
/// coverage would make almost nothing droppable, which would push every
/// compaction into the expensive summarize path and defeat the cheap trim
/// stage. Normalized to lowercase and stripped of surrounding punctuation, so
/// `"MySQL,"` and `mysql` compare equal.
pub fn entity_terms(text: &str) -> std::collections::BTreeSet<String> {
    text.split_whitespace()
        .filter_map(|raw| {
            let token = raw.trim_matches(|c: char| !c.is_alphanumeric());
            if token.is_empty() {
                return None;
            }
            let lower = token.to_lowercase();
            let has_digit = token.chars().any(|c| c.is_ascii_digit());
            // An interior capital (MySQL, PostgreSQL, camelCase) is a proper
            // noun or identifier regardless of position, so it is never a
            // stopword; a leading capital only counts if the word itself is not
            // a stopword.
            let interior_capital = token.chars().skip(1).any(|c| c.is_uppercase());
            let leading_capital = token.chars().next().is_some_and(|c| c.is_uppercase())
                && !CAPITALIZED_STOPWORDS.contains(&lower.as_str());
            (has_digit || interior_capital || leading_capital).then_some(lower)
        })
        .collect()
}

/// The information-bearing terms a turn carries: entities plus contrast and
/// decision markers.
///
/// Markers matter for *coverage* — a turn saying what was rejected or what does
/// not hold says something a flat fact does not — but deliberately not for
/// grounding, where a summarizer legitimately rewrites "chose" as "chosen" and
/// a marker mismatch would be a false hallucination report.
pub fn salient_terms(text: &str) -> std::collections::BTreeSet<String> {
    let mut terms = entity_terms(text);
    terms.extend(text.split_whitespace().filter_map(|raw| {
        let token = raw
            .trim_matches(|c: char| !c.is_alphanumeric())
            .to_lowercase();
        CONTRAST_MARKERS.contains(&token.as_str()).then_some(token)
    }));
    terms
}

/// Salient terms in `text` that `covering` does not carry.
pub fn uncovered_terms(text: &str, covering: &str) -> Vec<String> {
    let have = salient_terms(covering);
    salient_terms(text)
        .into_iter()
        .filter(|t| !have.contains(t))
        .collect()
}

/// Whether a turn is safely droppable given a semantically similar fact.
///
/// Cosine similarity measures **topical similarity, not coverage**, and the
/// redundancy drop treated the two as the same thing. "We chose Postgres over
/// MySQL because of write throughput" scores high against the fact "project uses
/// Postgres" — which contains neither the reason nor the rejected alternative —
/// and the turn was deleted anyway (car#703).
///
/// So similarity is necessary but not sufficient: the turn must also introduce
/// no salient term the fact lacks. This is deterministic and cheap, which
/// matters because this path runs on **every** turn via
/// `compact_conversation_heuristic`.
pub fn is_covered_by(turn_text: &str, fact_text: &str) -> bool {
    uncovered_terms(turn_text, fact_text).is_empty()
}

pub fn is_redundant(
    turn_embedding: Option<&[f32]>,
    fact_embeddings: &[&[f32]],
    threshold: f32,
) -> bool {
    let turn_emb = match turn_embedding {
        Some(e) if !e.is_empty() => e,
        _ => return false, // can't determine redundancy without embeddings
    };
    fact_embeddings
        .iter()
        .any(|fact_emb| cosine_similarity(turn_emb, fact_emb) > threshold)
}

/// Redundancy with a coverage guard: a turn is droppable only when some fact is
/// both semantically similar to it **and** carries every salient term it does.
///
/// Returns the index of the covering fact, or `None` when the turn must be
/// kept. Facts are supplied as `(text, embedding)` so the check can look at
/// content, which is precisely what the embedding-only version could not do.
pub fn redundant_against(
    turn_text: &str,
    turn_embedding: Option<&[f32]>,
    facts: &[(&str, Option<&[f32]>)],
    threshold: f32,
) -> Option<usize> {
    let turn_emb = match turn_embedding {
        Some(e) if !e.is_empty() => e,
        // Without an embedding we cannot establish similarity, and a turn is
        // never dropped on coverage alone — losing it would be unrecoverable.
        _ => return None,
    };
    facts.iter().position(|(fact_text, fact_emb)| {
        let similar = fact_emb
            .filter(|e| !e.is_empty())
            .is_some_and(|e| cosine_similarity(turn_emb, e) > threshold);
        similar && is_covered_by(turn_text, fact_text)
    })
}

/// Key facts from a compaction result that are **not** grounded in the text
/// they were extracted from.
///
/// `result.key_facts` comes from parsing a model response and is ingested as a
/// durable fact node with authority `"derived"` — so a hallucination here is not
/// a transient context error, it is retrieved as evidence indefinitely
/// (car#703). Grounding is checked the same deterministic way as coverage: an
/// extracted fact must not introduce salient terms absent from every source.
///
/// Returns `(fact, uncovered_terms)` pairs, so a caller can report *why* a fact
/// was rejected rather than silently dropping it.
pub fn ungrounded_facts(key_facts: &[String], sources: &[&str]) -> Vec<(String, Vec<String>)> {
    let corpus = sources.join(" ");
    let have = entity_terms(&corpus);
    key_facts
        .iter()
        .filter_map(|fact| {
            let missing: Vec<String> = entity_terms(fact)
                .into_iter()
                .filter(|t| !have.contains(t))
                .collect();
            (!missing.is_empty()).then(|| (fact.clone(), missing))
        })
        .collect()
}

// --- Semantic clustering ---

/// Cluster conversation turns by semantic similarity using embeddings.
/// Returns groups of node indices where each group shares a topic.
/// Falls back to sequential chunking if embeddings aren't available.
pub fn cluster_by_topic(
    turns: &[(NodeIndex, Option<&[f32]>)],
    similarity_threshold: f32,
    max_cluster_size: usize,
) -> Vec<Vec<NodeIndex>> {
    // If no embeddings, fall back to sequential chunks
    let has_embeddings = turns.iter().any(|(_, emb)| emb.is_some());
    if !has_embeddings {
        return turns
            .chunks(max_cluster_size)
            .map(|chunk| chunk.iter().map(|(nix, _)| *nix).collect())
            .collect();
    }

    // Greedy single-linkage clustering: assign each turn to the first cluster
    // whose centroid it's similar enough to, or start a new cluster
    let mut clusters: Vec<(Vec<f32>, Vec<NodeIndex>)> = Vec::new(); // (centroid, members)

    for &(nix, emb_opt) in turns {
        let emb = match emb_opt {
            Some(e) if !e.is_empty() => e,
            _ => {
                // No embedding — append to last cluster or start new one
                if let Some(last) = clusters.last_mut() {
                    if last.1.len() < max_cluster_size {
                        last.1.push(nix);
                        continue;
                    }
                }
                clusters.push((Vec::new(), vec![nix]));
                continue;
            }
        };

        let mut best_cluster = None;
        let mut best_sim = similarity_threshold;

        for (i, (centroid, members)) in clusters.iter().enumerate() {
            if members.len() >= max_cluster_size {
                continue;
            }
            if centroid.is_empty() {
                continue;
            }
            let sim = cosine_similarity(emb, centroid);
            if sim > best_sim {
                best_sim = sim;
                best_cluster = Some(i);
            }
        }

        if let Some(idx) = best_cluster {
            // Update centroid as running average
            let (centroid, members) = &mut clusters[idx];
            let n = members.len() as f32;
            for (i, &val) in emb.iter().enumerate() {
                if i < centroid.len() {
                    centroid[i] = (centroid[i] * n + val) / (n + 1.0);
                }
            }
            members.push(nix);
        } else {
            // New cluster
            clusters.push((emb.to_vec(), vec![nix]));
        }
    }

    clusters.into_iter().map(|(_, members)| members).collect()
}

// --- Prompts and parsing ---

/// Build a summarization prompt for a batch of conversation turns.
pub fn summarize_conversation_prompt(turns: &[&MemNode]) -> String {
    let turn_text: Vec<String> = turns.iter().map(|n| n.value.clone()).collect();

    format!(
        r#"Summarize the following conversation turns into a concise paragraph that preserves:
1. Key decisions made
2. Important facts stated
3. Action items or commitments
4. Questions that remain open

## Conversation
{turns}

Respond with ONLY a JSON object:
```json
{{
  "summary": "Concise paragraph summary preserving key information",
  "key_facts": ["fact1", "fact2"]
}}
```"#,
        turns = turn_text.join("\n"),
    )
}

/// Build a key-fact extraction prompt for already-summarized content.
pub fn extract_key_facts_prompt(summaries: &[&str]) -> String {
    format!(
        r#"Extract only the essential, atomic facts from these conversation summaries.
Each fact should be a single, self-contained statement.

## Summaries
{summaries}

Respond with ONLY a JSON object:
```json
{{
  "summary": "One-sentence combined summary",
  "key_facts": ["atomic fact 1", "atomic fact 2"]
}}
```"#,
        summaries = summaries.join("\n---\n"),
    )
}

/// Parse an inference response into a CompactionResult.
pub fn parse_compaction_result(response: &str) -> CompactionResult {
    if let Some(json_str) = extract_json_object(response) {
        if let Ok(result) = serde_json::from_str::<CompactionResult>(&json_str) {
            return result;
        }
    }
    CompactionResult {
        summary: response.trim().to_string(),
        key_facts: Vec::new(),
    }
}

// --- Heuristic fallbacks ---

/// Heuristic summarization — graph-informed, no inference needed.
///
/// Strategy: score each turn by importance, keep high-importance sentences,
/// summarize the rest with markers.
pub fn heuristic_summarize(turns: &[&MemNode]) -> CompactionResult {
    if turns.is_empty() {
        return CompactionResult {
            summary: String::new(),
            key_facts: Vec::new(),
        };
    }
    if turns.len() <= 2 {
        let summary = turns
            .iter()
            .map(|n| n.value.as_str())
            .collect::<Vec<_>>()
            .join(" ");
        return CompactionResult {
            summary,
            key_facts: Vec::new(),
        };
    }

    let first = &turns[0].value;
    let last = &turns[turns.len() - 1].value;
    let middle = &turns[1..turns.len() - 1];

    let mut key_sentences: Vec<String> = Vec::new();
    let mut key_facts: Vec<String> = Vec::new();

    for node in middle {
        for sentence in node.value.split(['.', '!', '?']) {
            let s = sentence.trim();
            if s.is_empty() {
                continue;
            }
            let lower = s.to_lowercase();
            if DECISION_KEYWORDS.iter().any(|kw| lower.contains(kw)) {
                key_sentences.push(format!("{}.", s));
                if STRONG_DECISION_KEYWORDS.iter().any(|kw| lower.contains(kw)) {
                    key_facts.push(format!("{}.", s));
                }
            }
        }
    }

    let summary = if key_sentences.is_empty() {
        format!(
            "{} [...{} turns summarized...] {}",
            first,
            middle.len(),
            last
        )
    } else {
        format!(
            "{} [...{} turns summarized: {}...] {}",
            first,
            middle.len(),
            key_sentences.join(" "),
            last,
        )
    };

    CompactionResult { summary, key_facts }
}

/// Heuristic key-fact extraction from summary text.
pub fn heuristic_extract_facts(summary_text: &str) -> Vec<String> {
    let mut facts = Vec::new();
    let fact_indicators = [
        "decided",
        "agreed",
        "confirmed",
        "is",
        "are",
        "was",
        "were",
        "must",
        "will",
        "should",
        "need",
    ];

    for sentence in summary_text.split(['.', '!', '?']) {
        let s = sentence.trim();
        if s.len() < 10 {
            continue;
        }
        let lower = s.to_lowercase();
        if fact_indicators.iter().any(|kw| lower.contains(kw)) && !lower.starts_with("[...") {
            facts.push(format!("{}.", s));
        }
    }
    facts
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::graph::{ContentType, FactMetadata, MemKind};
    use chrono::Utc;

    fn conv_node(text: &str) -> MemNode {
        MemNode {
            kind: MemKind::Conversation,
            layer: 3,
            key: "user".to_string(),
            value: text.to_string(),
            fact_id: None,
            scope: "global".to_string(),
            authority: "peer".to_string(),
            is_constraint: false,
            created_at: Utc::now(),
            expires_at: None,
            content_type: ContentType::NaturalLanguage,
            metadata: FactMetadata::default(),
        }
    }

    #[test]
    fn content_importance_scores_decisions() {
        assert!(content_importance("We decided to use PostgreSQL") > 0.0);
        assert!(content_importance("hello world") == 0.0);
        assert!(
            content_importance("We agreed on the budget of $500K")
                > content_importance("Ok sounds good")
        );
    }

    #[test]
    fn redundancy_detection() {
        // Identical normalized vectors → redundant
        let turn = [0.6f32, 0.8];
        let fact = [0.6f32, 0.8];
        assert!(is_redundant(Some(&turn), &[&fact], 0.85));

        // Orthogonal → not redundant
        let turn2 = [1.0f32, 0.0];
        let fact2 = [0.0f32, 1.0];
        assert!(!is_redundant(Some(&turn2), &[&fact2], 0.85));

        // No embedding → not redundant
        assert!(!is_redundant(None, &[&fact], 0.85));
    }

    #[test]
    fn cluster_without_embeddings_falls_back_to_chunks() {
        let turns: Vec<(NodeIndex, Option<&[f32]>)> =
            (0..10).map(|i| (NodeIndex::new(i), None)).collect();
        let clusters = cluster_by_topic(&turns, 0.7, 4);
        assert_eq!(clusters.len(), 3); // 10/4 = 3 chunks (4,4,2)
        assert_eq!(clusters[0].len(), 4);
        assert_eq!(clusters[1].len(), 4);
        assert_eq!(clusters[2].len(), 2);
    }

    #[test]
    fn cluster_with_embeddings_groups_similar() {
        // Two clusters: similar vectors grouped together
        let emb_a1 = [1.0f32, 0.0, 0.0];
        let emb_a2 = [0.95, 0.05, 0.0];
        let emb_b1 = [0.0, 1.0, 0.0];
        let emb_b2 = [0.0, 0.95, 0.05];

        let turns = vec![
            (NodeIndex::new(0), Some(emb_a1.as_slice())),
            (NodeIndex::new(1), Some(emb_b1.as_slice())),
            (NodeIndex::new(2), Some(emb_a2.as_slice())),
            (NodeIndex::new(3), Some(emb_b2.as_slice())),
        ];
        let clusters = cluster_by_topic(&turns, 0.7, 4);
        // Should form 2 clusters based on similarity
        assert_eq!(clusters.len(), 2);
    }

    #[test]
    fn heuristic_summarize_basic() {
        let nodes: Vec<MemNode> = (0..6).map(|i| conv_node(&format!("Turn {}", i))).collect();
        let refs: Vec<&MemNode> = nodes.iter().collect();
        let result = heuristic_summarize(&refs);
        assert!(!result.summary.is_empty());
        assert!(result.summary.contains("Turn 0"));
        assert!(result.summary.contains("Turn 5"));
        assert!(result.summary.contains("summarized"));
    }

    #[test]
    fn heuristic_summarize_preserves_decisions() {
        let nodes = [
            conv_node("user: Let's discuss the database."),
            conv_node("user: We decided to use PostgreSQL."),
            conv_node("assistant: I agreed. PostgreSQL it is."),
            conv_node("user: Now let's move on."),
        ];
        let refs: Vec<&MemNode> = nodes.iter().collect();
        let result = heuristic_summarize(&refs);
        assert!(result.summary.contains("decided") || result.summary.contains("PostgreSQL"));
        assert!(!result.key_facts.is_empty());
    }

    #[test]
    fn heuristic_summarize_short_input() {
        let nodes = [conv_node("Hello"), conv_node("World")];
        let refs: Vec<&MemNode> = nodes.iter().collect();
        let result = heuristic_summarize(&refs);
        assert!(result.summary.contains("Hello"));
        assert!(result.summary.contains("World"));
    }

    #[test]
    fn heuristic_extract_facts_basic() {
        let text = "We decided to use Rust. The deadline is Friday. Hello world.";
        let facts = heuristic_extract_facts(text);
        assert!(facts.iter().any(|f| f.contains("decided")));
        assert!(facts.iter().any(|f| f.contains("deadline")));
    }

    #[test]
    fn parse_compaction_result_json() {
        let response = r#"```json
{"summary": "They chose PostgreSQL.", "key_facts": ["Database is PostgreSQL"]}
```"#;
        let result = parse_compaction_result(response);
        assert_eq!(result.summary, "They chose PostgreSQL.");
        assert_eq!(result.key_facts, vec!["Database is PostgreSQL"]);
    }

    #[test]
    fn parse_compaction_result_fallback() {
        let response = "Just a plain text summary.";
        let result = parse_compaction_result(response);
        assert_eq!(result.summary, "Just a plain text summary.");
        assert!(result.key_facts.is_empty());
    }

    #[test]
    fn summarize_prompt_includes_all_turns() {
        let nodes = [conv_node("Turn A"), conv_node("Turn B")];
        let refs: Vec<&MemNode> = nodes.iter().collect();
        let prompt = summarize_conversation_prompt(&refs);
        assert!(prompt.contains("Turn A"));
        assert!(prompt.contains("Turn B"));
        assert!(prompt.contains("JSON"));
    }
}

#[cfg(test)]
mod fidelity_tests {
    use super::*;

    /// A unit vector pair with cosine ≈ 1.0, so similarity is never what
    /// decides these tests — coverage is.
    const SAME: [f32; 3] = [1.0, 0.0, 0.0];

    /// The worked example from car#703. Cosine puts this turn and this fact on
    /// the same topic, and they *are* on the same topic — but the fact carries
    /// neither the rejected alternative nor the reason, so deleting the turn
    /// loses both, permanently.
    #[test]
    fn a_decision_turn_is_not_covered_by_the_flat_fact_it_resembles() {
        let turn = "We chose Postgres over MySQL because of write throughput";
        let fact = "project uses Postgres";

        let missing = uncovered_terms(turn, fact);
        assert!(
            missing.contains(&"mysql".to_string()),
            "the rejected alternative must be missing: {missing:?}"
        );
        assert!(!is_covered_by(turn, fact));
        assert_eq!(
            redundant_against(turn, Some(&SAME), &[(fact, Some(&SAME[..]))], 0.85),
            None,
            "high similarity must not be enough to delete a turn the fact does not cover"
        );
    }

    /// The other half: a turn that genuinely says nothing the fact does not is
    /// still droppable, or the cheap trim stage stops working and every
    /// compaction falls through to the expensive summarize path.
    #[test]
    fn a_turn_the_fact_fully_covers_is_still_dropped() {
        let turn = "The project uses Postgres";
        let fact = "project uses Postgres for storage";
        assert!(is_covered_by(turn, fact));
        assert_eq!(
            redundant_against(turn, Some(&SAME), &[(fact, Some(&SAME[..]))], 0.85),
            Some(0)
        );
    }

    /// Coverage alone must never delete a turn — without an embedding there is
    /// no evidence the two are even about the same thing.
    #[test]
    fn coverage_without_similarity_never_drops_a_turn() {
        let turn = "uses Postgres";
        let fact = "the project uses Postgres and MySQL and everything else";
        assert!(is_covered_by(turn, fact));
        assert_eq!(
            redundant_against(turn, None, &[(fact, Some(&SAME[..]))], 0.85),
            None
        );
        assert_eq!(
            redundant_against(turn, Some(&SAME), &[(fact, None)], 0.85),
            None
        );
    }

    /// A dissimilar fact must not cover a turn even if it happens to contain
    /// the same salient tokens.
    #[test]
    fn low_similarity_blocks_the_drop() {
        let orthogonal: [f32; 3] = [0.0, 1.0, 0.0];
        let turn = "uses Postgres";
        let fact = "uses Postgres";
        assert_eq!(
            redundant_against(turn, Some(&SAME), &[(fact, Some(&orthogonal[..]))], 0.85),
            None
        );
    }

    #[test]
    fn negation_is_salient_so_a_contradicting_turn_survives() {
        let turn = "We are not using Redis";
        let fact = "the stack includes Redis";
        assert!(
            !is_covered_by(turn, fact),
            "dropping this would invert the meaning"
        );
    }

    #[test]
    fn numbers_are_salient() {
        assert!(!is_covered_by("the budget is 500000", "there is a budget"));
        assert!(is_covered_by(
            "the budget is 500000",
            "the budget is 500000 dollars"
        ));
    }

    /// Grounding: an extracted "fact" naming something no source mentions is a
    /// hallucination, and it would otherwise be ingested as a durable node with
    /// authority "derived" and retrieved as evidence indefinitely.
    #[test]
    fn a_fact_naming_something_no_source_mentions_is_ungrounded() {
        let sources = ["We chose Postgres for the write path"];
        let facts = vec![
            "The project uses Postgres".to_string(),
            "The project uses DynamoDB".to_string(),
        ];
        let ungrounded = ungrounded_facts(&facts, &sources);
        assert_eq!(ungrounded.len(), 1, "{ungrounded:?}");
        assert_eq!(ungrounded[0].0, "The project uses DynamoDB");
        assert!(ungrounded[0].1.contains(&"dynamodb".to_string()));
    }

    #[test]
    fn a_grounded_fact_passes() {
        let sources = ["We chose Postgres over MySQL for write throughput"];
        let facts = vec!["Postgres was chosen over MySQL".to_string()];
        assert!(ungrounded_facts(&facts, &sources).is_empty());
    }

    /// Grounding is checked against the sources *together* — a fact may
    /// legitimately combine two turns.
    #[test]
    fn a_fact_combining_two_sources_is_grounded() {
        let sources = ["We chose Postgres", "MySQL was rejected"];
        let facts = vec!["Postgres was chosen and MySQL rejected".to_string()];
        assert!(ungrounded_facts(&facts, &sources).is_empty());
    }
}