lean_ctx/core/
marginal_gate.rs

1//! Marginal Information Gate (#1308).
2//!
3//! Checks whether a tool response provides enough new information to justify
4//! delivery to the model. Based on COMI's MIG principle (arXiv 2602.01719):
5//! content should only be delivered if its information gain relative to what's
6//! already in context exceeds a threshold.
7
8use std::collections::HashSet;
9
10/// MIG decision.
11#[derive(Debug, Clone, PartialEq, Eq)]
12pub enum GateDecision {
13    /// Content provides enough new information — deliver it.
14    Pass,
15    /// Content is mostly redundant — suppress and return a stub.
16    Suppress {
17        novelty_ratio: u8,
18        reason: &'static str,
19    },
20}
21
22/// Configuration for the marginal information gate.
23#[derive(Debug, Clone)]
24pub struct GateConfig {
25    /// Minimum fraction of novel tokens required to pass. Range 0.0–1.0.
26    pub novelty_threshold: f64,
27    /// Minimum absolute novel tokens to pass regardless of ratio.
28    pub min_novel_tokens: usize,
29}
30
31impl Default for GateConfig {
32    fn default() -> Self {
33        Self {
34            novelty_threshold: 0.20,
35            min_novel_tokens: 50,
36        }
37    }
38}
39
40/// Check whether `response` provides sufficient new information
41/// given `already_delivered` content chunks.
42///
43/// Uses line-level deduplication: a response line that appears verbatim
44/// in any previously delivered chunk is considered redundant.
45pub fn check_information_gain(
46    response: &str,
47    already_delivered: &[&str],
48    config: &GateConfig,
49) -> GateDecision {
50    if already_delivered.is_empty() || response.is_empty() {
51        return GateDecision::Pass;
52    }
53
54    let delivered_lines: HashSet<&str> = already_delivered
55        .iter()
56        .flat_map(|chunk| chunk.lines())
57        .map(str::trim)
58        .filter(|l| !l.is_empty())
59        .collect();
60
61    let response_lines: Vec<&str> = response
62        .lines()
63        .map(str::trim)
64        .filter(|l| !l.is_empty())
65        .collect();
66
67    if response_lines.is_empty() {
68        return GateDecision::Pass;
69    }
70
71    let novel_count = response_lines
72        .iter()
73        .filter(|line| !delivered_lines.contains(*line))
74        .count();
75
76    let novelty_ratio = novel_count as f64 / response_lines.len() as f64;
77
78    if novel_count >= config.min_novel_tokens || novelty_ratio >= config.novelty_threshold {
79        return GateDecision::Pass;
80    }
81
82    let ratio_pct = (novelty_ratio * 100.0) as u8;
83    GateDecision::Suppress {
84        novelty_ratio: ratio_pct,
85        reason: "content mostly redundant with previously delivered context",
86    }
87}
88
89/// Format a suppression stub when the gate blocks delivery.
90pub fn suppression_stub(path: &str, total_lines: usize, novelty_pct: u8) -> String {
91    format!(
92        "[MIG: {path} ({total_lines} lines) — {novelty_pct}% novel, below threshold. \
93         Use ctx_read with lines= for specific sections.]"
94    )
95}
96
97#[cfg(test)]
98mod tests {
99    use super::*;
100
101    #[test]
102    fn pass_when_no_prior_delivery() {
103        let decision = check_information_gain("fn main() {}", &[], &GateConfig::default());
104        assert_eq!(decision, GateDecision::Pass);
105    }
106
107    #[test]
108    fn pass_when_content_is_novel() {
109        let prior = "fn alpha() { 1 }\nfn beta() { 2 }";
110        let response = "fn gamma() { 3 }\nfn delta() { 4 }\nfn epsilon() { 5 }";
111        let decision = check_information_gain(response, &[prior], &GateConfig::default());
112        assert_eq!(decision, GateDecision::Pass);
113    }
114
115    #[test]
116    fn suppress_when_mostly_redundant() {
117        let prior =
118            "line 1\nline 2\nline 3\nline 4\nline 5\nline 6\nline 7\nline 8\nline 9\nline 10";
119        let response =
120            "line 1\nline 2\nline 3\nline 4\nline 5\nline 6\nline 7\nline 8\nline 9\nline 10";
121        let config = GateConfig {
122            novelty_threshold: 0.20,
123            min_novel_tokens: 50,
124        };
125        let decision = check_information_gain(response, &[prior], &config);
126        match decision {
127            GateDecision::Suppress { novelty_ratio, .. } => {
128                assert_eq!(novelty_ratio, 0, "0% novel");
129            }
130            GateDecision::Pass => panic!("should have been suppressed"),
131        }
132    }
133
134    #[test]
135    fn pass_when_above_min_novel_tokens() {
136        let prior = "old line";
137        let novel_lines: Vec<String> = (0..60).map(|i| format!("new line {i}")).collect();
138        let response = novel_lines.join("\n");
139        let config = GateConfig {
140            novelty_threshold: 0.99,
141            min_novel_tokens: 50,
142        };
143        let decision = check_information_gain(&response, &[prior], &config);
144        assert_eq!(decision, GateDecision::Pass);
145    }
146
147    #[test]
148    fn suppression_stub_format() {
149        let stub = suppression_stub("src/db.py", 850, 5);
150        assert!(stub.contains("src/db.py"));
151        assert!(stub.contains("5% novel"));
152        assert!(stub.contains("below threshold"));
153    }
154}
lean_ctx/core/marginal_gate.rs

lean_ctx/core/
marginal_gate.rs
