lean_ctx/core/context_kernel/
dedup_wiring.rs

1//! Global content-deduplication wiring for context delivery hot paths.
2
3use std::collections::HashSet;
4use std::sync::{Mutex, MutexGuard, OnceLock};
5
6use super::context_dedup::{ContextDedup, DedupResult, format_unchanged_stub};
7
8static DEDUP: OnceLock<Mutex<ContextDedup>> = OnceLock::new();
9static SEEN_PATHS: OnceLock<Mutex<HashSet<String>>> = OnceLock::new();
10static STATS: OnceLock<Mutex<DedupStats>> = OnceLock::new();
11/// Action to take based on a content deduplication check.
12#[derive(Debug, Clone, PartialEq, Eq)]
13pub enum DedupAction {
14    /// Content is new and should be delivered in full.
15    DeliverFull,
16    /// Content is unchanged and can be replaced by a compact reference.
17    DeliverStub {
18        /// Compact reference to the content already present in context.
19        stub: String,
20    },
21    /// Content changed and should be delivered in full.
22    DeliverModified,
23}
24
25/// Cumulative content deduplication statistics for this process.
26#[derive(Debug, Clone, Copy, PartialEq, Default)]
27pub struct DedupStats {
28    /// Number of enabled deduplication checks.
29    pub total_checks: usize,
30    /// Number of checks that found unchanged content.
31    pub cache_hits: usize,
32    /// Number of checks that required full delivery.
33    pub cache_misses: usize,
34    /// Estimated tokens avoided by unchanged-content stubs.
35    pub tokens_saved: usize,
36    /// Fraction of enabled checks that found unchanged content.
37    pub hit_rate: f64,
38}
39
40/// Checks whether `content` changed since its last delivery at `path`.
41#[must_use]
42pub fn check_content(path: &str, content: &str) -> DedupAction {
43    check_content_enabled(
44        super::kernel_config::features().content_dedup,
45        path,
46        content,
47    )
48}
49
50/// Returns a snapshot of cumulative content deduplication statistics.
51#[must_use]
52pub fn dedup_stats() -> DedupStats {
53    let mut snapshot = *lock(stats());
54    snapshot.hit_rate = if snapshot.total_checks == 0 {
55        0.0
56    } else {
57        snapshot.cache_hits as f64 / snapshot.total_checks as f64
58    };
59    snapshot
60}
61
62/// Applies content deduplication, returning either full content or a stub.
63#[must_use]
64pub fn apply_dedup(path: &str, content: &str) -> String {
65    apply_dedup_enabled(
66        super::kernel_config::features().content_dedup,
67        path,
68        content,
69    )
70}
71
72fn apply_dedup_enabled(enabled: bool, path: &str, content: &str) -> String {
73    match check_content_enabled(enabled, path, content) {
74        DedupAction::DeliverStub { stub } => stub,
75        DedupAction::DeliverFull | DedupAction::DeliverModified => content.to_owned(),
76    }
77}
78
79/// Invalidates cached content for `path` after a write or external change.
80pub fn invalidate(path: &str) {
81    lock(dedup()).invalidate(path);
82    lock(seen_paths()).remove(path);
83}
84
85/// Clears cached content and all cumulative deduplication statistics.
86pub fn reset_dedup() {
87    lock(dedup()).clear();
88    lock(seen_paths()).clear();
89    *lock(stats()) = DedupStats::default();
90}
91
92fn check_content_enabled(enabled: bool, path: &str, content: &str) -> DedupAction {
93    if !enabled {
94        return DedupAction::DeliverFull;
95    }
96
97    let result = lock(dedup()).check_and_record(path, content);
98    match result {
99        DedupResult::Unchanged { hash, saved_tokens } => {
100            record_check(true, saved_tokens);
101            DedupAction::DeliverStub {
102                stub: format_unchanged_stub(path, &hash),
103            }
104        }
105        DedupResult::Fresh => {
106            let modified = !lock(seen_paths()).insert(path.to_owned());
107            record_check(false, 0);
108            if modified {
109                DedupAction::DeliverModified
110            } else {
111                DedupAction::DeliverFull
112            }
113        }
114    }
115}
116fn record_check(hit: bool, saved_tokens: usize) {
117    let mut current = lock(stats());
118    current.total_checks += 1;
119    if hit {
120        current.cache_hits += 1;
121        current.tokens_saved += saved_tokens;
122    } else {
123        current.cache_misses += 1;
124    }
125}
126
127fn dedup() -> &'static Mutex<ContextDedup> {
128    DEDUP.get_or_init(|| {
129        Mutex::new(ContextDedup::new(
130            super::kernel_config::features().dedup_capacity,
131        ))
132    })
133}
134
135fn seen_paths() -> &'static Mutex<HashSet<String>> {
136    SEEN_PATHS.get_or_init(|| Mutex::new(HashSet::new()))
137}
138
139fn stats() -> &'static Mutex<DedupStats> {
140    STATS.get_or_init(|| Mutex::new(DedupStats::default()))
141}
142
143fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
144    mutex
145        .lock()
146        .unwrap_or_else(std::sync::PoisonError::into_inner)
147}
148
149#[cfg(test)]
150mod tests {
151    use std::sync::{Mutex, MutexGuard};
152
153    use super::{
154        DedupAction, apply_dedup_enabled, check_content_enabled, dedup_stats, invalidate,
155        reset_dedup,
156    };
157
158    static TEST_LOCK: Mutex<()> = Mutex::new(());
159
160    fn isolated() -> MutexGuard<'static, ()> {
161        let guard = TEST_LOCK
162            .lock()
163            .unwrap_or_else(std::sync::PoisonError::into_inner);
164        reset_dedup();
165        guard
166    }
167
168    #[test]
169    fn new_content_delivers_full() {
170        let _guard = isolated();
171        assert_eq!(
172            check_content_enabled(true, "src/lib.rs", "content"),
173            DedupAction::DeliverFull
174        );
175    }
176
177    #[test]
178    fn repeated_content_delivers_stub() {
179        let _guard = isolated();
180        check_content_enabled(true, "src/lib.rs", "content");
181        assert!(matches!(
182            check_content_enabled(true, "src/lib.rs", "content"),
183            DedupAction::DeliverStub { .. }
184        ));
185    }
186
187    #[test]
188    fn modified_content_delivers_modified() {
189        let _guard = isolated();
190        check_content_enabled(true, "src/lib.rs", "before");
191        assert_eq!(
192            check_content_enabled(true, "src/lib.rs", "after"),
193            DedupAction::DeliverModified
194        );
195    }
196
197    #[test]
198    fn disabled_always_full() {
199        let _guard = isolated();
200        assert_eq!(
201            check_content_enabled(false, "src/lib.rs", "content"),
202            DedupAction::DeliverFull
203        );
204        assert_eq!(
205            check_content_enabled(false, "src/lib.rs", "content"),
206            DedupAction::DeliverFull
207        );
208        assert_eq!(dedup_stats().total_checks, 0);
209    }
210
211    #[test]
212    fn apply_dedup_returns_stub() {
213        let _guard = isolated();
214        assert_eq!(
215            apply_dedup_enabled(true, "src/lib.rs", "content"),
216            "content"
217        );
218        let stub = apply_dedup_enabled(true, "src/lib.rs", "content");
219        assert!(stub.contains("src/lib.rs unchanged"));
220    }
221
222    #[test]
223    fn invalidate_forces_full() {
224        let _guard = isolated();
225        check_content_enabled(true, "src/lib.rs", "content");
226        invalidate("src/lib.rs");
227        assert_eq!(
228            check_content_enabled(true, "src/lib.rs", "content"),
229            DedupAction::DeliverFull
230        );
231    }
232
233    #[test]
234    fn stats_track_hits() {
235        let _guard = isolated();
236        check_content_enabled(true, "a", "one");
237        check_content_enabled(true, "a", "one");
238        check_content_enabled(true, "a", "one");
239        check_content_enabled(true, "b", "two");
240        check_content_enabled(true, "b", "two");
241
242        let stats = dedup_stats();
243        assert_eq!(stats.total_checks, 5);
244        assert_eq!(stats.cache_hits, 3);
245        assert_eq!(stats.cache_misses, 2);
246        assert!((stats.hit_rate - 0.6).abs() < f64::EPSILON);
247    }
248}
lean_ctx/core/context_kernel/dedup_wiring.rs

lean_ctx/core/context_kernel/
dedup_wiring.rs
