context_forge/storage/
mod.rs

1use std::path::Path;
2use std::sync::Arc;
3
4/// Shared in-memory tantivy FTS index.
5pub(crate) mod fts_index;
6/// Turso-backed async FTS searcher.
7pub mod turso_searcher;
8/// Turso-backed async storage implementation.
9pub mod turso_storage;
10
11pub use turso_searcher::TursoSearcher;
12pub use turso_storage::TursoStorage;
13
14/// Create a paired storage + searcher backed by turso.
15///
16/// # Errors
17///
18/// Returns an error if the database cannot be opened or migrations fail.
19pub async fn open_storage(
20    db_path: &Path,
21    max_entries: usize,
22) -> crate::Result<(TursoStorage, TursoSearcher)> {
23    let storage = TursoStorage::open(db_path, max_entries).await?;
24    let db = Arc::clone(&storage.db);
25    let fts = Arc::clone(&storage.fts);
26    let searcher = TursoSearcher::new(db, fts);
27    Ok((storage, searcher))
28}
29
30#[cfg(test)]
31mod tests {
32    use std::path::Path;
33
34    use crate::engine::MATCH_ALL_QUERY;
35    use crate::entry::{kind, ContextEntry};
36    use crate::storage::open_storage;
37    use crate::traits::{ContextStorage, Searcher};
38
39    fn make_entry(id: &str, content: &str, timestamp: i64, kind: &str) -> ContextEntry {
40        ContextEntry {
41            id: id.into(),
42            content: content.into(),
43            timestamp,
44            kind: kind.to_owned(),
45            scope: None,
46            session_id: None,
47            token_count: None,
48            metadata: None,
49        }
50    }
51
52    fn make_scoped_entry(id: &str, content: &str, timestamp: i64, scope: &str) -> ContextEntry {
53        ContextEntry {
54            id: id.into(),
55            content: content.into(),
56            timestamp,
57            kind: kind::MANUAL.to_owned(),
58            scope: Some(scope.to_owned()),
59            session_id: None,
60            token_count: None,
61            metadata: None,
62        }
63    }
64
65    #[tokio::test]
66    async fn test_save_and_count() {
67        let (storage, _) = open_storage(Path::new(":memory:"), 100).await.unwrap();
68        let entry = make_entry("e1", "hello world", 1000, kind::MANUAL);
69        storage.save(&entry).await.unwrap();
70        assert_eq!(storage.count().await.unwrap(), 1);
71    }
72
73    #[tokio::test]
74    async fn test_save_and_get_top_k() {
75        let (storage, _) = open_storage(Path::new(":memory:"), 100).await.unwrap();
76        storage
77            .save(&make_entry("e1", "first", 100, kind::MANUAL))
78            .await
79            .unwrap();
80        storage
81            .save(&make_entry("e2", "second", 200, kind::SNAPSHOT))
82            .await
83            .unwrap();
84        storage
85            .save(&make_entry("e3", "third", 300, kind::SUMMARY))
86            .await
87            .unwrap();
88
89        let top2 = storage.get_top_k(2).await.unwrap();
90        assert_eq!(top2.len(), 2);
91        assert_eq!(top2[0].id, "e3"); // most recent
92        assert_eq!(top2[1].id, "e2");
93    }
94
95    #[tokio::test]
96    async fn test_save_and_get_all() {
97        let (storage, _) = open_storage(Path::new(":memory:"), 100).await.unwrap();
98        storage
99            .save(&make_entry("e1", "first", 100, kind::MANUAL))
100            .await
101            .unwrap();
102        storage
103            .save(&make_entry("e2", "second", 200, kind::MANUAL))
104            .await
105            .unwrap();
106        storage
107            .save(&make_entry("e3", "third", 300, kind::MANUAL))
108            .await
109            .unwrap();
110
111        let all = storage.get_all().await.unwrap();
112        assert_eq!(all.len(), 3);
113        assert_eq!(all[0].id, "e3");
114        assert_eq!(all[1].id, "e2");
115        assert_eq!(all[2].id, "e1");
116    }
117
118    #[tokio::test]
119    async fn test_delete() {
120        let (storage, _) = open_storage(Path::new(":memory:"), 100).await.unwrap();
121        storage
122            .save(&make_entry("e1", "hello", 1000, kind::MANUAL))
123            .await
124            .unwrap();
125
126        assert!(storage.delete("e1").await.unwrap());
127        assert!(!storage.delete("nonexistent").await.unwrap());
128        assert_eq!(storage.count().await.unwrap(), 0);
129    }
130
131    #[tokio::test]
132    async fn test_clear() {
133        let (storage, _) = open_storage(Path::new(":memory:"), 100).await.unwrap();
134        storage
135            .save(&make_entry("e1", "a", 100, kind::MANUAL))
136            .await
137            .unwrap();
138        storage
139            .save(&make_entry("e2", "b", 200, kind::MANUAL))
140            .await
141            .unwrap();
142        storage
143            .save(&make_entry("e3", "c", 300, kind::MANUAL))
144            .await
145            .unwrap();
146
147        let cleared = storage.clear().await.unwrap();
148        assert_eq!(cleared, 3);
149        assert_eq!(storage.count().await.unwrap(), 0);
150    }
151
152    #[tokio::test]
153    async fn test_clear_scope() {
154        let (storage, _) = open_storage(Path::new(":memory:"), 100).await.unwrap();
155        storage
156            .save(&make_scoped_entry("e1", "a", 100, "scope-a"))
157            .await
158            .unwrap();
159        storage
160            .save(&make_scoped_entry("e2", "b", 200, "scope-a"))
161            .await
162            .unwrap();
163        storage
164            .save(&make_scoped_entry("e3", "c", 300, "scope-b"))
165            .await
166            .unwrap();
167
168        let cleared = storage.clear_scope("scope-a").await.unwrap();
169        assert_eq!(cleared, 2);
170        assert_eq!(storage.count().await.unwrap(), 1);
171
172        let all = storage.get_all().await.unwrap();
173        assert_eq!(all[0].id, "e3");
174    }
175
176    #[tokio::test]
177    async fn test_clear_scope_no_match() {
178        let (storage, _) = open_storage(Path::new(":memory:"), 100).await.unwrap();
179        storage
180            .save(&make_scoped_entry("e1", "a", 100, "scope-a"))
181            .await
182            .unwrap();
183
184        let cleared = storage.clear_scope("scope-z").await.unwrap();
185        assert_eq!(cleared, 0);
186        assert_eq!(storage.count().await.unwrap(), 1);
187    }
188
189    #[tokio::test]
190    async fn test_lru_eviction() {
191        let (storage, _) = open_storage(Path::new(":memory:"), 2).await.unwrap();
192        storage
193            .save(&make_entry("e1", "oldest", 100, kind::MANUAL))
194            .await
195            .unwrap();
196        storage
197            .save(&make_entry("e2", "middle", 200, kind::MANUAL))
198            .await
199            .unwrap();
200        storage
201            .save(&make_entry("e3", "newest", 300, kind::MANUAL))
202            .await
203            .unwrap();
204
205        assert_eq!(storage.count().await.unwrap(), 2);
206
207        let all = storage.get_all().await.unwrap();
208        let ids: Vec<&str> = all.iter().map(|e| e.id.as_str()).collect();
209        assert!(
210            !ids.contains(&"e1"),
211            "oldest entry should have been evicted"
212        );
213        assert!(ids.contains(&"e2"));
214        assert!(ids.contains(&"e3"));
215    }
216
217    #[tokio::test]
218    async fn test_fts_search() {
219        let (storage, searcher) = open_storage(Path::new(":memory:"), 100).await.unwrap();
220        storage
221            .save(&make_entry(
222                "e1",
223                "rust programming language",
224                100,
225                kind::MANUAL,
226            ))
227            .await
228            .unwrap();
229        storage
230            .save(&make_entry("e2", "python scripting", 200, kind::MANUAL))
231            .await
232            .unwrap();
233        storage
234            .save(&make_entry("e3", "rust borrow checker", 300, kind::MANUAL))
235            .await
236            .unwrap();
237
238        let results = searcher.search("rust", None, 5).await.unwrap();
239        assert_eq!(results.len(), 2);
240        assert!(
241            results[0].score >= results[1].score,
242            "results should be ordered by descending score"
243        );
244    }
245
246    #[tokio::test]
247    async fn test_fts_search_no_results() {
248        let (storage, searcher) = open_storage(Path::new(":memory:"), 100).await.unwrap();
249        storage
250            .save(&make_entry("e1", "hello world", 100, kind::MANUAL))
251            .await
252            .unwrap();
253
254        let results = searcher.search("nonexistent", None, 5).await.unwrap();
255        assert!(results.is_empty());
256    }
257
258    #[tokio::test]
259    async fn test_fts_search_scoped() {
260        let (storage, searcher) = open_storage(Path::new(":memory:"), 100).await.unwrap();
261        storage
262            .save(&make_scoped_entry("e1", "rust programming", 100, "a"))
263            .await
264            .unwrap();
265        storage
266            .save(&make_scoped_entry("e2", "rust borrow checker", 200, "b"))
267            .await
268            .unwrap();
269
270        let results_a = searcher.search("rust", Some("a"), 5).await.unwrap();
271        assert_eq!(results_a.len(), 1);
272        assert_eq!(results_a[0].entry.id, "e1");
273
274        let results_b = searcher.search("rust", Some("b"), 5).await.unwrap();
275        assert_eq!(results_b.len(), 1);
276        assert_eq!(results_b[0].entry.id, "e2");
277
278        let results_all = searcher.search("rust", None, 5).await.unwrap();
279        assert_eq!(results_all.len(), 2);
280    }
281
282    #[tokio::test]
283    async fn test_new_entry_with_scope_and_metadata() {
284        let (storage, _) = open_storage(Path::new(":memory:"), 100).await.unwrap();
285
286        let metadata = serde_json::json!({"runtime": "codex", "model": "gpt-5.3-codex"});
287        let entry = ContextEntry {
288            id: "v3-entry".into(),
289            content: "entry with scope and metadata".into(),
290            timestamp: 1_700_000_100,
291            kind: kind::SUMMARY.to_owned(),
292            scope: Some("project:homelab-rs".into()),
293            session_id: Some("session-123".into()),
294            token_count: Some(6),
295            metadata: Some(metadata.clone()),
296        };
297
298        storage.save(&entry).await.unwrap();
299
300        let all = storage.get_all().await.unwrap();
301        assert_eq!(all.len(), 1);
302        let got = &all[0];
303        assert_eq!(got.id, entry.id);
304        assert_eq!(got.content, entry.content);
305        assert_eq!(got.timestamp, entry.timestamp);
306        assert_eq!(got.kind, entry.kind);
307        assert_eq!(got.scope, entry.scope);
308        assert_eq!(got.session_id, entry.session_id);
309        assert_eq!(got.token_count, entry.token_count);
310        assert_eq!(got.metadata, Some(metadata));
311    }
312
313    #[tokio::test]
314    async fn test_insert_or_replace() {
315        let (storage, _) = open_storage(Path::new(":memory:"), 100).await.unwrap();
316        storage
317            .save(&make_entry("e1", "original content", 100, kind::MANUAL))
318            .await
319            .unwrap();
320        storage
321            .save(&make_entry("e1", "updated content", 200, kind::SUMMARY))
322            .await
323            .unwrap();
324
325        assert_eq!(storage.count().await.unwrap(), 1);
326
327        let all = storage.get_all().await.unwrap();
328        assert_eq!(all[0].content, "updated content");
329    }
330
331    #[tokio::test]
332    async fn test_search_match_all_query() {
333        let (storage, searcher) = open_storage(Path::new(":memory:"), 100).await.unwrap();
334        storage
335            .save(&make_entry("e1", "first entry", 100, kind::MANUAL))
336            .await
337            .unwrap();
338        storage
339            .save(&make_entry("e2", "second entry", 200, kind::SNAPSHOT))
340            .await
341            .unwrap();
342        storage
343            .save(&make_entry("e3", "third entry", 300, kind::SUMMARY))
344            .await
345            .unwrap();
346
347        let results = searcher.search(MATCH_ALL_QUERY, None, 10).await.unwrap();
348        assert_eq!(results.len(), 3);
349
350        // Ordered by timestamp descending (newest first).
351        assert_eq!(results[0].entry.id, "e3");
352        assert_eq!(results[1].entry.id, "e2");
353        assert_eq!(results[2].entry.id, "e1");
354
355        // Match-all results all share the same fixed score.
356        for r in &results {
357            assert!((r.score - 1.0).abs() < f64::EPSILON);
358        }
359    }
360
361    #[tokio::test]
362    async fn test_search_match_all_query_scoped() {
363        let (storage, searcher) = open_storage(Path::new(":memory:"), 100).await.unwrap();
364        storage
365            .save(&make_scoped_entry("e1", "first entry", 100, "a"))
366            .await
367            .unwrap();
368        storage
369            .save(&make_scoped_entry("e2", "second entry", 200, "b"))
370            .await
371            .unwrap();
372
373        let results_a = searcher
374            .search(MATCH_ALL_QUERY, Some("a"), 10)
375            .await
376            .unwrap();
377        assert_eq!(results_a.len(), 1);
378        assert_eq!(results_a[0].entry.id, "e1");
379
380        let results_all = searcher.search(MATCH_ALL_QUERY, None, 10).await.unwrap();
381        assert_eq!(results_all.len(), 2);
382    }
383
384    #[tokio::test]
385    async fn search_with_fts5_operator_characters_does_not_error() {
386        let (storage, searcher) = open_storage(Path::new(":memory:"), 100).await.unwrap();
387        storage
388            .save(&make_entry("e1", "marco polo", 100, kind::MANUAL))
389            .await
390            .unwrap();
391
392        // Production bug: a query containing `"` and `.` previously caused
393        // `fts5: syntax error`. It must now succeed and find the entry.
394        let results = searcher
395            .search(r#"if I say "marco"."#, None, 10)
396            .await
397            .unwrap();
398
399        assert!(
400            results.iter().any(|r| r.entry.id == "e1"),
401            "expected 'marco polo' entry to be found despite FTS5 operator characters in the query"
402        );
403    }
404
405    #[tokio::test]
406    async fn test_fts_search_scores_are_nonzero() {
407        let (storage, searcher) = open_storage(Path::new(":memory:"), 100).await.unwrap();
408        storage
409            .save(&make_entry(
410                "e1",
411                "rust programming language",
412                100,
413                kind::MANUAL,
414            ))
415            .await
416            .unwrap();
417        storage
418            .save(&make_entry(
419                "e2",
420                "python scripting tools",
421                200,
422                kind::MANUAL,
423            ))
424            .await
425            .unwrap();
426
427        let results = searcher.search("rust", None, 5).await.unwrap();
428        assert!(!results.is_empty());
429        assert!(
430            results[0].score > 0.0,
431            "expected non-zero BM25 score from tantivy, got {}",
432            results[0].score
433        );
434    }
435
436    #[tokio::test]
437    async fn search_or_joins_terms_for_message_length_queries() {
438        let (storage, searcher) = open_storage(Path::new(":memory:"), 100).await.unwrap();
439        storage
440            .save(&make_entry("e1", "alpha one", 100, kind::MANUAL))
441            .await
442            .unwrap();
443        storage
444            .save(&make_entry("e2", "beta two", 200, kind::MANUAL))
445            .await
446            .unwrap();
447
448        // Under implicit AND, no entry contains all of "alpha", "beta", and
449        // "gamma", so this would return zero results. OR-join must return both.
450        let results = searcher.search("alpha beta gamma", None, 10).await.unwrap();
451
452        let ids: Vec<&str> = results.iter().map(|r| r.entry.id.as_str()).collect();
453        assert!(ids.contains(&"e1"), "expected 'alpha one' entry in results");
454        assert!(ids.contains(&"e2"), "expected 'beta two' entry in results");
455    }
456}
context_forge/storage/mod.rs

context_forge/storage/
mod.rs
