context_forge/lib.rs
1//! `context-forge` — a local-first persistent memory library for LLM applications.
2//!
3//! This crate provides turso + Tantivy BM25 retrieval, recency-decay scoring, and
4//! token-budget-aware context assembly with no network calls. It is intended to
5//! be embedded in larger applications (CLI tools, bots, agent runtimes) that need
6//! durable, searchable memory.
7//!
8//! # Quick start
9//!
10//! ```no_run
11//! use context_forge::{kind, ContextForge, Config, SaveOptions};
12//! use std::path::PathBuf;
13//!
14//! #[tokio::main]
15//! async fn main() -> Result<(), context_forge::Error> {
16//! let mut config = Config::default();
17//! config.db_path = PathBuf::from(":memory:");
18//!
19//! let cf = ContextForge::open(config).await?;
20//! cf.save("the deploy failure was caused by a missing env var", kind::SNAPSHOT, &SaveOptions::default()).await?;
21//!
22//! let hits = cf.query("deploy failure", None, 2048).await?;
23//! assert_eq!(hits.len(), 1);
24//! Ok(())
25//! }
26//! ```
27//!
28//! # Security
29//!
30//! **Retrieved entries are untrusted text.** Content persisted from past
31//! conversations may contain adversarial instructions (stored prompt
32//! injection) — whatever was saved into the store, including text that
33//! originated from another user or from a tool's output, comes back out
34//! verbatim on [`ContextForge::query`] (aside from save-time secret
35//! scrubbing, see below).
36//!
37//! Callers **MUST** present retrieved memory to models as quoted data
38//! (e.g. inside a fenced or otherwise clearly delimited context block
39//! labeled as history), **never** as system-level instructions, and
40//! **MUST NOT** execute or evaluate anything found in it.
41//!
42//! ## Save-time secret scrubbing
43//!
44//! [`ContextForge::save`] applies [`scrub_secrets`] to `content` before it
45//! is persisted, using the [`ScrubConfig`] supplied in [`Config::scrub`].
46//! This redacts common credential formats (cloud provider keys, API
47//! tokens, private key blocks, JWTs, bearer tokens) with
48//! `[REDACTED:<label>]` placeholders so they never reach the database or
49//! the search index. Scrubbing is **on by default** and can be disabled
50//! via `Config { scrub: ScrubConfig { enabled: false }, .. }` — this is an
51//! explicit, non-silent opt-out.
52//!
53//! Note that [`SaveOptions::metadata`] is **not** scrubbed (see its docs).
54
55#![warn(clippy::pedantic)]
56#![warn(missing_docs)]
57
58/// [`ContextForgeBuilder`] — the opinionated construction path for [`ContextForge`].
59pub mod builder;
60/// Engine and scrub configuration types (`Config`, `EvictionPolicy`, `ScrubConfig`).
61pub mod config;
62/// Local-LLM distillation trait and the optional `distill-http` implementation.
63pub mod distill;
64/// `ContextEngine`: search, recency decay, and token-budget assembly.
65pub mod engine;
66/// `ContextEntry`, `ScoredEntry`, and the `kind` constants module.
67pub mod entry;
68/// The crate's `Error` type.
69pub mod error;
70/// Save-time secret scrubbing (`scrub_secrets`, `ScrubConfig`).
71pub mod scrub;
72/// Session grouping helpers (`group_entries_by_session`, `SessionGroup`).
73pub mod session;
74/// Turso-backed storage and search implementations.
75pub mod storage;
76/// `ContextStorage` and `Searcher` traits, and the crate's `Result` alias.
77pub mod traits;
78
79/// Lexicon-based importance scoring — [`LexiconScorer`] trait, [`ConfigLexiconScorer`],
80/// [`DefaultEnglishScorer`], [`CompositeLexiconScorer`], [`LexiconAppender`], and
81/// [`LexiconProposal`].
82pub mod lexicon;
83
84/// Dense-vector embedding abstraction for semantic search.
85///
86/// The [`semantic::Embedder`] trait is always available. [`semantic::FasEmbedder`]
87/// (fastembed + ONNX Runtime, all-MiniLM-L6-v2) is compiled only with the
88/// `semantic` Cargo feature.
89pub mod semantic;
90
91/// Importance-detection pipeline (tokenizer, lexicon, scoring). Pure
92/// computation, no I/O. Enabled by the `analysis` feature (default).
93#[cfg(feature = "analysis")]
94pub mod analysis;
95
96#[cfg(feature = "parallel")]
97pub use analysis::with_thread_cap;
98
99use std::path::Path;
100
101// Re-export primary types at crate root for convenience.
102pub use builder::ContextForgeBuilder;
103pub use config::{Config, EvictionPolicy};
104pub use distill::{
105 merge_distilled, split_on_budget, ChunkingDistiller, DistilledMemory, Distiller, Fact,
106 FactKind, ReduceStrategy,
107};
108pub use engine::{ContextEngine, SaveOptions, MATCH_ALL_QUERY};
109pub use entry::{kind, ContextEntry, ScoredEntry};
110pub use error::Error;
111pub use lexicon::{
112 bootstrap_prompt, CompositeLexiconScorer, ConfigLexiconScorer, DefaultEnglishScorer,
113 LexiconAppender, LexiconConfig, LexiconPatterns, LexiconProposal, LexiconScorer,
114};
115pub use scrub::{scrub_secrets, ScrubConfig};
116pub use session::{group_entries_by_session, SessionGroup};
117pub use storage::{open_storage, TursoSearcher, TursoStorage};
118pub use traits::{ContextStorage, Result, Searcher};
119
120/// The documented entry point for `context-forge`.
121///
122/// `ContextForge` wires together a [`TursoStorage`] backend, a
123/// [`TursoSearcher`], and a [`ContextEngine`] behind a small,
124/// stable API surface. Advanced callers that need direct access to the
125/// underlying storage or searcher can construct those types directly and
126/// pass them to [`ContextEngine::new`] instead.
127pub struct ContextForge {
128 engine: ContextEngine,
129 scrub_config: ScrubConfig,
130}
131
132impl ContextForge {
133 /// Construct from already-built parts. Used by [`ContextForgeBuilder`].
134 pub(crate) fn from_parts(engine: ContextEngine, scrub_config: ScrubConfig) -> Self {
135 Self {
136 engine,
137 scrub_config,
138 }
139 }
140
141 /// Create a builder for `ContextForge`.
142 ///
143 /// The builder always pre-seeds [`DefaultEnglishScorer`] so plain-English
144 /// importance signals are active without any configuration. Use
145 /// [`ContextForgeBuilder::with_persona_scorer`] to stack a domain-specific
146 /// scorer on top. See also [`Self::open`] for the lower-level path that
147 /// wires no scorer.
148 #[must_use]
149 pub fn builder(config: Config) -> ContextForgeBuilder {
150 ContextForgeBuilder::new(config)
151 }
152
153 /// Open (or create) the database at `config.db_path`, run any pending
154 /// migrations, and build the engine.
155 ///
156 /// # Errors
157 ///
158 /// Returns an error if the database cannot be opened or migrations fail.
159 pub async fn open(config: Config) -> Result<Self> {
160 let db_path = config.db_path.clone();
161 let max_entries = config.max_entries;
162 let scrub_config = config.scrub.clone();
163 let (storage, searcher) = open_storage(Path::new(&db_path), max_entries).await?;
164 let engine = ContextEngine::new(Box::new(storage), Box::new(searcher), config);
165 Ok(Self {
166 engine,
167 scrub_config,
168 })
169 }
170
171 /// Save a new entry. Returns the generated entry ID.
172 ///
173 /// `kind` is a caller-defined classification (see [`mod@kind`] for
174 /// well-known values). Capacity enforcement (LRU eviction) is handled
175 /// atomically by the storage layer.
176 ///
177 /// Before persistence, `content` is passed through [`scrub_secrets`]
178 /// using this instance's [`ScrubConfig`] (see [`Config::scrub`]),
179 /// redacting common credential formats with `[REDACTED:<label>]`
180 /// placeholders. `opts.metadata` is stored verbatim and is **not**
181 /// scrubbed — see [`SaveOptions::metadata`].
182 ///
183 /// # Errors
184 ///
185 /// Returns an error if `content` is empty or if the underlying storage
186 /// write fails.
187 pub async fn save(&self, content: &str, kind: &str, opts: &SaveOptions) -> Result<String> {
188 let scrubbed = scrub_secrets(content, &self.scrub_config);
189 self.engine.save_snapshot(&scrubbed, kind, opts).await
190 }
191
192 /// Distill `transcript` into a summary and durable facts, then save
193 /// them as separate entries sharing `opts.scope` and
194 /// `opts.session_id`.
195 ///
196 /// `transcript` is passed through [`scrub_secrets`] **before** it is
197 /// sent to `distiller` (so secrets never reach a distillation
198 /// endpoint), and the summary/facts produced are scrubbed again before
199 /// persistence via the normal [`ContextForge::save`] path (defense in
200 /// depth).
201 ///
202 /// The summary is saved with `kind::SUMMARY`; each fact is saved with
203 /// `kind::FACT` and metadata `{"fact_kind": "<kind>", "source":
204 /// "distill"}`.
205 ///
206 /// The distilled output is bounded before any of it is saved: at most
207 /// [`MAX_FACTS`](crate::distill::MAX_FACTS) facts are kept, each fact's
208 /// text is truncated to at most
209 /// [`MAX_FACT_CHARS`](crate::distill::MAX_FACT_CHARS) characters, and the
210 /// summary is truncated to at most
211 /// [`MAX_SUMMARY_CHARS`](crate::distill::MAX_SUMMARY_CHARS) characters.
212 /// Excess facts and text beyond these limits are silently dropped or
213 /// truncated, since they are untrusted model-generated content.
214 ///
215 /// Returns the IDs of the saved entries: the summary's ID first,
216 /// followed by each fact's ID in order.
217 ///
218 /// # Errors
219 ///
220 /// Returns an error if distillation fails, or if any save fails.
221 pub async fn distill_and_save(
222 &self,
223 transcript: &str,
224 distiller: &dyn Distiller,
225 opts: &SaveOptions,
226 ) -> Result<Vec<String>> {
227 let scrubbed_transcript = scrub_secrets(transcript, &self.scrub_config);
228 let memory = tokio::task::block_in_place(|| distiller.distill(&scrubbed_transcript))?;
229 let memory = crate::distill::cap_distilled_memory(memory);
230
231 let mut ids = Vec::with_capacity(1 + memory.facts.len());
232
233 let summary_id = self.save(&memory.summary, kind::SUMMARY, opts).await?;
234 ids.push(summary_id);
235
236 for fact in &memory.facts {
237 let fact_kind_str = match fact.kind {
238 FactKind::Decision => "decision",
239 FactKind::Correction => "correction",
240 FactKind::Preference => "preference",
241 FactKind::State => "state",
242 };
243 let metadata = serde_json::json!({
244 "fact_kind": fact_kind_str,
245 "source": "distill",
246 });
247 let fact_opts = SaveOptions {
248 session_id: opts.session_id.clone(),
249 scope: opts.scope.clone(),
250 metadata: Some(metadata),
251 };
252 let fact_id = self.save(&fact.text, kind::FACT, &fact_opts).await?;
253 ids.push(fact_id);
254 }
255
256 Ok(ids)
257 }
258
259 /// Assemble entries matching `query` that fit within `token_budget`.
260 ///
261 /// `scope = None` searches every entry regardless of scope (global
262 /// recall). `scope = Some(s)` restricts the search to entries whose
263 /// `scope` equals `s`.
264 ///
265 /// `query` is treated as natural-language text: it is split into
266 /// alphanumeric terms which are OR-matched and ranked by bm25 relevance.
267 /// FTS5 operator syntax (`AND`, `OR`, `NEAR`, prefix `*`, quoted phrases,
268 /// column filters, etc.) is **not** interpreted — operator characters are
269 /// treated as term separators, so arbitrary user text never produces a
270 /// query syntax error. A query with no alphanumeric terms (empty or
271 /// punctuation-only) returns an empty result set rather than an error.
272 /// The special value [`MATCH_ALL_QUERY`] (`"*"`) matches every entry.
273 ///
274 /// # Errors
275 ///
276 /// Returns an error if the search or recency-weighting step fails.
277 pub async fn query(
278 &self,
279 query: &str,
280 scope: Option<&str>,
281 token_budget: usize,
282 ) -> Result<Vec<ContextEntry>> {
283 self.engine.assemble(query, scope, token_budget).await
284 }
285
286 /// Delete a single entry by id. Returns `true` if an entry was removed.
287 ///
288 /// # Errors
289 ///
290 /// Returns an error if the underlying storage delete fails.
291 pub async fn delete(&self, id: &str) -> Result<bool> {
292 self.engine.storage().delete(id).await
293 }
294
295 /// Remove all entries within a given scope. Returns the number of
296 /// entries removed.
297 ///
298 /// # Errors
299 ///
300 /// Returns an error if the underlying storage delete fails.
301 pub async fn clear_scope(&self, scope: &str) -> Result<usize> {
302 self.engine.storage().clear_scope(scope).await
303 }
304
305 /// Remove all entries. Returns the number of entries removed.
306 ///
307 /// # Errors
308 ///
309 /// Returns an error if the underlying storage delete fails.
310 pub async fn clear_all(&self) -> Result<usize> {
311 self.engine.storage().clear().await
312 }
313
314 /// Return the total number of stored entries.
315 ///
316 /// # Errors
317 ///
318 /// Returns an error if the underlying storage count fails.
319 pub async fn count(&self) -> Result<usize> {
320 self.engine.storage().count().await
321 }
322
323 /// Backfill embeddings for all entries that do not yet have one.
324 ///
325 /// Call this once after enabling semantic search on an existing database to
326 /// index historical entries. New entries are embedded automatically at save
327 /// time, so backfill is only needed for pre-existing data.
328 ///
329 /// `batch_size` controls how many entries are embedded per ONNX inference
330 /// call (recommended: 32). `progress` is called after each batch with
331 /// `(done, total)`. Returns the number of entries successfully embedded.
332 ///
333 /// Does nothing and returns `Ok(0)` if no embedder is configured or if all
334 /// entries already have embeddings.
335 ///
336 /// # Errors
337 ///
338 /// Returns an error if fetching unembedded entries fails or if the
339 /// embedding task panics.
340 #[cfg(feature = "semantic")]
341 pub async fn backfill_embeddings(
342 &self,
343 batch_size: usize,
344 progress: impl Fn(usize, usize),
345 ) -> Result<usize> {
346 self.engine.backfill_embeddings(batch_size, progress).await
347 }
348}
349
350#[cfg(test)]
351mod tests {
352 use super::*;
353 use std::path::PathBuf;
354
355 #[test]
356 fn context_entry_json_roundtrip() {
357 let entry = ContextEntry {
358 id: "e1".into(),
359 content: "hello world".into(),
360 timestamp: 1_700_000_000,
361 kind: kind::MANUAL.to_owned(),
362 scope: None,
363 session_id: None,
364 token_count: Some(3),
365 metadata: None,
366 };
367 let json = serde_json::to_string(&entry).unwrap();
368 let back: ContextEntry = serde_json::from_str(&json).unwrap();
369 assert_eq!(back.id, "e1");
370 assert_eq!(back.token_count, Some(3));
371 }
372
373 #[test]
374 fn error_display_messages() {
375 let invalid = Error::InvalidEntry("empty content".into());
376 assert_eq!(invalid.to_string(), "invalid entry: empty content");
377
378 let migration = Error::Migration("schema mismatch".into());
379 assert_eq!(migration.to_string(), "migration error: schema mismatch");
380
381 let distill = Error::Distill("model unavailable".into());
382 assert_eq!(distill.to_string(), "distillation error: model unavailable");
383 }
384
385 #[test]
386 fn core_config_json_roundtrip() {
387 let cfg = Config {
388 max_entries: 1000,
389 token_budget: 8192,
390 db_path: PathBuf::from("/tmp/cf.db"),
391 eviction_policy: EvictionPolicy::Lru,
392 recency_half_life_secs: 259_200.0,
393 ..Config::default()
394 };
395 let json = serde_json::to_string(&cfg).unwrap();
396 let back: Config = serde_json::from_str(&json).unwrap();
397 assert_eq!(back.max_entries, 1000);
398 assert_eq!(back.eviction_policy, EvictionPolicy::Lru);
399 }
400
401 #[test]
402 fn trait_objects_are_object_safe() {
403 // This test verifies that the traits compile as trait objects.
404 fn _assert_storage(_s: Box<dyn ContextStorage>) {}
405 fn _assert_searcher(_s: Box<dyn Searcher>) {}
406 }
407
408 #[test]
409 fn kind_constants_are_distinct() {
410 assert_ne!(kind::MANUAL, kind::SNAPSHOT);
411 assert_ne!(kind::MANUAL, kind::SUMMARY);
412 assert_ne!(kind::MANUAL, kind::FACT);
413 assert_ne!(kind::SNAPSHOT, kind::SUMMARY);
414 assert_ne!(kind::SNAPSHOT, kind::FACT);
415 assert_ne!(kind::SUMMARY, kind::FACT);
416 }
417
418 #[test]
419 fn scored_entry_json_roundtrip() {
420 let scored = ScoredEntry {
421 entry: ContextEntry {
422 id: "s1".into(),
423 content: "search hit".into(),
424 timestamp: 1_700_000_001,
425 kind: kind::SUMMARY.to_owned(),
426 scope: None,
427 session_id: None,
428 token_count: None,
429 metadata: None,
430 },
431 score: 0.95,
432 };
433 let json = serde_json::to_string(&scored).unwrap();
434 let back: ScoredEntry = serde_json::from_str(&json).unwrap();
435 assert_eq!(back.entry.id, "s1");
436 assert!((back.score - 0.95).abs() < f64::EPSILON);
437 }
438
439 #[test]
440 fn context_forge_is_send_sync() {
441 fn assert_send_sync<T: Send + Sync>() {}
442 assert_send_sync::<ContextForge>();
443 }
444
445 #[tokio::test]
446 async fn context_forge_open_save_query_roundtrip() {
447 let config = Config {
448 db_path: PathBuf::from(":memory:"),
449 ..Config::default()
450 };
451 let cf = ContextForge::open(config).await.unwrap();
452
453 let id = cf
454 .save("hello world", kind::MANUAL, &SaveOptions::default())
455 .await
456 .unwrap();
457 assert!(!id.is_empty());
458 assert_eq!(cf.count().await.unwrap(), 1);
459
460 let hits = cf.query("hello", None, 1000).await.unwrap();
461 assert_eq!(hits.len(), 1);
462 assert_eq!(hits[0].id, id);
463
464 assert!(cf.delete(&id).await.unwrap());
465 assert_eq!(cf.count().await.unwrap(), 0);
466 }
467
468 #[tokio::test]
469 async fn context_forge_clear_all() {
470 let config = Config {
471 db_path: PathBuf::from(":memory:"),
472 ..Config::default()
473 };
474 let cf = ContextForge::open(config).await.unwrap();
475
476 cf.save("a", kind::MANUAL, &SaveOptions::default())
477 .await
478 .unwrap();
479 cf.save("b", kind::MANUAL, &SaveOptions::default())
480 .await
481 .unwrap();
482
483 let cleared = cf.clear_all().await.unwrap();
484 assert_eq!(cleared, 2);
485 assert_eq!(cf.count().await.unwrap(), 0);
486 }
487
488 /// A stub [`Distiller`] for tests that records the transcript it was
489 /// called with and returns a fixed [`DistilledMemory`].
490 struct StubDistiller {
491 transcript: std::sync::Mutex<Option<String>>,
492 }
493
494 impl StubDistiller {
495 fn new() -> Self {
496 Self {
497 transcript: std::sync::Mutex::new(None),
498 }
499 }
500 }
501
502 impl Distiller for StubDistiller {
503 fn distill(&self, transcript: &str) -> Result<DistilledMemory> {
504 *self.transcript.lock().unwrap() = Some(transcript.to_owned());
505 Ok(DistilledMemory {
506 summary: "User decided to roll back the deploy.".to_owned(),
507 facts: vec![
508 Fact {
509 kind: FactKind::Decision,
510 text: "We decided to roll back the deploy.".to_owned(),
511 },
512 Fact {
513 kind: FactKind::Preference,
514 text: "The user prefers terse commit messages.".to_owned(),
515 },
516 ],
517 })
518 }
519 }
520
521 #[tokio::test(flavor = "multi_thread")]
522 async fn distill_and_save_scrubs_saves_and_returns_ids() {
523 let config = Config {
524 db_path: PathBuf::from(":memory:"),
525 ..Config::default()
526 };
527 let cf = ContextForge::open(config).await.unwrap();
528
529 let distiller = StubDistiller::new();
530 let transcript = "Here is a secret key=AKIAABCDEFGHIJKLMNOP end of transcript";
531
532 let opts = SaveOptions {
533 session_id: Some("sess-1".to_owned()),
534 scope: Some("project:test".to_owned()),
535 metadata: None,
536 };
537
538 let ids = cf
539 .distill_and_save(transcript, &distiller, &opts)
540 .await
541 .unwrap();
542
543 // Summary ID first, then one ID per fact.
544 assert_eq!(ids.len(), 3);
545 for id in &ids {
546 assert!(!id.is_empty());
547 }
548
549 // The transcript reaching the distiller was scrubbed.
550 let seen = distiller.transcript.lock().unwrap().clone().unwrap();
551 assert!(seen.contains("[REDACTED:aws-key]"));
552 assert!(!seen.contains("AKIAABCDEFGHIJKLMNOP"));
553
554 // Summary saved with kind::SUMMARY.
555 let summary = cf
556 .query("rollback OR rollback OR roll", None, 10_000)
557 .await
558 .unwrap();
559 let summary_entry = summary
560 .iter()
561 .find(|e| e.id == ids[0])
562 .expect("summary entry present");
563 assert_eq!(summary_entry.kind, kind::SUMMARY);
564 assert_eq!(summary_entry.scope.as_deref(), Some("project:test"));
565 assert_eq!(summary_entry.session_id.as_deref(), Some("sess-1"));
566
567 // Facts saved with kind::FACT and the right metadata.
568 let all = cf.query(MATCH_ALL_QUERY, None, 100_000).await.unwrap();
569 for fact_id in &ids[1..] {
570 let fact_entry = all
571 .iter()
572 .find(|e| &e.id == fact_id)
573 .expect("fact entry present");
574 assert_eq!(fact_entry.kind, kind::FACT);
575 let metadata = fact_entry.metadata.as_ref().expect("metadata present");
576 assert_eq!(metadata["source"], "distill");
577 assert!(metadata["fact_kind"].is_string());
578 }
579 }
580
581 /// A stub [`Distiller`] that returns a fixed number of facts, used to
582 /// verify that [`ContextForge::distill_and_save`] caps excess facts
583 /// before persisting them.
584 struct ManyFactsDistiller {
585 fact_count: usize,
586 }
587
588 impl Distiller for ManyFactsDistiller {
589 fn distill(&self, _transcript: &str) -> Result<DistilledMemory> {
590 let facts = (0..self.fact_count)
591 .map(|i| Fact {
592 kind: FactKind::State,
593 text: format!("fact number {i}"),
594 })
595 .collect();
596 Ok(DistilledMemory {
597 summary: "summary".to_owned(),
598 facts,
599 })
600 }
601 }
602
603 #[tokio::test(flavor = "multi_thread")]
604 async fn distill_and_save_caps_excess_facts() {
605 let config = Config {
606 db_path: PathBuf::from(":memory:"),
607 ..Config::default()
608 };
609 let cf = ContextForge::open(config).await.unwrap();
610
611 let distiller = ManyFactsDistiller {
612 fact_count: crate::distill::MAX_FACTS + 20,
613 };
614
615 let ids = cf
616 .distill_and_save("transcript", &distiller, &SaveOptions::default())
617 .await
618 .unwrap();
619
620 // Summary ID first, then one ID per capped fact.
621 assert_eq!(ids.len(), 1 + crate::distill::MAX_FACTS);
622 assert_eq!(cf.count().await.unwrap(), 1 + crate::distill::MAX_FACTS);
623 }
624}