context_forge/lexicon/
mod.rs

1//! Lexicon-based importance scoring.
2//!
3//! Provides [`LexiconScorer`] — a trait for domain-specific importance
4//! weighting injected at [`crate::ContextForge`] construction time via the
5//! builder API — and [`ConfigLexiconScorer`], the default TOML-driven
6//! implementation.
7//!
8//! The scorer runs inside [`crate::engine::ContextEngine::assemble`] after
9//! BM25 + recency decay, before the token-budget cut. Importance weighting
10//! must happen at this point to influence which entries survive the budget
11//! cut, not re-rank survivors after.
12//!
13//! ## Two-layer design
14//!
15//! The builder always pre-seeds a [`DefaultEnglishScorer`] that recognizes
16//! plain-English importance signals ("confirmed", "never mind", etc.). A
17//! persona scorer ([`ConfigLexiconScorer`] loaded from a TOML file) is
18//! optional and stacks on top via [`CompositeLexiconScorer`]. Both layers
19//! are additive — the engine applies the `-1.0` floor clamp after fusion.
20
21pub use self::appender::{LexiconAppender, LexiconProposal};
22pub use self::bootstrap::bootstrap_prompt;
23pub use self::config::{ConfigLexiconScorer, LexiconConfig, LexiconPatterns};
24pub use self::defaults::DefaultEnglishScorer;
25
26mod appender;
27mod bootstrap;
28mod config;
29mod defaults;
30
31use std::sync::Arc;
32
33use crate::entry::ContextEntry;
34
35/// Domain-specific importance scorer, injected at construction time.
36///
37/// Returns an additive boost in the range `(-1.0, +∞)`. The engine
38/// combines this with the BM25 + recency score as:
39/// `final = base * (1.0 + boost.max(-1.0))`.
40///
41/// A boost of `0.0` (the default) leaves scores unchanged. A boost of
42/// `1.0` doubles the base score. A boost of `-1.0` zeroes it (floor).
43///
44/// Implementations **must** be `Send + Sync` — the scorer runs inside
45/// `tokio::task::spawn_blocking` on the hot `assemble` path.
46pub trait LexiconScorer: Send + Sync {
47    /// Score a single entry given the current query string.
48    fn score(&self, entry: &ContextEntry, query: &str) -> f32;
49}
50
51/// Applies multiple [`LexiconScorer`]s in sequence and sums their boosts.
52///
53/// Used internally by the builder to combine the always-on
54/// [`DefaultEnglishScorer`] with any caller-provided persona scorer. Callers
55/// can also construct one directly when they need to compose scorers without
56/// the builder.
57///
58/// The `-1.0` floor clamp happens at the engine level after fusion, not here.
59pub struct CompositeLexiconScorer {
60    scorers: Vec<Arc<dyn LexiconScorer>>,
61}
62
63impl CompositeLexiconScorer {
64    /// Create a composite from a list of scorers.
65    #[must_use]
66    pub fn new(scorers: Vec<Arc<dyn LexiconScorer>>) -> Self {
67        Self { scorers }
68    }
69}
70
71impl LexiconScorer for CompositeLexiconScorer {
72    fn score(&self, entry: &ContextEntry, query: &str) -> f32 {
73        self.scorers.iter().map(|s| s.score(entry, query)).sum()
74    }
75}
context_forge/lexicon/mod.rs

context_forge/lexicon/
mod.rs
