context_forge/lexicon/
mod.rs

1//! Lexicon-based importance scoring.
2//!
3//! Provides [`LexiconScorer`] — a trait for domain-specific importance
4//! weighting injected at [`crate::ContextForge`] construction time via the
5//! builder API — and [`ConfigLexiconScorer`], the default TOML-driven
6//! implementation.
7//!
8//! The scorer runs inside [`crate::engine::ContextEngine::assemble`] after
9//! BM25 + recency decay, before the token-budget cut. Importance weighting
10//! must happen at this point to influence which entries survive the budget
11//! cut, not re-rank survivors after.
12//!
13//! ## Two-layer design
14//!
15//! Lexicon scoring is opt-in on the builder. [`DefaultEnglishScorer`] recognizes
16//! plain-English importance signals ("confirmed", "never mind", etc.) and is
17//! enabled via `with_default_english_scorer`. A persona scorer
18//! ([`ConfigLexiconScorer`] loaded from a TOML file) is added via
19//! `with_persona_scorer`. When both are set they stack via
20//! [`CompositeLexiconScorer`] — additive, with the engine applying the `-1.0`
21//! floor clamp after fusion.
22
23pub use self::appender::{LexiconAppender, LexiconProposal};
24pub use self::bootstrap::bootstrap_prompt;
25pub use self::config::{ConfigLexiconScorer, LexiconConfig, LexiconPatterns};
26pub use self::defaults::DefaultEnglishScorer;
27
28mod appender;
29mod bootstrap;
30mod config;
31mod defaults;
32
33use std::sync::Arc;
34
35use crate::entry::ContextEntry;
36
37/// Domain-specific importance scorer, injected at construction time.
38///
39/// Returns an additive boost in the range `(-1.0, +∞)`. The engine
40/// combines this with the BM25 + recency score as:
41/// `final = base * (1.0 + boost.max(-1.0))`.
42///
43/// A boost of `0.0` (the default) leaves scores unchanged. A boost of
44/// `1.0` doubles the base score. A boost of `-1.0` zeroes it (floor).
45///
46/// Implementations **must** be `Send + Sync` — the scorer runs inside
47/// `tokio::task::spawn_blocking` on the hot `assemble` path.
48pub trait LexiconScorer: Send + Sync {
49    /// Score a single entry given the current query string.
50    fn score(&self, entry: &ContextEntry, query: &str) -> f32;
51}
52
53/// Applies multiple [`LexiconScorer`]s in sequence and sums their boosts.
54///
55/// Used internally by the builder to combine the always-on
56/// [`DefaultEnglishScorer`] with any caller-provided persona scorer. Callers
57/// can also construct one directly when they need to compose scorers without
58/// the builder.
59///
60/// The `-1.0` floor clamp happens at the engine level after fusion, not here.
61pub struct CompositeLexiconScorer {
62    scorers: Vec<Arc<dyn LexiconScorer>>,
63}
64
65impl CompositeLexiconScorer {
66    /// Create a composite from a list of scorers.
67    #[must_use]
68    pub fn new(scorers: Vec<Arc<dyn LexiconScorer>>) -> Self {
69        Self { scorers }
70    }
71}
72
73impl LexiconScorer for CompositeLexiconScorer {
74    fn score(&self, entry: &ContextEntry, query: &str) -> f32 {
75        self.scorers.iter().map(|s| s.score(entry, query)).sum()
76    }
77}
context_forge/lexicon/mod.rs

context_forge/lexicon/
mod.rs
