scone_core/
recall.rs

1//! Hybrid retrieval (spec §7): BM25 + vectors fused by reciprocal-rank
2//! fusion, recency-weighted, and re-verified against SQLite truth before
3//! anything is returned (memory/bugs.md P-3).
4
5use std::collections::HashMap;
6
7use crate::Engine;
8use crate::auth::ScopedSpace;
9use crate::error::{Result, SconeError};
10
11const CANDIDATES_PER_GENERATOR: usize = 50;
12const RRF_K: f32 = 60.0;
13const W_FUSED: f32 = 0.8;
14const W_RECENCY: f32 = 0.2;
15const RECENCY_HALF_LIFE_DAYS: f32 = 30.0;
16
17#[derive(Debug, Clone)]
18pub struct RecallOpts {
19    pub limit: usize,
20    pub budget_bytes: Option<usize>,
21    /// Evaluate fact validity at this instant (ISO-8601). None = now.
22    /// Time travel: a past `as_of` serves the then-valid closed facts.
23    pub as_of: Option<String>,
24}
25
26impl Default for RecallOpts {
27    fn default() -> Self {
28        Self {
29            limit: 10,
30            budget_bytes: None,
31            as_of: None,
32        }
33    }
34}
35
36#[derive(Debug, Clone)]
37pub struct FactItem {
38    pub fact_id: i64,
39    pub subject: String,
40    pub predicate: String,
41    pub object: String,
42    pub confidence: f32,
43    pub valid_from: String,
44    pub valid_until: Option<String>,
45    pub status: String,
46}
47
48#[derive(Debug, Clone)]
49pub struct RecallItem {
50    pub chunk_id: i64,
51    pub episode_id: i64,
52    pub text: String,
53    pub score: f32,
54    pub source: Option<String>,
55    pub created_at: String,
56}
57
58#[derive(Debug, Default)]
59pub struct ContextPack {
60    /// Semantic facts, first-class and budget-first (spec §7).
61    pub facts: Vec<FactItem>,
62    pub items: Vec<RecallItem>,
63    /// Generators that could not contribute, stated loudly (spec §10).
64    pub degraded: Vec<String>,
65}
66
67impl Engine {
68    pub fn recall(
69        &mut self,
70        space: &ScopedSpace,
71        query: &str,
72        opts: &RecallOpts,
73    ) -> Result<ContextPack> {
74        if query.trim().is_empty() {
75            return Err(SconeError::InvalidInput("query is empty".into()));
76        }
77        // Staged index writes become visible here (flush-on-recall).
78        self.flush_indexes()?;
79        let mut degraded = Vec::new();
80
81        // Fact generator: entity/predicate/object term match, validity
82        // evaluated at `as_of` (spec §5 I2/I3 make this a WHERE clause).
83        let facts = self.recall_facts(space, query, opts)?;
84
85        // Candidate generation: each generator may degrade, never abort.
86        let fts_hits = match self
87            .fts
88            .search(space.id() as u64, query, CANDIDATES_PER_GENERATOR)
89        {
90            Ok(hits) => hits,
91            Err(SconeError::InvalidInput(msg)) => {
92                degraded.push(format!("fts: {msg}"));
93                Vec::new()
94            }
95            Err(other) => return Err(other),
96        };
97        let vec_hits = {
98            let q = self.embedder.embed(&[query])?;
99            match q.first() {
100                Some(qv) => self.vectors.search(qv, CANDIDATES_PER_GENERATOR)?,
101                None => Vec::new(),
102            }
103        };
104
105        // Reciprocal-rank fusion across both ranked lists.
106        let mut fused: HashMap<u64, f32> = HashMap::new();
107        for hits in [&fts_hits, &vec_hits] {
108            for (rank, (chunk_id, _)) in hits.iter().enumerate() {
109                *fused.entry(*chunk_id).or_insert(0.0) += 1.0 / (RRF_K + rank as f32 + 1.0);
110            }
111        }
112        let max_fused = fused
113            .values()
114            .cloned()
115            .fold(0.0f32, f32::max)
116            .max(f32::MIN_POSITIVE);
117
118        // Truth re-verification: candidates materialize FROM SQLite or not
119        // at all; vector hits are space-filtered here too.
120        let mut items = Vec::new();
121        {
122            let mut stmt = self.conn.prepare(
123                "SELECT c.episode_id, c.start_byte, c.end_byte, e.content, e.source,
124                        e.created_at,
125                        (julianday('now') - julianday(e.created_at)) AS age_days
126                 FROM chunks c JOIN episodes e ON e.id = c.episode_id
127                 WHERE c.id = ?1 AND e.space_id = ?2",
128            )?;
129            for (chunk_id, fused_score) in &fused {
130                let row = stmt.query_row(rusqlite::params![*chunk_id as i64, space.id()], |r| {
131                    Ok((
132                        r.get::<_, i64>(0)?,
133                        r.get::<_, i64>(1)?,
134                        r.get::<_, i64>(2)?,
135                        r.get::<_, String>(3)?,
136                        r.get::<_, Option<String>>(4)?,
137                        r.get::<_, String>(5)?,
138                        r.get::<_, f64>(6)?,
139                    ))
140                });
141                let (episode_id, start, end, content, source, created_at, age_days) = match row {
142                    Ok(r) => r,
143                    Err(rusqlite::Error::QueryReturnedNoRows) => continue,
144                    Err(e) => return Err(SconeError::Db(e)),
145                };
146                let (start, end) = (start as usize, end as usize);
147                let text = content.get(start..end).unwrap_or_default().to_owned();
148                let recency = (-(age_days.max(0.0) as f32) / RECENCY_HALF_LIFE_DAYS).exp();
149                let score = W_FUSED * (fused_score / max_fused) + W_RECENCY * recency;
150                items.push(RecallItem {
151                    chunk_id: *chunk_id as i64,
152                    episode_id,
153                    text,
154                    score,
155                    source,
156                    created_at,
157                });
158            }
159        }
160        items.sort_by(|a, b| b.score.total_cmp(&a.score));
161        items.truncate(opts.limit);
162
163        // Budget rule (pinned): the top item always survives; the budget
164        // truncates strictly after it.
165        if let Some(budget) = opts.budget_bytes {
166            let mut used = 0usize;
167            let mut kept = Vec::new();
168            for item in items {
169                if !kept.is_empty() && used + item.text.len() > budget {
170                    break;
171                }
172                used += item.text.len();
173                kept.push(item);
174            }
175            items = kept;
176        }
177
178        // Budget: facts are dense and land first (spec §7); chunks share
179        // what remains under the pinned top-item rule.
180        if let Some(budget) = opts.budget_bytes {
181            let facts_bytes: usize = facts
182                .iter()
183                .map(|f| f.subject.len() + f.predicate.len() + f.object.len())
184                .sum();
185            let chunk_budget = budget.saturating_sub(facts_bytes);
186            let mut used = 0usize;
187            let mut kept = Vec::new();
188            for item in items {
189                if !kept.is_empty() && used + item.text.len() > chunk_budget {
190                    break;
191                }
192                used += item.text.len();
193                kept.push(item);
194            }
195            items = kept;
196        }
197
198        Ok(ContextPack {
199            facts,
200            items,
201            degraded,
202        })
203    }
204
205    fn recall_facts(
206        &mut self,
207        space: &ScopedSpace,
208        query: &str,
209        opts: &RecallOpts,
210    ) -> Result<Vec<FactItem>> {
211        let terms: Vec<String> = query
212            .to_lowercase()
213            .split_whitespace()
214            .filter(|t| t.len() > 2)
215            .map(|t| format!("%{t}%"))
216            .collect();
217        if terms.is_empty() {
218            return Ok(Vec::new());
219        }
220        let as_of = opts
221            .as_of
222            .clone()
223            .unwrap_or_else(|| "now-sentinel".to_owned());
224        let mut found: Vec<FactItem> = Vec::new();
225        {
226            let mut stmt = self.conn.prepare(
227                "SELECT f.id, en.canonical, f.predicate, f.object, f.confidence,
228                        f.valid_from, f.valid_until, f.status
229                 FROM facts f
230                 JOIN entities en ON en.id = f.subject_entity
231                 WHERE f.space_id = ?1
232                   AND f.valid_from <= ?2
233                   AND (f.valid_until IS NULL OR f.valid_until > ?2)
234                   AND (en.canonical LIKE ?3 OR f.predicate LIKE ?3 OR f.object LIKE ?3
235                        OR EXISTS (SELECT 1 FROM entity_aliases a
236                                   WHERE a.entity_id = f.subject_entity AND a.alias LIKE ?3))
237                 ORDER BY f.confidence DESC, f.access_count DESC
238                 LIMIT ?4",
239            )?;
240            let now: String =
241                self.conn
242                    .query_row("SELECT strftime('%Y-%m-%dT%H:%M:%fZ','now')", [], |r| {
243                        r.get(0)
244                    })?;
245            let effective = if as_of == "now-sentinel" { now } else { as_of };
246            for term in &terms {
247                let rows = stmt.query_map(
248                    rusqlite::params![space.id(), effective, term, opts.limit as i64],
249                    |r| {
250                        Ok(FactItem {
251                            fact_id: r.get(0)?,
252                            subject: r.get(1)?,
253                            predicate: r.get(2)?,
254                            object: r.get(3)?,
255                            confidence: r.get(4)?,
256                            valid_from: r.get(5)?,
257                            valid_until: r.get(6)?,
258                            status: r.get(7)?,
259                        })
260                    },
261                )?;
262                for row in rows {
263                    let row = row?;
264                    if !found.iter().any(|f| f.fact_id == row.fact_id) {
265                        found.push(row);
266                    }
267                }
268            }
269        }
270        found.truncate(opts.limit);
271        // Reinforcement: recalled present-time facts strengthen (spec §7).
272        // Historical (as_of) browsing does not rewrite the present.
273        if opts.as_of.is_none() {
274            for f in &found {
275                self.conn.execute(
276                    "UPDATE facts SET access_count = access_count + 1,
277                            last_accessed = strftime('%Y-%m-%dT%H:%M:%fZ','now')
278                     WHERE id = ?1",
279                    [f.fact_id],
280                )?;
281            }
282        }
283        Ok(found)
284    }
285}
scone_core/recall.rs

scone_core/
recall.rs
