1use anyhow::Result;
2use rusqlite::{OptionalExtension, Transaction, params};
3use uuid::Uuid;
4
5use super::{SqliteStore, blob_to_embedding, cosine_blob, embedding_to_blob, parse_uuid};
6use crate::chunk;
7use crate::schema::MemoryType;
8
9#[derive(Debug, Clone, PartialEq)]
13pub struct ChunkWork {
14 pub memory_id: Uuid,
15 pub content_hash: String,
16 pub chunks: Vec<String>,
17}
18
19#[derive(Debug, Clone, PartialEq)]
20pub struct MemoryChunk {
21 pub ord: usize,
22 pub text: String,
23 pub embedding: Vec<f32>,
24 pub content_hash: String,
25}
26
27impl SqliteStore {
28 pub fn chunks_needing_embedding(&self, limit: usize) -> Result<Vec<ChunkWork>> {
33 Ok(self.scan_chunk_work(limit)?.0)
34 }
35
36 pub fn scan_chunk_work(&self, limit: usize) -> Result<(Vec<ChunkWork>, usize)> {
40 let conn = self.reader();
41 let mut stmt = conn.prepare(
42 "SELECT m.id, m.content, \
43 (SELECT c.content_hash FROM memory_chunks c WHERE c.memory_id = m.id LIMIT 1) \
44 FROM memories m \
45 WHERE m.tier = 'hot' AND m.superseded_by IS NULL AND length(m.content) >= ?1 \
46 ORDER BY m.rowid",
47 )?;
48 let mut rows = stmt.query(params![(2 * chunk::MIN_CHUNK_CHARS) as i64])?;
49 let mut out = Vec::new();
50 let mut pending = 0;
51 while let Some(row) = rows.next()? {
52 let content: String = row.get(1)?;
53 let stored: Option<String> = row.get(2)?;
54 let content_hash = chunk::content_hash(&content);
55 if stored.as_deref() == Some(content_hash.as_str()) {
56 continue;
57 }
58 let chunks = chunk::split(&content);
59 if chunks.is_empty() {
60 continue;
61 }
62 pending += 1;
63 if out.len() < limit {
64 out.push(ChunkWork {
65 memory_id: parse_uuid(&row.get::<_, String>(0)?)?,
66 content_hash,
67 chunks,
68 });
69 }
70 }
71 Ok((out, pending))
72 }
73
74 pub fn chunk_progress(&self) -> Result<(usize, Option<usize>)> {
78 let conn = self.reader();
79 let chunked: i64 = conn.query_row(
80 "SELECT COUNT(DISTINCT c.memory_id) FROM memory_chunks c \
81 JOIN memories m ON m.id = c.memory_id \
82 WHERE m.tier = 'hot' AND m.superseded_by IS NULL",
83 [],
84 |r| r.get(0),
85 )?;
86 let pending: Option<String> = conn
87 .query_row(
88 "SELECT value FROM index_meta WHERE key = 'chunks_pending'",
89 [],
90 |r| r.get(0),
91 )
92 .optional()?;
93 Ok((chunked as usize, pending.and_then(|v| v.parse().ok())))
94 }
95
96 pub(crate) fn set_chunks_pending(&self, pending: usize) -> Result<()> {
97 let conn = self.conn();
98 conn.execute(
99 "INSERT OR REPLACE INTO index_meta (key, value) VALUES ('chunks_pending', ?1)",
100 params![pending.to_string()],
101 )?;
102 Ok(())
103 }
104
105 pub fn store_chunks(
111 &self,
112 memory_id: Uuid,
113 content_hash: &str,
114 chunks: &[(usize, String, Vec<f32>)],
115 ) -> Result<bool> {
116 let id = memory_id.to_string();
117 self.with_txn(|tx| {
118 let content: Option<String> = tx
119 .query_row(
120 "SELECT content FROM memories \
121 WHERE id = ?1 AND tier = 'hot' AND superseded_by IS NULL",
122 params![id],
123 |r| r.get(0),
124 )
125 .optional()?;
126 if content.is_none_or(|c| chunk::content_hash(&c) != content_hash) {
127 return Ok(false);
128 }
129 Self::replace_chunk_rows(
130 tx,
131 &id,
132 content_hash,
133 chunks
134 .iter()
135 .map(|(ord, text, embedding)| (*ord, text.as_str(), embedding.as_slice())),
136 )?;
137 Self::append_outbox(tx, memory_id, "upsert")?;
138 Ok(true)
139 })
140 }
141
142 pub(crate) fn replace_chunk_rows<'a>(
144 tx: &Transaction<'_>,
145 memory_id: &str,
146 content_hash: &str,
147 chunks: impl IntoIterator<Item = (usize, &'a str, &'a [f32])>,
148 ) -> Result<()> {
149 tx.execute(
150 "DELETE FROM memory_chunks WHERE memory_id = ?1",
151 params![memory_id],
152 )?;
153 let mut insert = tx.prepare(
154 "INSERT INTO memory_chunks (memory_id, ord, text, embedding, content_hash) \
155 VALUES (?1, ?2, ?3, ?4, ?5)",
156 )?;
157 for (ord, text, embedding) in chunks {
158 let blob = (!embedding.is_empty()).then(|| embedding_to_blob(embedding));
159 insert.execute(params![memory_id, ord as i64, text, blob, content_hash])?;
160 }
161 Ok(())
162 }
163
164 pub fn chunks_for(&self, memory_id: Uuid) -> Result<Vec<MemoryChunk>> {
165 let conn = self.reader();
166 let mut stmt = conn.prepare(
167 "SELECT ord, text, embedding, content_hash FROM memory_chunks \
168 WHERE memory_id = ?1 ORDER BY ord",
169 )?;
170 let rows = stmt.query_map(params![memory_id.to_string()], |r| {
171 Ok(MemoryChunk {
172 ord: r.get::<_, i64>(0)? as usize,
173 text: r.get(1)?,
174 embedding: r
175 .get::<_, Option<Vec<u8>>>(2)?
176 .map(|b| blob_to_embedding(&b))
177 .unwrap_or_default(),
178 content_hash: r.get(3)?,
179 })
180 })?;
181 Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
182 }
183
184 pub fn delete_chunks(&self, memory_id: Uuid) -> Result<()> {
185 let conn = self.conn();
186 conn.execute(
187 "DELETE FROM memory_chunks WHERE memory_id = ?1",
188 params![memory_id.to_string()],
189 )?;
190 Ok(())
191 }
192
193 pub fn chunk_vec_key(&self, memory_id: Uuid, ord: usize) -> Result<u64> {
196 let conn = self.conn();
197 let id = memory_id.to_string();
198 conn.execute(
199 "INSERT OR IGNORE INTO chunk_vec_keys (memory_id, ord) VALUES (?1, ?2)",
200 params![id, ord as i64],
201 )?;
202 let key: i64 = conn.query_row(
203 "SELECT key FROM chunk_vec_keys WHERE memory_id = ?1 AND ord = ?2",
204 params![id, ord as i64],
205 |r| r.get(0),
206 )?;
207 Ok(key as u64)
208 }
209
210 pub fn chunk_vec_keys_for(&self, memory_id: Uuid) -> Result<Vec<(usize, u64)>> {
212 let conn = self.reader();
213 let mut stmt =
214 conn.prepare("SELECT ord, key FROM chunk_vec_keys WHERE memory_id = ?1 ORDER BY ord")?;
215 let rows = stmt.query_map(params![memory_id.to_string()], |r| {
216 Ok((r.get::<_, i64>(0)? as usize, r.get::<_, i64>(1)? as u64))
217 })?;
218 Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
219 }
220
221 pub fn delete_chunk_vec_key(&self, memory_id: Uuid, ord: usize) -> Result<()> {
224 let conn = self.conn();
225 conn.execute(
226 "DELETE FROM chunk_vec_keys WHERE memory_id = ?1 AND ord = ?2",
227 params![memory_id.to_string(), ord as i64],
228 )?;
229 Ok(())
230 }
231
232 pub fn chunk_for_vec_key(&self, key: u64) -> Result<Option<(Uuid, usize)>> {
233 let conn = self.reader();
234 let hit: Option<(String, i64)> = conn
235 .query_row(
236 "SELECT memory_id, ord FROM chunk_vec_keys WHERE key = ?1",
237 params![key as i64],
238 |r| Ok((r.get(0)?, r.get(1)?)),
239 )
240 .optional()?;
241 hit.map(|(id, ord)| Ok((Uuid::parse_str(&id)?, ord as usize)))
242 .transpose()
243 }
244
245 pub fn embedded_chunk_count(&self, dim: usize) -> Result<usize> {
248 let conn = self.reader();
249 let count: i64 = conn.query_row(
250 "SELECT COUNT(*) FROM memory_chunks c JOIN memories m ON m.id = c.memory_id \
251 WHERE m.tier = 'hot' AND m.superseded_by IS NULL \
252 AND c.embedding IS NOT NULL AND length(c.embedding) = ?1",
253 params![(dim * 4) as i64],
254 |r| r.get(0),
255 )?;
256 Ok(count as usize)
257 }
258
259 pub fn live_chunked_memories(&self) -> Result<Vec<(Uuid, String)>> {
262 let conn = self.reader();
263 let mut stmt = conn.prepare(
264 "SELECT m.id, m.content FROM memories m \
265 WHERE m.tier = 'hot' AND m.superseded_by IS NULL \
266 AND EXISTS (SELECT 1 FROM memory_chunks c WHERE c.memory_id = m.id)",
267 )?;
268 let rows = stmt.query_map([], |r| {
269 let id = parse_uuid(&r.get::<_, String>(0)?)?;
270 let content: String = r.get(1)?;
271 Ok((id, chunk::content_hash(&content)))
272 })?;
273 Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
274 }
275
276 pub fn purge_chunks_outside_live_set(&self) -> Result<()> {
280 const LIVE: &str = "SELECT id FROM memories WHERE tier = 'hot' AND superseded_by IS NULL";
281 self.with_txn(|tx| {
282 tx.execute(
283 &format!("DELETE FROM memory_chunks WHERE memory_id NOT IN ({LIVE})"),
284 [],
285 )?;
286 tx.execute(
287 &format!("DELETE FROM chunk_vec_keys WHERE memory_id NOT IN ({LIVE})"),
288 [],
289 )?;
290 Ok(())
291 })
292 }
293
294 pub(crate) fn chunk_window_text(&self, memory_id: Uuid, ord: usize) -> Result<Option<String>> {
295 let conn = self.reader();
296 let mut stmt = conn.prepare(
297 "SELECT text FROM memory_chunks WHERE memory_id = ?1 AND ord BETWEEN ?2 AND ?3 \
298 ORDER BY ord",
299 )?;
300 let texts = stmt
301 .query_map(
302 params![
303 memory_id.to_string(),
304 ord.saturating_sub(1) as i64,
305 ord as i64 + 1
306 ],
307 |r| r.get::<_, String>(0),
308 )?
309 .collect::<rusqlite::Result<Vec<_>>>()?;
310 Ok((!texts.is_empty()).then(|| texts.join(" ")))
311 }
312
313 pub fn filtered_chunk_search(
320 &self,
321 embedding: &[f32],
322 source: Option<&str>,
323 memory_type: Option<MemoryType>,
324 project_path: Option<&str>,
325 limit: usize,
326 ) -> Result<Vec<(Uuid, f32, usize)>> {
327 if limit == 0 {
328 return Ok(Vec::new());
329 }
330 let (predicate, args) = Self::scope_predicate(source, memory_type, project_path);
331 let conn = self.reader();
332 let mut stmt = conn.prepare(&format!(
333 "SELECT memory_id, ord, embedding FROM memory_chunks \
334 WHERE embedding IS NOT NULL AND memory_id IN ( \
335 SELECT id FROM memories \
336 WHERE tier = 'hot' AND superseded_by IS NULL{predicate}) \
337 ORDER BY memory_id, ord"
338 ))?;
339 let bound: Vec<&dyn rusqlite::ToSql> =
340 args.iter().map(|a| a as &dyn rusqlite::ToSql).collect();
341 let query_norm = embedding.iter().map(|x| x * x).sum::<f32>().sqrt();
342 let mut rows = stmt.query(bound.as_slice())?;
343 let mut top: Vec<(f32, String, usize)> = Vec::with_capacity(limit + 1);
344 let mut current: Option<(f32, String, usize)> = None;
345 let place = |best: (f32, String, usize), top: &mut Vec<(f32, String, usize)>| {
346 let at = top.partition_point(|(s, _, _)| *s >= best.0);
347 if at < limit {
348 top.insert(at, best);
349 top.truncate(limit);
350 }
351 };
352 while let Some(row) = rows.next()? {
353 let Some(blob) = row.get_ref(2)?.as_blob_or_null()? else {
354 continue;
355 };
356 let similarity = cosine_blob(embedding, query_norm, blob);
357 let memory_id = row.get_ref(0)?.as_str()?;
358 let ord = row.get::<_, i64>(1)? as usize;
359 match &mut current {
360 Some(best) if best.1 == memory_id => {
361 if similarity > best.0 {
362 best.0 = similarity;
363 best.2 = ord;
364 }
365 }
366 _ => {
367 if let Some(done) = current.take() {
368 place(done, &mut top);
369 }
370 current = Some((similarity, memory_id.to_string(), ord));
371 }
372 }
373 }
374 if let Some(done) = current.take() {
375 place(done, &mut top);
376 }
377 top.into_iter()
378 .map(|(similarity, id, ord)| Ok((Uuid::parse_str(&id)?, similarity, ord)))
379 .collect()
380 }
381}