1use anyhow::Result;
2use rusqlite::{OptionalExtension, params};
3use uuid::Uuid;
4
5use super::{SqliteStore, blob_to_embedding, cosine_blob, embedding_to_blob, parse_uuid};
6use crate::chunk;
7use crate::schema::MemoryType;
8
9#[derive(Debug, Clone, PartialEq)]
13pub struct ChunkWork {
14 pub memory_id: Uuid,
15 pub content_hash: String,
16 pub chunks: Vec<String>,
17}
18
19#[derive(Debug, Clone, PartialEq)]
20pub struct MemoryChunk {
21 pub ord: usize,
22 pub text: String,
23 pub embedding: Vec<f32>,
24 pub content_hash: String,
25}
26
27impl SqliteStore {
28 pub fn chunks_needing_embedding(&self, limit: usize) -> Result<Vec<ChunkWork>> {
33 Ok(self.scan_chunk_work(limit)?.0)
34 }
35
36 pub fn scan_chunk_work(&self, limit: usize) -> Result<(Vec<ChunkWork>, usize)> {
40 let conn = self.reader();
41 let mut stmt = conn.prepare(
42 "SELECT m.id, m.content, \
43 (SELECT c.content_hash FROM memory_chunks c WHERE c.memory_id = m.id LIMIT 1) \
44 FROM memories m \
45 WHERE m.tier = 'hot' AND m.superseded_by IS NULL AND length(m.content) >= ?1 \
46 ORDER BY m.rowid",
47 )?;
48 let mut rows = stmt.query(params![(2 * chunk::MIN_CHUNK_CHARS) as i64])?;
49 let mut out = Vec::new();
50 let mut pending = 0;
51 while let Some(row) = rows.next()? {
52 let content: String = row.get(1)?;
53 let stored: Option<String> = row.get(2)?;
54 let content_hash = chunk::content_hash(&content);
55 if stored.as_deref() == Some(content_hash.as_str()) {
56 continue;
57 }
58 let chunks = chunk::split(&content);
59 if chunks.is_empty() {
60 continue;
61 }
62 pending += 1;
63 if out.len() < limit {
64 out.push(ChunkWork {
65 memory_id: parse_uuid(&row.get::<_, String>(0)?)?,
66 content_hash,
67 chunks,
68 });
69 }
70 }
71 Ok((out, pending))
72 }
73
74 pub fn chunk_progress(&self) -> Result<(usize, Option<usize>)> {
78 let conn = self.reader();
79 let chunked: i64 = conn.query_row(
80 "SELECT COUNT(DISTINCT c.memory_id) FROM memory_chunks c \
81 JOIN memories m ON m.id = c.memory_id \
82 WHERE m.tier = 'hot' AND m.superseded_by IS NULL",
83 [],
84 |r| r.get(0),
85 )?;
86 let pending: Option<String> = conn
87 .query_row(
88 "SELECT value FROM index_meta WHERE key = 'chunks_pending'",
89 [],
90 |r| r.get(0),
91 )
92 .optional()?;
93 Ok((chunked as usize, pending.and_then(|v| v.parse().ok())))
94 }
95
96 pub(crate) fn set_chunks_pending(&self, pending: usize) -> Result<()> {
97 let conn = self.conn();
98 conn.execute(
99 "INSERT OR REPLACE INTO index_meta (key, value) VALUES ('chunks_pending', ?1)",
100 params![pending.to_string()],
101 )?;
102 Ok(())
103 }
104
105 pub fn store_chunks(
111 &self,
112 memory_id: Uuid,
113 content_hash: &str,
114 chunks: &[(usize, String, Vec<f32>)],
115 ) -> Result<bool> {
116 let id = memory_id.to_string();
117 self.with_txn(|tx| {
118 let content: Option<String> = tx
119 .query_row(
120 "SELECT content FROM memories \
121 WHERE id = ?1 AND tier = 'hot' AND superseded_by IS NULL",
122 params![id],
123 |r| r.get(0),
124 )
125 .optional()?;
126 if content.is_none_or(|c| chunk::content_hash(&c) != content_hash) {
127 return Ok(false);
128 }
129 tx.execute(
130 "DELETE FROM memory_chunks WHERE memory_id = ?1",
131 params![id],
132 )?;
133 let mut insert = tx.prepare(
134 "INSERT INTO memory_chunks (memory_id, ord, text, embedding, content_hash) \
135 VALUES (?1, ?2, ?3, ?4, ?5)",
136 )?;
137 for (ord, text, embedding) in chunks {
138 let blob = (!embedding.is_empty()).then(|| embedding_to_blob(embedding));
139 insert.execute(params![id, *ord as i64, text, blob, content_hash])?;
140 }
141 Self::append_outbox(tx, memory_id, "upsert")?;
142 Ok(true)
143 })
144 }
145
146 pub fn chunks_for(&self, memory_id: Uuid) -> Result<Vec<MemoryChunk>> {
147 let conn = self.reader();
148 let mut stmt = conn.prepare(
149 "SELECT ord, text, embedding, content_hash FROM memory_chunks \
150 WHERE memory_id = ?1 ORDER BY ord",
151 )?;
152 let rows = stmt.query_map(params![memory_id.to_string()], |r| {
153 Ok(MemoryChunk {
154 ord: r.get::<_, i64>(0)? as usize,
155 text: r.get(1)?,
156 embedding: r
157 .get::<_, Option<Vec<u8>>>(2)?
158 .map(|b| blob_to_embedding(&b))
159 .unwrap_or_default(),
160 content_hash: r.get(3)?,
161 })
162 })?;
163 Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
164 }
165
166 pub fn delete_chunks(&self, memory_id: Uuid) -> Result<()> {
167 let conn = self.conn();
168 conn.execute(
169 "DELETE FROM memory_chunks WHERE memory_id = ?1",
170 params![memory_id.to_string()],
171 )?;
172 Ok(())
173 }
174
175 pub fn chunk_vec_key(&self, memory_id: Uuid, ord: usize) -> Result<u64> {
178 let conn = self.conn();
179 let id = memory_id.to_string();
180 conn.execute(
181 "INSERT OR IGNORE INTO chunk_vec_keys (memory_id, ord) VALUES (?1, ?2)",
182 params![id, ord as i64],
183 )?;
184 let key: i64 = conn.query_row(
185 "SELECT key FROM chunk_vec_keys WHERE memory_id = ?1 AND ord = ?2",
186 params![id, ord as i64],
187 |r| r.get(0),
188 )?;
189 Ok(key as u64)
190 }
191
192 pub fn chunk_vec_keys_for(&self, memory_id: Uuid) -> Result<Vec<(usize, u64)>> {
194 let conn = self.reader();
195 let mut stmt =
196 conn.prepare("SELECT ord, key FROM chunk_vec_keys WHERE memory_id = ?1 ORDER BY ord")?;
197 let rows = stmt.query_map(params![memory_id.to_string()], |r| {
198 Ok((r.get::<_, i64>(0)? as usize, r.get::<_, i64>(1)? as u64))
199 })?;
200 Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
201 }
202
203 pub fn delete_chunk_vec_key(&self, memory_id: Uuid, ord: usize) -> Result<()> {
206 let conn = self.conn();
207 conn.execute(
208 "DELETE FROM chunk_vec_keys WHERE memory_id = ?1 AND ord = ?2",
209 params![memory_id.to_string(), ord as i64],
210 )?;
211 Ok(())
212 }
213
214 pub fn chunk_for_vec_key(&self, key: u64) -> Result<Option<(Uuid, usize)>> {
215 let conn = self.reader();
216 let hit: Option<(String, i64)> = conn
217 .query_row(
218 "SELECT memory_id, ord FROM chunk_vec_keys WHERE key = ?1",
219 params![key as i64],
220 |r| Ok((r.get(0)?, r.get(1)?)),
221 )
222 .optional()?;
223 hit.map(|(id, ord)| Ok((Uuid::parse_str(&id)?, ord as usize)))
224 .transpose()
225 }
226
227 pub fn embedded_chunk_count(&self, dim: usize) -> Result<usize> {
230 let conn = self.reader();
231 let count: i64 = conn.query_row(
232 "SELECT COUNT(*) FROM memory_chunks c JOIN memories m ON m.id = c.memory_id \
233 WHERE m.tier = 'hot' AND m.superseded_by IS NULL \
234 AND c.embedding IS NOT NULL AND length(c.embedding) = ?1",
235 params![(dim * 4) as i64],
236 |r| r.get(0),
237 )?;
238 Ok(count as usize)
239 }
240
241 pub fn live_chunked_memories(&self) -> Result<Vec<(Uuid, String)>> {
244 let conn = self.reader();
245 let mut stmt = conn.prepare(
246 "SELECT m.id, m.content FROM memories m \
247 WHERE m.tier = 'hot' AND m.superseded_by IS NULL \
248 AND EXISTS (SELECT 1 FROM memory_chunks c WHERE c.memory_id = m.id)",
249 )?;
250 let rows = stmt.query_map([], |r| {
251 let id = parse_uuid(&r.get::<_, String>(0)?)?;
252 let content: String = r.get(1)?;
253 Ok((id, chunk::content_hash(&content)))
254 })?;
255 Ok(rows.collect::<rusqlite::Result<Vec<_>>>()?)
256 }
257
258 pub fn purge_chunks_outside_live_set(&self) -> Result<()> {
262 const LIVE: &str = "SELECT id FROM memories WHERE tier = 'hot' AND superseded_by IS NULL";
263 self.with_txn(|tx| {
264 tx.execute(
265 &format!("DELETE FROM memory_chunks WHERE memory_id NOT IN ({LIVE})"),
266 [],
267 )?;
268 tx.execute(
269 &format!("DELETE FROM chunk_vec_keys WHERE memory_id NOT IN ({LIVE})"),
270 [],
271 )?;
272 Ok(())
273 })
274 }
275
276 pub(crate) fn chunk_window_text(&self, memory_id: Uuid, ord: usize) -> Result<Option<String>> {
277 let conn = self.reader();
278 let mut stmt = conn.prepare(
279 "SELECT text FROM memory_chunks WHERE memory_id = ?1 AND ord BETWEEN ?2 AND ?3 \
280 ORDER BY ord",
281 )?;
282 let texts = stmt
283 .query_map(
284 params![
285 memory_id.to_string(),
286 ord.saturating_sub(1) as i64,
287 ord as i64 + 1
288 ],
289 |r| r.get::<_, String>(0),
290 )?
291 .collect::<rusqlite::Result<Vec<_>>>()?;
292 Ok((!texts.is_empty()).then(|| texts.join(" ")))
293 }
294
295 pub fn filtered_chunk_search(
302 &self,
303 embedding: &[f32],
304 source: Option<&str>,
305 memory_type: Option<MemoryType>,
306 project_path: Option<&str>,
307 limit: usize,
308 ) -> Result<Vec<(Uuid, f32, usize)>> {
309 if limit == 0 {
310 return Ok(Vec::new());
311 }
312 let (predicate, args) = Self::scope_predicate(source, memory_type, project_path);
313 let conn = self.reader();
314 let mut stmt = conn.prepare(&format!(
315 "SELECT memory_id, ord, embedding FROM memory_chunks \
316 WHERE embedding IS NOT NULL AND memory_id IN ( \
317 SELECT id FROM memories \
318 WHERE tier = 'hot' AND superseded_by IS NULL{predicate}) \
319 ORDER BY memory_id, ord"
320 ))?;
321 let bound: Vec<&dyn rusqlite::ToSql> =
322 args.iter().map(|a| a as &dyn rusqlite::ToSql).collect();
323 let query_norm = embedding.iter().map(|x| x * x).sum::<f32>().sqrt();
324 let mut rows = stmt.query(bound.as_slice())?;
325 let mut top: Vec<(f32, String, usize)> = Vec::with_capacity(limit + 1);
326 let mut current: Option<(f32, String, usize)> = None;
327 let place = |best: (f32, String, usize), top: &mut Vec<(f32, String, usize)>| {
328 let at = top.partition_point(|(s, _, _)| *s >= best.0);
329 if at < limit {
330 top.insert(at, best);
331 top.truncate(limit);
332 }
333 };
334 while let Some(row) = rows.next()? {
335 let Some(blob) = row.get_ref(2)?.as_blob_or_null()? else {
336 continue;
337 };
338 let similarity = cosine_blob(embedding, query_norm, blob);
339 let memory_id = row.get_ref(0)?.as_str()?;
340 let ord = row.get::<_, i64>(1)? as usize;
341 match &mut current {
342 Some(best) if best.1 == memory_id => {
343 if similarity > best.0 {
344 best.0 = similarity;
345 best.2 = ord;
346 }
347 }
348 _ => {
349 if let Some(done) = current.take() {
350 place(done, &mut top);
351 }
352 current = Some((similarity, memory_id.to_string(), ord));
353 }
354 }
355 }
356 if let Some(done) = current.take() {
357 place(done, &mut top);
358 }
359 top.into_iter()
360 .map(|(similarity, id, ord)| Ok((Uuid::parse_str(&id)?, similarity, ord)))
361 .collect()
362 }
363}