Skip to main content

second_brain_core/sqlite_store/
cold.rs

1use std::borrow::Cow;
2use std::collections::{HashMap, HashSet};
3use std::sync::Arc;
4use std::time::Duration;
5
6use anyhow::{Context, Result, anyhow, bail, ensure};
7use chrono::{DateTime, Utc};
8use rusqlite::{Connection, OptionalExtension, Transaction, params};
9use uuid::Uuid;
10
11use super::raw_sources::ARCHIVED_RAW;
12use super::stored_redaction::unparseable;
13use super::{MEMORY_COLS, RedactionPhase, SqliteStore, blob_to_embedding, row_to_memory};
14use crate::cold::{
15    COLD_AFTER_DAYS, ColdRecord, ColdTier, MAX_RECORD_BYTES, SEGMENT_BYTES, UnreadableSegment,
16    stub_digest, stub_for, stub_matches,
17};
18use crate::schema::{Memory, Tier};
19
20/// Rows read per candidate query; a segment takes them until it holds
21/// `SEGMENT_BYTES`.
22const CANDIDATE_BATCH: usize = 4096;
23
24/// What one call to `seal_cold_segment` did.
25#[derive(Debug, Clone, PartialEq, Eq)]
26pub enum ColdSeal {
27    Sealed(SealedSegment),
28    /// No archived raw is old enough.
29    Nothing,
30    /// No archive key is available, so every row stays hot.
31    NoKey,
32    /// The one-time stored-redaction pass has not finished, and content must
33    /// not reach a segment before it was redacted.
34    AwaitingRedaction,
35}
36
37#[derive(Debug, Clone, PartialEq, Eq)]
38pub struct SealedSegment {
39    pub segment: String,
40    pub rows: u64,
41    pub bytes: u64,
42}
43
44#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)]
45pub struct ColdStats {
46    pub segments: u64,
47    pub rows: u64,
48    pub segment_bytes: u64,
49    pub unreadable: Vec<UnreadableSegment>,
50}
51
52#[derive(Debug, Clone, Default, PartialEq, Eq)]
53pub struct ColdPassReport {
54    pub sealed_segments: u64,
55    pub sealed_rows: u64,
56    pub sealed_bytes: u64,
57    /// Segment files no manifest named any more.
58    pub removed_files: u64,
59    pub removed_partial_files: u64,
60    /// Segments no row pointed at; their files go on the next pass.
61    pub retired_segments: u64,
62    /// Segments copied without the records no row pointed at.
63    pub rewritten_segments: u64,
64    pub unreadable: Vec<UnreadableSegment>,
65    /// Why sealing stopped: `Nothing`, `NoKey` or `AwaitingRedaction`.
66    pub stopped: Option<ColdSeal>,
67}
68
69struct CandidateKey {
70    at: Vec<u8>,
71    id: Vec<u8>,
72}
73
74struct Candidate {
75    record: ColdRecord,
76    embedding_blob: Option<Vec<u8>>,
77    updated_at: String,
78}
79
80fn unreadable_known(tier: &ColdTier, known: &HashSet<String>) -> Vec<UnreadableSegment> {
81    tier.unreadable()
82        .into_iter()
83        .filter(|u| known.contains(&u.segment))
84        .collect()
85}
86
87fn manifest_on(conn: &Connection, segment: &str) -> Result<Option<String>> {
88    Ok(conn
89        .query_row(
90            "SELECT manifest FROM archive_manifest WHERE segment_id = ?1",
91            params![segment],
92            |r| r.get(0),
93        )
94        .optional()?)
95}
96
97fn insert_manifest(tx: &Transaction<'_>, segment: &str, manifest: &str) -> Result<()> {
98    tx.execute(
99        "INSERT INTO archive_manifest (segment_id, manifest, created_at) VALUES (?1, ?2, ?3)",
100        params![segment, manifest, Utc::now().to_rfc3339()],
101    )?;
102    Ok(())
103}
104
105fn retire_manifest_on(tx: &Transaction<'_>, segment: &str) -> Result<bool> {
106    Ok(tx.execute(
107        "DELETE FROM archive_manifest WHERE segment_id = ?1 \
108         AND NOT EXISTS (SELECT 1 FROM memories WHERE archived_segment = ?1)",
109        params![segment],
110    )? > 0)
111}
112
113fn cold_row(conn: &Connection, id: Uuid) -> Result<Option<(String, Option<String>)>> {
114    Ok(conn
115        .query_row(
116            "SELECT content, archived_segment FROM memories WHERE id = ?1",
117            params![id.to_string()],
118            |r| Ok((r.get(0)?, r.get(1)?)),
119        )
120        .optional()?)
121}
122
123impl SqliteStore {
124    /// Gives the store the cold tier its cold rows live in. Only the first
125    /// call takes effect; returns whether this one did.
126    pub fn attach_cold_tier(&self, tier: Arc<ColdTier>) -> bool {
127        self.cold.set(tier).is_ok()
128    }
129
130    pub fn cold_tier(&self) -> Option<&Arc<ColdTier>> {
131        self.cold.get()
132    }
133
134    fn require_cold(&self) -> Result<&Arc<ColdTier>> {
135        self.cold
136            .get()
137            .ok_or_else(|| anyhow!("this store holds cold rows but has no cold tier attached"))
138    }
139
140    /// Reads a cold record and checks it against the row's stub. No store
141    /// connection may be held across this: opening a segment can look up
142    /// keys in the store, so `manifest` loads through a connection of its
143    /// own, or through the caller's write transaction.
144    fn cold_record_with(
145        &self,
146        id: Uuid,
147        segment: &str,
148        stub: &str,
149        manifest: impl FnOnce() -> Result<Option<String>>,
150    ) -> Result<ColdRecord> {
151        let record = self.require_cold()?.read(segment, id, manifest)?;
152        ensure!(
153            stub_digest(stub).is_some(),
154            "cold memory {id} has no stub where its content was"
155        );
156        ensure!(
157            stub_matches(stub, &record.content),
158            "cold segment {segment} holds text for memory {id} that does not match its stub"
159        );
160        Ok(record)
161    }
162
163    fn cold_record_read(&self, id: Uuid, segment: &str, stub: &str) -> Result<ColdRecord> {
164        self.cold_record_with(id, segment, stub, || manifest_on(&self.reader(), segment))
165    }
166
167    /// The full text and embedding of memory `id` from its cold segment, or
168    /// None when the row is not cold. A segment that fails to open is an
169    /// error, never an absent record.
170    pub fn cold_record(&self, id: Uuid) -> Result<Option<ColdRecord>> {
171        let row = cold_row(&self.reader(), id)?;
172        match row {
173            Some((stub, Some(segment))) => self.cold_record_read(id, &segment, &stub).map(Some),
174            _ => Ok(None),
175        }
176    }
177
178    /// `get_memory_with_embedding` with a cold row's content and embedding
179    /// read back from its segment. Store readers otherwise see the stub.
180    pub fn get_memory_full(&self, id: Uuid) -> Result<Option<Memory>> {
181        let memory = self
182            .reader()
183            .query_row(
184                &format!("SELECT {MEMORY_COLS}, embedding FROM memories WHERE id = ?1"),
185                params![id.to_string()],
186                |r| row_to_memory(r, true),
187            )
188            .optional()?;
189        let Some(mut memory) = memory else {
190            return Ok(None);
191        };
192        if let Some(segment) = memory.archived_segment.clone() {
193            let record = self.cold_record_read(id, &segment, &memory.content)?;
194            memory.content = record.content;
195            memory.embedding = record.embedding;
196        }
197        Ok(Some(memory))
198    }
199
200    /// The form of `memory` a sync record carries: never its segment, which
201    /// is local, for a cold row its full text and embedding read back from
202    /// the segment, since a peer that took the stub would lose the text, and
203    /// always its row's author and scope.
204    pub(crate) fn sync_form<'m>(
205        &self,
206        tx: &Connection,
207        memory: &'m Memory,
208    ) -> Result<Cow<'m, Memory>> {
209        let form = self.full_text_form(tx, memory)?;
210        Self::with_row_attribution(tx, form)
211    }
212
213    pub(crate) fn full_text_form<'m>(
214        &self,
215        tx: &Connection,
216        memory: &'m Memory,
217    ) -> Result<Cow<'m, Memory>> {
218        let Some(segment) = &memory.archived_segment else {
219            return Ok(Cow::Borrowed(memory));
220        };
221        let mut full = memory.clone();
222        full.archived_segment = None;
223        if stub_digest(&memory.content).is_some() {
224            let record = self
225                .cold_record_with(memory.id, segment, &memory.content, || {
226                    manifest_on(tx, segment)
227                })
228                .with_context(|| format!("logging cold memory {} for sync", memory.id))?;
229            full.content = record.content;
230            full.embedding = record.embedding;
231        }
232        Ok(Cow::Owned(full))
233    }
234
235    /// `memory` with the content and segment its row holds now, when either
236    /// side is cold: a caller's copy can predate a seal or a segment rewrite.
237    pub(crate) fn as_stored_now<'m>(
238        tx: &Connection,
239        memory: &'m Memory,
240    ) -> Result<Cow<'m, Memory>> {
241        let Some((content, segment)) = cold_row(tx, memory.id)? else {
242            return Ok(Cow::Borrowed(memory));
243        };
244        if segment.is_none() && memory.archived_segment.is_none() {
245            return Ok(Cow::Borrowed(memory));
246        }
247        let mut now = memory.clone();
248        now.content = content;
249        now.archived_segment = segment;
250        Ok(Cow::Owned(now))
251    }
252
253    /// Fits an incoming synced row, its peer segment id already dropped, to
254    /// this store's cold tier: an archived row whose text matches the local
255    /// cold row's stub keeps the stub, so the update applies without
256    /// bringing the text back into the hot store.
257    pub(crate) fn keep_cold_stub(tx: &Transaction<'_>, incoming: &mut Memory) -> Result<()> {
258        if incoming.tier != Tier::Archived {
259            return Ok(());
260        }
261        if let Some((stub, Some(segment))) = cold_row(tx, incoming.id)?
262            && stub_matches(&stub, &incoming.content)
263        {
264            incoming.content = stub;
265            incoming.embedding.clear();
266            incoming.archived_segment = Some(segment);
267        }
268        Ok(())
269    }
270
271    /// Archived raws old enough to seal, oldest first after `cursor`, up to
272    /// one segment's worth; `cursor` moves past every row looked at, so a
273    /// pass reads each row once. A row too large for a block, or one whose
274    /// values do not convert, stays hot.
275    fn cold_candidates(
276        &self,
277        older_than: DateTime<Utc>,
278        cursor: &mut Option<CandidateKey>,
279    ) -> Result<Vec<Candidate>> {
280        let mut out = Vec::new();
281        let mut bytes = 0;
282        loop {
283            let keys = self.candidate_keys(older_than, cursor.as_ref())?;
284            if keys.is_empty() {
285                return Ok(out);
286            }
287            for key in keys {
288                let id = std::str::from_utf8(&key.id)
289                    .ok()
290                    .and_then(|id| Uuid::parse_str(id).ok());
291                *cursor = Some(key);
292                let Some(id) = id else {
293                    continue;
294                };
295                let Some(candidate) = self.candidate(id)? else {
296                    continue;
297                };
298                bytes += candidate.record.content.len()
299                    + candidate.embedding_blob.as_ref().map_or(0, Vec::len);
300                out.push(candidate);
301                if bytes >= SEGMENT_BYTES {
302                    return Ok(out);
303                }
304            }
305        }
306    }
307
308    fn candidate_keys(
309        &self,
310        older_than: DateTime<Utc>,
311        after: Option<&CandidateKey>,
312    ) -> Result<Vec<CandidateKey>> {
313        let conn = self.reader();
314        let mut stmt = conn.prepare(&format!(
315            "SELECT CAST(COALESCE(session_at, created_at) AS BLOB) AS at, CAST(id AS BLOB) AS bid \
316             FROM memories \
317             WHERE {ARCHIVED_RAW} AND archived_segment IS NULL \
318             AND COALESCE(session_at, created_at) < ?1 \
319             AND (?2 IS NULL OR (at, bid) > (?2, ?3)) \
320             ORDER BY at, bid LIMIT ?4"
321        ))?;
322        let keys = stmt
323            .query_map(
324                params![
325                    older_than.to_rfc3339(),
326                    after.map(|k| &k.at),
327                    after.map(|k| &k.id),
328                    CANDIDATE_BATCH as i64
329                ],
330                |r| {
331                    Ok(CandidateKey {
332                        at: r.get(0)?,
333                        id: r.get(1)?,
334                    })
335                },
336            )?
337            .collect::<rusqlite::Result<Vec<_>>>()?;
338        Ok(keys)
339    }
340
341    fn candidate(&self, id: Uuid) -> Result<Option<Candidate>> {
342        let conn = self.reader();
343        let row = conn
344            .query_row(
345                &format!(
346                    "SELECT content, embedding, updated_at FROM memories \
347                     WHERE id = ?1 AND {ARCHIVED_RAW} AND archived_segment IS NULL \
348                     AND length(CAST(content AS BLOB)) + COALESCE(length(embedding), 0) <= ?2"
349                ),
350                params![id.to_string(), MAX_RECORD_BYTES as i64],
351                |r| {
352                    Ok((
353                        r.get::<_, String>(0)?,
354                        r.get::<_, Option<Vec<u8>>>(1)?,
355                        r.get::<_, String>(2)?,
356                    ))
357                },
358            )
359            .optional();
360        let (content, embedding_blob, updated_at) = match row {
361            Ok(Some(row)) => row,
362            Ok(None) => return Ok(None),
363            Err(e) if unparseable(&e) => {
364                tracing::debug!("cold tier: memory {id} stays hot: {e}");
365                return Ok(None);
366            }
367            Err(e) => return Err(e.into()),
368        };
369        Ok(Some(Candidate {
370            record: ColdRecord {
371                id,
372                content,
373                embedding: embedding_blob
374                    .as_deref()
375                    .map(blob_to_embedding)
376                    .unwrap_or_default(),
377            },
378            embedding_blob,
379            updated_at,
380        }))
381    }
382
383    /// Moves up to one segment's worth of archived raws older than
384    /// `older_than` into a new cold segment. Each row keeps a stub of its
385    /// content and loses its embedding; `updated_at`, the sync log and the
386    /// index outbox are left alone, because archiving is local. A row that
387    /// changed after it was read, its embedding included, stays as it is.
388    pub fn seal_cold_segment(&self, older_than: DateTime<Utc>) -> Result<ColdSeal> {
389        let tier = self.require_cold()?;
390        let _pass = tier.pass_guard();
391        self.seal_locked(tier, older_than, &mut None)
392    }
393
394    fn seal_locked(
395        &self,
396        tier: &ColdTier,
397        older_than: DateTime<Utc>,
398        cursor: &mut Option<CandidateKey>,
399    ) -> Result<ColdSeal> {
400        if self.redaction_phase()? != RedactionPhase::Done {
401            return Ok(ColdSeal::AwaitingRedaction);
402        }
403        let candidates = self.cold_candidates(older_than, cursor)?;
404        if candidates.is_empty() {
405            return Ok(ColdSeal::Nothing);
406        }
407        let records: Vec<ColdRecord> = candidates.iter().map(|c| c.record.clone()).collect();
408        let Some((segment, manifest)) = tier.write_segment(&records)? else {
409            return Ok(ColdSeal::NoKey);
410        };
411        drop(records);
412        let committed = self.with_txn(|tx| {
413            insert_manifest(tx, &segment, &manifest)?;
414            let mut stmt = tx.prepare(&format!(
415                "UPDATE memories SET content = ?2, embedding = NULL, archived_segment = ?3 \
416                 WHERE id = ?1 AND {ARCHIVED_RAW} AND archived_segment IS NULL \
417                 AND updated_at = ?4 AND content = ?5 AND embedding IS ?6"
418            ))?;
419            let mut rows = 0u64;
420            for c in &candidates {
421                rows += stmt.execute(params![
422                    c.record.id.to_string(),
423                    stub_for(&c.record.content),
424                    segment,
425                    c.updated_at,
426                    c.record.content,
427                    c.embedding_blob,
428                ])? as u64;
429            }
430            drop(stmt);
431            if rows == 0 {
432                retire_manifest_on(tx, &segment)?;
433            }
434            Ok(rows)
435        });
436        match committed {
437            Ok(0) => {
438                tier.remove_segment(&segment)?;
439                Ok(ColdSeal::Nothing)
440            }
441            Ok(rows) => {
442                let bytes = tier.segment_len(&segment).unwrap_or(0);
443                Ok(ColdSeal::Sealed(SealedSegment {
444                    segment,
445                    rows,
446                    bytes,
447                }))
448            }
449            // A commit that reports an error may still have landed, so the
450            // file is left for the sweep, which keeps any a manifest names.
451            Err(e) => Err(e),
452        }
453    }
454
455    pub fn cold_manifest(&self, segment: &str) -> Result<Option<String>> {
456        manifest_on(&self.reader(), segment)
457    }
458
459    fn manifest_ids(&self) -> Result<HashSet<String>> {
460        let conn = self.reader();
461        let mut stmt = conn.prepare("SELECT segment_id FROM archive_manifest")?;
462        let ids = stmt
463            .query_map([], |r| r.get(0))?
464            .collect::<rusqlite::Result<HashSet<String>>>()?;
465        Ok(ids)
466    }
467
468    /// Opens every segment's header so one that would fail a recall shows
469    /// up in status first. Returns the segments that failed.
470    pub fn check_cold_segments(&self) -> Result<Vec<UnreadableSegment>> {
471        let tier = self.require_cold()?;
472        let known = self.manifest_ids()?;
473        for segment in &known {
474            if let Err(e) = tier.check(segment, || manifest_on(&self.reader(), segment)) {
475                tracing::warn!("{e:#}");
476            }
477        }
478        Ok(unreadable_known(tier, &known))
479    }
480
481    fn cold_rows_per_segment(&self) -> Result<HashMap<String, usize>> {
482        let conn = self.reader();
483        let mut stmt = conn.prepare(
484            "SELECT archived_segment, COUNT(*) FROM memories \
485             WHERE archived_segment IS NOT NULL GROUP BY archived_segment",
486        )?;
487        let counts = stmt
488            .query_map([], |r| {
489                Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)? as usize))
490            })?
491            .collect::<rusqlite::Result<HashMap<_, _>>>()?;
492        Ok(counts)
493    }
494
495    /// Deletes segment files no manifest names: a write whose rows never
496    /// committed, or a segment an earlier pass retired.
497    pub(super) fn sweep_cold_files(&self, tier: &ColdTier) -> Result<(u64, u64)> {
498        let known = self.manifest_ids()?;
499        let (files, partial) = tier.files()?;
500        let mut removed = 0;
501        for segment in files.iter().filter(|s| !known.contains(*s)) {
502            tier.remove_segment(segment)?;
503            removed += 1;
504        }
505        let mut partials = 0;
506        for path in partial {
507            std::fs::remove_file(&path).with_context(|| format!("removing {}", path.display()))?;
508            partials += 1;
509        }
510        Ok((removed, partials))
511    }
512
513    /// Retires the manifest of every segment no row points at any more, and
514    /// rewrites a segment some of whose records no row points at, so text a
515    /// synced edit or a hard forget replaced does not stay on disk.
516    pub(super) fn retire_dead_records(&self, tier: &ColdTier) -> Result<(u64, u64)> {
517        let live = self.cold_rows_per_segment()?;
518        let (mut retired, mut rewritten) = (0, 0);
519        for segment in self.manifest_ids()? {
520            let listed = match tier.record_count(&segment, || manifest_on(&self.reader(), &segment))
521            {
522                Ok(n) => n,
523                Err(e) => {
524                    tracing::warn!("cold segment {segment}: {e:#}");
525                    continue;
526                }
527            };
528            match live.get(&segment).copied().unwrap_or(0) {
529                0 => {
530                    retired +=
531                        u64::from(self.with_durable_txn(|tx| retire_manifest_on(tx, &segment))?)
532                }
533                n if n < listed => match self.rewrite_segment(tier, &segment) {
534                    Ok(true) => rewritten += 1,
535                    Ok(false) => {}
536                    Err(e) => tracing::warn!("rewriting cold segment {segment}: {e:#}"),
537                },
538                _ => {}
539            }
540        }
541        Ok((retired, rewritten))
542    }
543
544    /// Copies the records rows still point at into a new segment and moves
545    /// those rows to it. False when no key is available to seal the copy.
546    fn rewrite_segment(&self, tier: &ColdTier, segment: &str) -> Result<bool> {
547        let rows: Vec<(String, String)> = {
548            let conn = self.reader();
549            let mut stmt =
550                conn.prepare("SELECT id, content FROM memories WHERE archived_segment = ?1")?;
551            stmt.query_map(params![segment], |r| Ok((r.get(0)?, r.get(1)?)))?
552                .collect::<rusqlite::Result<_>>()?
553        };
554        let mut records = Vec::with_capacity(rows.len());
555        for (id, stub) in &rows {
556            records.push(self.cold_record_read(Uuid::parse_str(id)?, segment, stub)?);
557        }
558        let Some((copy, manifest)) = tier.write_segment(&records)? else {
559            return Ok(false);
560        };
561        drop(records);
562        self.with_durable_txn(|tx| {
563            insert_manifest(tx, &copy, &manifest)?;
564            let mut stmt = tx.prepare(
565                "UPDATE memories SET archived_segment = ?3 \
566                 WHERE id = ?1 AND archived_segment = ?2 AND content = ?4",
567            )?;
568            for (id, stub) in &rows {
569                stmt.execute(params![id, segment, copy, stub])?;
570            }
571            drop(stmt);
572            retire_manifest_on(tx, segment)?;
573            Ok(())
574        })?;
575        Ok(true)
576    }
577
578    /// One full cold pass: deletes files earlier passes retired, retires or
579    /// rewrites segments holding records no row points at, seals until
580    /// nothing is old enough (pausing between segments so other writers get
581    /// the writer lock), then checks every segment.
582    pub fn run_cold_pass(&self, now: DateTime<Utc>, pause: Duration) -> Result<ColdPassReport> {
583        let tier = self.require_cold()?;
584        let _pass = tier.pass_guard();
585        let mut report = ColdPassReport::default();
586        (report.removed_files, report.removed_partial_files) = self.sweep_cold_files(tier)?;
587        (report.retired_segments, report.rewritten_segments) = self.retire_dead_records(tier)?;
588        let older_than = now - chrono::Duration::days(COLD_AFTER_DAYS);
589        let mut cursor = None;
590        let sealing = loop {
591            match self.seal_locked(tier, older_than, &mut cursor) {
592                Ok(ColdSeal::Sealed(sealed)) => {
593                    report.sealed_segments += 1;
594                    report.sealed_rows += sealed.rows;
595                    report.sealed_bytes += sealed.bytes;
596                    std::thread::sleep(pause);
597                }
598                Ok(stopped) => {
599                    report.stopped = Some(stopped);
600                    break Ok(());
601                }
602                Err(e) => break Err(e),
603            }
604        };
605        report.unreadable = self.check_cold_segments()?;
606        sealing?;
607        Ok(report)
608    }
609
610    /// Fails when the store holds cold rows but no cold tier is attached,
611    /// since none of them can then be read.
612    pub fn cold_stats(&self) -> Result<ColdStats> {
613        let (segments, rows) = {
614            let conn = self.reader();
615            let segments: i64 =
616                conn.query_row("SELECT COUNT(*) FROM archive_manifest", [], |r| r.get(0))?;
617            let rows: i64 = conn.query_row(
618                "SELECT COUNT(*) FROM memories WHERE archived_segment IS NOT NULL",
619                [],
620                |r| r.get(0),
621            )?;
622            (segments as u64, rows as u64)
623        };
624        let Some(tier) = self.cold.get() else {
625            if rows > 0 {
626                bail!("{rows} cold rows but no cold tier attached to read them");
627            }
628            return Ok(ColdStats {
629                segments,
630                rows,
631                ..ColdStats::default()
632            });
633        };
634        let segment_bytes = tier
635            .files()?
636            .0
637            .iter()
638            .filter_map(|s| tier.segment_len(s))
639            .sum();
640        Ok(ColdStats {
641            segments,
642            rows,
643            segment_bytes,
644            unreadable: unreadable_known(tier, &self.manifest_ids()?),
645        })
646    }
647}