Skip to main content

second_brain_core/cold/
mod.rs

1mod envelope_keys;
2mod segment;
3
4use std::collections::{BTreeMap, HashMap};
5use std::path::{Path, PathBuf};
6use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
7
8use anyhow::{Context, Result, anyhow};
9use sha2::{Digest, Sha256};
10
11pub use crate::keys::KeyId;
12use uuid::Uuid;
13pub use zeroize::Zeroizing;
14
15pub use envelope_keys::EnvelopeArchiveKeys;
16pub use segment::{BlockEntry, ColdRecord, SegmentManifest, SegmentSummary};
17use segment::{OpenSegment, seal};
18
19/// Archived raws whose session time (creation time when unknown) is older
20/// than this move to the cold tier.
21pub const COLD_AFTER_DAYS: i64 = 30;
22/// About this much record plaintext goes into one segment file.
23pub const SEGMENT_BYTES: usize = 8 << 20;
24pub const BLOCK_BYTES: usize = 256 << 10;
25/// A larger row stays hot: it would fill a block alone, and a reader refuses
26/// blocks of more than 64 MiB.
27pub const MAX_RECORD_BYTES: usize = 32 << 20;
28const STUB_MARKER: &str = "\u{1e}sha256:";
29const SEGMENT_SUFFIX: &str = "seg";
30const MANIFEST_CACHE: usize = 32;
31
32pub fn dir_for_db(db_path: &Path) -> PathBuf {
33    db_path.with_extension("cold")
34}
35
36/// Names the `archive-kek` subkey a segment is sealed under: the identity
37/// key epoch and that epoch's key id.
38#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
39pub struct KekLabel {
40    pub epoch: u32,
41    pub key_id: KeyId,
42}
43
44pub struct Kek {
45    pub label: KekLabel,
46    pub key: Zeroizing<[u8; 32]>,
47}
48
49/// Where segment sealing gets its key-encryption key. The cold tier never
50/// reads key material itself; the daemon hands it the identity key's
51/// `archive-kek` subkey through this.
52pub trait ArchiveKeys: Send + Sync {
53    /// The subkey new segments are sealed under, or None while this device
54    /// holds no identity key, which leaves archived raws hot.
55    fn current(&self) -> Result<Option<Kek>>;
56    /// The subkey of the epoch `label` names, or None when this device does
57    /// not hold that epoch.
58    fn by_label(&self, label: &KekLabel) -> Result<Option<Zeroizing<[u8; 32]>>>;
59}
60
61/// The stub a cold row keeps as its content: the text's first
62/// `store::CONTENT_PREFIX_CHARS` characters, which ingest's prefix dedup
63/// matches on, and the digest of the whole, so a row can be compared and its
64/// segment record checked without the key.
65pub fn stub_for(content: &str) -> String {
66    let lead: String = content
67        .chars()
68        .take(crate::store::CONTENT_PREFIX_CHARS)
69        .collect();
70    format!("{lead}{STUB_MARKER}{}", content_digest(content))
71}
72
73/// SQL for the leading text of the stub in `column`, which equals the
74/// `content_prefix` of the text the stub stands for.
75pub(crate) fn stub_lead_sql(column: &str) -> String {
76    format!(
77        "substr({column}, 1, length({column}) - {})",
78        STUB_MARKER.chars().count() + 64
79    )
80}
81
82/// Whether `stub` stands for exactly `content`.
83pub fn stub_matches(stub: &str, content: &str) -> bool {
84    stub_digest(stub) == Some(content_digest(content).as_str())
85}
86
87pub fn content_digest(content: &str) -> String {
88    crate::keys::hex(&Sha256::digest(content.as_bytes()))
89}
90
91/// The digest a cold row's stub carries, or None when `stub` is not one.
92pub fn stub_digest(stub: &str) -> Option<&str> {
93    let (_, digest) = stub.rsplit_once(STUB_MARKER)?;
94    (digest.len() == 64 && digest.bytes().all(|b| b.is_ascii_hexdigit())).then_some(digest)
95}
96
97/// A segment that failed to open or to yield a record, kept for status.
98#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
99pub struct UnreadableSegment {
100    pub segment: String,
101    pub error: String,
102}
103
104/// The local cold tier: segment files in one directory beside the database,
105/// opened with keys from an `ArchiveKeys`. Manifests live in the store; this
106/// caches their parsed form, which never changes once written.
107pub struct ColdTier {
108    dir: PathBuf,
109    keys: Arc<dyn ArchiveKeys>,
110    manifests: Mutex<HashMap<String, Arc<SegmentManifest>>>,
111    summaries: Mutex<HashMap<String, SegmentSummary>>,
112    last_block: Mutex<Option<(String, u32, Arc<Vec<ColdRecord>>)>>,
113    unreadable: Mutex<BTreeMap<(String, Option<u32>), String>>,
114    pass: Mutex<()>,
115}
116
117fn lock<T>(m: &Mutex<T>) -> MutexGuard<'_, T> {
118    m.lock().unwrap_or_else(PoisonError::into_inner)
119}
120
121impl ColdTier {
122    pub fn open(dir: impl Into<PathBuf>, keys: Arc<dyn ArchiveKeys>) -> Result<Self> {
123        let dir = dir.into();
124        create_private_dir(&dir).with_context(|| format!("creating {}", dir.display()))?;
125        Ok(Self {
126            dir,
127            keys,
128            manifests: Mutex::new(HashMap::new()),
129            summaries: Mutex::new(HashMap::new()),
130            last_block: Mutex::new(None),
131            unreadable: Mutex::new(BTreeMap::new()),
132            pass: Mutex::new(()),
133        })
134    }
135
136    pub fn dir(&self) -> &Path {
137        &self.dir
138    }
139
140    /// Held for a whole pass: a segment file exists before its manifest
141    /// commits, so a second pass's sweep could otherwise delete it.
142    pub(crate) fn pass_guard(&self) -> MutexGuard<'_, ()> {
143        lock(&self.pass)
144    }
145
146    fn path(&self, segment: &str) -> PathBuf {
147        self.dir.join(format!("{segment}.{SEGMENT_SUFFIX}"))
148    }
149
150    /// Seals `records` into a new segment file, durably on disk before this
151    /// returns. None when no key is available yet. Returns the segment id and
152    /// the manifest JSON the store must keep for it.
153    pub fn write_segment(&self, records: &[ColdRecord]) -> Result<Option<(String, String)>> {
154        let Some(kek) = self.keys.current()? else {
155            return Ok(None);
156        };
157        let segment = Uuid::new_v4();
158        let (bytes, manifest) = seal(segment, records, &kek, BLOCK_BYTES)?;
159        let id = segment.to_string();
160        let path = self.path(&id);
161        write_private_atomic(&path, &bytes)
162            .with_context(|| format!("writing {}", path.display()))?;
163        Ok(Some((id, serde_json::to_string(&manifest)?)))
164    }
165
166    /// The record for `memory` in `segment`. `manifest` loads the stored
167    /// manifest JSON when it is not cached. A failure is remembered for
168    /// status until the part that failed (the segment's header or one block)
169    /// next reads cleanly.
170    pub fn read(
171        &self,
172        segment: &str,
173        memory: Uuid,
174        manifest: impl FnOnce() -> Result<Option<String>>,
175    ) -> Result<ColdRecord> {
176        self.read_inner(segment, memory, manifest)
177            .with_context(|| format!("cold segment {segment}"))
178    }
179
180    fn read_inner(
181        &self,
182        segment: &str,
183        memory: Uuid,
184        load: impl FnOnce() -> Result<Option<String>>,
185    ) -> Result<ColdRecord> {
186        let manifest = self.failed(segment, None, self.manifest(segment, load))?;
187        let block = self.failed(
188            segment,
189            None,
190            manifest
191                .records
192                .get(&memory)
193                .copied()
194                .ok_or_else(|| anyhow!("memory {memory} is not in the segment's manifest")),
195        )?;
196        let records = self.block(segment, &manifest, block)?;
197        let found = records
198            .iter()
199            .find(|r| r.id == memory)
200            .cloned()
201            .ok_or_else(|| anyhow!("memory {memory} is not in block {block}"));
202        self.noted(segment, Some(block), found)
203    }
204
205    /// Opens the segment and unwraps its key without reading any block, to
206    /// find a segment that would fail before a recall needs it. A clean
207    /// check clears only header failures, never a bad block's.
208    pub fn check(
209        &self,
210        segment: &str,
211        manifest: impl FnOnce() -> Result<Option<String>>,
212    ) -> Result<()> {
213        let result = self
214            .summary(segment, manifest)
215            .and_then(|s| self.open_segment(segment, &s).map(|_| ()));
216        self.noted(segment, None, result)
217            .with_context(|| format!("cold segment {segment}"))
218    }
219
220    /// Records a failure without clearing an earlier one on success, for
221    /// steps whose success says nothing about the header or a block.
222    fn failed<T>(&self, segment: &str, block: Option<u32>, result: Result<T>) -> Result<T> {
223        if let Err(e) = &result {
224            lock(&self.unreadable).insert((segment.to_string(), block), format!("{e:#}"));
225        }
226        result
227    }
228
229    fn noted<T>(&self, segment: &str, block: Option<u32>, result: Result<T>) -> Result<T> {
230        let key = (segment.to_string(), block);
231        let mut unreadable = lock(&self.unreadable);
232        match &result {
233            Ok(_) => {
234                unreadable.remove(&key);
235            }
236            Err(e) => {
237                unreadable.insert(key, format!("{e:#}"));
238            }
239        }
240        result
241    }
242
243    pub fn record_count(
244        &self,
245        segment: &str,
246        manifest: impl FnOnce() -> Result<Option<String>>,
247    ) -> Result<usize> {
248        Ok(self.summary(segment, manifest)?.records)
249    }
250
251    fn summary(
252        &self,
253        segment: &str,
254        manifest: impl FnOnce() -> Result<Option<String>>,
255    ) -> Result<SegmentSummary> {
256        if let Some(s) = lock(&self.summaries).get(segment) {
257            return Ok(*s);
258        }
259        let summary = self.manifest(segment, manifest)?.summary()?;
260        lock(&self.summaries).insert(segment.to_string(), summary);
261        Ok(summary)
262    }
263
264    /// Every memory the segment's manifest lists.
265    pub fn record_ids(
266        &self,
267        segment: &str,
268        manifest: impl FnOnce() -> Result<Option<String>>,
269    ) -> Result<Vec<Uuid>> {
270        let manifest = self.manifest(segment, manifest)?;
271        let mut ids: Vec<(u32, Uuid)> = manifest.records.iter().map(|(id, b)| (*b, *id)).collect();
272        ids.sort();
273        Ok(ids.into_iter().map(|(_, id)| id).collect())
274    }
275
276    fn manifest(
277        &self,
278        segment: &str,
279        load: impl FnOnce() -> Result<Option<String>>,
280    ) -> Result<Arc<SegmentManifest>> {
281        if let Some(m) = lock(&self.manifests).get(segment) {
282            return Ok(m.clone());
283        }
284        let text = load()?.ok_or_else(|| anyhow!("the store holds no manifest for it"))?;
285        let parsed: Arc<SegmentManifest> =
286            Arc::new(serde_json::from_str(&text).context("parsing its manifest")?);
287        let mut cache = lock(&self.manifests);
288        if cache.len() >= MANIFEST_CACHE {
289            cache.clear();
290        }
291        cache.insert(segment.to_string(), parsed.clone());
292        Ok(parsed)
293    }
294
295    fn open_segment(&self, segment: &str, summary: &SegmentSummary) -> Result<OpenSegment> {
296        let id = Uuid::parse_str(segment).context("segment ids are UUIDs")?;
297        let label = summary.label;
298        let key = self.keys.by_label(&label)?.ok_or_else(|| {
299            anyhow!(
300                "this device holds no archive key for epoch {} (key id {})",
301                label.epoch,
302                label.key_id
303            )
304        })?;
305        OpenSegment::open(&self.path(segment), id, summary, &key)
306    }
307
308    fn block(
309        &self,
310        segment: &str,
311        manifest: &SegmentManifest,
312        index: u32,
313    ) -> Result<Arc<Vec<ColdRecord>>> {
314        if let Some((s, i, records)) = &*lock(&self.last_block)
315            && s == segment
316            && *i == index
317        {
318            return Ok(records.clone());
319        }
320        let mut open = self.noted(
321            segment,
322            None,
323            manifest
324                .summary()
325                .and_then(|s| self.open_segment(segment, &s)),
326        )?;
327        let records = Arc::new(self.noted(segment, Some(index), open.block(manifest, index))?);
328        *lock(&self.last_block) = Some((segment.to_string(), index, records.clone()));
329        Ok(records)
330    }
331
332    pub fn segment_len(&self, segment: &str) -> Option<u64> {
333        std::fs::metadata(self.path(segment)).ok().map(|m| m.len())
334    }
335
336    pub fn remove_segment(&self, segment: &str) -> Result<()> {
337        lock(&self.manifests).remove(segment);
338        lock(&self.summaries).remove(segment);
339        lock(&self.unreadable).retain(|(s, _), _| s != segment);
340        {
341            let mut last = lock(&self.last_block);
342            if last.as_ref().is_some_and(|(s, _, _)| s == segment) {
343                *last = None;
344            }
345        }
346        match std::fs::remove_file(self.path(segment)) {
347            Ok(()) => Ok(()),
348            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
349            Err(e) => Err(e).with_context(|| format!("removing cold segment {segment}")),
350        }
351    }
352
353    /// Segment ids with a file in the directory, and leftover temporary
354    /// files from a write that never finished.
355    pub fn files(&self) -> Result<(Vec<String>, Vec<PathBuf>)> {
356        let mut segments = Vec::new();
357        let mut partial = Vec::new();
358        let suffix = format!(".{SEGMENT_SUFFIX}");
359        for entry in std::fs::read_dir(&self.dir)? {
360            let path = entry?.path();
361            let name = path
362                .file_name()
363                .and_then(|n| n.to_str())
364                .unwrap_or_default();
365            if let Some(id) = name.strip_suffix(&suffix) {
366                segments.push(id.to_string());
367            } else if name.ends_with(".tmp") {
368                partial.push(path);
369            }
370        }
371        segments.sort();
372        Ok((segments, partial))
373    }
374
375    /// One entry per segment, with its header failure if it has one.
376    pub fn unreadable(&self) -> Vec<UnreadableSegment> {
377        let mut listed: Vec<UnreadableSegment> = Vec::new();
378        for ((segment, _), error) in lock(&self.unreadable).iter() {
379            if listed.last().is_some_and(|u| u.segment == *segment) {
380                continue;
381            }
382            listed.push(UnreadableSegment {
383                segment: segment.clone(),
384                error: error.clone(),
385            });
386        }
387        listed
388    }
389}
390
391fn create_private_dir(dir: &Path) -> std::io::Result<()> {
392    let mut builder = std::fs::DirBuilder::new();
393    builder.recursive(true);
394    #[cfg(unix)]
395    std::os::unix::fs::DirBuilderExt::mode(&mut builder, crate::private_files::PRIVATE_DIR_MODE);
396    builder.create(dir)
397}
398
399fn write_private_atomic(target: &Path, bytes: &[u8]) -> std::io::Result<()> {
400    use std::io::Write;
401    let mut tmp_name = target.file_name().unwrap_or_default().to_os_string();
402    tmp_name.push(".tmp");
403    let tmp = target.with_file_name(tmp_name);
404    let mut options = std::fs::OpenOptions::new();
405    options.write(true).create(true).truncate(true);
406    #[cfg(unix)]
407    std::os::unix::fs::OpenOptionsExt::mode(&mut options, crate::private_files::PRIVATE_FILE_MODE);
408    let mut file = options.open(&tmp)?;
409    file.write_all(bytes)?;
410    file.sync_all()?;
411    std::fs::rename(&tmp, target)?;
412    match target.parent() {
413        Some(dir) => std::fs::File::open(dir)?.sync_all(),
414        None => Ok(()),
415    }
416}
417
418#[cfg(test)]
419mod tests {
420    use super::*;
421
422    #[test]
423    fn a_stub_carries_the_digest_of_the_text_it_stands_for() {
424        let stub = stub_for("the daemon port is 7200");
425        assert_eq!(
426            stub_digest(&stub),
427            Some(content_digest("the daemon port is 7200").as_str())
428        );
429        assert!(stub_matches(&stub, "the daemon port is 7200"));
430        assert!(!stub_matches(&stub, "the daemon port is 7300"));
431        assert_eq!(stub_digest("the daemon port is 7200"), None);
432        assert_eq!(stub_digest("text\u{1e}sha256:abc"), None);
433    }
434
435    #[test]
436    fn a_stub_keeps_the_leading_characters_ingest_dedup_compares() {
437        let long: String = "é".repeat(crate::store::CONTENT_PREFIX_CHARS + 50);
438        let stub = stub_for(&long);
439        let lead: String = long
440            .chars()
441            .take(crate::store::CONTENT_PREFIX_CHARS)
442            .collect();
443        assert!(stub.starts_with(&lead));
444        assert!(!stub.starts_with(&long));
445        assert!(stub_for("short").starts_with("short"));
446    }
447}