Skip to main content

second_brain_core/cold/
mod.rs

1mod envelope_keys;
2mod segment;
3
4use std::collections::{BTreeMap, HashMap};
5use std::path::{Path, PathBuf};
6use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
7
8use anyhow::{Context, Result, anyhow};
9use sha2::{Digest, Sha256};
10
11pub use crate::keys::KeyId;
12use uuid::Uuid;
13pub use zeroize::Zeroizing;
14
15pub use envelope_keys::EnvelopeArchiveKeys;
16pub use segment::{BlockEntry, ColdRecord, SegmentManifest, SegmentSummary};
17use segment::{OpenSegment, seal};
18
19/// Archived raws whose session time (creation time when unknown) is older
20/// than this move to the cold tier.
21pub const COLD_AFTER_DAYS: i64 = 30;
22/// About this much record plaintext goes into one segment file.
23pub const SEGMENT_BYTES: usize = 8 << 20;
24pub const BLOCK_BYTES: usize = 256 << 10;
25/// A larger row stays hot: it would fill a block alone, and a reader refuses
26/// blocks of more than 64 MiB.
27pub const MAX_RECORD_BYTES: usize = 32 << 20;
28const STUB_MARKER: &str = "\u{1e}sha256:";
29const SEGMENT_SUFFIX: &str = "seg";
30const MANIFEST_CACHE: usize = 32;
31
32pub fn dir_for_db(db_path: &Path) -> PathBuf {
33    db_path.with_extension("cold")
34}
35
36/// Names the `archive-kek` subkey a segment is sealed under: the identity
37/// key epoch and that epoch's key id.
38#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
39pub struct KekLabel {
40    pub epoch: u32,
41    pub key_id: KeyId,
42}
43
44pub struct Kek {
45    pub label: KekLabel,
46    pub key: Zeroizing<[u8; 32]>,
47}
48
49/// Where segment sealing gets its key-encryption key. The cold tier never
50/// reads key material itself; the daemon hands it the identity key's
51/// `archive-kek` subkey through this.
52pub trait ArchiveKeys: Send + Sync {
53    /// The subkey new segments are sealed under, or None while this device
54    /// holds no identity key, which leaves archived raws hot.
55    fn current(&self) -> Result<Option<Kek>>;
56    /// The subkey of the epoch `label` names, or None when this device does
57    /// not hold that epoch.
58    fn by_label(&self, label: &KekLabel) -> Result<Option<Zeroizing<[u8; 32]>>>;
59}
60
61/// The stub a cold row keeps as its content: the text's first
62/// `store::CONTENT_PREFIX_CHARS` characters, which ingest's prefix dedup
63/// matches on, and the digest of the whole, so a row can be compared and its
64/// segment record checked without the key.
65pub fn stub_for(content: &str) -> String {
66    let lead: String = content
67        .chars()
68        .take(crate::store::CONTENT_PREFIX_CHARS)
69        .collect();
70    format!("{lead}{STUB_MARKER}{}", content_digest(content))
71}
72
73/// SQL for the leading text of the stub in `column`, which equals the
74/// `content_prefix` of the text the stub stands for.
75pub(crate) fn stub_lead_sql(column: &str) -> String {
76    format!(
77        "substr({column}, 1, length({column}) - {})",
78        STUB_MARKER.chars().count() + 64
79    )
80}
81
82/// Whether `stub` stands for exactly `content`.
83pub fn stub_matches(stub: &str, content: &str) -> bool {
84    stub_digest(stub) == Some(content_digest(content).as_str())
85}
86
87pub fn content_digest(content: &str) -> String {
88    crate::keys::hex(&Sha256::digest(content.as_bytes()))
89}
90
91/// The digest a cold row's stub carries, or None when `stub` is not one.
92pub fn stub_digest(stub: &str) -> Option<&str> {
93    let (_, digest) = stub.rsplit_once(STUB_MARKER)?;
94    (digest.len() == 64 && digest.bytes().all(|b| b.is_ascii_hexdigit())).then_some(digest)
95}
96
97/// A segment that failed to open or to yield a record, kept for status.
98#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
99pub struct UnreadableSegment {
100    pub segment: String,
101    pub error: String,
102}
103
104/// The local cold tier: segment files in one directory beside the database,
105/// opened with keys from an `ArchiveKeys`. Manifests live in the store; this
106/// caches their parsed form, which never changes once written.
107pub struct ColdTier {
108    dir: PathBuf,
109    keys: Arc<dyn ArchiveKeys>,
110    manifests: Mutex<HashMap<String, Arc<SegmentManifest>>>,
111    summaries: Mutex<HashMap<String, SegmentSummary>>,
112    last_block: Mutex<Option<(String, u32, Arc<Vec<ColdRecord>>)>>,
113    unreadable: Mutex<BTreeMap<(String, Option<u32>), String>>,
114    pass: Mutex<()>,
115}
116
117fn lock<T>(m: &Mutex<T>) -> MutexGuard<'_, T> {
118    m.lock().unwrap_or_else(PoisonError::into_inner)
119}
120
121impl ColdTier {
122    pub fn open(dir: impl Into<PathBuf>, keys: Arc<dyn ArchiveKeys>) -> Result<Self> {
123        let dir = dir.into();
124        create_private_dir(&dir).with_context(|| format!("creating {}", dir.display()))?;
125        Ok(Self {
126            dir,
127            keys,
128            manifests: Mutex::new(HashMap::new()),
129            summaries: Mutex::new(HashMap::new()),
130            last_block: Mutex::new(None),
131            unreadable: Mutex::new(BTreeMap::new()),
132            pass: Mutex::new(()),
133        })
134    }
135
136    pub fn dir(&self) -> &Path {
137        &self.dir
138    }
139
140    /// Held for a whole pass: a segment file exists before its manifest
141    /// commits, so a second pass's sweep could otherwise delete it.
142    pub(crate) fn pass_guard(&self) -> MutexGuard<'_, ()> {
143        lock(&self.pass)
144    }
145
146    /// The pass guard without waiting: None while a cold pass, which can
147    /// run for minutes, holds it. A guard poisoned by a panicked pass is
148    /// taken, the way every other lock in the tier is.
149    pub fn try_hold_passes(&self) -> Option<MutexGuard<'_, ()>> {
150        match self.pass.try_lock() {
151            Ok(guard) => Some(guard),
152            Err(std::sync::TryLockError::Poisoned(poisoned)) => Some(poisoned.into_inner()),
153            Err(std::sync::TryLockError::WouldBlock) => None,
154        }
155    }
156
157    fn path(&self, segment: &str) -> PathBuf {
158        self.dir.join(format!("{segment}.{SEGMENT_SUFFIX}"))
159    }
160
161    /// Seals `records` into a new segment file, durably on disk before this
162    /// returns. None when no key is available yet. Returns the segment id and
163    /// the manifest JSON the store must keep for it.
164    pub fn write_segment(&self, records: &[ColdRecord]) -> Result<Option<(String, String)>> {
165        let Some(kek) = self.keys.current()? else {
166            return Ok(None);
167        };
168        let segment = Uuid::new_v4();
169        let (bytes, manifest) = seal(segment, records, &kek, BLOCK_BYTES)?;
170        let id = segment.to_string();
171        let path = self.path(&id);
172        write_private_atomic(&path, &bytes)
173            .with_context(|| format!("writing {}", path.display()))?;
174        Ok(Some((id, serde_json::to_string(&manifest)?)))
175    }
176
177    /// The record for `memory` in `segment`. `manifest` loads the stored
178    /// manifest JSON when it is not cached. A failure is remembered for
179    /// status until the part that failed (the segment's header or one block)
180    /// next reads cleanly.
181    pub fn read(
182        &self,
183        segment: &str,
184        memory: Uuid,
185        manifest: impl FnOnce() -> Result<Option<String>>,
186    ) -> Result<ColdRecord> {
187        self.read_inner(segment, memory, manifest)
188            .with_context(|| format!("cold segment {segment}"))
189    }
190
191    fn read_inner(
192        &self,
193        segment: &str,
194        memory: Uuid,
195        load: impl FnOnce() -> Result<Option<String>>,
196    ) -> Result<ColdRecord> {
197        let manifest = self.failed(segment, None, self.manifest(segment, load))?;
198        let block = self.failed(
199            segment,
200            None,
201            manifest
202                .records
203                .get(&memory)
204                .copied()
205                .ok_or_else(|| anyhow!("memory {memory} is not in the segment's manifest")),
206        )?;
207        let records = self.block(segment, &manifest, block)?;
208        let found = records
209            .iter()
210            .find(|r| r.id == memory)
211            .cloned()
212            .ok_or_else(|| anyhow!("memory {memory} is not in block {block}"));
213        self.noted(segment, Some(block), found)
214    }
215
216    /// Opens the segment and unwraps its key without reading any block, to
217    /// find a segment that would fail before a recall needs it. A clean
218    /// check clears only header failures, never a bad block's.
219    pub fn check(
220        &self,
221        segment: &str,
222        manifest: impl FnOnce() -> Result<Option<String>>,
223    ) -> Result<()> {
224        let result = self
225            .summary(segment, manifest)
226            .and_then(|s| self.open_segment(segment, &s).map(|_| ()));
227        self.noted(segment, None, result)
228            .with_context(|| format!("cold segment {segment}"))
229    }
230
231    /// Records a failure without clearing an earlier one on success, for
232    /// steps whose success says nothing about the header or a block.
233    fn failed<T>(&self, segment: &str, block: Option<u32>, result: Result<T>) -> Result<T> {
234        if let Err(e) = &result {
235            lock(&self.unreadable).insert((segment.to_string(), block), format!("{e:#}"));
236        }
237        result
238    }
239
240    fn noted<T>(&self, segment: &str, block: Option<u32>, result: Result<T>) -> Result<T> {
241        let key = (segment.to_string(), block);
242        let mut unreadable = lock(&self.unreadable);
243        match &result {
244            Ok(_) => {
245                unreadable.remove(&key);
246            }
247            Err(e) => {
248                unreadable.insert(key, format!("{e:#}"));
249            }
250        }
251        result
252    }
253
254    pub fn record_count(
255        &self,
256        segment: &str,
257        manifest: impl FnOnce() -> Result<Option<String>>,
258    ) -> Result<usize> {
259        Ok(self.summary(segment, manifest)?.records)
260    }
261
262    fn summary(
263        &self,
264        segment: &str,
265        manifest: impl FnOnce() -> Result<Option<String>>,
266    ) -> Result<SegmentSummary> {
267        if let Some(s) = lock(&self.summaries).get(segment) {
268            return Ok(*s);
269        }
270        let summary = self.manifest(segment, manifest)?.summary()?;
271        lock(&self.summaries).insert(segment.to_string(), summary);
272        Ok(summary)
273    }
274
275    /// Every memory the segment's manifest lists.
276    pub fn record_ids(
277        &self,
278        segment: &str,
279        manifest: impl FnOnce() -> Result<Option<String>>,
280    ) -> Result<Vec<Uuid>> {
281        let manifest = self.manifest(segment, manifest)?;
282        let mut ids: Vec<(u32, Uuid)> = manifest.records.iter().map(|(id, b)| (*b, *id)).collect();
283        ids.sort();
284        Ok(ids.into_iter().map(|(_, id)| id).collect())
285    }
286
287    fn manifest(
288        &self,
289        segment: &str,
290        load: impl FnOnce() -> Result<Option<String>>,
291    ) -> Result<Arc<SegmentManifest>> {
292        if let Some(m) = lock(&self.manifests).get(segment) {
293            return Ok(m.clone());
294        }
295        let text = load()?.ok_or_else(|| anyhow!("the store holds no manifest for it"))?;
296        let parsed: Arc<SegmentManifest> =
297            Arc::new(serde_json::from_str(&text).context("parsing its manifest")?);
298        let mut cache = lock(&self.manifests);
299        if cache.len() >= MANIFEST_CACHE {
300            cache.clear();
301        }
302        cache.insert(segment.to_string(), parsed.clone());
303        Ok(parsed)
304    }
305
306    fn open_segment(&self, segment: &str, summary: &SegmentSummary) -> Result<OpenSegment> {
307        let id = Uuid::parse_str(segment).context("segment ids are UUIDs")?;
308        let label = summary.label;
309        let key = self.keys.by_label(&label)?.ok_or_else(|| {
310            anyhow!(
311                "this device holds no archive key for epoch {} (key id {})",
312                label.epoch,
313                label.key_id
314            )
315        })?;
316        OpenSegment::open(&self.path(segment), id, summary, &key)
317    }
318
319    fn block(
320        &self,
321        segment: &str,
322        manifest: &SegmentManifest,
323        index: u32,
324    ) -> Result<Arc<Vec<ColdRecord>>> {
325        if let Some((s, i, records)) = &*lock(&self.last_block)
326            && s == segment
327            && *i == index
328        {
329            return Ok(records.clone());
330        }
331        let mut open = self.noted(
332            segment,
333            None,
334            manifest
335                .summary()
336                .and_then(|s| self.open_segment(segment, &s)),
337        )?;
338        let records = Arc::new(self.noted(segment, Some(index), open.block(manifest, index))?);
339        *lock(&self.last_block) = Some((segment.to_string(), index, records.clone()));
340        Ok(records)
341    }
342
343    pub fn segment_len(&self, segment: &str) -> Option<u64> {
344        std::fs::metadata(self.path(segment)).ok().map(|m| m.len())
345    }
346
347    pub fn remove_segment(&self, segment: &str) -> Result<()> {
348        lock(&self.manifests).remove(segment);
349        lock(&self.summaries).remove(segment);
350        lock(&self.unreadable).retain(|(s, _), _| s != segment);
351        {
352            let mut last = lock(&self.last_block);
353            if last.as_ref().is_some_and(|(s, _, _)| s == segment) {
354                *last = None;
355            }
356        }
357        match std::fs::remove_file(self.path(segment)) {
358            Ok(()) => Ok(()),
359            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
360            Err(e) => Err(e).with_context(|| format!("removing cold segment {segment}")),
361        }
362    }
363
364    /// Segment ids with a file in the directory, and leftover temporary
365    /// files from a write that never finished.
366    pub fn files(&self) -> Result<(Vec<String>, Vec<PathBuf>)> {
367        let mut segments = Vec::new();
368        let mut partial = Vec::new();
369        let suffix = format!(".{SEGMENT_SUFFIX}");
370        for entry in std::fs::read_dir(&self.dir)? {
371            let path = entry?.path();
372            let name = path
373                .file_name()
374                .and_then(|n| n.to_str())
375                .unwrap_or_default();
376            if let Some(id) = name.strip_suffix(&suffix) {
377                segments.push(id.to_string());
378            } else if name.ends_with(".tmp") {
379                partial.push(path);
380            }
381        }
382        segments.sort();
383        Ok((segments, partial))
384    }
385
386    /// One entry per segment, with its header failure if it has one.
387    pub fn unreadable(&self) -> Vec<UnreadableSegment> {
388        let mut listed: Vec<UnreadableSegment> = Vec::new();
389        for ((segment, _), error) in lock(&self.unreadable).iter() {
390            if listed.last().is_some_and(|u| u.segment == *segment) {
391                continue;
392            }
393            listed.push(UnreadableSegment {
394                segment: segment.clone(),
395                error: error.clone(),
396            });
397        }
398        listed
399    }
400}
401
402fn create_private_dir(dir: &Path) -> std::io::Result<()> {
403    let mut builder = std::fs::DirBuilder::new();
404    builder.recursive(true);
405    #[cfg(unix)]
406    std::os::unix::fs::DirBuilderExt::mode(&mut builder, crate::private_files::PRIVATE_DIR_MODE);
407    builder.create(dir)
408}
409
410fn write_private_atomic(target: &Path, bytes: &[u8]) -> std::io::Result<()> {
411    use std::io::Write;
412    let mut tmp_name = target.file_name().unwrap_or_default().to_os_string();
413    tmp_name.push(".tmp");
414    let tmp = target.with_file_name(tmp_name);
415    let mut options = std::fs::OpenOptions::new();
416    options.write(true).create(true).truncate(true);
417    #[cfg(unix)]
418    std::os::unix::fs::OpenOptionsExt::mode(&mut options, crate::private_files::PRIVATE_FILE_MODE);
419    let mut file = options.open(&tmp)?;
420    file.write_all(bytes)?;
421    file.sync_all()?;
422    std::fs::rename(&tmp, target)?;
423    match target.parent() {
424        Some(dir) => std::fs::File::open(dir)?.sync_all(),
425        None => Ok(()),
426    }
427}
428
429#[cfg(test)]
430mod tests {
431    use super::*;
432
433    struct NoKeys;
434
435    impl ArchiveKeys for NoKeys {
436        fn current(&self) -> Result<Option<Kek>> {
437            Ok(None)
438        }
439
440        fn by_label(&self, _label: &KekLabel) -> Result<Option<Zeroizing<[u8; 32]>>> {
441            Ok(None)
442        }
443    }
444
445    #[test]
446    fn holding_passes_does_not_wait_for_a_running_pass_and_survives_a_panicked_one() {
447        let dir = tempfile::tempdir().unwrap();
448        let tier = Arc::new(ColdTier::open(dir.path().join("cold"), Arc::new(NoKeys)).unwrap());
449        {
450            let _pass = tier.pass_guard();
451            assert!(tier.try_hold_passes().is_none());
452        }
453        assert!(tier.try_hold_passes().is_some());
454
455        let panicking = tier.clone();
456        let _ = std::thread::spawn(move || {
457            let _pass = panicking.pass_guard();
458            panic!("a pass failed");
459        })
460        .join();
461        assert!(tier.try_hold_passes().is_some());
462    }
463
464    #[test]
465    fn a_stub_carries_the_digest_of_the_text_it_stands_for() {
466        let stub = stub_for("the daemon port is 7200");
467        assert_eq!(
468            stub_digest(&stub),
469            Some(content_digest("the daemon port is 7200").as_str())
470        );
471        assert!(stub_matches(&stub, "the daemon port is 7200"));
472        assert!(!stub_matches(&stub, "the daemon port is 7300"));
473        assert_eq!(stub_digest("the daemon port is 7200"), None);
474        assert_eq!(stub_digest("text\u{1e}sha256:abc"), None);
475    }
476
477    #[test]
478    fn a_stub_keeps_the_leading_characters_ingest_dedup_compares() {
479        let long: String = "é".repeat(crate::store::CONTENT_PREFIX_CHARS + 50);
480        let stub = stub_for(&long);
481        let lead: String = long
482            .chars()
483            .take(crate::store::CONTENT_PREFIX_CHARS)
484            .collect();
485        assert!(stub.starts_with(&lead));
486        assert!(!stub.starts_with(&long));
487        assert!(stub_for("short").starts_with("short"));
488    }
489}