second-brain-core 0.13.2

Core library for second-brain: SQLite storage with vector and lexical indexes, BGE embeddings, and hybrid retrieval
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
mod envelope_keys;
mod segment;

use std::collections::{BTreeMap, HashMap};
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};

use anyhow::{Context, Result, anyhow};
use sha2::{Digest, Sha256};

pub use crate::keys::KeyId;
use uuid::Uuid;
pub use zeroize::Zeroizing;

pub use envelope_keys::EnvelopeArchiveKeys;
pub use segment::{BlockEntry, ColdRecord, SegmentManifest, SegmentSummary};
use segment::{OpenSegment, seal};

/// Archived raws whose session time (creation time when unknown) is older
/// than this move to the cold tier.
pub const COLD_AFTER_DAYS: i64 = 30;
/// About this much record plaintext goes into one segment file.
pub const SEGMENT_BYTES: usize = 8 << 20;
pub const BLOCK_BYTES: usize = 256 << 10;
/// A larger row stays hot: it would fill a block alone, and a reader refuses
/// blocks of more than 64 MiB.
pub const MAX_RECORD_BYTES: usize = 32 << 20;
const STUB_MARKER: &str = "\u{1e}sha256:";
const SEGMENT_SUFFIX: &str = "seg";
const MANIFEST_CACHE: usize = 32;

pub fn dir_for_db(db_path: &Path) -> PathBuf {
    db_path.with_extension("cold")
}

/// Names the `archive-kek` subkey a segment is sealed under: the identity
/// key epoch and that epoch's key id.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct KekLabel {
    pub epoch: u32,
    pub key_id: KeyId,
}

pub struct Kek {
    pub label: KekLabel,
    pub key: Zeroizing<[u8; 32]>,
}

/// Where segment sealing gets its key-encryption key. The cold tier never
/// reads key material itself; the daemon hands it the identity key's
/// `archive-kek` subkey through this.
pub trait ArchiveKeys: Send + Sync {
    /// The subkey new segments are sealed under, or None while this device
    /// holds no identity key, which leaves archived raws hot.
    fn current(&self) -> Result<Option<Kek>>;
    /// The subkey of the epoch `label` names, or None when this device does
    /// not hold that epoch.
    fn by_label(&self, label: &KekLabel) -> Result<Option<Zeroizing<[u8; 32]>>>;
}

/// The stub a cold row keeps as its content: the text's first
/// `store::CONTENT_PREFIX_CHARS` characters, which ingest's prefix dedup
/// matches on, and the digest of the whole, so a row can be compared and its
/// segment record checked without the key.
pub fn stub_for(content: &str) -> String {
    let lead: String = content
        .chars()
        .take(crate::store::CONTENT_PREFIX_CHARS)
        .collect();
    format!("{lead}{STUB_MARKER}{}", content_digest(content))
}

/// SQL for the leading text of the stub in `column`, which equals the
/// `content_prefix` of the text the stub stands for.
pub(crate) fn stub_lead_sql(column: &str) -> String {
    format!(
        "substr({column}, 1, length({column}) - {})",
        STUB_MARKER.chars().count() + 64
    )
}

/// Whether `stub` stands for exactly `content`.
pub fn stub_matches(stub: &str, content: &str) -> bool {
    stub_digest(stub) == Some(content_digest(content).as_str())
}

pub fn content_digest(content: &str) -> String {
    crate::keys::hex(&Sha256::digest(content.as_bytes()))
}

/// The digest a cold row's stub carries, or None when `stub` is not one.
pub fn stub_digest(stub: &str) -> Option<&str> {
    let (_, digest) = stub.rsplit_once(STUB_MARKER)?;
    (digest.len() == 64 && digest.bytes().all(|b| b.is_ascii_hexdigit())).then_some(digest)
}

/// A segment that failed to open or to yield a record, kept for status.
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
pub struct UnreadableSegment {
    pub segment: String,
    pub error: String,
}

/// The local cold tier: segment files in one directory beside the database,
/// opened with keys from an `ArchiveKeys`. Manifests live in the store; this
/// caches their parsed form, which never changes once written.
pub struct ColdTier {
    dir: PathBuf,
    keys: Arc<dyn ArchiveKeys>,
    manifests: Mutex<HashMap<String, Arc<SegmentManifest>>>,
    summaries: Mutex<HashMap<String, SegmentSummary>>,
    last_block: Mutex<Option<(String, u32, Arc<Vec<ColdRecord>>)>>,
    unreadable: Mutex<BTreeMap<(String, Option<u32>), String>>,
    pass: Mutex<()>,
}

fn lock<T>(m: &Mutex<T>) -> MutexGuard<'_, T> {
    m.lock().unwrap_or_else(PoisonError::into_inner)
}

impl ColdTier {
    pub fn open(dir: impl Into<PathBuf>, keys: Arc<dyn ArchiveKeys>) -> Result<Self> {
        let dir = dir.into();
        create_private_dir(&dir).with_context(|| format!("creating {}", dir.display()))?;
        Ok(Self {
            dir,
            keys,
            manifests: Mutex::new(HashMap::new()),
            summaries: Mutex::new(HashMap::new()),
            last_block: Mutex::new(None),
            unreadable: Mutex::new(BTreeMap::new()),
            pass: Mutex::new(()),
        })
    }

    pub fn dir(&self) -> &Path {
        &self.dir
    }

    /// Held for a whole pass: a segment file exists before its manifest
    /// commits, so a second pass's sweep could otherwise delete it.
    pub(crate) fn pass_guard(&self) -> MutexGuard<'_, ()> {
        lock(&self.pass)
    }

    fn path(&self, segment: &str) -> PathBuf {
        self.dir.join(format!("{segment}.{SEGMENT_SUFFIX}"))
    }

    /// Seals `records` into a new segment file, durably on disk before this
    /// returns. None when no key is available yet. Returns the segment id and
    /// the manifest JSON the store must keep for it.
    pub fn write_segment(&self, records: &[ColdRecord]) -> Result<Option<(String, String)>> {
        let Some(kek) = self.keys.current()? else {
            return Ok(None);
        };
        let segment = Uuid::new_v4();
        let (bytes, manifest) = seal(segment, records, &kek, BLOCK_BYTES)?;
        let id = segment.to_string();
        let path = self.path(&id);
        write_private_atomic(&path, &bytes)
            .with_context(|| format!("writing {}", path.display()))?;
        Ok(Some((id, serde_json::to_string(&manifest)?)))
    }

    /// The record for `memory` in `segment`. `manifest` loads the stored
    /// manifest JSON when it is not cached. A failure is remembered for
    /// status until the part that failed (the segment's header or one block)
    /// next reads cleanly.
    pub fn read(
        &self,
        segment: &str,
        memory: Uuid,
        manifest: impl FnOnce() -> Result<Option<String>>,
    ) -> Result<ColdRecord> {
        self.read_inner(segment, memory, manifest)
            .with_context(|| format!("cold segment {segment}"))
    }

    fn read_inner(
        &self,
        segment: &str,
        memory: Uuid,
        load: impl FnOnce() -> Result<Option<String>>,
    ) -> Result<ColdRecord> {
        let manifest = self.failed(segment, None, self.manifest(segment, load))?;
        let block = self.failed(
            segment,
            None,
            manifest
                .records
                .get(&memory)
                .copied()
                .ok_or_else(|| anyhow!("memory {memory} is not in the segment's manifest")),
        )?;
        let records = self.block(segment, &manifest, block)?;
        let found = records
            .iter()
            .find(|r| r.id == memory)
            .cloned()
            .ok_or_else(|| anyhow!("memory {memory} is not in block {block}"));
        self.noted(segment, Some(block), found)
    }

    /// Opens the segment and unwraps its key without reading any block, to
    /// find a segment that would fail before a recall needs it. A clean
    /// check clears only header failures, never a bad block's.
    pub fn check(
        &self,
        segment: &str,
        manifest: impl FnOnce() -> Result<Option<String>>,
    ) -> Result<()> {
        let result = self
            .summary(segment, manifest)
            .and_then(|s| self.open_segment(segment, &s).map(|_| ()));
        self.noted(segment, None, result)
            .with_context(|| format!("cold segment {segment}"))
    }

    /// Records a failure without clearing an earlier one on success, for
    /// steps whose success says nothing about the header or a block.
    fn failed<T>(&self, segment: &str, block: Option<u32>, result: Result<T>) -> Result<T> {
        if let Err(e) = &result {
            lock(&self.unreadable).insert((segment.to_string(), block), format!("{e:#}"));
        }
        result
    }

    fn noted<T>(&self, segment: &str, block: Option<u32>, result: Result<T>) -> Result<T> {
        let key = (segment.to_string(), block);
        let mut unreadable = lock(&self.unreadable);
        match &result {
            Ok(_) => {
                unreadable.remove(&key);
            }
            Err(e) => {
                unreadable.insert(key, format!("{e:#}"));
            }
        }
        result
    }

    pub fn record_count(
        &self,
        segment: &str,
        manifest: impl FnOnce() -> Result<Option<String>>,
    ) -> Result<usize> {
        Ok(self.summary(segment, manifest)?.records)
    }

    fn summary(
        &self,
        segment: &str,
        manifest: impl FnOnce() -> Result<Option<String>>,
    ) -> Result<SegmentSummary> {
        if let Some(s) = lock(&self.summaries).get(segment) {
            return Ok(*s);
        }
        let summary = self.manifest(segment, manifest)?.summary()?;
        lock(&self.summaries).insert(segment.to_string(), summary);
        Ok(summary)
    }

    /// Every memory the segment's manifest lists.
    pub fn record_ids(
        &self,
        segment: &str,
        manifest: impl FnOnce() -> Result<Option<String>>,
    ) -> Result<Vec<Uuid>> {
        let manifest = self.manifest(segment, manifest)?;
        let mut ids: Vec<(u32, Uuid)> = manifest.records.iter().map(|(id, b)| (*b, *id)).collect();
        ids.sort();
        Ok(ids.into_iter().map(|(_, id)| id).collect())
    }

    fn manifest(
        &self,
        segment: &str,
        load: impl FnOnce() -> Result<Option<String>>,
    ) -> Result<Arc<SegmentManifest>> {
        if let Some(m) = lock(&self.manifests).get(segment) {
            return Ok(m.clone());
        }
        let text = load()?.ok_or_else(|| anyhow!("the store holds no manifest for it"))?;
        let parsed: Arc<SegmentManifest> =
            Arc::new(serde_json::from_str(&text).context("parsing its manifest")?);
        let mut cache = lock(&self.manifests);
        if cache.len() >= MANIFEST_CACHE {
            cache.clear();
        }
        cache.insert(segment.to_string(), parsed.clone());
        Ok(parsed)
    }

    fn open_segment(&self, segment: &str, summary: &SegmentSummary) -> Result<OpenSegment> {
        let id = Uuid::parse_str(segment).context("segment ids are UUIDs")?;
        let label = summary.label;
        let key = self.keys.by_label(&label)?.ok_or_else(|| {
            anyhow!(
                "this device holds no archive key for epoch {} (key id {})",
                label.epoch,
                label.key_id
            )
        })?;
        OpenSegment::open(&self.path(segment), id, summary, &key)
    }

    fn block(
        &self,
        segment: &str,
        manifest: &SegmentManifest,
        index: u32,
    ) -> Result<Arc<Vec<ColdRecord>>> {
        if let Some((s, i, records)) = &*lock(&self.last_block)
            && s == segment
            && *i == index
        {
            return Ok(records.clone());
        }
        let mut open = self.noted(
            segment,
            None,
            manifest
                .summary()
                .and_then(|s| self.open_segment(segment, &s)),
        )?;
        let records = Arc::new(self.noted(segment, Some(index), open.block(manifest, index))?);
        *lock(&self.last_block) = Some((segment.to_string(), index, records.clone()));
        Ok(records)
    }

    pub fn segment_len(&self, segment: &str) -> Option<u64> {
        std::fs::metadata(self.path(segment)).ok().map(|m| m.len())
    }

    pub fn remove_segment(&self, segment: &str) -> Result<()> {
        lock(&self.manifests).remove(segment);
        lock(&self.summaries).remove(segment);
        lock(&self.unreadable).retain(|(s, _), _| s != segment);
        {
            let mut last = lock(&self.last_block);
            if last.as_ref().is_some_and(|(s, _, _)| s == segment) {
                *last = None;
            }
        }
        match std::fs::remove_file(self.path(segment)) {
            Ok(()) => Ok(()),
            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
            Err(e) => Err(e).with_context(|| format!("removing cold segment {segment}")),
        }
    }

    /// Segment ids with a file in the directory, and leftover temporary
    /// files from a write that never finished.
    pub fn files(&self) -> Result<(Vec<String>, Vec<PathBuf>)> {
        let mut segments = Vec::new();
        let mut partial = Vec::new();
        let suffix = format!(".{SEGMENT_SUFFIX}");
        for entry in std::fs::read_dir(&self.dir)? {
            let path = entry?.path();
            let name = path
                .file_name()
                .and_then(|n| n.to_str())
                .unwrap_or_default();
            if let Some(id) = name.strip_suffix(&suffix) {
                segments.push(id.to_string());
            } else if name.ends_with(".tmp") {
                partial.push(path);
            }
        }
        segments.sort();
        Ok((segments, partial))
    }

    /// One entry per segment, with its header failure if it has one.
    pub fn unreadable(&self) -> Vec<UnreadableSegment> {
        let mut listed: Vec<UnreadableSegment> = Vec::new();
        for ((segment, _), error) in lock(&self.unreadable).iter() {
            if listed.last().is_some_and(|u| u.segment == *segment) {
                continue;
            }
            listed.push(UnreadableSegment {
                segment: segment.clone(),
                error: error.clone(),
            });
        }
        listed
    }
}

fn create_private_dir(dir: &Path) -> std::io::Result<()> {
    let mut builder = std::fs::DirBuilder::new();
    builder.recursive(true);
    #[cfg(unix)]
    std::os::unix::fs::DirBuilderExt::mode(&mut builder, crate::private_files::PRIVATE_DIR_MODE);
    builder.create(dir)
}

fn write_private_atomic(target: &Path, bytes: &[u8]) -> std::io::Result<()> {
    use std::io::Write;
    let mut tmp_name = target.file_name().unwrap_or_default().to_os_string();
    tmp_name.push(".tmp");
    let tmp = target.with_file_name(tmp_name);
    let mut options = std::fs::OpenOptions::new();
    options.write(true).create(true).truncate(true);
    #[cfg(unix)]
    std::os::unix::fs::OpenOptionsExt::mode(&mut options, crate::private_files::PRIVATE_FILE_MODE);
    let mut file = options.open(&tmp)?;
    file.write_all(bytes)?;
    file.sync_all()?;
    std::fs::rename(&tmp, target)?;
    match target.parent() {
        Some(dir) => std::fs::File::open(dir)?.sync_all(),
        None => Ok(()),
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn a_stub_carries_the_digest_of_the_text_it_stands_for() {
        let stub = stub_for("the daemon port is 7200");
        assert_eq!(
            stub_digest(&stub),
            Some(content_digest("the daemon port is 7200").as_str())
        );
        assert!(stub_matches(&stub, "the daemon port is 7200"));
        assert!(!stub_matches(&stub, "the daemon port is 7300"));
        assert_eq!(stub_digest("the daemon port is 7200"), None);
        assert_eq!(stub_digest("text\u{1e}sha256:abc"), None);
    }

    #[test]
    fn a_stub_keeps_the_leading_characters_ingest_dedup_compares() {
        let long: String = "é".repeat(crate::store::CONTENT_PREFIX_CHARS + 50);
        let stub = stub_for(&long);
        let lead: String = long
            .chars()
            .take(crate::store::CONTENT_PREFIX_CHARS)
            .collect();
        assert!(stub.starts_with(&lead));
        assert!(!stub.starts_with(&long));
        assert!(stub_for("short").starts_with("short"));
    }
}