lantern 0.2.2

Local-first, provenance-aware semantic search for agent activity
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116

use std::fs;
use std::path::Path;

use flate2::read::GzDecoder;
use lantern::ingest::ingest_path;
use lantern::search::{SearchOptions, search};
use lantern::stash::stash;
use lantern::store::Store;
use tar::Archive;
use tempfile::tempdir;

fn seed_store(root: &Path) -> Store {
    let mut store = Store::initialize(&root.join("store")).unwrap();
    let data = root.join("data");
    fs::create_dir_all(&data).unwrap();
    fs::write(data.join("a.md"), "alpha beta gamma").unwrap();
    fs::write(data.join("b.txt"), "needle buried in plain text").unwrap();
    ingest_path(&mut store, &data).unwrap();
    store
}

fn archive_entries(archive_path: &Path) -> Vec<String> {
    let file = fs::File::open(archive_path).unwrap();
    let gz = GzDecoder::new(file);
    let mut tar = Archive::new(gz);
    tar.entries()
        .unwrap()
        .map(|entry| {
            entry
                .unwrap()
                .path()
                .unwrap()
                .to_string_lossy()
                .into_owned()
        })
        .collect()
}

#[test]
fn stash_creates_archive_under_stashes_subdir() {
    let root = tempdir().unwrap();
    let mut store = seed_store(root.path());

    let report = stash(&mut store).unwrap();
    let archive = Path::new(&report.archive_path);

    assert!(archive.exists(), "archive should exist on disk");
    assert!(report.archive_bytes > 0);
    assert_eq!(report.files, vec!["lantern.db".to_string()]);

    let expected_dir = root.path().join("store/stashes");
    assert!(archive.starts_with(&expected_dir));
    let file_name = archive.file_name().unwrap().to_string_lossy().to_string();
    assert!(
        file_name.starts_with("lantern-") && file_name.ends_with(".tar.gz"),
        "unexpected archive name: {file_name}"
    );
}

#[test]
fn archive_contents_round_trip_to_a_usable_database() {
    let root = tempdir().unwrap();
    let mut store = seed_store(root.path());
    let report = stash(&mut store).unwrap();

    let entries = archive_entries(Path::new(&report.archive_path));
    assert_eq!(entries, vec!["lantern.db".to_string()]);

    // Extract into a fresh dir and confirm the db opens and answers queries.
    let restore_dir = root.path().join("restore");
    fs::create_dir_all(&restore_dir).unwrap();
    let mut archive = Archive::new(GzDecoder::new(
        fs::File::open(&report.archive_path).unwrap(),
    ));
    archive.unpack(&restore_dir).unwrap();
    assert!(restore_dir.join("lantern.db").exists());

    let restored = Store::open(&restore_dir).unwrap();
    let hits = search(&restored, "needle", SearchOptions::default()).unwrap();
    assert_eq!(hits.len(), 1);
    assert!(hits[0].uri.ends_with("/b.txt"));
}

#[test]
fn store_remains_fully_usable_after_stash() {
    let root = tempdir().unwrap();
    let mut store = seed_store(root.path());
    stash(&mut store).unwrap();

    // Search still works on the live store.
    let hits = search(&store, "alpha", SearchOptions::default()).unwrap();
    assert_eq!(hits.len(), 1);

    // A subsequent ingest is still writable and searchable.
    let new_file = root.path().join("data/c.md");
    fs::write(&new_file, "fresh marker after stash").unwrap();
    ingest_path(&mut store, &new_file).unwrap();
    let hits = search(&store, "marker", SearchOptions::default()).unwrap();
    assert_eq!(hits.len(), 1);
}

#[test]
fn back_to_back_stashes_do_not_overwrite_each_other() {
    let root = tempdir().unwrap();
    let mut store = seed_store(root.path());

    let first = stash(&mut store).unwrap();
    let second = stash(&mut store).unwrap();
    let third = stash(&mut store).unwrap();

    assert_ne!(first.archive_path, second.archive_path);
    assert_ne!(second.archive_path, third.archive_path);
    assert!(Path::new(&first.archive_path).exists());
    assert!(Path::new(&second.archive_path).exists());
    assert!(Path::new(&third.archive_path).exists());
}