lantern 0.2.2

Local-first, provenance-aware semantic search for agent activity
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117

use std::fs;

use lantern::forget::forget;
use lantern::ingest::ingest_path;
use lantern::search::{SearchOptions, search};
use lantern::store::Store;
use tempfile::tempdir;

fn setup_store_with(files: &[(&str, &str)]) -> (tempfile::TempDir, Store) {
    let root = tempdir().unwrap();
    let mut store = Store::initialize(&root.path().join("store")).unwrap();
    let data = root.path().join("data");
    fs::create_dir_all(&data).unwrap();
    for (name, body) in files {
        fs::write(data.join(name), body).unwrap();
    }
    ingest_path(&mut store, &data).unwrap();
    (root, store)
}

fn counts(store: &Store) -> (i64, i64, i64) {
    let conn = store.conn();
    let sources: i64 = conn
        .query_row("SELECT COUNT(*) FROM sources", [], |r| r.get(0))
        .unwrap();
    let chunks: i64 = conn
        .query_row("SELECT COUNT(*) FROM chunks", [], |r| r.get(0))
        .unwrap();
    let fts: i64 = conn
        .query_row("SELECT COUNT(*) FROM chunks_fts", [], |r| r.get(0))
        .unwrap();
    (sources, chunks, fts)
}

#[test]
fn empty_pattern_is_rejected() {
    let (_root, mut store) = setup_store_with(&[("a.md", "content")]);
    let err = forget(&mut store, "", true).unwrap_err();
    assert!(err.to_string().contains("must not be empty"));
    let err = forget(&mut store, "   ", true).unwrap_err();
    assert!(err.to_string().contains("must not be empty"));
}

#[test]
fn no_matches_reports_empty_and_changes_nothing() {
    let (_root, mut store) = setup_store_with(&[("a.md", "content")]);
    let before = counts(&store);

    let report = forget(&mut store, "does-not-match", true).unwrap();
    assert!(report.removed.is_empty());
    assert!(report.applied);
    assert_eq!(counts(&store), before);
}

#[test]
fn removes_matching_source_chunks_and_fts_entries() {
    let (_root, mut store) = setup_store_with(&[
        ("apples.md", "apple notes with needle keyword"),
        ("bananas.md", "unrelated banana words"),
    ]);
    let (src_before, chunk_before, fts_before) = counts(&store);
    assert_eq!(src_before, 2);
    assert!(chunk_before >= 2);
    assert_eq!(chunk_before, fts_before);

    let report = forget(&mut store, "apples", true).unwrap();
    assert_eq!(report.removed.len(), 1);
    assert!(report.removed[0].uri.ends_with("/apples.md"));
    assert!(report.removed[0].chunks > 0);

    let (src_after, chunk_after, fts_after) = counts(&store);
    assert_eq!(src_after, 1);
    assert_eq!(chunk_after, chunk_before - report.removed[0].chunks);
    assert_eq!(
        chunk_after, fts_after,
        "fts index must stay aligned with chunks"
    );

    // Search for a term that only existed in the removed file should now miss.
    let hits = search(&store, "needle", SearchOptions::default()).unwrap();
    assert!(hits.is_empty());
    // Words from the surviving file still resolve.
    let hits = search(&store, "banana", SearchOptions::default()).unwrap();
    assert_eq!(hits.len(), 1);
}

#[test]
fn dry_run_preserves_state_but_still_lists_candidates() {
    let (_root, mut store) = setup_store_with(&[("apples.md", "one"), ("bananas.md", "two")]);
    let before = counts(&store);

    let report = forget(&mut store, "apples", false).unwrap();
    assert!(!report.applied);
    assert_eq!(report.removed.len(), 1);
    assert!(report.removed[0].uri.ends_with("/apples.md"));

    assert_eq!(
        counts(&store),
        before,
        "dry-run must not modify the database"
    );
}

#[test]
fn substring_pattern_matches_any_source_containing_it() {
    let (_root, mut store) = setup_store_with(&[
        ("alpha-1.md", "first"),
        ("alpha-2.md", "second"),
        ("beta.md", "third"),
    ]);

    let report = forget(&mut store, "alpha", true).unwrap();
    assert_eq!(report.removed.len(), 2);

    let (src_after, _, _) = counts(&store);
    assert_eq!(src_after, 1);
}