context-forge 0.5.0

Local-first persistent memory for LLM applications - SQLite + FTS5 BM25 retrieval, recency decay, token-budget context assembly, secret scrubbing, and optional local-LLM distillation.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128

//! Integration test for Phase 3 secret scrubbing: proves that a secret
//! present in `save`-d content never reaches the on-disk `SQLite` database
//! (table or FTS index), and that the opt-out path stores it verbatim.

use context_forge::{kind, Config, ContextForge, SaveOptions};
use rusqlite::Connection;

/// A syntactically valid (fake) GitHub personal access token: `ghp_` + 36
/// alphanumeric characters.
fn fake_github_token() -> String {
    format!("ghp_{}", "a".repeat(36))
}

#[test]
fn github_token_is_scrubbed_before_persistence() {
    let dir = tempfile::tempdir().expect("create temp dir");
    let db_path = dir.path().join("scrub-test.db");

    let token = fake_github_token();
    let content = format!("here is my token: {token} please remember it");

    {
        let mut config = Config::default();
        config.db_path = db_path.clone();
        let cf = ContextForge::open(config).expect("open db");
        cf.save(&content, kind::MANUAL, &SaveOptions::default())
            .expect("save scrubbed entry");
    }

    // Re-open the raw SQLite file directly, bypassing the library entirely.
    let conn = Connection::open(&db_path).expect("open raw sqlite connection");

    // (a) No stored content matches the raw token string.
    let entries_match: i64 = conn
        .query_row(
            "SELECT COUNT(*) FROM entries WHERE content LIKE ?1",
            [format!("%{token}%")],
            |row| row.get(0),
        )
        .expect("query entries table");
    assert_eq!(
        entries_match, 0,
        "raw GitHub token must not appear in the entries table"
    );

    // (b) The stored content contains the redaction placeholder.
    let redacted_match: i64 = conn
        .query_row(
            "SELECT COUNT(*) FROM entries WHERE content LIKE '%[REDACTED:github-token]%'",
            [],
            |row| row.get(0),
        )
        .expect("query entries table for redaction marker");
    assert_eq!(
        redacted_match, 1,
        "entries table must contain the [REDACTED:github-token] placeholder"
    );

    // (c) The FTS index does not contain the raw token either.
    let fts_match: i64 = conn
        .query_row(
            "SELECT COUNT(*) FROM entries_fts WHERE content LIKE ?1",
            [format!("%{token}%")],
            |row| row.get(0),
        )
        .expect("query entries_fts table");
    assert_eq!(
        fts_match, 0,
        "raw GitHub token must not appear in the FTS index"
    );

    // An FTS MATCH on a fragment of the token must return nothing.
    let fts_match_query_result = conn
        .query_row(
            "SELECT COUNT(*) FROM entries_fts WHERE entries_fts MATCH ?1",
            [format!("{token}*")],
            |row| row.get::<_, i64>(0),
        )
        .unwrap_or(0);
    assert_eq!(
        fts_match_query_result, 0,
        "FTS MATCH on the raw token must return no rows"
    );
}

#[test]
fn opt_out_stores_token_verbatim() {
    let dir = tempfile::tempdir().expect("create temp dir");
    let db_path = dir.path().join("scrub-opt-out.db");

    let token = fake_github_token();
    let content = format!("here is my token: {token} please remember it");

    {
        let mut config = Config::default();
        config.db_path = db_path.clone();
        config.scrub.enabled = false;
        let cf = ContextForge::open(config).expect("open db");
        cf.save(&content, kind::MANUAL, &SaveOptions::default())
            .expect("save unscrubbed entry");
    }

    let conn = Connection::open(&db_path).expect("open raw sqlite connection");

    let entries_match: i64 = conn
        .query_row(
            "SELECT COUNT(*) FROM entries WHERE content LIKE ?1",
            [format!("%{token}%")],
            |row| row.get(0),
        )
        .expect("query entries table");
    assert_eq!(
        entries_match, 1,
        "with scrubbing disabled, the raw token must be stored verbatim"
    );

    let redacted_match: i64 = conn
        .query_row(
            "SELECT COUNT(*) FROM entries WHERE content LIKE '%[REDACTED:github-token]%'",
            [],
            |row| row.get(0),
        )
        .expect("query entries table for redaction marker");
    assert_eq!(
        redacted_match, 0,
        "with scrubbing disabled, no redaction placeholder should be present"
    );
}