remem-ai 0.5.148

Local-first coding agent memory for Claude Code and OpenAI Codex
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124

use std::path::Path;

use anyhow::Result;

use super::types::CORPUS_NAME;
use super::{
    run::rendered_line_contains_title_and_label, run_sandbox_eval, InjectionEvalMetadata,
    InjectionEvalOptions, InjectionEvalReport, InjectionMetricSummary, InjectionRateMetric,
};

#[test]
fn injection_eval_exercises_session_start_render_path() -> Result<()> {
    let report = run_sandbox_eval(InjectionEvalOptions::default())?;

    assert!(!report.metadata.real_db_touched);
    assert_eq!(report.metadata.boundary, "context::render_context_output");
    assert!(
        report.metrics.expected_memory_recall.is_perfect(),
        "{:#?}",
        report
    );
    assert!(
        report.metrics.forbidden_memory_exclusion.is_perfect(),
        "{:#?}",
        report
    );
    assert!(
        report.metrics.abstention_false_positive_bound.is_perfect(),
        "{:#?}",
        report
    );
    assert!(
        report.metrics.stale_anchor_labeling.is_perfect(),
        "{:#?}",
        report
    );
    assert!(
        report.metrics.user_prompt_submit_memory_recall.is_perfect(),
        "{:#?}",
        report
    );
    assert!(
        report
            .metrics
            .user_prompt_submit_abstention_false_positive_bound
            .is_perfect(),
        "{:#?}",
        report
    );
    assert!(report.metrics.all_checks_passed, "{:#?}", report);
    assert!(report.metadata.memories_loaded > 0);
    assert!(report.metadata.core_count > 0 || report.metadata.index_count > 0);
    assert!(report.failing_examples.is_empty());
    assert!(!Path::new(&report.metadata.data_dir).exists());
    Ok(())
}

#[test]
fn injection_eval_display_includes_metrics() {
    let report = InjectionEvalReport {
        metadata: InjectionEvalMetadata {
            corpus: CORPUS_NAME.to_string(),
            boundary: "context::render_context_output".to_string(),
            storage: "temporary sqlite".to_string(),
            data_dir: "/tmp/example".to_string(),
            data_dir_kept: false,
            real_db_touched: false,
            project: "/tmp/remem-injection-eval/repo".to_string(),
            host: "codex-cli".to_string(),
            branch: "main".to_string(),
            output_chars: 100,
            memories_loaded: 2,
            core_count: 1,
            index_count: 1,
            lesson_count: 0,
            preference_count: 0,
            session_count: 0,
            workstream_count: 0,
            truncated: false,
        },
        metrics: InjectionMetricSummary {
            expected_memory_recall: InjectionRateMetric::new(2, 2),
            forbidden_memory_exclusion: InjectionRateMetric::new(3, 3),
            abstention_false_positive_bound: InjectionRateMetric::new(1, 1),
            stale_anchor_labeling: InjectionRateMetric::new(1, 1),
            user_prompt_submit_memory_recall: InjectionRateMetric::new(1, 1),
            user_prompt_submit_abstention_false_positive_bound: InjectionRateMetric::new(1, 1),
            all_checks_passed: true,
        },
        cases: vec![],
        failing_examples: vec![],
    };

    let rendered = format!("{report}");

    assert!(rendered.contains("=== remem eval-injection"));
    assert!(rendered.contains("expected_memory_recall: 2/2"));
    assert!(rendered.contains("forbidden_memory_exclusion: 3/3"));
    assert!(rendered.contains("abstention_false_positive_bound: 1/1"));
    assert!(rendered.contains("stale_anchor_labeling: 1/1"));
    assert!(rendered.contains("user_prompt_submit_memory_recall: 1/1"));
    assert!(rendered.contains("user_prompt_submit_abstention_false_positive_bound: 1/1"));
    assert!(rendered.contains("all_checks_passed: true"));
}

#[test]
fn stale_anchor_eval_requires_label_on_target_memory_line() {
    let output = "\
**#8 Stale source anchor decision** (src=memory:#8;status=active;staleness=fresh;source_anchor=tracked) | **#9 Other memory** (src=memory:#9;status=active;staleness=fresh;source_anchor=verify-before-trust)
";

    assert!(!rendered_line_contains_title_and_label(
        output,
        8,
        "Stale source anchor decision",
        "source_anchor=verify-before-trust"
    ));
    assert!(rendered_line_contains_title_and_label(
        "**#8 Stale source anchor decision** (src=memory:#8;source_anchor=verify-before-trust)",
        8,
        "Stale source anchor decision",
        "source_anchor=verify-before-trust"
    ));
}