1use std::ffi::OsString;
2use std::fmt::{self, Display};
3use std::path::{Path, PathBuf};
4use std::sync::OnceLock;
5use std::time::{SystemTime, UNIX_EPOCH};
6
7use anyhow::{anyhow, Context, Result};
8use serde::{Deserialize, Serialize};
9use tokio::sync::{oneshot, Mutex};
10
11const PROJECT: &str = "/remem/eval-e2e";
12const CORPUS_NAME: &str = "builtin-coding-agent-life-v1";
13
14#[derive(Debug, Clone, Copy)]
15pub struct E2eEvalOptions {
16 pub k: usize,
17 pub keep_data_dir: bool,
18}
19
20impl Default for E2eEvalOptions {
21 fn default() -> Self {
22 Self {
23 k: 5,
24 keep_data_dir: false,
25 }
26 }
27}
28
29#[derive(Debug, Serialize)]
30pub struct E2eEvalReport {
31 pub metadata: E2eEvalMetadata,
32 pub api_metrics: E2eMetricSummary,
33 pub keyword_baseline: E2eMetricSummary,
34 pub queries: Vec<E2eQueryReport>,
35}
36
37#[derive(Debug, Serialize)]
38pub struct E2eEvalMetadata {
39 pub commit: Option<String>,
40 pub command: String,
41 pub corpus: String,
42 pub corpus_items: usize,
43 pub query_count: usize,
44 pub data_dir: String,
45 pub data_dir_kept: bool,
46 pub api_base_url: String,
47 pub config: E2eEvalConfig,
48}
49
50#[derive(Debug, Serialize)]
51pub struct E2eEvalConfig {
52 pub boundary: String,
53 pub project: String,
54 pub k: usize,
55}
56
57#[derive(Debug, Serialize, Clone, PartialEq)]
58pub struct E2eMetricSummary {
59 pub total_queries: usize,
60 pub hit_count: usize,
61 pub hit_rate: f64,
62 pub mrr: f64,
63}
64
65#[derive(Debug, Serialize)]
66pub struct E2eQueryReport {
67 pub id: String,
68 pub query: String,
69 pub expected_topic_key: String,
70 pub api_rank: Option<usize>,
71 pub keyword_baseline_rank: Option<usize>,
72 pub api_result_topic_keys: Vec<String>,
73 pub keyword_baseline_topic_keys: Vec<String>,
74}
75
76#[derive(Clone, Copy)]
77struct CorpusMemory {
78 topic_key: &'static str,
79 title: &'static str,
80 text: &'static str,
81 memory_type: &'static str,
82}
83
84#[derive(Clone, Copy)]
85struct EvalQuery {
86 id: &'static str,
87 query: &'static str,
88 expected_topic_key: &'static str,
89}
90
91const CORPUS: &[CorpusMemory] = &[
92 CorpusMemory {
93 topic_key: "eval-migration-locking",
94 title: "SQLite migration locking fix",
95 text: "Root cause: concurrent schema migrations raced on startup. Fix: serialize migration execution and verify with cargo test migrate::tests.",
96 memory_type: "bugfix",
97 },
98 CorpusMemory {
99 topic_key: "eval-raw-archive-fallback",
100 title: "Raw archive fallback for sparse recall",
101 text: "When curated search is sparse, remem should attach raw archive hits so literal chat content remains discoverable without treating raw rows as curated memories.",
102 memory_type: "decision",
103 },
104 CorpusMemory {
105 topic_key: "eval-codex-hook-timeout",
106 title: "Codex hook stdin timeout",
107 text: "Codex hook stdin reads must allow normal startup latency and then fall back to CLI values instead of failing the whole capture path.",
108 memory_type: "discovery",
109 },
110 CorpusMemory {
111 topic_key: "eval-project-scope-discipline",
112 title: "Project scope discipline",
113 text: "Global memories must be explicitly requested; project memories should not leak into unrelated workspaces during context injection or retrieval.",
114 memory_type: "preference",
115 },
116];
117
118const QUERIES: &[EvalQuery] = &[
119 EvalQuery {
120 id: "migration-race",
121 query: "schema migration race serialize startup",
122 expected_topic_key: "eval-migration-locking",
123 },
124 EvalQuery {
125 id: "raw-fallback",
126 query: "sparse curated search raw archive fallback",
127 expected_topic_key: "eval-raw-archive-fallback",
128 },
129 EvalQuery {
130 id: "codex-timeout",
131 query: "Codex hook stdin timeout fallback CLI values",
132 expected_topic_key: "eval-codex-hook-timeout",
133 },
134 EvalQuery {
135 id: "scope-leak",
136 query: "global memories explicit project leak context retrieval",
137 expected_topic_key: "eval-project-scope-discipline",
138 },
139];
140
141#[derive(Serialize)]
142struct ApiSaveRequest<'a> {
143 text: &'a str,
144 title: &'a str,
145 project: &'a str,
146 topic_key: &'a str,
147 memory_type: &'a str,
148 scope: &'a str,
149 local_copy_enabled: bool,
150}
151
152#[derive(Deserialize)]
153struct ApiSaveResponse {
154 id: i64,
155}
156
157#[derive(Deserialize)]
158struct ApiSearchResponse {
159 data: Vec<ApiMemoryItem>,
160}
161
162#[derive(Deserialize)]
163struct ApiMemoryItem {
164 topic_key: Option<String>,
165}
166
167pub async fn run_sandbox_eval(options: E2eEvalOptions) -> Result<E2eEvalReport> {
168 let _env_guard = env_lock().lock().await;
169 let k = options.k.max(1);
170 let data_dir = unique_temp_data_dir();
171 std::fs::create_dir_all(&data_dir)
172 .with_context(|| format!("create eval data dir {}", data_dir.display()))?;
173 let _restore = EnvRestore::set("REMEM_DATA_DIR", data_dir.as_os_str().to_os_string());
174 crate::api::ensure_api_token().context("create sandbox API token")?;
175 let api_token = crate::api::load_api_token().context("load sandbox API token")?;
176
177 let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
178 .await
179 .context("bind sandbox eval API listener")?;
180 let addr = listener
181 .local_addr()
182 .context("read sandbox API listener addr")?;
183 let base_url = format!("http://{}", addr);
184 let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>();
185 let app = crate::api::build_router(addr.port()).with_state(crate::api::DbState);
186 let server = tokio::spawn(async move {
187 axum::serve(listener, app)
188 .with_graceful_shutdown(async {
189 let _ = shutdown_rx.await;
190 })
191 .await
192 });
193
194 let client = reqwest::Client::new();
195 let run_result = run_api_boundary_eval(&client, &base_url, &api_token, k).await;
196 let _ = shutdown_tx.send(());
197 let server_result = server.await.context("join sandbox eval API server")?;
198 server_result.context("sandbox eval API server failed")?;
199
200 let result = run_result.map(|mut report| {
201 report.metadata.commit = build_git_commit();
202 report.metadata.command = format!("remem eval-e2e --k {}", k);
203 report.metadata.data_dir = data_dir.display().to_string();
204 report.metadata.data_dir_kept = options.keep_data_dir;
205 report.metadata.api_base_url = base_url;
206 report
207 });
208
209 cleanup_data_dir_after_eval(&data_dir, options.keep_data_dir, result)
210}
211
212async fn run_api_boundary_eval(
213 client: &reqwest::Client,
214 base_url: &str,
215 api_token: &str,
216 k: usize,
217) -> Result<E2eEvalReport> {
218 wait_for_status(client, base_url, api_token).await?;
219 let mut saved_ids = Vec::with_capacity(CORPUS.len());
220 for memory in CORPUS {
221 let saved = save_memory_via_api(client, base_url, api_token, memory).await?;
222 saved_ids.push(saved.id);
223 }
224
225 let mut query_reports = Vec::with_capacity(QUERIES.len());
226 for query in QUERIES {
227 let api_topic_keys =
228 search_topic_keys_via_api(client, base_url, api_token, query.query, k).await?;
229 let keyword_topic_keys = keyword_baseline_topic_keys(query.query, k);
230 query_reports.push(E2eQueryReport {
231 id: query.id.to_string(),
232 query: query.query.to_string(),
233 expected_topic_key: query.expected_topic_key.to_string(),
234 api_rank: one_based_rank(&api_topic_keys, query.expected_topic_key),
235 keyword_baseline_rank: one_based_rank(&keyword_topic_keys, query.expected_topic_key),
236 api_result_topic_keys: api_topic_keys,
237 keyword_baseline_topic_keys: keyword_topic_keys,
238 });
239 }
240
241 Ok(E2eEvalReport {
242 metadata: E2eEvalMetadata {
243 commit: None,
244 command: String::new(),
245 corpus: CORPUS_NAME.to_string(),
246 corpus_items: saved_ids.len(),
247 query_count: QUERIES.len(),
248 data_dir: String::new(),
249 data_dir_kept: false,
250 api_base_url: String::new(),
251 config: E2eEvalConfig {
252 boundary: "REST API /api/v1/memories + /api/v1/search".to_string(),
253 project: PROJECT.to_string(),
254 k,
255 },
256 },
257 api_metrics: summarize_ranks(query_reports.iter().map(|query| query.api_rank)),
258 keyword_baseline: summarize_ranks(
259 query_reports
260 .iter()
261 .map(|query| query.keyword_baseline_rank),
262 ),
263 queries: query_reports,
264 })
265}
266
267async fn wait_for_status(client: &reqwest::Client, base_url: &str, api_token: &str) -> Result<()> {
268 let url = format!("{}/api/v1/status", base_url);
269 let mut last_error = None;
270 for _ in 0..20 {
271 match client.get(&url).bearer_auth(api_token).send().await {
272 Ok(response) if response.status().is_success() => return Ok(()),
273 Ok(response) => last_error = Some(anyhow!("status returned {}", response.status())),
274 Err(error) => last_error = Some(error.into()),
275 }
276 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
277 }
278 Err(last_error.unwrap_or_else(|| anyhow!("status endpoint did not respond")))
279}
280
281async fn save_memory_via_api(
282 client: &reqwest::Client,
283 base_url: &str,
284 api_token: &str,
285 memory: &CorpusMemory,
286) -> Result<ApiSaveResponse> {
287 let request = ApiSaveRequest {
288 text: memory.text,
289 title: memory.title,
290 project: PROJECT,
291 topic_key: memory.topic_key,
292 memory_type: memory.memory_type,
293 scope: "project",
294 local_copy_enabled: false,
295 };
296 let response = client
297 .post(format!("{}/api/v1/memories", base_url))
298 .bearer_auth(api_token)
299 .json(&request)
300 .send()
301 .await
302 .context("POST /api/v1/memories failed")?;
303 let status = response.status();
304 if !status.is_success() {
305 let body = response.text().await.unwrap_or_default();
306 return Err(anyhow!("POST /api/v1/memories returned {status}: {body}"));
307 }
308 response
309 .json::<ApiSaveResponse>()
310 .await
311 .context("parse save memory response")
312}
313
314async fn search_topic_keys_via_api(
315 client: &reqwest::Client,
316 base_url: &str,
317 api_token: &str,
318 query: &str,
319 k: usize,
320) -> Result<Vec<String>> {
321 let limit = k.to_string();
322 let response = client
323 .get(format!("{}/api/v1/search", base_url))
324 .bearer_auth(api_token)
325 .query(&[
326 ("query", query),
327 ("project", PROJECT),
328 ("limit", limit.as_str()),
329 ])
330 .send()
331 .await
332 .context("GET /api/v1/search failed")?;
333 let status = response.status();
334 if !status.is_success() {
335 let body = response.text().await.unwrap_or_default();
336 return Err(anyhow!("GET /api/v1/search returned {status}: {body}"));
337 }
338 let search = response
339 .json::<ApiSearchResponse>()
340 .await
341 .context("parse search response")?;
342 Ok(search
343 .data
344 .into_iter()
345 .filter_map(|item| item.topic_key)
346 .collect())
347}
348
349fn keyword_baseline_topic_keys(query: &str, k: usize) -> Vec<String> {
350 let query_tokens = tokenize(query);
351 let mut scored: Vec<(usize, &'static str)> = CORPUS
352 .iter()
353 .map(|memory| {
354 let text = format!("{} {}", memory.title, memory.text);
355 let doc_tokens = tokenize(&text);
356 let score = query_tokens
357 .iter()
358 .filter(|token| doc_tokens.contains(*token))
359 .count();
360 (score, memory.topic_key)
361 })
362 .collect();
363 scored.sort_by(|a, b| b.0.cmp(&a.0).then_with(|| a.1.cmp(b.1)));
364 scored
365 .into_iter()
366 .filter(|(score, _)| *score > 0)
367 .take(k)
368 .map(|(_, topic_key)| topic_key.to_string())
369 .collect()
370}
371
372fn tokenize(text: &str) -> Vec<String> {
373 let mut tokens: Vec<String> = text
374 .to_lowercase()
375 .split(|ch: char| !ch.is_ascii_alphanumeric())
376 .filter(|token| token.len() > 2)
377 .map(str::to_string)
378 .collect();
379 tokens.sort();
380 tokens.dedup();
381 tokens
382}
383
384fn one_based_rank(topic_keys: &[String], expected_topic_key: &str) -> Option<usize> {
385 topic_keys
386 .iter()
387 .position(|topic_key| topic_key == expected_topic_key)
388 .map(|index| index + 1)
389}
390
391fn summarize_ranks(ranks: impl Iterator<Item = Option<usize>>) -> E2eMetricSummary {
392 let mut total_queries = 0usize;
393 let mut hit_count = 0usize;
394 let mut reciprocal_sum = 0.0;
395 for rank in ranks {
396 total_queries += 1;
397 if let Some(rank) = rank {
398 hit_count += 1;
399 reciprocal_sum += 1.0 / rank as f64;
400 }
401 }
402 E2eMetricSummary {
403 total_queries,
404 hit_count,
405 hit_rate: if total_queries == 0 {
406 0.0
407 } else {
408 hit_count as f64 / total_queries as f64
409 },
410 mrr: if total_queries == 0 {
411 0.0
412 } else {
413 reciprocal_sum / total_queries as f64
414 },
415 }
416}
417
418fn env_lock() -> &'static Mutex<()> {
419 static LOCK: OnceLock<Mutex<()>> = OnceLock::new();
420 LOCK.get_or_init(|| Mutex::new(()))
421}
422
423struct EnvRestore {
424 key: &'static str,
425 previous: Option<OsString>,
426}
427
428impl EnvRestore {
429 fn set(key: &'static str, value: OsString) -> Self {
430 let previous = std::env::var_os(key);
431 std::env::set_var(key, value);
432 Self { key, previous }
433 }
434}
435
436impl Drop for EnvRestore {
437 fn drop(&mut self) {
438 if let Some(previous) = self.previous.as_ref() {
439 std::env::set_var(self.key, previous);
440 } else {
441 std::env::remove_var(self.key);
442 }
443 }
444}
445
446fn unique_temp_data_dir() -> PathBuf {
447 let nanos = SystemTime::now()
448 .duration_since(UNIX_EPOCH)
449 .map(|duration| duration.as_nanos())
450 .unwrap_or(0);
451 std::env::temp_dir().join(format!("remem-e2e-eval-{}-{}", std::process::id(), nanos))
452}
453
454fn build_git_commit() -> Option<String> {
455 option_env!("REMEM_BUILD_GIT_COMMIT")
456 .map(str::trim)
457 .filter(|sha| !sha.is_empty())
458 .map(str::to_string)
459}
460
461fn cleanup_data_dir_after_eval<T>(
462 data_dir: &Path,
463 keep_data_dir: bool,
464 result: Result<T>,
465) -> Result<T> {
466 if keep_data_dir {
467 return result;
468 }
469
470 let cleanup = std::fs::remove_dir_all(data_dir)
471 .with_context(|| format!("remove eval data dir {}", data_dir.display()));
472 match (result, cleanup) {
473 (Ok(value), Ok(())) => Ok(value),
474 (Ok(_), Err(err)) => Err(err),
475 (Err(err), Ok(())) => Err(err),
476 (Err(err), Err(cleanup_err)) => {
477 crate::log::warn(
478 "eval-e2e",
479 &format!("cleanup failed after eval error: {}", cleanup_err),
480 );
481 Err(err)
482 }
483 }
484}
485
486impl Display for E2eEvalReport {
487 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
488 writeln!(
489 f,
490 "=== remem eval-e2e ({}, {} queries, k={}) ===",
491 self.metadata.corpus, self.metadata.query_count, self.metadata.config.k
492 )?;
493 writeln!(f, "boundary: {}", self.metadata.config.boundary)?;
494 writeln!(f, "data_dir: {}", self.metadata.data_dir)?;
495 writeln!(f, "data_dir_kept: {}", self.metadata.data_dir_kept)?;
496 writeln!(
497 f,
498 "api: hit_rate={:.1}% mrr={:.3} ({}/{})",
499 self.api_metrics.hit_rate * 100.0,
500 self.api_metrics.mrr,
501 self.api_metrics.hit_count,
502 self.api_metrics.total_queries
503 )?;
504 writeln!(
505 f,
506 "keyword_baseline: hit_rate={:.1}% mrr={:.3} ({}/{})",
507 self.keyword_baseline.hit_rate * 100.0,
508 self.keyword_baseline.mrr,
509 self.keyword_baseline.hit_count,
510 self.keyword_baseline.total_queries
511 )?;
512 for query in &self.queries {
513 writeln!(
514 f,
515 "- {} api_rank={:?} baseline_rank={:?}",
516 query.id, query.api_rank, query.keyword_baseline_rank
517 )?;
518 }
519 Ok(())
520 }
521}
522
523#[cfg(test)]
524mod tests {
525 use super::*;
526
527 #[test]
528 fn summarize_ranks_reports_hit_rate_and_mrr() {
529 let got = summarize_ranks([Some(1), Some(4), None].into_iter());
530 assert_eq!(
531 got,
532 E2eMetricSummary {
533 total_queries: 3,
534 hit_count: 2,
535 hit_rate: 2.0 / 3.0,
536 mrr: (1.0 + 0.25) / 3.0,
537 }
538 );
539 }
540
541 #[test]
542 fn keyword_baseline_ranks_expected_memory() {
543 let got = keyword_baseline_topic_keys("raw archive fallback sparse search", 2);
544 assert_eq!(
545 got.first().map(String::as_str),
546 Some("eval-raw-archive-fallback")
547 );
548 }
549
550 #[test]
551 fn keyword_baseline_excludes_zero_score_memories() {
552 let got = keyword_baseline_topic_keys("xqzv jjjj qqqq", 5);
553 assert!(got.is_empty());
554 }
555
556 #[test]
557 fn cleanup_removes_data_dir_even_when_eval_fails() -> Result<()> {
558 let data_dir = unique_temp_data_dir();
559 std::fs::create_dir_all(&data_dir)?;
560
561 let result: Result<()> =
562 cleanup_data_dir_after_eval(&data_dir, false, Err(anyhow!("forced failure")));
563
564 let err = result.expect_err("original eval error should be returned");
565 assert!(err.to_string().contains("forced failure"));
566 assert!(!data_dir.exists());
567 Ok(())
568 }
569
570 #[test]
571 fn cleanup_keeps_data_dir_when_requested() -> Result<()> {
572 let data_dir = unique_temp_data_dir();
573 std::fs::create_dir_all(&data_dir)?;
574
575 cleanup_data_dir_after_eval(&data_dir, true, Ok(()))?;
576
577 assert!(data_dir.exists());
578 std::fs::remove_dir_all(&data_dir)?;
579 Ok(())
580 }
581}