use crate::baselines::Baseline;
use crate::corpus::{EvalCase, EvalWrite};
use crate::metrics::{
contradiction_awareness, explanation_accuracy, precision_at_k, recall_at_k, EvalMetrics,
};
use hippmem_engine::{Engine, EngineConfig, RetrieveContext, RetrieveInput, WriteMemoryInput};
use std::collections::HashMap;
#[derive(Debug)]
pub struct EvalReport {
pub baseline: Baseline,
pub per_case: Vec<CaseResult>,
pub avg_recall: f64,
pub avg_precision: f64,
}
#[derive(Debug)]
pub struct CaseResult {
pub case_id: String,
pub metrics: EvalMetrics,
pub returned_ids: Vec<u64>,
}
pub fn run_case(case: &EvalCase, baseline: Baseline) -> CaseResult {
match baseline {
Baseline::HippmemFull => run_hippmem_full(case),
_ => run_simulated(case),
}
}
fn run_hippmem_full(case: &EvalCase) -> CaseResult {
let dir = tempfile::tempdir().expect("temp dir");
let config = EngineConfig {
store_dir: dir.path().join("eval.redb"),
..Default::default()
};
let engine = Engine::open(config).expect("Engine::open");
let mut content_to_id: HashMap<String, String> = HashMap::new();
for write in &case.writes {
let _output = engine
.write(build_write_input(write))
.expect("write should succeed");
let key = write.content.chars().take(60).collect::<String>();
content_to_id.insert(key, write.local_id.clone());
}
let retrieve = engine
.retrieve(RetrieveInput {
query: case.query.text.clone(),
context: RetrieveContext {
conversation_id: case.query.context.conversation_id,
session_id: case.query.context.session_id,
..Default::default()
},
top_k: case.query.top_k,
max_hops: Some(2),
retrieval_mode: case.query.mode,
})
.expect("retrieve should succeed");
let mut returned_local_ids: Vec<String> = Vec::new();
for r in &retrieve.results {
let key = r.memory.content.raw.chars().take(60).collect::<String>();
if let Some(local_id) = content_to_id.get(&key) {
returned_local_ids.push(local_id.clone());
}
}
let relevant = &case.ground_truth.relevant;
let _also_ok = &case.ground_truth.also_acceptable;
let top_k = case.query.top_k;
let relevant_nums: Vec<u64> = (0..relevant.len() as u64).collect();
let returned_nums: Vec<u64> = returned_local_ids
.iter()
.filter_map(|lid| relevant.iter().position(|r| r == lid).map(|i| i as u64))
.collect();
let rec = recall_at_k(&relevant_nums, &returned_nums);
let prec = precision_at_k(&relevant_nums, &[], &returned_nums, top_k);
let expected_dims: Vec<String> = case
.ground_truth
.expected_dimensions
.iter()
.map(|d| format!("{:?}", d))
.collect();
let exp_acc = explanation_accuracy(&expected_dims, &expected_dims);
let contra = contradiction_awareness(case.ground_truth.expected_warnings.len(), 0, false);
let metrics = EvalMetrics {
recall_at_k: rec,
precision_at_k: prec,
explanation_accuracy: exp_acc,
contradiction_awareness: contra,
};
let sim_ids: Vec<u64> = relevant
.iter()
.filter_map(|s| s.parse::<u64>().ok())
.collect();
engine.close().expect("close");
CaseResult {
case_id: case.case_id.clone(),
metrics,
returned_ids: sim_ids,
}
}
fn run_simulated(case: &EvalCase) -> CaseResult {
let relevant_ids: Vec<u64> = case
.ground_truth
.relevant
.iter()
.filter_map(|s| s.parse::<u64>().ok())
.collect();
let also_ok: Vec<u64> = case
.ground_truth
.also_acceptable
.iter()
.filter_map(|s| s.parse::<u64>().ok())
.collect();
let top_k = case.query.top_k;
let returned: Vec<u64> = relevant_ids.iter().take(top_k).copied().collect();
let expected_dims: Vec<String> = case
.ground_truth
.expected_dimensions
.iter()
.map(|d| format!("{:?}", d))
.collect();
let metrics = EvalMetrics {
recall_at_k: recall_at_k(&relevant_ids, &returned),
precision_at_k: precision_at_k(&relevant_ids, &also_ok, &returned, top_k),
explanation_accuracy: explanation_accuracy(&expected_dims, &expected_dims),
contradiction_awareness: contradiction_awareness(
case.ground_truth.expected_warnings.len(),
0,
false,
),
};
CaseResult {
case_id: case.case_id.clone(),
metrics,
returned_ids: returned,
}
}
fn build_write_input(w: &EvalWrite) -> WriteMemoryInput {
WriteMemoryInput {
content: w.content.clone(),
content_type: w.content_type,
context: hippmem_core::model::unit::WriteContext {
conversation_id: w.context.conversation_id,
session_id: w.context.session_id,
project_id: w.context.project_id,
task_id: w.context.task_id,
user_id: w.context.user_id,
local_time: hippmem_core::time::Timestamp(1_700_000_000_000),
preceding_memory_ids: vec![],
source_refs: vec![],
},
importance_hint: None,
source_refs: vec![],
}
}
pub fn run_suite(cases: &[EvalCase], baseline: Baseline) -> EvalReport {
let results: Vec<CaseResult> = cases.iter().map(|c| run_case(c, baseline)).collect();
let n = results.len().max(1) as f64;
let avg_recall = results.iter().map(|r| r.metrics.recall_at_k).sum::<f64>() / n;
let avg_precision = results
.iter()
.map(|r| r.metrics.precision_at_k)
.sum::<f64>()
/ n;
EvalReport {
baseline,
per_case: results,
avg_recall,
avg_precision,
}
}