#![allow(clippy::expect_used, clippy::panic)]
use std::{
collections::HashSet,
path::Path,
time::{Duration, Instant},
};
use okc::{config::OkcConfig, service::OkcService};
use serde::Deserialize;
use tempfile::TempDir;
const EVALUATION_JSON: &str = include_str!("fixtures/search-eval-v1.json");
const LATENCY_SAMPLES_PER_QUERY: usize = 25;
const MAX_P95_LATENCY_MICROS: u64 = 900;
const FAILURE_CLASSES: &[&str] = &[
"lexical_normalization",
"typo_fuzzy_matching",
"ranking",
"semantic_recall",
"graph_expansion",
"filter_interaction",
];
#[derive(Debug, Deserialize)]
struct EvaluationCorpus {
version: u32,
corpus: String,
proposal_gate: ProposalGate,
queries: Vec<EvaluationQuery>,
}
#[derive(Debug, Deserialize)]
struct ProposalGate {
minimum_absolute_recall_at_5_gain: f64,
minimum_absolute_mrr_at_10_gain: f64,
maximum_p95_latency_regression_percent: f64,
maximum_exact_query_recall_regression: f64,
}
#[derive(Debug, Deserialize)]
struct EvaluationQuery {
id: String,
category: String,
query: String,
#[serde(default)]
path_prefix: Option<String>,
#[serde(default)]
types: Vec<String>,
#[serde(default)]
tags: Vec<String>,
expected_relevant: Vec<String>,
#[serde(default)]
expect_empty: bool,
#[serde(default)]
failure_class: Option<String>,
}
#[derive(Debug)]
struct QueryResult {
id: String,
category: String,
retrieved: Vec<String>,
expected: Vec<String>,
expect_empty: bool,
failure_class: Option<String>,
}
#[derive(Debug)]
struct Metrics {
recall_at_5: f64,
recall_at_10: f64,
mrr_at_10: f64,
zero_required_evidence_rate: f64,
intentional_zero_hit_accuracy: f64,
}
#[test]
fn production_lexical_search_baseline_v1() {
let corpus: EvaluationCorpus =
serde_json::from_str(EVALUATION_JSON).expect("parse search evaluation corpus");
assert_eq!(corpus.version, 1);
assert_eq!(corpus.corpus, "tests/fixtures/simple");
validate_proposal_gate(&corpus.proposal_gate);
validate_corpus(&corpus.queries);
let repo = copy_fixture(Path::new(&corpus.corpus));
let config = OkcConfig {
roots: vec![repo.path().to_path_buf()],
db_path: repo.path().join("evaluation.db"),
..Default::default()
};
let mut service = OkcService::open(&config).expect("open evaluation service");
service.scan().expect("scan evaluation corpus");
let mut results = Vec::new();
let mut latencies = Vec::new();
for query in &corpus.queries {
let response = run_query(&service, query);
results.push(QueryResult {
id: query.id.clone(),
category: query.category.clone(),
retrieved: response.results.into_iter().map(|item| item.path).collect(),
expected: query.expected_relevant.clone(),
expect_empty: query.expect_empty,
failure_class: query.failure_class.clone(),
});
for _ in 0..LATENCY_SAMPLES_PER_QUERY {
let started = Instant::now();
let _ = run_query(&service, query);
latencies.push(started.elapsed());
}
}
let metrics = calculate_metrics(&results);
let p50 = percentile(&mut latencies.clone(), 0.50);
let p95 = percentile(&mut latencies, 0.95);
print_report(&results, &metrics, p50, p95);
assert_eq!(results.len(), 10);
assert!(metrics.recall_at_5 >= 0.7777);
assert!(metrics.recall_at_10 >= 0.7777);
assert!(metrics.mrr_at_10 >= 0.7221);
assert!(metrics.zero_required_evidence_rate <= 0.2223);
assert_eq!(metrics.intentional_zero_hit_accuracy, 1.0);
let typo = results
.iter()
.find(|result| result.id == "typo-monthly-revenue")
.expect("versioned typo judgment");
assert_eq!(recall_at(typo, 5), 1.0);
assert!(
p95 <= Duration::from_micros(MAX_P95_LATENCY_MICROS),
"p95 {:?} exceeds the p95 latency cap of {}us (baseline 432us)",
p95,
MAX_P95_LATENCY_MICROS
);
assert!(results
.iter()
.filter(|result| result.category == "exact")
.all(|result| recall_at(result, 10) == 1.0));
assert!(results.iter().all(|result| {
result.expected.is_empty() || recall_at(result, 10) == 1.0 || result.failure_class.is_some()
}));
}
fn run_query(
service: &OkcService,
query: &EvaluationQuery,
) -> okc::model::document::SearchResponse {
service
.search(
&query.query,
query.path_prefix.as_deref(),
(!query.types.is_empty()).then_some(query.types.as_slice()),
(!query.tags.is_empty()).then_some(query.tags.as_slice()),
10,
)
.unwrap_or_else(|error| panic!("search evaluation query '{}' failed: {error}", query.id))
}
fn calculate_metrics(results: &[QueryResult]) -> Metrics {
let judged = results
.iter()
.filter(|result| !result.expected.is_empty())
.collect::<Vec<_>>();
let intentional_empty = results
.iter()
.filter(|result| result.expect_empty)
.collect::<Vec<_>>();
Metrics {
recall_at_5: mean(judged.iter().map(|result| recall_at(result, 5))),
recall_at_10: mean(judged.iter().map(|result| recall_at(result, 10))),
mrr_at_10: mean(judged.iter().map(|result| reciprocal_rank(result, 10))),
zero_required_evidence_rate: mean(judged.iter().map(|result| {
f64::from(
!result
.retrieved
.iter()
.any(|path| result.expected.contains(path)),
)
})),
intentional_zero_hit_accuracy: mean(
intentional_empty
.iter()
.map(|result| f64::from(result.retrieved.is_empty())),
),
}
}
fn recall_at(result: &QueryResult, cutoff: usize) -> f64 {
let relevant = result.expected.iter().collect::<HashSet<_>>();
let found = result
.retrieved
.iter()
.take(cutoff)
.filter(|path| relevant.contains(path))
.count();
found as f64 / relevant.len() as f64
}
fn reciprocal_rank(result: &QueryResult, cutoff: usize) -> f64 {
result
.retrieved
.iter()
.take(cutoff)
.position(|path| result.expected.contains(path))
.map_or(0.0, |index| 1.0 / (index + 1) as f64)
}
fn mean(values: impl Iterator<Item = f64>) -> f64 {
let values = values.collect::<Vec<_>>();
if values.is_empty() {
0.0
} else {
values.iter().sum::<f64>() / values.len() as f64
}
}
fn percentile(samples: &mut [Duration], quantile: f64) -> Duration {
samples.sort_unstable();
let index = ((samples.len() - 1) as f64 * quantile).ceil() as usize;
samples[index]
}
fn print_report(results: &[QueryResult], metrics: &Metrics, p50: Duration, p95: Duration) {
println!("search-eval-v1");
println!("recall_at_5={:.4}", metrics.recall_at_5);
println!("recall_at_10={:.4}", metrics.recall_at_10);
println!("mrr_at_10={:.4}", metrics.mrr_at_10);
println!(
"zero_required_evidence_rate={:.4}",
metrics.zero_required_evidence_rate
);
println!(
"intentional_zero_hit_accuracy={:.4}",
metrics.intentional_zero_hit_accuracy
);
println!("latency_p50_us={}", p50.as_micros());
println!("latency_p95_us={}", p95.as_micros());
for class in FAILURE_CLASSES {
let count = results
.iter()
.filter(|result| {
!result.expected.is_empty()
&& recall_at(result, 10) < 1.0
&& result.failure_class.as_deref() == Some(*class)
})
.count();
println!("failure_count_{class}={count}");
}
for result in results {
let observed_failure = if !result.expected.is_empty() && recall_at(result, 10) < 1.0 {
result.failure_class.as_deref().unwrap_or("unclassified")
} else {
"none"
};
println!(
"query={} category={} expected={:?} retrieved={:?} observed_failure={}",
result.id, result.category, result.expected, result.retrieved, observed_failure
);
}
}
fn validate_proposal_gate(gate: &ProposalGate) {
assert_eq!(gate.minimum_absolute_recall_at_5_gain, 0.10);
assert_eq!(gate.minimum_absolute_mrr_at_10_gain, 0.05);
assert_eq!(gate.maximum_p95_latency_regression_percent, 25.0);
assert_eq!(gate.maximum_exact_query_recall_regression, 0.0);
}
fn validate_corpus(queries: &[EvaluationQuery]) {
let mut ids = HashSet::new();
let categories = queries
.iter()
.map(|query| query.category.as_str())
.collect::<HashSet<_>>();
for required in [
"exact",
"paraphrase",
"zero_hit",
"graph_assisted",
"metadata_filtered",
] {
assert!(
categories.contains(required),
"missing query category {required}"
);
}
for query in queries {
assert!(ids.insert(&query.id), "duplicate query id {}", query.id);
assert!(
!query.expect_empty || query.expected_relevant.is_empty(),
"intentional empty query {} must not declare relevant documents",
query.id
);
if let Some(class) = query.failure_class.as_deref() {
assert!(
FAILURE_CLASSES.contains(&class),
"unknown failure class {class}"
);
}
}
}
fn copy_fixture(source: &Path) -> TempDir {
let destination = TempDir::new().expect("create search evaluation temp directory");
copy_dir_all(source, destination.path()).expect("copy search evaluation fixture");
destination
}
fn copy_dir_all(source: &Path, destination: &Path) -> std::io::Result<()> {
std::fs::create_dir_all(destination)?;
for entry in std::fs::read_dir(source)? {
let entry = entry?;
let target = destination.join(entry.file_name());
if entry.file_type()?.is_dir() {
copy_dir_all(&entry.path(), &target)?;
} else {
std::fs::copy(entry.path(), target)?;
}
}
Ok(())
}
#[cfg(test)]
mod metric_tests {
use super::*;
#[test]
fn metrics_use_judged_relevance_and_ignore_intentional_empty_queries() {
let results = vec![
QueryResult {
id: "ranked".to_string(),
category: "exact".to_string(),
retrieved: vec!["other.md".to_string(), "relevant.md".to_string()],
expected: vec!["relevant.md".to_string()],
expect_empty: false,
failure_class: None,
},
QueryResult {
id: "empty".to_string(),
category: "zero_hit".to_string(),
retrieved: Vec::new(),
expected: Vec::new(),
expect_empty: true,
failure_class: None,
},
];
let metrics = calculate_metrics(&results);
assert_eq!(metrics.recall_at_5, 1.0);
assert_eq!(metrics.recall_at_10, 1.0);
assert_eq!(metrics.mrr_at_10, 0.5);
assert_eq!(metrics.zero_required_evidence_rate, 0.0);
assert_eq!(metrics.intentional_zero_hit_accuracy, 1.0);
}
}