1use std::collections::BTreeMap;
2use std::fmt::{Display, Formatter, Result as FmtResult};
3
4use anyhow::{bail, Context, Result};
5use rusqlite::Connection;
6use serde::Serialize;
7
8use super::golden::{self, CategoryEvaluation, GoldenDataset, MetricAverages};
9use crate::retrieval::search::SearchWeights;
10
11mod usage_shadow;
12
13pub const DEFAULT_DATASET_PATH: &str = "eval/golden.json";
14pub const DEFAULT_REPORT_PATH: &str = "eval/weight-grid/report.json";
15const EPSILON: f64 = 0.000_001;
16const MIN_RECALL_AT_K_DEFAULT_FLIP_DELTA: f64 = 0.05;
17
18#[derive(Debug, Clone)]
19pub struct WeightGridOptions {
20 pub dataset_path: String,
21 pub k: usize,
22}
23
24impl Default for WeightGridOptions {
25 fn default() -> Self {
26 Self {
27 dataset_path: DEFAULT_DATASET_PATH.to_string(),
28 k: 5,
29 }
30 }
31}
32
33#[derive(Debug, Clone, Serialize)]
34pub struct WeightGridReport {
35 pub version: String,
36 pub dataset_path: String,
37 pub k: usize,
38 pub scoring: WeightGridScoring,
39 pub default_weights: SearchWeights,
40 pub default_rank: usize,
41 pub default_score: f64,
42 pub best: WeightGridCandidate,
43 pub recommendation: WeightGridRecommendation,
44 pub checks: WeightGridChecks,
45 pub usage_shadow: usage_shadow::UsageShadowReport,
46 pub candidates: Vec<WeightGridCandidate>,
47}
48
49#[derive(Debug, Clone, Serialize)]
50pub struct WeightGridScoring {
51 pub evidence_recall_weight: f64,
52 pub hit_weight: f64,
53 pub ndcg_weight: f64,
54 pub mrr_weight: f64,
55 pub abstention_weight: f64,
56}
57
58impl Default for WeightGridScoring {
59 fn default() -> Self {
60 Self {
61 evidence_recall_weight: 4.0,
62 hit_weight: 3.0,
63 ndcg_weight: 2.0,
64 mrr_weight: 1.0,
65 abstention_weight: 1.0,
66 }
67 }
68}
69
70#[derive(Debug, Clone, Serialize)]
71#[serde(rename_all = "snake_case")]
72pub enum WeightGridRecommendation {
73 KeepShippedDefaults,
74 CandidateImprovesSecondaryMetricOnlyKeepDefaults,
75 CandidateOutperformsDefaultsNeedsDecision,
76}
77
78#[derive(Debug, Clone, Serialize)]
79pub struct WeightGridChecks {
80 pub fixture_corpus_used: bool,
81 pub default_weights_in_grid: bool,
82 pub best_preserves_abstention: bool,
83 pub best_preserves_scored_query_count: bool,
84 pub best_meets_recall_at_k_default_flip_gate: bool,
85}
86
87#[derive(Debug, Clone, Serialize)]
88pub struct WeightGridCandidate {
89 pub rank: usize,
90 pub weights: SearchWeights,
91 pub score: f64,
92 pub distance_from_defaults: f64,
93 pub deltas_vs_default: WeightGridDeltas,
94 pub overall: CategoryEvaluation,
95 pub by_slice: BTreeMap<String, CategoryEvaluation>,
96}
97
98#[derive(Debug, Clone, Default, Serialize)]
99pub struct WeightGridDeltas {
100 pub hit_at_k: f64,
101 pub mrr_at_10: f64,
102 pub precision_at_k: f64,
103 pub recall_at_k: f64,
104 pub ndcg_at_10: f64,
105 pub evidence_recall_at_k: f64,
106 pub abstention_pass_rate: f64,
107}
108
109pub fn run_weight_grid(options: WeightGridOptions) -> Result<WeightGridReport> {
110 let dataset = golden::load_dataset(&options.dataset_path)?;
111 run_weight_grid_dataset(
112 dataset,
113 options.dataset_path,
114 options.k,
115 default_candidate_grid(),
116 )
117}
118
119fn run_weight_grid_dataset(
120 dataset: GoldenDataset,
121 dataset_path: String,
122 requested_k: usize,
123 candidates: Vec<SearchWeights>,
124) -> Result<WeightGridReport> {
125 if !dataset.has_fixture_corpus() {
126 bail!("weight grid eval requires a fixture-backed golden dataset");
127 }
128 if candidates.is_empty() {
129 bail!("weight grid requires at least one candidate");
130 }
131
132 let k = requested_k.max(1);
133 let conn = Connection::open_in_memory().context("open in-memory weight grid eval DB")?;
134 crate::migrate::run_migrations(&conn).context("migrate weight grid eval DB")?;
135 golden::run::seed_fixture_corpus(&conn, &dataset.corpus)?;
136
137 let default_weights = SearchWeights::default();
138 let scoring = WeightGridScoring::default();
139 let mut evaluated = Vec::with_capacity(candidates.len());
140 for weights in candidates {
141 evaluated.push(evaluate_candidate(&conn, &dataset, k, weights, &scoring)?);
142 }
143 let default_index = evaluated
144 .iter()
145 .position(|candidate| candidate.weights == default_weights)
146 .context("default search weights were not included in the grid")?;
147 let default_snapshot = evaluated[default_index].clone();
148 let default_score = default_snapshot.score;
149 let default_overall = default_snapshot.overall.clone();
150
151 for candidate in &mut evaluated {
152 candidate.distance_from_defaults = weight_distance(candidate.weights, default_weights);
153 candidate.deltas_vs_default = build_candidate_deltas(&default_overall, &candidate.overall);
154 }
155 evaluated.sort_by(compare_candidates);
156 for (index, candidate) in evaluated.iter_mut().enumerate() {
157 candidate.rank = index + 1;
158 }
159
160 let default_rank = evaluated
161 .iter()
162 .find(|candidate| candidate.weights == default_weights)
163 .map(|candidate| candidate.rank)
164 .context("default search weights disappeared after sorting")?;
165 let best = evaluated
166 .first()
167 .cloned()
168 .context("weight grid produced no evaluated candidates")?;
169 let best_meets_recall_at_k_default_flip_gate =
170 candidate_meets_recall_at_k_default_flip_gate(&best);
171 let recommendation = if best.weights == default_weights || best.score <= default_score + EPSILON
172 {
173 WeightGridRecommendation::KeepShippedDefaults
174 } else if best_meets_recall_at_k_default_flip_gate {
175 WeightGridRecommendation::CandidateOutperformsDefaultsNeedsDecision
176 } else {
177 WeightGridRecommendation::CandidateImprovesSecondaryMetricOnlyKeepDefaults
178 };
179 let checks = WeightGridChecks {
180 fixture_corpus_used: true,
181 default_weights_in_grid: true,
182 best_preserves_abstention: best.overall.abstention_passed
183 >= default_overall.abstention_passed,
184 best_preserves_scored_query_count: best.overall.scored_queries
185 >= default_overall.scored_queries,
186 best_meets_recall_at_k_default_flip_gate,
187 };
188 let usage_shadow = usage_shadow::build_usage_shadow_report(&conn, &dataset, k)?;
189
190 Ok(WeightGridReport {
191 version: "2026-06-23".to_string(),
192 dataset_path,
193 k,
194 scoring,
195 default_weights,
196 default_rank,
197 default_score,
198 best,
199 recommendation,
200 checks,
201 usage_shadow,
202 candidates: evaluated,
203 })
204}
205
206fn evaluate_candidate(
207 conn: &Connection,
208 dataset: &GoldenDataset,
209 k: usize,
210 weights: SearchWeights,
211 scoring: &WeightGridScoring,
212) -> Result<WeightGridCandidate> {
213 weights.validate()?;
214 let mut overall = golden::run::CategoryAccumulator::default();
215 let mut by_slice = BTreeMap::<String, golden::run::CategoryAccumulator>::new();
216 let fetch_limit = k.max(10) as i64;
217
218 for query in &dataset.queries {
219 let results = crate::retrieval::search::search_with_branch_weights(
220 conn,
221 Some(&query.query),
222 query.project.as_deref(),
223 query.memory_type.as_deref(),
224 fetch_limit,
225 0,
226 false,
227 query.branch.as_deref(),
228 weights,
229 )?;
230 let query_tokens = golden::run::estimate_query_tokens(&query.query);
231 let evaluation = golden::run::evaluate_query(query, &results, k, query_tokens, 0.0);
232 golden::run::record_bucket(&mut overall, query, &evaluation);
233 golden::run::record_bucket(
234 by_slice.entry(query.slice_label().to_string()).or_default(),
235 query,
236 &evaluation,
237 );
238 }
239
240 let overall = golden::run::bucket_evaluation(overall);
241 let score = candidate_score(&overall, scoring);
242 Ok(WeightGridCandidate {
243 rank: 0,
244 weights,
245 score,
246 distance_from_defaults: 0.0,
247 deltas_vs_default: WeightGridDeltas::default(),
248 overall,
249 by_slice: by_slice
250 .into_iter()
251 .map(|(slice, bucket)| (slice, golden::run::bucket_evaluation(bucket)))
252 .collect(),
253 })
254}
255
256fn default_candidate_grid() -> Vec<SearchWeights> {
257 let default = SearchWeights::default();
258 let mut candidates = Vec::new();
259 for fts in [2.0, default.fts, 3.0] {
260 for vector in [2.5, default.vector, 3.5] {
261 for entity in [1.0, default.entity, 1.5] {
262 for temporal in [0.75, default.temporal, 1.25] {
263 for like_fallback in [0.1, default.like_fallback] {
264 push_unique(
265 &mut candidates,
266 SearchWeights {
267 fts,
268 vector,
269 entity,
270 temporal,
271 like_fallback,
272 ..default
273 },
274 );
275 }
276 }
277 }
278 }
279 }
280 for min_evidence_confidence in [0.0, 0.5, default.min_evidence_confidence, 0.75, 1.0] {
281 push_unique(
282 &mut candidates,
283 SearchWeights {
284 min_evidence_confidence,
285 ..default
286 },
287 );
288 }
289 for fact in [0.0, default.fact, 1.8] {
290 push_unique(&mut candidates, SearchWeights { fact, ..default });
291 }
292 for usage in [0.25, 0.75, 1.5] {
293 push_unique(&mut candidates, SearchWeights { usage, ..default });
294 }
295 candidates
296}
297
298fn push_unique(candidates: &mut Vec<SearchWeights>, weights: SearchWeights) {
299 if !candidates.contains(&weights) {
300 candidates.push(weights);
301 }
302}
303
304fn compare_candidates(
305 left: &WeightGridCandidate,
306 right: &WeightGridCandidate,
307) -> std::cmp::Ordering {
308 right
309 .score
310 .total_cmp(&left.score)
311 .then_with(|| {
312 left.distance_from_defaults
313 .total_cmp(&right.distance_from_defaults)
314 })
315 .then_with(|| left.weights.fts.total_cmp(&right.weights.fts))
316 .then_with(|| left.weights.vector.total_cmp(&right.weights.vector))
317 .then_with(|| left.weights.entity.total_cmp(&right.weights.entity))
318 .then_with(|| left.weights.graph.total_cmp(&right.weights.graph))
319 .then_with(|| left.weights.temporal.total_cmp(&right.weights.temporal))
320 .then_with(|| left.weights.fact.total_cmp(&right.weights.fact))
321 .then_with(|| {
322 left.weights
323 .like_fallback
324 .total_cmp(&right.weights.like_fallback)
325 })
326 .then_with(|| left.weights.usage.total_cmp(&right.weights.usage))
327 .then_with(|| {
328 left.weights
329 .usage_recency_half_life_days
330 .total_cmp(&right.weights.usage_recency_half_life_days)
331 })
332 .then_with(|| {
333 left.weights
334 .min_evidence_confidence
335 .total_cmp(&right.weights.min_evidence_confidence)
336 })
337}
338
339fn candidate_score(overall: &CategoryEvaluation, scoring: &WeightGridScoring) -> f64 {
340 let metric_score = overall.metrics.as_ref().map_or(0.0, |metrics| {
341 scoring.evidence_recall_weight * metrics.evidence_recall_at_k
342 + scoring.hit_weight * metrics.hit_at_k
343 + scoring.ndcg_weight * metrics.ndcg_at_10
344 + scoring.mrr_weight * metrics.mrr_at_10
345 });
346 metric_score + scoring.abstention_weight * abstention_pass_rate(overall)
347}
348
349fn abstention_pass_rate(evaluation: &CategoryEvaluation) -> f64 {
350 if evaluation.abstention_queries == 0 {
351 1.0
352 } else {
353 evaluation.abstention_passed as f64 / evaluation.abstention_queries as f64
354 }
355}
356
357fn candidate_meets_recall_at_k_default_flip_gate(candidate: &WeightGridCandidate) -> bool {
358 candidate.deltas_vs_default.recall_at_k >= MIN_RECALL_AT_K_DEFAULT_FLIP_DELTA
359 || candidate.deltas_vs_default.evidence_recall_at_k >= MIN_RECALL_AT_K_DEFAULT_FLIP_DELTA
360}
361
362fn build_candidate_deltas(
363 default_overall: &CategoryEvaluation,
364 candidate_overall: &CategoryEvaluation,
365) -> WeightGridDeltas {
366 WeightGridDeltas {
367 hit_at_k: metric_average_delta(
368 default_overall.metrics.as_ref(),
369 candidate_overall.metrics.as_ref(),
370 |m| m.hit_at_k,
371 ),
372 mrr_at_10: metric_average_delta(
373 default_overall.metrics.as_ref(),
374 candidate_overall.metrics.as_ref(),
375 |m| m.mrr_at_10,
376 ),
377 precision_at_k: metric_average_delta(
378 default_overall.metrics.as_ref(),
379 candidate_overall.metrics.as_ref(),
380 |m| m.precision_at_k,
381 ),
382 recall_at_k: metric_average_delta(
383 default_overall.metrics.as_ref(),
384 candidate_overall.metrics.as_ref(),
385 |m| m.recall_at_k,
386 ),
387 ndcg_at_10: metric_average_delta(
388 default_overall.metrics.as_ref(),
389 candidate_overall.metrics.as_ref(),
390 |m| m.ndcg_at_10,
391 ),
392 evidence_recall_at_k: metric_average_delta(
393 default_overall.metrics.as_ref(),
394 candidate_overall.metrics.as_ref(),
395 |m| m.evidence_recall_at_k,
396 ),
397 abstention_pass_rate: abstention_pass_rate(candidate_overall)
398 - abstention_pass_rate(default_overall),
399 }
400}
401
402fn metric_average_delta(
403 default: Option<&MetricAverages>,
404 candidate: Option<&MetricAverages>,
405 value: impl Fn(&MetricAverages) -> f64,
406) -> f64 {
407 match (default, candidate) {
408 (Some(default), Some(candidate)) => value(candidate) - value(default),
409 (None, Some(candidate)) => value(candidate),
410 (Some(default), None) => -value(default),
411 (None, None) => 0.0,
412 }
413}
414
415fn weight_distance(candidate: SearchWeights, default: SearchWeights) -> f64 {
416 (candidate.fts - default.fts).abs()
417 + (candidate.vector - default.vector).abs()
418 + (candidate.entity - default.entity).abs()
419 + (candidate.graph - default.graph).abs()
420 + (candidate.temporal - default.temporal).abs()
421 + (candidate.fact - default.fact).abs()
422 + (candidate.like_fallback - default.like_fallback).abs()
423 + (candidate.usage - default.usage).abs()
424 + (candidate.usage_recency_half_life_days - default.usage_recency_half_life_days).abs()
425 + f64::from((candidate.max_vector_distance - default.max_vector_distance).abs())
426 + (candidate.rrf_k - default.rrf_k).abs()
427 + (candidate.min_evidence_confidence - default.min_evidence_confidence).abs()
428}
429
430impl Display for WeightGridReport {
431 fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult {
432 writeln!(
433 f,
434 "remem weight grid — {} candidates, k={}, default_rank={}, recommendation={:?}",
435 self.candidates.len(),
436 self.k,
437 self.default_rank,
438 self.recommendation
439 )?;
440 writeln!(f, "dataset: {}", self.dataset_path)?;
441 writeln!(
442 f,
443 "default score={:.4}, best score={:.4}",
444 self.default_score, self.best.score
445 )?;
446 writeln!(f)?;
447 writeln!(f, "--- Top Candidates ---")?;
448 for candidate in self.candidates.iter().take(10) {
449 writeln!(
450 f,
451 " #{:02} score={:.4} dist={:.2} fts={:.2} vector={:.2} entity={:.2} temporal={:.2} fact={:.2} like={:.2} usage={:.2} confidence={:.2}",
452 candidate.rank,
453 candidate.score,
454 candidate.distance_from_defaults,
455 candidate.weights.fts,
456 candidate.weights.vector,
457 candidate.weights.entity,
458 candidate.weights.temporal,
459 candidate.weights.fact,
460 candidate.weights.like_fallback,
461 candidate.weights.usage,
462 candidate.weights.min_evidence_confidence
463 )?;
464 }
465 Ok(())
466 }
467}
468
469#[cfg(test)]
470mod tests {
471 use super::*;
472 use crate::eval::golden::{EvidenceRef, GoldenMemory, GoldenQuery};
473
474 #[test]
475 fn empty_candidate_evaluation_rejects_non_finite_weights() -> Result<()> {
476 let conn = Connection::open_in_memory()?;
477 let dataset = GoldenDataset {
478 version: None,
479 description: None,
480 corpus: vec![],
481 queries: vec![],
482 };
483 let error = evaluate_candidate(
484 &conn,
485 &dataset,
486 5,
487 SearchWeights {
488 fact: f64::NAN,
489 ..SearchWeights::default()
490 },
491 &WeightGridScoring::default(),
492 )
493 .expect_err("empty evaluations must still reject non-finite weights");
494 assert!(error.to_string().contains("fact"), "{error:#}");
495 Ok(())
496 }
497
498 #[test]
499 fn grid_report_includes_defaults_and_ranks_candidates() -> Result<()> {
500 let default = SearchWeights::default();
501 let dataset = GoldenDataset {
502 version: Some("test".to_string()),
503 description: None,
504 corpus: vec![GoldenMemory {
505 project: "/repo-a".to_string(),
506 topic_key: Some("sqlcipher-store".to_string()),
507 title: "SQLCipher store".to_string(),
508 content: "SQLCipher encrypts the local memory database at rest.".to_string(),
509 memory_type: "architecture".to_string(),
510 branch: Some("main".to_string()),
511 scope: "project".to_string(),
512 status: "active".to_string(),
513 files: None,
514 created_at_epoch: Some(1),
515 access_count: None,
516 last_accessed_epoch: None,
517 search_context: None,
518 }],
519 queries: vec![GoldenQuery {
520 id: "q1".to_string(),
521 query: "local database encryption".to_string(),
522 category: "retrieval".to_string(),
523 slice: Some("paraphrase".to_string()),
524 hop_path: None,
525 project: Some("/repo-a".to_string()),
526 branch: Some("main".to_string()),
527 memory_type: None,
528 relevant_ids: vec![],
529 evidence_refs: vec![EvidenceRef {
530 topic_key: Some("sqlcipher-store".to_string()),
531 text_contains: Some("encrypts the local memory database".to_string()),
532 ..EvidenceRef::default()
533 }],
534 expect_abstain: false,
535 false_premise: false,
536 notes: None,
537 }],
538 };
539 let report = run_weight_grid_dataset(
540 dataset,
541 "test-golden.json".to_string(),
542 5,
543 vec![
544 default,
545 SearchWeights {
546 vector: default.vector + 0.5,
547 ..default
548 },
549 ],
550 )?;
551
552 assert_eq!(report.candidates.len(), 2);
553 assert!(report.checks.default_weights_in_grid);
554 assert!(report.default_rank >= 1);
555 assert!(report
556 .candidates
557 .iter()
558 .any(|candidate| candidate.weights == default));
559 assert!(!report.usage_shadow.default_usage_weight_zero);
560 assert_eq!(report.usage_shadow.baseline_usage_weight, 0.0);
561 assert!(report
562 .usage_shadow
563 .comparisons
564 .iter()
565 .all(|comparison| comparison.usage_weight > 0.0));
566 assert_eq!(report.candidates[0].rank, 1);
567 Ok(())
568 }
569}