lens_core/baseline/
mod.rs

1//! # Baseline Competitors Module
2//!
3//! Implements BM25+proximity and hybrid lexical+dense baseline competitors
4//! as specified in TODO.md Step 3 - Baseline fortification.
5//!
6//! Competitors:
7//! - BM25+proximity (lexical with positional awareness)
8//! - Hybrid lexical+dense (traditional hybrid search)
9//! - Tuned variants optimized for code search
10//!
11//! Gate: retain ≥ +3 pp margin across SWE-bench Verified and CoIR, under SLA
12
13pub mod bm25_proximity;
14pub mod hybrid_lexical_dense;
15pub mod baseline_orchestrator;
16pub mod competitive_benchmarking;
17
18pub use bm25_proximity::{BM25ProximitySearcher, ProximityConfig};
19pub use hybrid_lexical_dense::{HybridSearcher, HybridConfig};
20pub use baseline_orchestrator::{BaselineOrchestrator, BaselineConfig};
21pub use competitive_benchmarking::{CompetitiveBenchmark, BenchmarkResult};
22
23use anyhow::Result;
24use serde::{Deserialize, Serialize};
25use std::collections::HashMap;
26
27/// Baseline search system trait
28#[async_trait::async_trait]
29pub trait BaselineSearcher: Send + Sync {
30    /// Get system name for identification
31    fn system_name(&self) -> &str;
32    
33    /// Search with the baseline system
34    async fn search(&self, query: &str, intent: &str, language: &str, max_results: usize) -> Result<Vec<SearchResult>>;
35    
36    /// Get system configuration for reproducibility
37    fn get_config(&self) -> BaselineSystemConfig;
38    
39    /// Warm up the system (optional)
40    async fn warmup(&self) -> Result<()> {
41        Ok(())
42    }
43    
44    /// Get system statistics
45    async fn get_statistics(&self) -> Result<SystemStatistics>;
46}
47
48/// Search result from baseline system
49#[derive(Debug, Clone, Serialize, Deserialize)]
50pub struct SearchResult {
51    pub file_path: String,
52    pub score: f32,
53    pub snippet: String,
54    pub rank: usize,
55    pub metadata: ResultMetadata,
56}
57
58#[derive(Debug, Clone, Serialize, Deserialize)]
59pub struct ResultMetadata {
60    pub line_number: Option<usize>,
61    pub function_name: Option<String>,
62    pub class_name: Option<String>,
63    pub language: String,
64    pub file_size: usize,
65    pub last_modified: Option<chrono::DateTime<chrono::Utc>>,
66    pub scoring_breakdown: ScoringBreakdown,
67}
68
69#[derive(Debug, Clone, Serialize, Deserialize)]
70pub struct ScoringBreakdown {
71    pub lexical_score: f32,
72    pub semantic_score: Option<f32>,
73    pub proximity_score: Option<f32>,
74    pub recency_score: Option<f32>,
75    pub popularity_score: Option<f32>,
76    pub final_score: f32,
77}
78
79/// Baseline system configuration
80#[derive(Debug, Clone, Serialize, Deserialize)]
81pub struct BaselineSystemConfig {
82    pub system_name: String,
83    pub version: String,
84    pub parameters: HashMap<String, serde_json::Value>,
85    pub index_config: IndexConfig,
86    pub scoring_config: ScoringConfig,
87}
88
89#[derive(Debug, Clone, Serialize, Deserialize)]
90pub struct IndexConfig {
91    pub tokenizer: String,
92    pub stemming: bool,
93    pub stop_words: bool,
94    pub n_grams: Vec<usize>,
95    pub case_sensitive: bool,
96    pub special_characters: bool,
97}
98
99#[derive(Debug, Clone, Serialize, Deserialize)]
100pub struct ScoringConfig {
101    pub bm25_k1: f32,
102    pub bm25_b: f32,
103    pub proximity_weight: Option<f32>,
104    pub semantic_weight: Option<f32>,
105    pub recency_weight: Option<f32>,
106    pub normalization: String,
107}
108
109/// System performance statistics
110#[derive(Debug, Clone, Serialize, Deserialize)]
111pub struct SystemStatistics {
112    pub queries_processed: usize,
113    pub average_latency_ms: f32,
114    pub p95_latency_ms: f32,
115    pub p99_latency_ms: f32,
116    pub cache_hit_rate: f32,
117    pub index_size_mb: f32,
118    pub memory_usage_mb: f32,
119}
120
121/// Performance comparison result
122#[derive(Debug, Clone, Serialize, Deserialize)]
123pub struct PerformanceComparison {
124    pub baseline_system: String,
125    pub lens_system: String,
126    pub metrics: HashMap<String, MetricComparison>,
127    pub statistical_significance: HashMap<String, f32>,
128    pub margin_analysis: MarginAnalysis,
129    pub sla_compliance: SlaCompliance,
130}
131
132#[derive(Debug, Clone, Serialize, Deserialize)]
133pub struct MetricComparison {
134    pub baseline_value: f32,
135    pub lens_value: f32,
136    pub improvement_pp: f32,
137    pub improvement_percentage: f32,
138    pub confidence_interval: (f32, f32),
139    pub statistical_significance: f32,
140}
141
142#[derive(Debug, Clone, Serialize, Deserialize)]
143pub struct MarginAnalysis {
144    pub required_margin_pp: f32,
145    pub achieved_margin_pp: f32,
146    pub margin_maintained: bool,
147    pub risk_factors: Vec<String>,
148}
149
150#[derive(Debug, Clone, Serialize, Deserialize)]
151pub struct SlaCompliance {
152    pub sla_threshold_ms: f32,
153    pub baseline_p99_ms: f32,
154    pub lens_p99_ms: f32,
155    pub both_systems_compliant: bool,
156    pub comparative_advantage: f32,
157}
158
159/// Create all baseline competitors
160pub async fn create_baseline_competitors() -> Result<Vec<Box<dyn BaselineSearcher>>> {
161    let mut competitors = Vec::new();
162    
163    // BM25 + Proximity competitor
164    let bm25_proximity = bm25_proximity::BM25ProximitySearcher::new(
165        bm25_proximity::ProximityConfig::optimized_for_code()
166    ).await?;
167    competitors.push(Box::new(bm25_proximity) as Box<dyn BaselineSearcher>);
168    
169    // Hybrid Lexical + Dense competitor
170    let hybrid = hybrid_lexical_dense::HybridSearcher::new(
171        hybrid_lexical_dense::HybridConfig::balanced_hybrid()
172    ).await?;
173    competitors.push(Box::new(hybrid) as Box<dyn BaselineSearcher>);
174    
175    // Additional tuned variants
176    let bm25_tuned = bm25_proximity::BM25ProximitySearcher::new(
177        bm25_proximity::ProximityConfig::high_precision()
178    ).await?;
179    competitors.push(Box::new(bm25_tuned) as Box<dyn BaselineSearcher>);
180    
181    let hybrid_tuned = hybrid_lexical_dense::HybridSearcher::new(
182        hybrid_lexical_dense::HybridConfig::code_optimized()
183    ).await?;
184    competitors.push(Box::new(hybrid_tuned) as Box<dyn BaselineSearcher>);
185    
186    Ok(competitors)
187}
188
189/// Validate minimum performance margin
190pub fn validate_performance_margin(comparison: &PerformanceComparison) -> Result<bool> {
191    const MINIMUM_MARGIN_PP: f32 = 3.0; // ≥ +3 pp margin requirement
192    
193    let key_metrics = vec!["ndcg_at_10", "recall_at_50"];
194    
195    for metric_name in &key_metrics {
196        if let Some(metric) = comparison.metrics.get(*metric_name) {
197            if metric.improvement_pp < MINIMUM_MARGIN_PP {
198                tracing::warn!(
199                    "Insufficient margin for {}: {:.1}pp < {:.1}pp required",
200                    metric_name, metric.improvement_pp, MINIMUM_MARGIN_PP
201                );
202                return Ok(false);
203            }
204        }
205    }
206    
207    // Check SLA compliance for both systems
208    if !comparison.sla_compliance.both_systems_compliant {
209        tracing::warn!("SLA compliance failed for comparison");
210        return Ok(false);
211    }
212    
213    // Verify statistical significance
214    for (metric_name, &p_value) in &comparison.statistical_significance {
215        if key_metrics.contains(&metric_name.as_str()) && p_value > 0.05 {
216            tracing::warn!(
217                "No statistical significance for {}: p={:.3} > 0.05",
218                metric_name, p_value
219            );
220            return Ok(false);
221        }
222    }
223    
224    Ok(true)
225}
226
227#[cfg(test)]
228mod tests {
229    use super::*;
230
231    #[test]
232    fn test_performance_comparison_validation() {
233        let mut metrics = HashMap::new();
234        metrics.insert("ndcg_at_10".to_string(), MetricComparison {
235            baseline_value: 0.65,
236            lens_value: 0.68,
237            improvement_pp: 3.5, // Above 3pp requirement
238            improvement_percentage: 5.4,
239            confidence_interval: (3.1, 3.9),
240            statistical_significance: 0.001,
241        });
242        
243        let comparison = PerformanceComparison {
244            baseline_system: "BM25+Proximity".to_string(),
245            lens_system: "Lens+Semantic".to_string(),
246            metrics,
247            statistical_significance: [("ndcg_at_10".to_string(), 0.001)].into(),
248            margin_analysis: MarginAnalysis {
249                required_margin_pp: 3.0,
250                achieved_margin_pp: 3.5,
251                margin_maintained: true,
252                risk_factors: vec![],
253            },
254            sla_compliance: SlaCompliance {
255                sla_threshold_ms: 150.0,
256                baseline_p99_ms: 145.0,
257                lens_p99_ms: 147.0,
258                both_systems_compliant: true,
259                comparative_advantage: 2.0,
260            },
261        };
262        
263        assert!(validate_performance_margin(&comparison).unwrap());
264    }
265
266    #[test]
267    fn test_insufficient_margin_detection() {
268        let mut metrics = HashMap::new();
269        metrics.insert("ndcg_at_10".to_string(), MetricComparison {
270            baseline_value: 0.65,
271            lens_value: 0.67,
272            improvement_pp: 2.0, // Below 3pp requirement
273            improvement_percentage: 3.1,
274            confidence_interval: (1.5, 2.5),
275            statistical_significance: 0.001,
276        });
277        
278        let comparison = PerformanceComparison {
279            baseline_system: "BM25+Proximity".to_string(),
280            lens_system: "Lens+Semantic".to_string(),
281            metrics,
282            statistical_significance: [("ndcg_at_10".to_string(), 0.001)].into(),
283            margin_analysis: MarginAnalysis {
284                required_margin_pp: 3.0,
285                achieved_margin_pp: 2.0,
286                margin_maintained: false,
287                risk_factors: vec!["Narrow margin".to_string()],
288            },
289            sla_compliance: SlaCompliance {
290                sla_threshold_ms: 150.0,
291                baseline_p99_ms: 145.0,
292                lens_p99_ms: 147.0,
293                both_systems_compliant: true,
294                comparative_advantage: 2.0,
295            },
296        };
297        
298        assert!(!validate_performance_margin(&comparison).unwrap());
299    }
300}
lens_core/baseline/mod.rs

lens_core/baseline/
mod.rs
