Skip to main content

fetchium_core/rank/
quality.rs

1//! Response Quality Estimator (RQE) — confidence scoring for result sets.
2//!
3//! Novel algorithm: After ranking, estimate the overall quality/confidence of
4//! the result set before presenting to the user. Uses 5 signals:
5//!
6//! 1. **Source Diversity** — How many unique backends contributed? (0-1)
7//! 2. **Score Distribution** — Are top scores well-separated from noise? (0-1)
8//! 3. **Result Agreement** — Do top results cover the same topic? (0-1)
9//! 4. **Coverage Completeness** — Are all query aspects addressed? (0-1)
10//! 5. **Freshness Signal** — Do results have recent publication dates? (0-1)
11//!
12//! Outputs an overall quality score [0.0, 1.0] and optional warnings.
13
14use crate::types::ResultItem;
15use std::collections::HashSet;
16
17/// Quality assessment of a result set.
18#[derive(Debug, Clone)]
19pub struct QualityAssessment {
20    /// Overall quality score [0.0, 1.0].
21    pub score: f64,
22    /// Confidence level (derived from score).
23    pub confidence: ConfidenceLevel,
24    /// Individual signal scores.
25    pub signals: QualitySignals,
26    /// Warnings for the user (empty = no issues).
27    pub warnings: Vec<String>,
28}
29
30/// Signal breakdown.
31#[derive(Debug, Clone)]
32pub struct QualitySignals {
33    /// How many unique backends contributed results.
34    pub source_diversity: f64,
35    /// How well-separated the top scores are from noise.
36    pub score_separation: f64,
37    /// How much the top results agree on topic.
38    pub result_agreement: f64,
39    /// How well the results cover all query terms.
40    pub coverage: f64,
41    /// Freshness of results (recent dates).
42    pub freshness: f64,
43}
44
45/// Confidence level — maps from the numeric score.
46#[derive(Debug, Clone, Copy, PartialEq, Eq)]
47pub enum ConfidenceLevel {
48    /// Score >= 0.8 — results are comprehensive and reliable.
49    High,
50    /// Score >= 0.5 — results are decent but may have gaps.
51    Medium,
52    /// Score >= 0.3 — results are sparse; user should verify.
53    Low,
54    /// Score < 0.3 — very few relevant results found.
55    VeryLow,
56}
57
58impl ConfidenceLevel {
59    fn from_score(score: f64) -> Self {
60        if score >= 0.8 {
61            Self::High
62        } else if score >= 0.5 {
63            Self::Medium
64        } else if score >= 0.3 {
65            Self::Low
66        } else {
67            Self::VeryLow
68        }
69    }
70
71    /// Human-readable label.
72    pub fn label(&self) -> &'static str {
73        match self {
74            Self::High => "high",
75            Self::Medium => "medium",
76            Self::Low => "low",
77            Self::VeryLow => "very low",
78        }
79    }
80}
81
82impl std::fmt::Display for ConfidenceLevel {
83    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
84        write!(f, "{}", self.label())
85    }
86}
87
88/// Assess the quality of a ranked result set.
89///
90/// Should be called after ranking but before output formatting.
91pub fn assess_quality(results: &[ResultItem], query: &str) -> QualityAssessment {
92    if results.is_empty() {
93        return QualityAssessment {
94            score: 0.0,
95            confidence: ConfidenceLevel::VeryLow,
96            signals: QualitySignals {
97                source_diversity: 0.0,
98                score_separation: 0.0,
99                result_agreement: 0.0,
100                coverage: 0.0,
101                freshness: 0.0,
102            },
103            warnings: vec!["No results found.".into()],
104        };
105    }
106
107    let diversity = source_diversity_score(results);
108    let separation = score_separation(results);
109    let agreement = result_agreement(results);
110    let coverage = coverage_score(results, query);
111    let freshness = freshness_score(results);
112
113    // Weighted combination
114    let score = diversity * 0.20
115        + separation * 0.25
116        + agreement * 0.20
117        + coverage * 0.25
118        + freshness * 0.10;
119
120    let score = score.clamp(0.0, 1.0);
121    let confidence = ConfidenceLevel::from_score(score);
122
123    let mut warnings = Vec::new();
124
125    if diversity < 0.3 {
126        warnings
127            .push("Results come from very few sources. Consider broadening your search.".into());
128    }
129    if separation < 0.2 {
130        warnings
131            .push("Top results have similar relevance scores — ranking may be unreliable.".into());
132    }
133    if coverage < 0.3 {
134        warnings.push("Some query terms are not well-covered in results.".into());
135    }
136    if results.len() < 3 {
137        warnings.push(format!("Only {} result(s) found.", results.len()));
138    }
139
140    QualityAssessment {
141        score,
142        confidence,
143        signals: QualitySignals {
144            source_diversity: diversity,
145            score_separation: separation,
146            result_agreement: agreement,
147            coverage,
148            freshness,
149        },
150        warnings,
151    }
152}
153
154/// Source diversity: what fraction of available backend types contributed?
155fn source_diversity_score(results: &[ResultItem]) -> f64 {
156    let unique_backends: HashSet<String> = results.iter().map(|r| r.backend.to_string()).collect();
157
158    // Scale: 1 backend = 0.2, 2 = 0.4, 3 = 0.6, 4 = 0.8, 5+ = 1.0
159    let n = unique_backends.len();
160    (n as f64 / 5.0).min(1.0)
161}
162
163/// Score separation: how well-separated are the top results from the rest?
164///
165/// Uses the gap between the best score and the median score.
166/// A large gap means the ranker is confident about the top results.
167fn score_separation(results: &[ResultItem]) -> f64 {
168    let mut scores: Vec<f64> = results.iter().filter_map(|r| r.score).collect();
169
170    if scores.len() < 2 {
171        return 0.5; // neutral if we can't compute separation
172    }
173
174    scores.sort_by(|a, b| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal));
175
176    let top = scores[0];
177    let median_idx = scores.len() / 2;
178    let median = scores[median_idx];
179
180    // Gap between top and median, normalized
181    let gap = top - median;
182
183    // Also check gap between #1 and #2
184    let top2_gap = if scores.len() >= 2 {
185        top - scores[1]
186    } else {
187        0.0
188    };
189
190    // Combined: larger gaps = more confidence in ranking
191    (gap * 2.0 + top2_gap).min(1.0)
192}
193
194/// Result agreement: do the top results talk about the same topic?
195///
196/// Uses term overlap between the top-3 snippets. High overlap = agreement.
197fn result_agreement(results: &[ResultItem]) -> f64 {
198    let top_n = results.iter().take(5).collect::<Vec<_>>();
199
200    if top_n.len() < 2 {
201        return 1.0; // single result trivially agrees
202    }
203
204    // Extract term sets from each result
205    let term_sets: Vec<HashSet<String>> = top_n
206        .iter()
207        .map(|r| {
208            let text = format!("{} {}", r.title, r.snippet).to_lowercase();
209            text.split(|c: char| !c.is_alphanumeric())
210                .filter(|t| t.len() >= 3)
211                .map(|t| t.to_string())
212                .collect()
213        })
214        .collect();
215
216    // Average pairwise Jaccard similarity
217    let mut total_sim = 0.0;
218    let mut pairs = 0;
219
220    for i in 0..term_sets.len() {
221        for j in (i + 1)..term_sets.len() {
222            let intersection = term_sets[i].intersection(&term_sets[j]).count();
223            let union = term_sets[i].union(&term_sets[j]).count();
224            if union > 0 {
225                total_sim += intersection as f64 / union as f64;
226            }
227            pairs += 1;
228        }
229    }
230
231    if pairs == 0 {
232        return 0.5;
233    }
234
235    // Scale: Jaccard of 0.1 = 0.5, 0.3 = 1.0
236    let avg_jaccard = total_sim / pairs as f64;
237    (avg_jaccard * 3.33).min(1.0)
238}
239
240/// Coverage: are all query terms represented in the results?
241fn coverage_score(results: &[ResultItem], query: &str) -> f64 {
242    let query_terms: Vec<String> = query
243        .to_lowercase()
244        .split(|c: char| !c.is_alphanumeric())
245        .filter(|t| t.len() >= 2)
246        .map(|t| t.to_string())
247        .collect();
248
249    if query_terms.is_empty() {
250        return 1.0;
251    }
252
253    // Build a combined text from all results
254    let combined: String = results
255        .iter()
256        .map(|r| format!("{} {}", r.title, r.snippet))
257        .collect::<Vec<_>>()
258        .join(" ")
259        .to_lowercase();
260
261    // Check what fraction of query terms appear in the results
262    let covered = query_terms
263        .iter()
264        .filter(|term| combined.contains(term.as_str()))
265        .count();
266
267    covered as f64 / query_terms.len() as f64
268}
269
270/// Freshness: what fraction of results have recent publication dates?
271fn freshness_score(results: &[ResultItem]) -> f64 {
272    let total = results.len();
273    if total == 0 {
274        return 0.5;
275    }
276
277    // Count results with any date information
278    let with_date = results
279        .iter()
280        .filter(|r| r.published_date.is_some())
281        .count();
282
283    if with_date == 0 {
284        return 0.5; // neutral if no date info
285    }
286
287    // Having dates is a positive signal (backends that provide dates
288    // tend to be more structured/reliable)
289    let date_ratio = with_date as f64 / total as f64;
290
291    // Scale: 0% with dates = 0.3, 50% = 0.6, 100% = 1.0
292    0.3 + date_ratio * 0.7
293}
294
295/// Format quality assessment as a one-line summary for CLI display.
296pub fn quality_line(assessment: &QualityAssessment) -> String {
297    format!(
298        "Quality: {:.0}% ({}) — diversity={:.0}% coverage={:.0}% agreement={:.0}%",
299        assessment.score * 100.0,
300        assessment.confidence.label(),
301        assessment.signals.source_diversity * 100.0,
302        assessment.signals.coverage * 100.0,
303        assessment.signals.result_agreement * 100.0,
304    )
305}
306
307#[cfg(test)]
308mod tests {
309    use super::*;
310    use crate::types::BackendId;
311
312    fn make_item(title: &str, snippet: &str, backend: BackendId, score: f64) -> ResultItem {
313        ResultItem {
314            title: title.into(),
315            url: format!("https://{}.com", title.to_lowercase().replace(' ', "-")),
316            snippet: snippet.into(),
317            rank: 0,
318            backend,
319            score: Some(score),
320            published_date: None,
321        }
322    }
323
324    #[test]
325    fn empty_results_very_low_confidence() {
326        let assessment = assess_quality(&[], "test query");
327        assert_eq!(assessment.confidence, ConfidenceLevel::VeryLow);
328        assert_eq!(assessment.score, 0.0);
329        assert!(!assessment.warnings.is_empty());
330    }
331
332    #[test]
333    fn diverse_sources_higher_quality() {
334        let results = vec![
335            make_item(
336                "Rust Docs",
337                "Rust programming guide",
338                BackendId::DuckDuckGo,
339                0.9,
340            ),
341            make_item(
342                "Rust Wikipedia",
343                "Rust is a language",
344                BackendId::Wikipedia,
345                0.8,
346            ),
347            make_item(
348                "Rust Reddit",
349                "Rust discussion forum",
350                BackendId::Reddit,
351                0.7,
352            ),
353            make_item("Rust HN", "Rust on Hacker News", BackendId::HackerNews, 0.6),
354            make_item("Rust GitHub", "Rust repositories", BackendId::Github, 0.5),
355        ];
356
357        let diverse = assess_quality(&results, "rust programming");
358
359        let single_source = vec![
360            make_item("A", "rust stuff", BackendId::DuckDuckGo, 0.9),
361            make_item("B", "more rust", BackendId::DuckDuckGo, 0.8),
362            make_item("C", "even more rust", BackendId::DuckDuckGo, 0.7),
363        ];
364        let uniform = assess_quality(&single_source, "rust programming");
365
366        assert!(
367            diverse.signals.source_diversity > uniform.signals.source_diversity,
368            "Diverse sources should score higher: diverse={} uniform={}",
369            diverse.signals.source_diversity,
370            uniform.signals.source_diversity
371        );
372    }
373
374    #[test]
375    fn good_coverage_higher_score() {
376        let results = vec![
377            make_item(
378                "Rust Async",
379                "Rust async programming with tokio and futures",
380                BackendId::DuckDuckGo,
381                0.9,
382            ),
383            make_item(
384                "Async Patterns",
385                "Patterns for async rust code",
386                BackendId::Wikipedia,
387                0.8,
388            ),
389        ];
390
391        let assessment = assess_quality(&results, "rust async programming");
392        assert!(
393            assessment.signals.coverage > 0.5,
394            "All query terms should be covered: {}",
395            assessment.signals.coverage
396        );
397    }
398
399    #[test]
400    fn poor_coverage_generates_warning() {
401        let results = vec![make_item(
402            "Dogs",
403            "Cute dogs and puppies",
404            BackendId::DuckDuckGo,
405            0.3,
406        )];
407
408        let assessment = assess_quality(&results, "quantum computing research");
409        assert!(
410            assessment.signals.coverage < 0.5,
411            "Irrelevant results should have low coverage"
412        );
413    }
414
415    #[test]
416    fn confidence_levels_from_score() {
417        assert_eq!(ConfidenceLevel::from_score(0.9), ConfidenceLevel::High);
418        assert_eq!(ConfidenceLevel::from_score(0.6), ConfidenceLevel::Medium);
419        assert_eq!(ConfidenceLevel::from_score(0.35), ConfidenceLevel::Low);
420        assert_eq!(ConfidenceLevel::from_score(0.1), ConfidenceLevel::VeryLow);
421    }
422
423    #[test]
424    fn quality_line_formatting() {
425        let results = vec![make_item(
426            "Test",
427            "test content",
428            BackendId::DuckDuckGo,
429            0.8,
430        )];
431        let assessment = assess_quality(&results, "test");
432        let line = quality_line(&assessment);
433        assert!(line.contains("Quality:"));
434        assert!(line.contains("%"));
435    }
436
437    #[test]
438    fn single_result_not_panic() {
439        let results = vec![make_item(
440            "Only Result",
441            "some content",
442            BackendId::Wikipedia,
443            0.5,
444        )];
445        let assessment = assess_quality(&results, "query");
446        assert!(assessment.score > 0.0);
447    }
448
449    #[test]
450    fn freshness_with_dates() {
451        let mut results = vec![
452            make_item("A", "content", BackendId::DuckDuckGo, 0.8),
453            make_item("B", "content", BackendId::Wikipedia, 0.7),
454        ];
455        results[0].published_date = Some("2024-01-15".into());
456        results[1].published_date = Some("2024-02-20".into());
457
458        let with_dates = assess_quality(&results, "test");
459
460        let no_dates = vec![
461            make_item("C", "content", BackendId::DuckDuckGo, 0.8),
462            make_item("D", "content", BackendId::Wikipedia, 0.7),
463        ];
464        let without_dates = assess_quality(&no_dates, "test");
465
466        assert!(
467            with_dates.signals.freshness > without_dates.signals.freshness,
468            "Results with dates should score higher freshness"
469        );
470    }
471}