fetchium-core 1.0.0

Fetchium core library — search, extract, rank, validate, cache
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
//! Adaptive Backend Selector (ABS) — smart per-query backend routing.
//!
//! Novel algorithm: Instead of querying all backends for every query, ABS
//! dynamically selects the most promising backends based on:
//!
//! 1. **Query intent** — Academic queries prefer arXiv + Scholar; code queries
//!    prefer GitHub + StackOverflow; news queries prefer HackerNews + Reddit.
//! 2. **Historical success rates** — Backends that have returned good results
//!    for similar queries in the past are preferred.
//! 3. **Current health** — Backends with open circuit breakers are skipped.
//!
//! Uses a **UCB1 multi-armed bandit** to balance exploration (trying backends
//! that haven't been used recently) vs exploitation (using backends known to work).

use crate::rank::fusion::QueryIntent;
use crate::types::BackendId;
use dashmap::DashMap;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;

/// Configuration for the Adaptive Backend Selector.
#[derive(Debug, Clone)]
pub struct AbsConfig {
    /// Minimum number of backends to always include.
    pub min_backends: usize,
    /// Maximum number of backends to select.
    pub max_backends: usize,
    /// UCB1 exploration parameter (higher = more exploration).
    pub exploration_factor: f64,
    /// Whether to always include at least one "reliable" backend.
    pub always_include_reliable: bool,
}

impl Default for AbsConfig {
    fn default() -> Self {
        Self {
            min_backends: 3,
            max_backends: 8,
            exploration_factor: 1.41, // sqrt(2), standard UCB1
            always_include_reliable: true,
        }
    }
}

/// Per-backend performance statistics.
#[derive(Debug)]
struct BackendStats {
    /// Total number of times this backend was selected.
    selections: AtomicU64,
    /// Total number of successful queries (returned >= 1 result).
    successes: AtomicU64,
    /// Total reward (sum of quality scores from results).
    total_reward: AtomicU64, // stored as reward * 1000 to avoid floating point atomics
}

impl BackendStats {
    fn new() -> Self {
        Self {
            selections: AtomicU64::new(0),
            successes: AtomicU64::new(0),
            total_reward: AtomicU64::new(0),
        }
    }

    fn success_rate(&self) -> f64 {
        let sel = self.selections.load(Ordering::Relaxed);
        if sel == 0 {
            return 0.5; // optimistic prior
        }
        self.successes.load(Ordering::Relaxed) as f64 / sel as f64
    }

    fn avg_reward(&self) -> f64 {
        let sel = self.selections.load(Ordering::Relaxed);
        if sel == 0 {
            return 0.5; // optimistic prior
        }
        (self.total_reward.load(Ordering::Relaxed) as f64 / 1000.0) / sel as f64
    }
}

/// Intent-to-backend affinity matrix.
///
/// Maps query intents to their preferred backends with affinity scores.
fn intent_affinity(intent: &QueryIntent, backend: &BackendId) -> f64 {
    // SearXNG is a META-SEARCH engine (aggregates Google, Bing, Brave, DDG, Wikipedia,
    // StackOverflow, GitHub, ArXiv). It should ALWAYS be selected because it provides
    // the broadest coverage of any single backend.
    if *backend == BackendId::Searxng {
        return 0.85; // High for all intents — it covers everything
    }

    // Premium backends also get consistently high scores — they provide
    // pre-scored, AI-optimized results when API keys are available.
    match backend {
        BackendId::Tavily | BackendId::Serper | BackendId::Exa | BackendId::Firecrawl => {
            return 0.80; // Premium backends always valuable
        }
        _ => {}
    }

    match intent {
        QueryIntent::Academic | QueryIntent::Data => match backend {
            BackendId::Arxiv => 0.95,
            BackendId::GoogleScholar => 0.90,
            BackendId::Wikipedia => 0.70,
            BackendId::StackOverflow => 0.60,
            BackendId::Github => 0.50,
            _ => 0.30,
        },
        QueryIntent::Code => match backend {
            BackendId::Github => 0.95,
            BackendId::StackOverflow => 0.90,
            BackendId::HackerNews => 0.50,
            BackendId::Reddit => 0.40,
            _ => 0.30,
        },
        QueryIntent::HowTo => match backend {
            BackendId::StackOverflow => 0.95,
            BackendId::Github => 0.80,
            BackendId::DuckDuckGo => 0.80,
            BackendId::Google => 0.75,
            BackendId::Brave => 0.70,
            BackendId::Bing => 0.65,
            BackendId::Wikipedia => 0.15,
            BackendId::Arxiv => 0.10,
            BackendId::Reddit => 0.35,
            _ => 0.40,
        },
        QueryIntent::CurrentEvents => match backend {
            BackendId::HackerNews => 0.90,
            BackendId::Reddit => 0.85,
            BackendId::Google => 0.75,
            BackendId::Bing => 0.65,
            BackendId::DuckDuckGo => 0.60,
            BackendId::Brave => 0.55,
            BackendId::StackOverflow => 0.30,
            BackendId::Github => 0.15,
            BackendId::Wikipedia => 0.10, // Wikipedia almost never useful for temporal queries
            BackendId::Arxiv => 0.10,     // ArXiv returns old papers, not news
            _ => 0.30,
        },
        QueryIntent::Comparison => match backend {
            BackendId::DuckDuckGo => 0.85,
            BackendId::Google => 0.80,
            BackendId::Brave => 0.75,
            BackendId::Bing => 0.70,
            BackendId::StackOverflow => 0.68,
            BackendId::Wikipedia => 0.60,
            BackendId::Reddit => 0.20,
            BackendId::HackerNews => 0.20,
            BackendId::Arxiv => 0.05,
            _ => 0.40,
        },
        QueryIntent::Factual => match backend {
            BackendId::Wikipedia => 0.95,
            BackendId::DuckDuckGo => 0.70,
            BackendId::Google => 0.65,
            BackendId::Brave => 0.60,
            BackendId::Github => 0.20,
            _ => 0.40,
        },
        QueryIntent::Informational => match backend {
            BackendId::Wikipedia => 0.95,
            BackendId::DuckDuckGo => 0.80,
            BackendId::Google => 0.75,
            BackendId::Brave => 0.70,
            BackendId::Bing => 0.65,
            BackendId::StackOverflow => 0.55,
            BackendId::Reddit => 0.45,
            BackendId::HackerNews => 0.40,
            BackendId::Arxiv => 0.35,
            BackendId::Github => 0.10,
            _ => 0.40,
        },
        QueryIntent::Verification => match backend {
            BackendId::Wikipedia => 0.90,
            BackendId::Arxiv => 0.85,
            BackendId::GoogleScholar => 0.80,
            BackendId::DuckDuckGo => 0.60,
            _ => 0.35,
        },
        QueryIntent::Opinion => match backend {
            BackendId::Reddit => 0.90,
            BackendId::HackerNews => 0.85,
            BackendId::StackOverflow => 0.80,
            BackendId::DuckDuckGo => 0.70,
            BackendId::Google => 0.65,
            BackendId::Brave => 0.60,
            BackendId::Wikipedia => 0.40,
            BackendId::Arxiv => 0.15,
            _ => 0.50,
        },
        QueryIntent::Casual => match backend {
            BackendId::DuckDuckGo => 0.85,
            BackendId::Google => 0.85,
            BackendId::Brave => 0.80,
            BackendId::Bing => 0.75,
            BackendId::Reddit => 0.70,
            BackendId::Wikipedia => 0.30,
            BackendId::HackerNews => 0.20,
            BackendId::StackOverflow => 0.10,
            BackendId::Github => 0.05,
            BackendId::Arxiv => 0.05,
            BackendId::GoogleScholar => 0.05,
            _ => 0.40,
        },
        _ => match backend {
            BackendId::DuckDuckGo => 0.70,
            BackendId::Google => 0.70,
            BackendId::StackOverflow => 0.65,
            BackendId::Bing => 0.60,
            BackendId::Brave => 0.60,
            BackendId::Wikipedia => 0.55,
            _ => 0.50,
        },
    }
}

/// Hard precision guardrails: exclude backends that are consistently off-intent.
///
/// This runs before ranking so noisy sources are never selected for clearly
/// mismatched intents (for example StackOverflow for medical or policy queries).
fn intent_backend_allowed(intent: &QueryIntent, backend: &BackendId) -> bool {
    match intent {
        QueryIntent::CurrentEvents => !matches!(
            backend,
            BackendId::StackOverflow | BackendId::Github | BackendId::Arxiv
        ),
        QueryIntent::Factual | QueryIntent::Informational => {
            // ArXiv returns papers for ANY search term — flooding non-academic queries
            // with irrelevant results. Only allow ArXiv for Academic/Verification intents.
            !matches!(
                backend,
                BackendId::StackOverflow | BackendId::Github | BackendId::Arxiv
            )
        }
        QueryIntent::Verification => {
            !matches!(backend, BackendId::StackOverflow | BackendId::Github)
        }
        QueryIntent::HowTo => !matches!(backend, BackendId::Arxiv | BackendId::GoogleScholar),
        QueryIntent::Comparison => !matches!(
            backend,
            BackendId::Arxiv | BackendId::GoogleScholar | BackendId::Reddit | BackendId::HackerNews
        ),
        QueryIntent::Academic | QueryIntent::Data => !matches!(backend, BackendId::Reddit),
        QueryIntent::Opinion => !matches!(backend, BackendId::Arxiv),
        QueryIntent::Casual => {
            // Casual queries should never hit specialist backends
            !matches!(
                backend,
                BackendId::StackOverflow
                    | BackendId::Github
                    | BackendId::Arxiv
                    | BackendId::GoogleScholar
            )
        }
        _ => true,
    }
}

/// Adaptive Backend Selector using UCB1 multi-armed bandit.
#[derive(Clone)]
pub struct AdaptiveBackendSelector {
    stats: Arc<DashMap<String, BackendStats>>,
    total_rounds: Arc<AtomicU64>,
    config: AbsConfig,
}

/// Selection result with metadata.
#[derive(Debug, Clone)]
pub struct BackendSelection {
    /// The selected backends, ordered by expected value.
    pub backends: Vec<BackendId>,
    /// UCB1 scores for each selected backend (parallel with `backends`).
    pub scores: Vec<f64>,
}

impl AdaptiveBackendSelector {
    /// Create a new selector.
    pub fn new(config: AbsConfig) -> Self {
        Self {
            stats: Arc::new(DashMap::new()),
            total_rounds: Arc::new(AtomicU64::new(0)),
            config,
        }
    }

    /// Select backends for a query based on intent and historical performance.
    pub fn select(
        &self,
        intent: &QueryIntent,
        available_backends: &[BackendId],
        unhealthy_backends: &[BackendId],
    ) -> BackendSelection {
        let total = self.total_rounds.fetch_add(1, Ordering::Relaxed) + 1;

        // Filter out unhealthy backends
        let healthy: Vec<&BackendId> = available_backends
            .iter()
            .filter(|b| !unhealthy_backends.contains(b))
            .filter(|b| intent_backend_allowed(intent, b))
            .collect();

        if healthy.is_empty() {
            return BackendSelection {
                backends: vec![],
                scores: vec![],
            };
        }

        // Score each backend with UCB1 + intent affinity
        let mut scored: Vec<(BackendId, f64)> = healthy
            .iter()
            .map(|&backend| {
                let key = backend.to_string();
                let ucb1 = self.ucb1_score(&key, total);
                let affinity = intent_affinity(intent, backend);
                let combined = ucb1 * 0.4 + affinity * 0.6; // 60% intent, 40% history
                (backend.clone(), combined)
            })
            .collect();

        // Sort by combined score (descending)
        scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));

        // Select top N backends (bounded by config)
        let n = scored
            .len()
            .min(self.config.max_backends)
            .max(self.config.min_backends.min(scored.len()));

        let mut backends: Vec<BackendId> = scored[..n].iter().map(|(b, _)| b.clone()).collect();
        let mut scores: Vec<f64> = scored[..n].iter().map(|(_, s)| *s).collect();

        // Ensure at least one reliable backend is included — intent-aware selection.
        // CurrentEvents → DuckDuckGo/HackerNews (not Wikipedia which returns static articles)
        // Code → StackOverflow/DuckDuckGo
        // Academic → Arxiv/Wikipedia
        // Default → Wikipedia/DuckDuckGo
        if self.config.always_include_reliable {
            let reliable: &[BackendId] = match intent {
                QueryIntent::CurrentEvents => &[BackendId::DuckDuckGo, BackendId::HackerNews],
                QueryIntent::Code => &[BackendId::StackOverflow, BackendId::DuckDuckGo],
                QueryIntent::HowTo => &[BackendId::StackOverflow, BackendId::DuckDuckGo],
                QueryIntent::Academic | QueryIntent::Data => {
                    &[BackendId::Arxiv, BackendId::Wikipedia]
                }
                _ => &[BackendId::Wikipedia, BackendId::DuckDuckGo],
            };
            let has_reliable = backends.iter().any(|b| reliable.contains(b));
            if !has_reliable {
                if let Some(reliable_b) = reliable
                    .iter()
                    .find(|b| healthy.contains(b) && !backends.contains(b))
                {
                    if backends.len() >= self.config.max_backends {
                        backends.pop();
                        scores.pop();
                    }
                    backends.push(reliable_b.clone());
                    scores.push(0.5);
                }
            }
        }

        BackendSelection { backends, scores }
    }

    /// Report the outcome of using a backend.
    pub fn report_outcome(&self, backend: &BackendId, result_count: usize, quality: f64) {
        let key = backend.to_string();
        let entry = self.stats.entry(key).or_insert_with(BackendStats::new);
        entry.selections.fetch_add(1, Ordering::Relaxed);
        if result_count > 0 {
            entry.successes.fetch_add(1, Ordering::Relaxed);
        }
        let reward_int = (quality.clamp(0.0, 1.0) * 1000.0) as u64;
        entry.total_reward.fetch_add(reward_int, Ordering::Relaxed);
    }

    /// Get performance summary for all backends.
    pub fn summary(&self) -> Vec<(String, f64, u64)> {
        self.stats
            .iter()
            .map(|entry| {
                let key = entry.key().clone();
                let rate = entry.value().success_rate();
                let count = entry.value().selections.load(Ordering::Relaxed);
                (key, rate, count)
            })
            .collect()
    }

    /// UCB1 score for a backend.
    ///
    /// UCB1 = avg_reward + C * sqrt(ln(total_rounds) / selections)
    fn ucb1_score(&self, key: &str, total_rounds: u64) -> f64 {
        match self.stats.get(key) {
            Some(stats) => {
                let selections = stats.selections.load(Ordering::Relaxed);
                if selections == 0 {
                    return 1.0; // moderate exploration bonus for unseen
                }
                let avg = stats.avg_reward();
                let exploration = self.config.exploration_factor
                    * ((total_rounds as f64).ln() / selections as f64).sqrt();
                (avg + exploration).min(1.0)
            }
            None => 1.0, // unseen backend — moderate exploration bonus
        }
    }
}

impl Default for AdaptiveBackendSelector {
    fn default() -> Self {
        Self::new(AbsConfig::default())
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn all_backends() -> Vec<BackendId> {
        vec![
            BackendId::DuckDuckGo,
            BackendId::Google,
            BackendId::Bing,
            BackendId::Wikipedia,
            BackendId::HackerNews,
            BackendId::Arxiv,
            BackendId::Github,
            BackendId::Reddit,
            BackendId::StackOverflow,
            BackendId::Brave,
        ]
    }

    #[test]
    fn academic_query_prefers_arxiv() {
        let abs = AdaptiveBackendSelector::default();
        let selection = abs.select(&QueryIntent::Academic, &all_backends(), &[]);

        // arXiv should be in the top 3 for academic queries
        let top3: Vec<_> = selection.backends.iter().take(3).collect();
        assert!(
            top3.contains(&&BackendId::Arxiv),
            "Academic query should include arXiv in top 3, got: {:?}",
            top3
        );
    }

    #[test]
    fn code_query_prefers_github() {
        let abs = AdaptiveBackendSelector::default();
        let selection = abs.select(&QueryIntent::Code, &all_backends(), &[]);

        let top3: Vec<_> = selection.backends.iter().take(3).collect();
        assert!(
            top3.contains(&&BackendId::Github),
            "Code query should include GitHub in top 3, got: {:?}",
            top3
        );
    }

    #[test]
    fn unhealthy_backends_excluded() {
        let abs = AdaptiveBackendSelector::default();
        let unhealthy = vec![BackendId::Google, BackendId::Bing];
        let selection = abs.select(&QueryIntent::Factual, &all_backends(), &unhealthy);

        assert!(!selection.backends.contains(&BackendId::Google));
        assert!(!selection.backends.contains(&BackendId::Bing));
    }

    #[test]
    fn respects_max_backends() {
        let config = AbsConfig {
            max_backends: 3,
            min_backends: 1,
            ..AbsConfig::default()
        };
        let abs = AdaptiveBackendSelector::new(config);
        let selection = abs.select(&QueryIntent::Factual, &all_backends(), &[]);

        assert!(selection.backends.len() <= 3);
    }

    #[test]
    fn learning_improves_selection() {
        let abs = AdaptiveBackendSelector::default();

        // Report that Wikipedia consistently gives great results
        for _ in 0..20 {
            abs.report_outcome(&BackendId::Wikipedia, 10, 0.95);
        }
        // Report that Bing consistently fails
        for _ in 0..20 {
            abs.report_outcome(&BackendId::Bing, 0, 0.0);
        }

        let selection = abs.select(&QueryIntent::Factual, &all_backends(), &[]);
        let wiki_pos = selection
            .backends
            .iter()
            .position(|b| *b == BackendId::Wikipedia);
        let bing_pos = selection
            .backends
            .iter()
            .position(|b| *b == BackendId::Bing);

        // Wikipedia should rank higher than Bing after learning
        match (wiki_pos, bing_pos) {
            (Some(w), Some(b)) => assert!(w < b, "Wikipedia should rank above Bing after learning"),
            (Some(_), None) => {} // Bing might be excluded entirely — that's fine
            _ => panic!("Wikipedia should be selected"),
        }
    }

    #[test]
    fn reliable_backend_always_included() {
        let config = AbsConfig {
            always_include_reliable: true,
            max_backends: 3,
            min_backends: 1,
            ..AbsConfig::default()
        };
        let abs = AdaptiveBackendSelector::new(config);

        // Factual intent: reliable fallback is Wikipedia/DuckDuckGo
        let backends = vec![
            BackendId::Arxiv,
            BackendId::Github,
            BackendId::HackerNews,
            BackendId::Wikipedia, // reliable for Factual
        ];

        let selection = abs.select(&QueryIntent::Factual, &backends, &[]);
        assert!(
            selection.backends.contains(&BackendId::Wikipedia)
                || selection.backends.contains(&BackendId::DuckDuckGo),
            "Should include at least one reliable backend for Factual"
        );

        // Code intent: reliable fallback is StackOverflow/DuckDuckGo
        let code_backends = vec![
            BackendId::Arxiv,
            BackendId::Github,
            BackendId::HackerNews,
            BackendId::StackOverflow, // reliable for Code
        ];

        let selection = abs.select(&QueryIntent::Code, &code_backends, &[]);
        assert!(
            selection.backends.contains(&BackendId::StackOverflow)
                || selection.backends.contains(&BackendId::DuckDuckGo),
            "Should include at least one reliable backend for Code"
        );
    }

    #[test]
    fn empty_available_returns_empty() {
        let abs = AdaptiveBackendSelector::default();
        let selection = abs.select(&QueryIntent::Factual, &[], &[]);
        assert!(selection.backends.is_empty());
    }

    #[test]
    fn factual_excludes_code_backends() {
        let abs = AdaptiveBackendSelector::default();
        let backends = vec![
            BackendId::StackOverflow,
            BackendId::Github,
            BackendId::Wikipedia,
            BackendId::DuckDuckGo,
            BackendId::Searxng,
        ];
        let selection = abs.select(&QueryIntent::Factual, &backends, &[]);
        assert!(!selection.backends.contains(&BackendId::StackOverflow));
        assert!(!selection.backends.contains(&BackendId::Github));
    }

    #[test]
    fn current_events_excludes_static_dev_sources() {
        let abs = AdaptiveBackendSelector::default();
        let backends = vec![
            BackendId::StackOverflow,
            BackendId::Github,
            BackendId::Arxiv,
            BackendId::HackerNews,
            BackendId::Searxng,
            BackendId::DuckDuckGo,
        ];
        let selection = abs.select(&QueryIntent::CurrentEvents, &backends, &[]);
        assert!(!selection.backends.contains(&BackendId::StackOverflow));
        assert!(!selection.backends.contains(&BackendId::Github));
        assert!(!selection.backends.contains(&BackendId::Arxiv));
    }
}