use async_trait::async_trait;
use log::{error, info, warn};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use crate::domain::models::search_result::SearchResult;
use crate::domain::services::relevance_scorer::{DateParserComponent, RelevanceScorer};
use crate::engines::engine_client::EngineClient;
use crate::search::client::{BaiduSearchEngine, BingSearchEngine, SogouSearchEngine};
use crate::search::dedup::ResultDeduplicator;
use crate::search::engine_trait::{SearchEngine, SearchRequest};
use crate::search::error::SearchError;
use crate::search::response::{Response, ResponseItem};
use crate::search::rrf::RRFFuser;
use crate::search::types::{EngineHealth, SearchEngineType};
mod fanout;
pub struct SmartSearchEngine {
baidu_engine: BaiduSearchEngine,
bing_engine: BingSearchEngine,
sogou_engine: SogouSearchEngine,
rrf_fuser: RRFFuser,
engine_timeout: Duration,
gates: [OutboundGate; 3],
extra_engine: Option<Arc<dyn SearchEngine>>,
}
struct OutboundGate {
min_interval: Duration,
last: Mutex<Option<Instant>>,
}
impl OutboundGate {
fn new(min_interval: Duration) -> Self {
Self {
min_interval,
last: Mutex::new(None),
}
}
async fn wait_turn(&self) {
let target = {
let mut last = self.last.lock().unwrap();
let now = Instant::now();
let start = match *last {
Some(prev) => prev + self.min_interval,
None => now,
};
*last = Some(start.max(now));
start.max(now)
};
let wait = target.saturating_duration_since(Instant::now());
if wait > Duration::ZERO {
tokio::time::sleep(wait).await;
}
}
}
const DEFAULT_ENGINE_TIMEOUT_SECS: u64 = 30;
const GATE_BING: usize = 0;
const GATE_BAIDU: usize = 1;
const GATE_SOGOU: usize = 2;
async fn with_timeout<F, T>(timeout: Duration, fut: F) -> Result<T, SearchError>
where
F: std::future::Future<Output = Result<T, SearchError>>,
{
match tokio::time::timeout(timeout, fut).await {
Ok(result) => result,
Err(_) => {
warn!(
"{}",
crate::i18n::tr_log_args(
"search-smart-engine-timeout",
&[("timeout", format!("{timeout:?}"),)],
)
);
Err(SearchError::SmartRoutingTimeout(timeout.as_secs()))
}
}
}
fn build_base_request(request: &SearchRequest) -> SearchRequest {
let mut base = SearchRequest::new(&request.query).with_limit(request.limit);
if request.offset > 0 {
base = base.with_offset(request.offset);
}
if let Some(lang) = &request.lang {
base = base.with_lang(lang);
}
if let Some(country) = &request.country {
base = base.with_country(country);
}
base
}
impl SmartSearchEngine {
pub fn new(engine_client: Arc<EngineClient>) -> Self {
Self {
baidu_engine: BaiduSearchEngine::new(engine_client.clone()),
bing_engine: BingSearchEngine::new(engine_client.clone()),
sogou_engine: SogouSearchEngine::new(engine_client),
rrf_fuser: RRFFuser::default(),
engine_timeout: Duration::from_secs(DEFAULT_ENGINE_TIMEOUT_SECS),
gates: [
OutboundGate::new(Duration::from_secs(1)), OutboundGate::new(Duration::from_secs(2)), OutboundGate::new(Duration::from_secs(2)), ],
extra_engine: None,
}
}
#[must_use]
pub fn with_additional_engine(mut self, engine: Arc<dyn SearchEngine>) -> Self {
self.extra_engine = Some(engine);
self
}
#[must_use]
pub fn with_engine_timeout(mut self, secs: u64) -> Self {
self.engine_timeout = Duration::from_secs(secs);
self
}
fn item_to_search_result(item: &ResponseItem) -> SearchResult {
SearchResult {
title: item.title.clone(),
url: item.url.clone(),
description: Some(item.description.clone()),
engine: item.engine.name().to_string(),
score: 0.0,
published_time: None,
}
}
fn apply_relevance_scoring(results: &mut [SearchResult], query: &str) {
let scorer = RelevanceScorer::for_query(query);
let date_parser = DateParserComponent::with_defaults();
let mut components: Vec<(f64, f64)> = Vec::with_capacity(results.len());
for result in results.iter_mut() {
let relevance_score =
scorer.calculate_score(&result.title, result.description.as_deref(), &result.url);
if let Some(ref description) = result.description {
if let Some(published_date) =
RelevanceScorer::extract_published_date_with_parser(description, &date_parser)
{
result.published_time = Some(published_date);
}
}
let freshness_score = if let Some(published_time) = result.published_time {
RelevanceScorer::calculate_freshness_score(published_time)
} else {
0.5
};
components.push((relevance_score, freshness_score));
}
let max_rrf = results.iter().map(|r| r.score).fold(0.0_f64, f64::max);
let min_rrf = results
.iter()
.map(|r| r.score)
.fold(f64::INFINITY, f64::min);
let max_rel = components.iter().map(|c| c.0).fold(0.0_f64, f64::max);
let min_rel = components.iter().map(|c| c.0).fold(f64::INFINITY, f64::min);
let normalize = |v: f64, min: f64, max: f64| {
if max > min {
(v - min) / (max - min)
} else {
0.0
}
};
for (result, (relevance_score, freshness_score)) in results.iter_mut().zip(&components) {
let rrf_norm = normalize(result.score, min_rrf, max_rrf);
let rel_norm = normalize(*relevance_score, min_rel, max_rel);
result.score = rrf_norm * 0.4 + rel_norm * 0.42 + freshness_score * 0.18;
}
results.sort_by(|a, b| {
b.score
.partial_cmp(&a.score)
.unwrap_or(std::cmp::Ordering::Equal)
});
}
}
#[async_trait]
impl SearchEngine for SmartSearchEngine {
fn name(&self) -> &'static str {
"smart"
}
fn engine_type(&self) -> SearchEngineType {
SearchEngineType::Smart
}
fn health(&self) -> EngineHealth {
EngineHealth::Healthy
}
async fn search(&self, request: &SearchRequest) -> Result<Response<ResponseItem>, SearchError> {
let query = &request.query;
let limit = request.limit;
info!(
"{}",
crate::i18n::tr_log_args(
"search-smart-start",
&[
("query", query.as_str().to_string()),
("limit", (limit as i64).to_string()),
("lang", format!("{:?}", request.lang),),
("country", format!("{:?}", request.country),),
("offset", (request.offset as i64).to_string()),
],
)
);
let base_request = build_base_request(request);
let (bing_result, baidu_result, sogou_result, extra_result) =
self.fan_out_engines(&base_request).await;
let mut bing_results = fanout::collect_named_results("Bing", bing_result);
let mut baidu_results = fanout::collect_named_results("Baidu", baidu_result);
let mut sogou_results = fanout::collect_named_results("Sogou", sogou_result);
let mut extra_results = fanout::collect_extra_results(self, extra_result);
if bing_results.is_empty()
&& baidu_results.is_empty()
&& sogou_results.is_empty()
&& extra_results.is_empty()
{
error!("All search engines failed or returned empty results");
return Err(SearchError::AllEnginesFailed);
}
info!(
"Collected results: bing={}, baidu={}, sogou={}, extra={}",
bing_results.len(),
baidu_results.len(),
sogou_results.len(),
extra_results.len()
);
let mut deduplicator = ResultDeduplicator::with_default_config();
bing_results = fanout::dedup_engine_results(&mut deduplicator, bing_results);
baidu_results = fanout::dedup_engine_results(&mut deduplicator, baidu_results);
sogou_results = fanout::dedup_engine_results(&mut deduplicator, sogou_results);
extra_results = deduplicator.deduplicate(extra_results);
let mut fused = self.rrf_fuser.fuse(vec![
bing_results,
baidu_results,
sogou_results,
extra_results,
]);
info!("RRF fused {} unique results", fused.len());
let mut cross_dedup = ResultDeduplicator::with_default_config();
let before = fused.len();
fused = cross_dedup.deduplicate(fused);
if fused.len() != before {
info!(
"{}",
crate::i18n::tr_log_args(
"search-smart-cross-dedup",
&[("count", ((before - fused.len()) as i64).to_string())],
)
);
}
Self::apply_relevance_scoring(&mut fused, query);
fused.truncate(limit as usize);
info!("Returning {} final results", fused.len());
let items = fanout::to_response_items(fused);
Ok(Response {
items,
total_results: None,
engine: SearchEngineType::Smart,
})
}
}
pub fn create_smart_search(engine_client: Arc<EngineClient>) -> Arc<dyn SearchEngine> {
Arc::new(SmartSearchEngine::new(engine_client))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_smart_engine_type() {
let engine_client = Arc::new(EngineClient::new());
let engine = SmartSearchEngine::new(engine_client);
assert_eq!(engine.engine_type(), SearchEngineType::Smart);
assert_eq!(engine.name(), "smart");
assert_eq!(engine.health(), EngineHealth::Healthy);
}
#[test]
fn test_create_smart_search_factory() {
let engine_client = Arc::new(EngineClient::new());
let engine = create_smart_search(engine_client);
assert_eq!(engine.engine_type(), SearchEngineType::Smart);
assert_eq!(engine.name(), "smart");
}
#[test]
fn test_item_to_search_result() {
let item = ResponseItem {
title: "Test Title".to_string(),
url: "https://example.com".to_string(),
description: "Test desc".to_string(),
engine: SearchEngineType::Bing,
};
let result = SmartSearchEngine::item_to_search_result(&item);
assert_eq!(result.title, "Test Title");
assert_eq!(result.url, "https://example.com");
assert_eq!(result.description, Some("Test desc".to_string()));
assert_eq!(result.engine, "Bing");
assert_eq!(result.score, 0.0);
}
#[test]
fn test_apply_relevance_scoring_sorts_by_score() {
let mut results = vec![
SearchResult {
title: "Rust Programming".to_string(),
url: "https://example.com/rust".to_string(),
description: Some("Learn Rust".to_string()),
engine: "bing".to_string(),
score: 0.01, published_time: None,
},
SearchResult {
title: "Rust Programming Guide Tutorial".to_string(),
url: "https://example.com/rust-guide".to_string(),
description: Some("Complete Rust guide".to_string()),
engine: "baidu".to_string(),
score: 0.005, published_time: None,
},
];
SmartSearchEngine::apply_relevance_scoring(&mut results, "Rust Programming");
assert!(
results[0].score >= results[1].score,
"results should be sorted by score descending"
);
assert!(results[0].score > 0.0);
}
#[test]
fn test_apply_relevance_scoring_consensus_beats_single_engine_relevance() {
let mut results = vec![
SearchResult {
title: "Rust Programming".to_string(),
url: "https://example.com/rust".to_string(),
description: None,
engine: "bing".to_string(),
score: 1.0 / 61.0, published_time: None,
},
SearchResult {
title: "Guide to Programming".to_string(),
url: "https://example.com/guide".to_string(),
description: None,
engine: "baidu".to_string(),
score: 1.0 / 61.0 + 1.0 / 62.0, published_time: None,
},
SearchResult {
title: "Unrelated Cooking Tips".to_string(),
url: "https://example.com/cooking".to_string(),
description: None,
engine: "sogou".to_string(),
score: 1.0 / 65.0, published_time: None,
},
];
SmartSearchEngine::apply_relevance_scoring(&mut results, "Rust Programming");
let top = &results[0];
assert!(
top.url == "https://example.com/guide",
"multi-engine consensus result must outrank single-engine max-relevance result, got order: {:?}",
results.iter().map(|r| (&r.url, r.score)).collect::<Vec<_>>()
);
}
#[test]
fn test_apply_relevance_scoring_single_result_no_panic() {
let mut results = vec![SearchResult {
title: "Only Result".to_string(),
url: "https://example.com/only".to_string(),
description: None,
engine: "bing".to_string(),
score: 0.016,
published_time: None,
}];
SmartSearchEngine::apply_relevance_scoring(&mut results, "Only Result");
assert_eq!(results.len(), 1);
assert!(results[0].score.is_finite());
}
#[test]
fn test_apply_relevance_scoring_extracts_date_from_description() {
let mut results = vec![SearchResult {
title: "Recent Article".to_string(),
url: "https://example.com/recent".to_string(),
description: Some("Published on 2025-08-01, a comprehensive guide".to_string()),
engine: "bing".to_string(),
score: 0.01,
published_time: None,
}];
SmartSearchEngine::apply_relevance_scoring(&mut results, "guide");
assert!(
results[0].published_time.is_some(),
"date should be extracted from description"
);
assert!(results[0].score > 0.0);
}
#[test]
fn test_apply_relevance_scoring_no_description_uses_default_freshness() {
let mut results = vec![SearchResult {
title: "No Description".to_string(),
url: "https://example.com/no-desc".to_string(),
description: None,
engine: "bing".to_string(),
score: 0.01,
published_time: None,
}];
SmartSearchEngine::apply_relevance_scoring(&mut results, "test");
assert!(results[0].published_time.is_none());
assert!(results[0].score > 0.0);
}
#[tokio::test]
async fn test_with_timeout_degrades_on_slow_future() {
use std::time::Duration;
let slow = async {
tokio::time::sleep(Duration::from_millis(1200)).await;
Ok::<_, SearchError>(1u8)
};
let result = with_timeout(Duration::from_secs(1), slow).await;
assert!(
matches!(result, Err(SearchError::SmartRoutingTimeout(1))),
"slow future must degrade to SmartRoutingTimeout(1), got {result:?}"
);
let fast = async { Ok::<_, SearchError>(2u8) };
assert_eq!(with_timeout(Duration::from_secs(1), fast).await.unwrap(), 2);
}
#[test]
fn test_build_base_request_carries_locale_and_offset() {
let mut request = SearchRequest::new("rust");
request.limit = 20;
request.offset = 10;
request.lang = Some("en".to_string());
request.country = Some("US".to_string());
let base = build_base_request(&request);
assert_eq!(base.query, "rust");
assert_eq!(base.limit, 20);
assert_eq!(base.offset, 10);
assert_eq!(base.lang.as_deref(), Some("en"));
assert_eq!(base.country.as_deref(), Some("US"));
let plain = build_base_request(&SearchRequest::new("go"));
assert_eq!(plain.offset, 0);
assert!(plain.lang.is_none());
assert!(plain.country.is_none());
}
#[test]
fn test_cross_engine_title_dedup_merges_same_title() {
let mut dedup = ResultDeduplicator::with_default_config();
let merged = vec![
SearchResult {
title: "Rust Async Programming Guide".to_string(),
url: "https://a.example/rust".to_string(),
description: None,
engine: "bing".to_string(),
score: 0.9,
published_time: None,
},
SearchResult {
title: "Rust Async Programming Guide".to_string(),
url: "https://b.example/rust-async".to_string(),
description: None,
engine: "baidu".to_string(),
score: 0.8,
published_time: None,
},
];
let out = dedup.deduplicate(merged);
assert_eq!(
out.len(),
1,
"same-title cross-engine duplicates must merge"
);
}
#[tokio::test]
async fn test_outbound_gate_enforces_min_interval() {
let min_interval = Duration::from_millis(120);
let gate = OutboundGate::new(min_interval);
let start = Instant::now();
gate.wait_turn().await;
assert!(start.elapsed() < min_interval, "first turn must not wait");
let first_record = gate.last.lock().unwrap().expect("first turn must record");
let second_start = Instant::now();
gate.wait_turn().await;
let waited = second_start.elapsed();
let expected = min_interval.saturating_sub(second_start.duration_since(first_record));
assert!(
waited + Duration::from_millis(1) >= expected,
"second turn must wait out the min interval (waited {waited:?}, expected ≥ {expected:?})"
);
}
struct StubExtraEngine {
engine_type: SearchEngineType,
name: &'static str,
items: Vec<ResponseItem>,
}
#[async_trait]
impl SearchEngine for StubExtraEngine {
fn name(&self) -> &'static str {
self.name
}
fn engine_type(&self) -> SearchEngineType {
self.engine_type
}
fn health(&self) -> EngineHealth {
EngineHealth::Healthy
}
async fn search(
&self,
_request: &SearchRequest,
) -> Result<Response<ResponseItem>, SearchError> {
Ok(Response {
items: self.items.clone(),
total_results: Some(self.items.len() as u64),
engine: self.engine_type,
})
}
}
#[tokio::test]
async fn test_additional_engine_participates_in_fusion() {
let stub = StubExtraEngine {
engine_type: SearchEngineType::Searxng,
name: "SearxNG",
items: vec![ResponseItem {
title: "SearXNG result".to_string(),
url: "https://searx-result.example/1".to_string(),
description: "from the extra engine".to_string(),
engine: SearchEngineType::Searxng,
}],
};
let engine = SmartSearchEngine::new(Arc::new(EngineClient::new()))
.with_engine_timeout(1)
.with_additional_engine(Arc::new(stub));
let response = engine
.search(&SearchRequest::new("rust").with_limit(5))
.await
.expect("aggregation must succeed with the extra engine only");
assert_eq!(response.engine, SearchEngineType::Smart);
let fused = response
.items
.iter()
.find(|item| item.url == "https://searx-result.example/1")
.expect("extra engine result must be fused");
assert_eq!(fused.engine, SearchEngineType::Searxng);
assert_eq!(fused.title, "SearXNG result");
}
}