use super::provider::SearchProvider;
use crate::types::{ErrorCode, ErrorInfo, Failure, FileQuery, Outcome, Validate};
use async_stream::stream;
use futures::{Stream, StreamExt};
use std::collections::HashSet;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use tokio::sync::{mpsc, Semaphore};
#[derive(Debug, Clone, Default)]
pub struct Options {
pub timeout: Option<Duration>,
pub deadline: Option<Duration>,
pub limit: Option<usize>,
pub offset: Option<usize>,
pub concurrency: Option<usize>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProviderMetric {
pub provider: String,
pub results: usize,
pub failures: usize,
pub elapsed: Duration,
}
#[derive(Clone, Default)]
pub struct Metrics {
entries: Arc<Mutex<Vec<ProviderMetric>>>,
}
impl Metrics {
pub fn new() -> Self {
Self::default()
}
pub fn record(&self, metric: ProviderMetric) {
if let Ok(mut entries) = self.entries.lock() {
entries.push(metric);
}
}
pub fn snapshot(&self) -> Vec<ProviderMetric> {
let mut entries = self
.entries
.lock()
.map(|entries| entries.clone())
.unwrap_or_default();
entries.sort_by(|a, b| a.provider.cmp(&b.provider));
entries
}
pub fn is_empty(&self) -> bool {
self.entries
.lock()
.map(|entries| entries.is_empty())
.unwrap_or(true)
}
}
pub fn fanout(
query: FileQuery,
providers: Vec<Arc<dyn SearchProvider>>,
options: Options,
) -> impl Stream<Item = Outcome> + 'static {
fanout_with_metrics(query, providers, options, &Metrics::new())
}
pub fn fanout_with_metrics(
query: FileQuery,
providers: Vec<Arc<dyn SearchProvider>>,
options: Options,
metrics: &Metrics,
) -> impl Stream<Item = Outcome> + 'static {
let metrics = metrics.clone();
stream! {
let (tx, mut rx) = mpsc::channel::<Outcome>(64);
let concurrency = options
.concurrency
.unwrap_or(Semaphore::MAX_PERMITS)
.max(1);
let semaphore = Arc::new(Semaphore::new(concurrency));
let names: Vec<String> = providers.iter().map(|p| p.name().to_owned()).collect();
let finished: Arc<Mutex<HashSet<String>>> = Arc::new(Mutex::new(HashSet::new()));
for provider in providers {
let tx = tx.clone();
let query = query.clone();
let timeout = options.timeout;
let semaphore = semaphore.clone();
let metrics = metrics.clone();
let finished = finished.clone();
tokio::spawn(async move {
let start = Instant::now();
let _permit = semaphore.acquire_owned().await;
let name = provider.name().to_string();
let mut result_count = 0usize;
let mut failure_count = 0usize;
let mut results = provider.search(&query);
let forward = async {
while let Some(item) = results.next().await {
let outcome = match item {
Ok(mut result) => {
if result.provider.is_empty() {
result.provider = name.clone();
}
if let Err(error) = result.validate() {
failure_count += 1;
let _ = tx
.send(Outcome::Failure(Failure {
provider: name.clone(),
error: ErrorInfo {
code: ErrorCode::malformed(),
message: format!(
"provider returned an invalid result: {error}"
),
},
}))
.await;
continue;
}
result_count += 1;
Outcome::Result(result)
}
Err(error) => {
failure_count += 1;
let _ = tx
.send(Outcome::Failure(Failure {
provider: name.clone(),
error: error.into(),
}))
.await;
return;
}
};
if tx.send(outcome).await.is_err() {
return;
}
}
};
if let Some(duration) = timeout {
if tokio::time::timeout(duration, forward).await.is_err() {
failure_count += 1;
let _ = tx
.send(Outcome::Failure(Failure {
provider: name.clone(),
error: ErrorInfo {
code: ErrorCode::timeout(),
message: format!("provider timed out after {duration:?}"),
},
}))
.await;
}
} else {
forward.await;
}
metrics.record(ProviderMetric {
provider: name.clone(),
results: result_count,
failures: failure_count,
elapsed: start.elapsed(),
});
if let Ok(mut finished) = finished.lock() {
finished.insert(name);
}
});
}
drop(tx);
let deadline = options
.deadline
.map(|after| tokio::time::Instant::now() + after);
let mut yielded = 0usize;
let mut skipped = 0usize;
let offset = options.offset.unwrap_or(0);
let mut hit_deadline = false;
loop {
let next = match deadline {
Some(at) => tokio::time::timeout_at(at, rx.recv()).await,
None => Ok::<_, tokio::time::error::Elapsed>(rx.recv().await),
};
let outcome = match next {
Ok(Some(outcome)) => outcome,
Ok(None) => break,
Err(_) => {
hit_deadline = true;
break;
}
};
if matches!(outcome, Outcome::Result(_)) {
if skipped < offset {
skipped += 1;
continue;
}
if options.limit.is_some_and(|limit| yielded >= limit) {
break;
}
yielded += 1;
}
yield outcome;
}
if hit_deadline {
let done = finished.lock().map(|done| done.clone()).unwrap_or_default();
let budget = options.deadline.unwrap_or_default();
for name in names {
if !done.contains(&name) {
yield Outcome::Failure(Failure {
provider: name,
error: ErrorInfo {
code: ErrorCode::timeout(),
message: format!(
"search deadline of {budget:?} exceeded before the provider finished"
),
},
});
}
}
}
}
}