use super::provider::SearchProvider;
use crate::types::{ErrorCode, ErrorInfo, Failure, FileQuery, Outcome, Validate};
use async_stream::stream;
use futures::{Stream, StreamExt};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use tokio::sync::{mpsc, Semaphore};
#[derive(Debug, Clone, Default)]
pub struct Options {
pub timeout: Option<Duration>,
pub limit: Option<usize>,
pub offset: Option<usize>,
pub concurrency: Option<usize>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProviderMetric {
pub provider: String,
pub results: usize,
pub failures: usize,
pub elapsed: Duration,
}
#[derive(Clone, Default)]
pub struct Metrics {
entries: Arc<Mutex<Vec<ProviderMetric>>>,
}
impl Metrics {
pub fn new() -> Self {
Self::default()
}
pub fn record(&self, metric: ProviderMetric) {
if let Ok(mut entries) = self.entries.lock() {
entries.push(metric);
}
}
pub fn snapshot(&self) -> Vec<ProviderMetric> {
let mut entries = self
.entries
.lock()
.map(|entries| entries.clone())
.unwrap_or_default();
entries.sort_by(|a, b| a.provider.cmp(&b.provider));
entries
}
pub fn is_empty(&self) -> bool {
self.entries
.lock()
.map(|entries| entries.is_empty())
.unwrap_or(true)
}
}
pub fn fanout(
query: FileQuery,
providers: Vec<Arc<dyn SearchProvider>>,
options: Options,
) -> impl Stream<Item = Outcome> + 'static {
fanout_with_metrics(query, providers, options, &Metrics::new())
}
pub fn fanout_with_metrics(
query: FileQuery,
providers: Vec<Arc<dyn SearchProvider>>,
options: Options,
metrics: &Metrics,
) -> impl Stream<Item = Outcome> + 'static {
let metrics = metrics.clone();
stream! {
let (tx, mut rx) = mpsc::channel::<Outcome>(64);
let concurrency = options
.concurrency
.unwrap_or(Semaphore::MAX_PERMITS)
.max(1);
let semaphore = Arc::new(Semaphore::new(concurrency));
for provider in providers {
let tx = tx.clone();
let query = query.clone();
let timeout = options.timeout;
let semaphore = semaphore.clone();
let metrics = metrics.clone();
tokio::spawn(async move {
let start = Instant::now();
let _permit = semaphore.acquire_owned().await;
let name = provider.name().to_string();
let mut result_count = 0usize;
let mut failure_count = 0usize;
let mut results = provider.search(&query);
let forward = async {
while let Some(item) = results.next().await {
let outcome = match item {
Ok(mut result) => {
if result.provider.is_empty() {
result.provider = name.clone();
}
if let Err(error) = result.validate() {
failure_count += 1;
let _ = tx
.send(Outcome::Failure(Failure {
provider: name.clone(),
error: ErrorInfo {
code: ErrorCode::malformed(),
message: format!(
"provider returned an invalid result: {error}"
),
},
}))
.await;
continue;
}
result_count += 1;
Outcome::Result(result)
}
Err(error) => {
failure_count += 1;
let _ = tx
.send(Outcome::Failure(Failure {
provider: name.clone(),
error: error.into(),
}))
.await;
return;
}
};
if tx.send(outcome).await.is_err() {
return;
}
}
};
if let Some(duration) = timeout {
if tokio::time::timeout(duration, forward).await.is_err() {
failure_count += 1;
let _ = tx
.send(Outcome::Failure(Failure {
provider: name.clone(),
error: ErrorInfo {
code: ErrorCode::timeout(),
message: format!("provider timed out after {duration:?}"),
},
}))
.await;
}
} else {
forward.await;
}
metrics.record(ProviderMetric {
provider: name,
results: result_count,
failures: failure_count,
elapsed: start.elapsed(),
});
});
}
drop(tx);
let mut yielded = 0usize;
let mut skipped = 0usize;
let offset = options.offset.unwrap_or(0);
while let Some(outcome) = rx.recv().await {
if matches!(outcome, Outcome::Result(_)) {
if skipped < offset {
skipped += 1;
continue;
}
if options.limit.is_some_and(|limit| yielded >= limit) {
break;
}
yielded += 1;
}
yield outcome;
}
}
}