use crate::merge::{merge, MergeOptions, MergedResult};
use crate::search::{
fanout, fanout_with_metrics, Capabilities, Metrics, Options, ProviderCapabilities,
ProviderMetric, ProviderProbe, ResultStream, SearchProvider,
};
use crate::types::{Failure, FileQuery, Outcome};
use futures::{Stream, StreamExt};
use std::collections::HashMap;
use std::pin::Pin;
use std::sync::Arc;
use std::time::Duration;
#[cfg(feature = "fetch")]
use crate::fetch::{FetchError, FetchRegistry, Fetched, ProgressFn};
#[cfg(feature = "fetch")]
use std::path::PathBuf;
pub struct SearchReport {
pub results: Vec<MergedResult>,
pub failures: Vec<Failure>,
pub skipped: Vec<String>,
pub metrics: Vec<ProviderMetric>,
}
pub struct SearchStream {
pub skipped: Vec<String>,
outcomes: Pin<Box<dyn Stream<Item = Outcome> + Send>>,
}
impl SearchStream {
pub fn into_outcomes(self) -> Pin<Box<dyn Stream<Item = Outcome> + Send>> {
self.outcomes
}
}
impl Stream for SearchStream {
type Item = Outcome;
fn poll_next(
mut self: Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Option<Self::Item>> {
self.outcomes.as_mut().poll_next(cx)
}
}
pub struct Engine {
providers: Vec<Arc<dyn SearchProvider>>,
options: Options,
provider_weights: HashMap<String, f64>,
capabilities: Option<Arc<CapabilityCache>>,
#[cfg(feature = "fetch")]
fetchers: FetchRegistry,
}
impl Engine {
#[cfg(feature = "fetch")]
pub fn new(providers: Vec<Arc<dyn SearchProvider>>) -> Self {
Self {
providers,
options: Options::default(),
provider_weights: HashMap::new(),
capabilities: None,
fetchers: FetchRegistry::with_defaults(),
}
}
#[cfg(not(feature = "fetch"))]
pub fn new(providers: Vec<Arc<dyn SearchProvider>>) -> Self {
Self {
providers,
options: Options::default(),
provider_weights: HashMap::new(),
capabilities: None,
}
}
pub fn with_options(mut self, options: Options) -> Self {
self.options = options;
self
}
pub fn with_weights(mut self, provider_weights: HashMap<String, f64>) -> Self {
self.provider_weights = provider_weights;
self
}
#[cfg(feature = "fetch")]
pub fn with_fetchers(mut self, fetchers: FetchRegistry) -> Self {
self.fetchers = fetchers;
self
}
pub fn providers(&self) -> &[Arc<dyn SearchProvider>] {
&self.providers
}
pub fn with_capability_cache(mut self) -> Self {
if self.capabilities.is_some() {
return self;
}
let cache = Arc::new(CapabilityCache::default());
self.capabilities = Some(cache.clone());
self.providers = std::mem::take(&mut self.providers)
.into_iter()
.map(|inner| {
let key = Arc::as_ptr(&inner) as *const () as usize;
Arc::new(CapabilityCachedProvider {
inner,
key,
cache: cache.clone(),
}) as Arc<dyn SearchProvider>
})
.collect();
self
}
pub fn invalidate_capabilities(&self) {
if let Some(cache) = &self.capabilities {
cache.clear();
}
for provider in &self.providers {
provider.invalidate_capabilities();
}
}
pub async fn search(&self, query: FileQuery) -> SearchReport {
let (providers, skipped) = self.capable(&query).await;
let metrics = Metrics::new();
let outcomes = collect(fanout_with_metrics(
query,
providers,
self.options.clone(),
&metrics,
))
.await;
let results: Vec<crate::types::FileResult> = outcomes
.iter()
.filter_map(|outcome| outcome.as_result().cloned())
.collect();
let failures = outcomes
.iter()
.filter_map(|outcome| outcome.as_failure().cloned())
.collect();
let results = merge(
results,
&MergeOptions {
provider_weights: self.provider_weights.clone(),
},
);
SearchReport {
results,
failures,
skipped,
metrics: metrics.snapshot(),
}
}
pub async fn search_stream(&self, query: FileQuery) -> SearchStream {
let (providers, skipped) = self.capable(&query).await;
SearchStream {
skipped,
outcomes: Box::pin(fanout(query, providers, self.options.clone())),
}
}
async fn capable(&self, query: &FileQuery) -> (Vec<Arc<dyn SearchProvider>>, Vec<String>) {
let checks = self.providers.iter().map(|provider| {
let provider = provider.clone();
let probe = query.clone();
async move {
let accepts = provider.capabilities().await.accepts(&probe);
(accepts, provider)
}
});
let evaluated = futures::future::join_all(checks).await;
let mut providers = Vec::new();
let mut skipped = Vec::new();
for (accepts, provider) in evaluated {
if accepts {
providers.push(provider);
} else {
skipped.push(provider.name().to_owned());
}
}
(providers, skipped)
}
#[cfg(feature = "fetch")]
pub async fn fetch(
&self,
result: &MergedResult,
destination: impl Into<PathBuf>,
) -> Result<Fetched, FetchError> {
self.fetchers.fetch_merged(result, destination).await
}
#[cfg(feature = "fetch")]
pub async fn fetch_with_progress(
&self,
result: &MergedResult,
destination: impl Into<PathBuf>,
progress: &ProgressFn<'_>,
) -> Result<Fetched, FetchError> {
self.fetchers
.fetch_merged_with_progress(result, destination, progress)
.await
}
pub async fn provider_capabilities(&self) -> Vec<ProviderCapabilities> {
let timeout = self
.options
.timeout
.unwrap_or_else(|| Duration::from_secs(10));
crate::search::provider_capabilities(self.providers.clone(), timeout).await
}
pub async fn probe(&self, query: FileQuery) -> Vec<ProviderProbe> {
let timeout = self
.options
.timeout
.unwrap_or_else(|| Duration::from_secs(10));
crate::search::probe(query, self.providers.clone(), timeout).await
}
}
async fn collect(
stream: impl futures::Stream<Item = crate::types::Outcome>,
) -> Vec<crate::types::Outcome> {
futures::pin_mut!(stream);
let mut outcomes = Vec::new();
while let Some(outcome) = stream.next().await {
outcomes.push(outcome);
}
outcomes
}
#[derive(Default)]
struct CapabilityCache {
entries: std::sync::Mutex<HashMap<usize, Capabilities>>,
}
impl CapabilityCache {
fn get(&self, key: usize) -> Option<Capabilities> {
self.entries
.lock()
.ok()
.and_then(|entries| entries.get(&key).cloned())
}
fn insert(&self, key: usize, capabilities: Capabilities) {
if let Ok(mut entries) = self.entries.lock() {
entries.insert(key, capabilities);
}
}
fn remove(&self, key: usize) {
if let Ok(mut entries) = self.entries.lock() {
entries.remove(&key);
}
}
fn clear(&self) {
if let Ok(mut entries) = self.entries.lock() {
entries.clear();
}
}
}
struct CapabilityCachedProvider {
inner: Arc<dyn SearchProvider>,
key: usize,
cache: Arc<CapabilityCache>,
}
#[async_trait::async_trait]
impl SearchProvider for CapabilityCachedProvider {
fn name(&self) -> &str {
self.inner.name()
}
async fn capabilities(&self) -> Capabilities {
if let Some(capabilities) = self.cache.get(self.key) {
return capabilities;
}
let capabilities = self.inner.capabilities().await;
self.cache.insert(self.key, capabilities.clone());
capabilities
}
fn invalidate_capabilities(&self) {
self.cache.remove(self.key);
self.inner.invalidate_capabilities();
}
fn search(&self, query: &FileQuery) -> ResultStream<'_> {
self.inner.search(query)
}
}