use rayon::prelude::*;
use crate::error::Result;
use crate::http::{FetchConfig, Fetcher, Page};
use crate::parse::{Article, ExtractOptions, extract_with};
use crate::rank::{Context, SlimConfig, slim};
use crate::search::{Router, SearchConfig, SearchOutcome, SearchResult};
#[derive(Debug, Clone)]
pub struct Pipeline {
router: Router,
extract: ExtractOptions,
slim: SlimConfig,
}
#[derive(Debug, Clone, serde::Serialize)]
pub struct Research {
pub query: String,
pub context: Context,
pub results: Vec<SearchResult>,
pub articles: Vec<Article>,
pub failures: Vec<(String, String)>,
}
impl Pipeline {
pub fn new() -> Result<Self> {
Self::builder().build()
}
pub fn builder() -> PipelineBuilder {
PipelineBuilder::default()
}
pub fn fetcher(&self) -> &Fetcher {
self.router.fetcher()
}
pub async fn search(&self, query: &str) -> SearchOutcome {
self.router.search(query).await
}
pub async fn fetch(&self, urls: &[String]) -> Vec<Result<Page>> {
self.fetcher().fetch_many(urls).await
}
pub async fn read(&self, urls: &[String]) -> Vec<Result<Article>> {
let pages = self.fetch(urls).await;
pages
.into_par_iter()
.map(|page| {
let page = page?;
extract_with(&page.body, Some(&page.final_url), &self.extract)
})
.collect()
}
pub async fn research(&self, query: &str, max_sources: usize) -> Research {
let outcome = self.search(query).await;
let mut failures: Vec<(String, String)> =
outcome.failures.iter().map(|(p, m)| (format!("search:{p}"), m.clone())).collect();
let urls: Vec<String> =
outcome.results.iter().take(max_sources).map(|r| r.url.clone()).collect();
let read = self.read(&urls).await;
let mut articles = Vec::with_capacity(read.len());
for (url, result) in urls.iter().zip(read) {
match result {
Ok(article) if !article.units.is_empty() => articles.push(article),
Ok(_) => failures.push((format!("extract:{url}"), "no content extracted".into())),
Err(e) => failures.push((format!("fetch:{url}"), e.to_string())),
}
}
let context = slim(query, &articles, &self.slim);
Research { query: query.to_string(), context, results: outcome.results, articles, failures }
}
}
#[derive(Debug, Clone, Default)]
pub struct PipelineBuilder {
fetch: FetchConfig,
search: SearchConfig,
extract: ExtractOptions,
slim: SlimConfig,
}
impl PipelineBuilder {
pub fn fetch(mut self, cfg: FetchConfig) -> Self {
self.fetch = cfg;
self
}
pub fn search(mut self, cfg: SearchConfig) -> Self {
self.search = cfg;
self
}
pub fn extract(mut self, cfg: ExtractOptions) -> Self {
self.extract = cfg;
self
}
pub fn slim(mut self, cfg: SlimConfig) -> Self {
self.slim = cfg;
self
}
pub fn build(self) -> Result<Pipeline> {
let fetcher = Fetcher::with_config(self.fetch)?;
Ok(Pipeline {
router: Router::new(fetcher, self.search),
extract: self.extract,
slim: self.slim,
})
}
}
impl Default for ExtractOptions {
fn default() -> Self {
ExtractOptions::new()
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::search::Provider;
#[test]
fn builder_wires_every_stage() {
let pipeline = Pipeline::builder()
.fetch(FetchConfig { concurrency: 4, ..Default::default() })
.search(SearchConfig {
providers: vec![Provider::Wikipedia("ko".into())],
limit: 3,
per_provider: 5,
contact_email: None,
})
.slim(SlimConfig::with_budget(512))
.build()
.unwrap();
assert_eq!(pipeline.fetcher().config().concurrency, 4);
assert_eq!(pipeline.slim.max_tokens, 512);
}
#[test]
fn default_pipeline_builds() {
assert!(Pipeline::new().is_ok());
}
#[test]
fn invalid_fetch_config_fails_the_build() {
let err =
Pipeline::builder().fetch(FetchConfig { concurrency: 0, ..Default::default() }).build();
assert!(err.is_err());
}
}