1use rayon::prelude::*;
10
11use crate::error::Result;
12use crate::http::{FetchConfig, Fetcher, Page};
13use crate::parse::{Article, ExtractOptions, extract_with};
14use crate::rank::{Context, SlimConfig, slim};
15use crate::search::{Router, SearchConfig, SearchOutcome, SearchResult};
16
17#[derive(Debug, Clone)]
19pub struct Pipeline {
20 router: Router,
21 extract: ExtractOptions,
22 slim: SlimConfig,
23}
24
25#[derive(Debug, Clone, serde::Serialize)]
27pub struct Research {
28 pub query: String,
30 pub context: Context,
32 pub results: Vec<SearchResult>,
34 pub articles: Vec<Article>,
36 pub failures: Vec<(String, String)>,
38}
39
40impl Pipeline {
41 pub fn new() -> Result<Self> {
43 Self::builder().build()
44 }
45
46 pub fn builder() -> PipelineBuilder {
48 PipelineBuilder::default()
49 }
50
51 pub fn fetcher(&self) -> &Fetcher {
53 self.router.fetcher()
54 }
55
56 pub async fn search(&self, query: &str) -> SearchOutcome {
58 self.router.search(query).await
59 }
60
61 pub async fn fetch(&self, urls: &[String]) -> Vec<Result<Page>> {
63 self.fetcher().fetch_many(urls).await
64 }
65
66 pub async fn read(&self, urls: &[String]) -> Vec<Result<Article>> {
68 let pages = self.fetch(urls).await;
69 pages
72 .into_par_iter()
73 .map(|page| {
74 let page = page?;
75 extract_with(&page.body, Some(&page.final_url), &self.extract)
76 })
77 .collect()
78 }
79
80 pub async fn research(&self, query: &str, max_sources: usize) -> Research {
85 let outcome = self.search(query).await;
86 let mut failures: Vec<(String, String)> =
87 outcome.failures.iter().map(|(p, m)| (format!("search:{p}"), m.clone())).collect();
88
89 let urls: Vec<String> =
90 outcome.results.iter().take(max_sources).map(|r| r.url.clone()).collect();
91 let read = self.read(&urls).await;
92
93 let mut articles = Vec::with_capacity(read.len());
94 for (url, result) in urls.iter().zip(read) {
95 match result {
96 Ok(article) if !article.units.is_empty() => articles.push(article),
99 Ok(_) => failures.push((format!("extract:{url}"), "no content extracted".into())),
100 Err(e) => failures.push((format!("fetch:{url}"), e.to_string())),
101 }
102 }
103
104 let context = slim(query, &articles, &self.slim);
105 Research { query: query.to_string(), context, results: outcome.results, articles, failures }
106 }
107}
108
109#[derive(Debug, Clone, Default)]
111pub struct PipelineBuilder {
112 fetch: FetchConfig,
113 search: SearchConfig,
114 extract: ExtractOptions,
115 slim: SlimConfig,
116}
117
118impl PipelineBuilder {
119 pub fn fetch(mut self, cfg: FetchConfig) -> Self {
121 self.fetch = cfg;
122 self
123 }
124
125 pub fn search(mut self, cfg: SearchConfig) -> Self {
127 self.search = cfg;
128 self
129 }
130
131 pub fn extract(mut self, cfg: ExtractOptions) -> Self {
133 self.extract = cfg;
134 self
135 }
136
137 pub fn slim(mut self, cfg: SlimConfig) -> Self {
139 self.slim = cfg;
140 self
141 }
142
143 pub fn build(self) -> Result<Pipeline> {
145 let fetcher = Fetcher::with_config(self.fetch)?;
146 Ok(Pipeline {
147 router: Router::new(fetcher, self.search),
148 extract: self.extract,
149 slim: self.slim,
150 })
151 }
152}
153
154impl Default for ExtractOptions {
155 fn default() -> Self {
156 ExtractOptions::new()
157 }
158}
159
160#[cfg(test)]
161mod tests {
162 use super::*;
163 use crate::search::Provider;
164
165 #[test]
166 fn builder_wires_every_stage() {
167 let pipeline = Pipeline::builder()
168 .fetch(FetchConfig { concurrency: 4, ..Default::default() })
169 .search(SearchConfig {
170 providers: vec![Provider::Wikipedia("ko".into())],
171 limit: 3,
172 per_provider: 5,
173 contact_email: None,
174 })
175 .slim(SlimConfig::with_budget(512))
176 .build()
177 .unwrap();
178 assert_eq!(pipeline.fetcher().config().concurrency, 4);
179 assert_eq!(pipeline.slim.max_tokens, 512);
180 }
181
182 #[test]
183 fn default_pipeline_builds() {
184 assert!(Pipeline::new().is_ok());
185 }
186
187 #[test]
188 fn invalid_fetch_config_fails_the_build() {
189 let err =
190 Pipeline::builder().fetch(FetchConfig { concurrency: 0, ..Default::default() }).build();
191 assert!(err.is_err());
192 }
193}