Skip to main content

rustai_core/
pipeline.rs

1//! The end-to-end pipeline: a question in, a context window out.
2//!
3//! Each stage is usable on its own — [`Pipeline::search`], [`Pipeline::read`],
4//! [`crate::parse::extract`], [`crate::rank::slim`] — but [`Pipeline::research`]
5//! is the reason the crate exists. It runs the network stages on tokio and the
6//! CPU stages on rayon, which is the split that keeps the memory ceiling flat:
7//! no stage ever holds more than the documents currently in flight.
8
9use rayon::prelude::*;
10
11use crate::error::Result;
12use crate::http::{FetchConfig, Fetcher, Page};
13use crate::parse::{Article, ExtractOptions, extract_with};
14use crate::rank::{Context, SlimConfig, slim};
15use crate::search::{Router, SearchConfig, SearchOutcome, SearchResult};
16
17/// A configured pipeline: fetcher, providers, extractor and slimmer.
18#[derive(Debug, Clone)]
19pub struct Pipeline {
20    router: Router,
21    extract: ExtractOptions,
22    slim: SlimConfig,
23}
24
25/// Everything one research run produced.
26#[derive(Debug, Clone, serde::Serialize)]
27pub struct Research {
28    /// The query as asked.
29    pub query: String,
30    /// The context to hand to a model.
31    pub context: Context,
32    /// Search hits, best first, including ones that failed to fetch.
33    pub results: Vec<SearchResult>,
34    /// Successfully extracted articles, in the order they were read.
35    pub articles: Vec<Article>,
36    /// `(stage, message)` for anything that went wrong without stopping the run.
37    pub failures: Vec<(String, String)>,
38}
39
40impl Pipeline {
41    /// Build a pipeline with default settings everywhere.
42    pub fn new() -> Result<Self> {
43        Self::builder().build()
44    }
45
46    /// Start configuring a pipeline.
47    pub fn builder() -> PipelineBuilder {
48        PipelineBuilder::default()
49    }
50
51    /// The underlying fetcher, for reusing its connection pool.
52    pub fn fetcher(&self) -> &Fetcher {
53        self.router.fetcher()
54    }
55
56    /// Run a search across the configured providers.
57    pub async fn search(&self, query: &str) -> SearchOutcome {
58        self.router.search(query).await
59    }
60
61    /// Fetch URLs concurrently. Order matches the input.
62    pub async fn fetch(&self, urls: &[String]) -> Vec<Result<Page>> {
63        self.fetcher().fetch_many(urls).await
64    }
65
66    /// Fetch and extract, dropping nothing: failures come back in place.
67    pub async fn read(&self, urls: &[String]) -> Vec<Result<Article>> {
68        let pages = self.fetch(urls).await;
69        // Fetching was I/O-bound and is done; extraction is pure CPU, so it
70        // moves to rayon rather than blocking a tokio worker.
71        pages
72            .into_par_iter()
73            .map(|page| {
74                let page = page?;
75                extract_with(&page.body, Some(&page.final_url), &self.extract)
76            })
77            .collect()
78    }
79
80    /// Search, read the top results, and compress them into a context window.
81    ///
82    /// `max_sources` caps how many result pages are actually fetched, which is
83    /// the only knob that meaningfully changes how long a run takes.
84    pub async fn research(&self, query: &str, max_sources: usize) -> Research {
85        let outcome = self.search(query).await;
86        let mut failures: Vec<(String, String)> =
87            outcome.failures.iter().map(|(p, m)| (format!("search:{p}"), m.clone())).collect();
88
89        let urls: Vec<String> =
90            outcome.results.iter().take(max_sources).map(|r| r.url.clone()).collect();
91        let read = self.read(&urls).await;
92
93        let mut articles = Vec::with_capacity(read.len());
94        for (url, result) in urls.iter().zip(read) {
95            match result {
96                // A page that yielded nothing usable is a failure, not an
97                // empty source: keeping it would only dilute the ranking.
98                Ok(article) if !article.units.is_empty() => articles.push(article),
99                Ok(_) => failures.push((format!("extract:{url}"), "no content extracted".into())),
100                Err(e) => failures.push((format!("fetch:{url}"), e.to_string())),
101            }
102        }
103
104        let context = slim(query, &articles, &self.slim);
105        Research { query: query.to_string(), context, results: outcome.results, articles, failures }
106    }
107}
108
109/// Builder for [`Pipeline`].
110#[derive(Debug, Clone, Default)]
111pub struct PipelineBuilder {
112    fetch: FetchConfig,
113    search: SearchConfig,
114    extract: ExtractOptions,
115    slim: SlimConfig,
116}
117
118impl PipelineBuilder {
119    /// Replace the fetch configuration.
120    pub fn fetch(mut self, cfg: FetchConfig) -> Self {
121        self.fetch = cfg;
122        self
123    }
124
125    /// Replace the search configuration.
126    pub fn search(mut self, cfg: SearchConfig) -> Self {
127        self.search = cfg;
128        self
129    }
130
131    /// Replace the extraction configuration.
132    pub fn extract(mut self, cfg: ExtractOptions) -> Self {
133        self.extract = cfg;
134        self
135    }
136
137    /// Replace the slimming configuration.
138    pub fn slim(mut self, cfg: SlimConfig) -> Self {
139        self.slim = cfg;
140        self
141    }
142
143    /// Build the pipeline.
144    pub fn build(self) -> Result<Pipeline> {
145        let fetcher = Fetcher::with_config(self.fetch)?;
146        Ok(Pipeline {
147            router: Router::new(fetcher, self.search),
148            extract: self.extract,
149            slim: self.slim,
150        })
151    }
152}
153
154impl Default for ExtractOptions {
155    fn default() -> Self {
156        ExtractOptions::new()
157    }
158}
159
160#[cfg(test)]
161mod tests {
162    use super::*;
163    use crate::search::Provider;
164
165    #[test]
166    fn builder_wires_every_stage() {
167        let pipeline = Pipeline::builder()
168            .fetch(FetchConfig { concurrency: 4, ..Default::default() })
169            .search(SearchConfig {
170                providers: vec![Provider::Wikipedia("ko".into())],
171                limit: 3,
172                per_provider: 5,
173                contact_email: None,
174            })
175            .slim(SlimConfig::with_budget(512))
176            .build()
177            .unwrap();
178        assert_eq!(pipeline.fetcher().config().concurrency, 4);
179        assert_eq!(pipeline.slim.max_tokens, 512);
180    }
181
182    #[test]
183    fn default_pipeline_builds() {
184        assert!(Pipeline::new().is_ok());
185    }
186
187    #[test]
188    fn invalid_fetch_config_fails_the_build() {
189        let err =
190            Pipeline::builder().fetch(FetchConfig { concurrency: 0, ..Default::default() }).build();
191        assert!(err.is_err());
192    }
193}