rustai_core/lib.rs
1//! A zero-cost local research pipeline: collect, denoise and compress the open
2//! web into something a small local model can actually read.
3//!
4//! No API key, no per-call billing, and no browser-grade DOM in the hot path.
5//! The four stages are independent and each is useful on its own:
6//!
7//! | Stage | Module | What it does |
8//! |---|---|---|
9//! | Search | [`search`] | free providers, queried concurrently, fused by rank |
10//! | Fetch | [`http`] | Chrome TLS/JA3 fingerprint, robots-aware, capped and paced |
11//! | Denoise | [`parse`] / [`denoise`] | find the article, drop the furniture, emit Markdown |
12//! | Slim | [`rank`] | BM25 + information density + MMR, under a token budget |
13//!
14//! # The whole pipeline
15//!
16//! ```no_run
17//! # async fn run() -> rustai_core::Result<()> {
18//! use rustai_core::pipeline::Pipeline;
19//!
20//! let pipeline = Pipeline::new()?;
21//! let research = pipeline.research("what is BM25", 5).await;
22//!
23//! println!("{}", research.context.markdown);
24//! for (stage, why) in &research.failures {
25//! eprintln!("skipped {stage}: {why}");
26//! }
27//! # Ok(()) }
28//! ```
29//!
30//! # Just the denoiser
31//!
32//! No network, no runtime, no configuration:
33//!
34//! ```
35//! let html = r#"
36//! <html><body>
37//! <nav><a href="/a">A</a><a href="/b">B</a><a href="/c">C</a></nav>
38//! <article>
39//! <h1>Arena parsers</h1>
40//! <p>Every node lives in one contiguous buffer, so traversal is a
41//! pointer bump rather than a chase across the heap.</p>
42//! </article>
43//! <footer>© 2026. All rights reserved.</footer>
44//! </body></html>"#;
45//!
46//! let article = rustai_core::parse::extract(html, Some("https://example.com/post"))?;
47//!
48//! assert_eq!(article.title(), Some("Arena parsers"));
49//! assert!(article.markdown.contains("# Arena parsers"));
50//! assert!(!article.text.contains("All rights reserved"));
51//! # Ok::<(), rustai_core::Error>(())
52//! ```
53//!
54//! # Just the slimmer
55//!
56//! Rank blocks you already have against a question, under a token budget that
57//! covers the rendered output — headers and breadcrumbs included, not just the
58//! blocks themselves:
59//!
60//! ```
61//! use rustai_core::rank::{SlimConfig, slim};
62//!
63//! let article = rustai_core::parse::extract(
64//! "<article><h1>Numbers</h1>\
65//! <p>Streaming 200 documents held 4.8 MiB resident, at 520 documents per second.</p>\
66//! <p>Paprika is best bloomed in fat before the liquid goes into the pot.</p>\
67//! </article>",
68//! Some("https://example.com/post"),
69//! )?;
70//!
71//! let context = slim("how much memory", &[article], &SlimConfig::with_budget(64));
72//!
73//! assert!(context.markdown.contains("4.8 MiB"));
74//! assert!(context.tokens <= 64);
75//! # Ok::<(), rustai_core::Error>(())
76//! ```
77//!
78//! # Being a good citizen
79//!
80//! [`http::Fetcher`] honours `robots.txt` and `Crawl-delay`, spaces requests to
81//! a host, caps concurrency and body size, and backs off exponentially on
82//! retry — all by default. Browser impersonation exists so that ordinary
83//! reading is not misclassified as abuse; it is not a licence to ignore a
84//! site's terms.
85//!
86//! # Cargo features
87//!
88//! | Feature | Default | Effect |
89//! |---|---|---|
90//! | `impersonate` | yes | Chrome TLS/JA3 and HTTP/2 fingerprint emulation |
91//! | `python` | no | PyO3 bindings; enabled by maturin |
92//! | `browser` | no | headless-Chrome fallback for JS-gated pages |
93//! | `nightly-simd` | no | SIMD tokenisation in `tl`; **needs a nightly toolchain** |
94
95#![deny(missing_docs)]
96
97pub mod chunk;
98pub mod denoise;
99pub mod error;
100pub mod http;
101pub mod parse;
102pub mod pipeline;
103pub mod rank;
104pub mod search;
105pub mod text;
106
107#[cfg(feature = "python")]
108mod py;
109
110pub use error::{Error, Result};