celestia_spider/lib.rs
1//! # celestia-spider
2//!
3//! A high-performance Rust web browsing library and AI Intermediate Representation (IR) generator.
4//! Built around a streaming builder architecture (`Browser::builder()`) with support for static HTTP fetching,
5//! dynamic JavaScript rendering, anti-bot stealth mode via Headless Chrome, and one-off single page crawling utilities (`crawl_single_page`).
6//!
7//! ## Interleaved Queue-Based Streaming Architecture
8//!
9//! Pushes starting landing page onto navigation queue, pops tasks one by one to fetch HTML, extract `PageIR` markdown,
10//! execute analysis callbacks, export payloads to disk/storage sinks, scan HTML for same-domain child links, and push
11//! discovered tasks back onto the queue until draining completes.
12//!
13//! ## Quick Stealth Dynamic Rendering Example
14//!
15//! ```rust,no_run
16//! use std::time::Duration;
17//! use celestia_spider::{Browser, RenderMode, WaitUntil, TimeoutStrategy};
18//!
19//! #[tokio::main]
20//! async fn main() -> Result<(), String> {
21//! let browser = Browser::builder()
22//! .start_url("https://example.com")
23//! .render_mode(RenderMode::Dynamic)
24//! .stealth(true)
25//! .wait_until(WaitUntil::NetworkIdle)
26//! .render_timeout(Duration::from_secs(10))
27//! .timeout_strategy(TimeoutStrategy::ExtractPartial)
28//! .on_page(|page_ir| async move {
29//! println!("Analyzing URL: {}", page_ir.url);
30//! Ok(())
31//! })
32//! .build()?;
33//!
34//! let summary = browser.run().await?;
35//! println!("Pages Processed: {}", summary.pages_processed);
36//! Ok(())
37//! }
38//! ```
39//!
40//! ## Single Page Crawl Utility Example
41//!
42//! ```rust,no_run
43//! use celestia_spider::{crawl_single_page, RenderOptions, RenderMode};
44//!
45//! #[tokio::main]
46//! async fn main() -> Result<(), String> {
47//! let options = RenderOptions {
48//! render_mode: RenderMode::Static,
49//! ..Default::default()
50//! };
51//! let page_ir = crawl_single_page("https://example.com", &options).await?;
52//! println!("Title: {}", page_ir.title);
53//! println!("Markdown IR:\n{}", page_ir.markdown_ir);
54//! Ok(())
55//! }
56//! ```
57//!
58//! ## UI / GUI Integration (Tauri, Electron, egui, ...)
59//!
60//! For UI hosts, [`CrawlSession`] is the integration surface: a serde-serializable
61//! [`CrawlConfig`], a broadcast [`CrawlerEvent`] stream, and a
62//! [`crate::CrawlSessionHandle`] with `cancel` / `pause` / `resume` / `join`.
63//!
64//! ```rust,no_run
65//! use celestia_spider::{CrawlConfig, CrawlSession, CrawlerEvent};
66//!
67//! #[tokio::main]
68//! async fn main() -> Result<(), String> {
69//! let session = CrawlSession::spawn(CrawlConfig {
70//! urls: vec!["https://example.com".into()],
71//! max_depth: 2,
72//! ..Default::default()
73//! })?;
74//! let mut events = session.take_events().unwrap();
75//!
76//! // In a real UI: forward each event to the window (Tauri emit, IPC, ...).
77//! tokio::spawn(async move {
78//! while let Ok(event) = events.recv().await {
79//! if let CrawlerEvent::Finished { summary } = event {
80//! println!("done: {} pages", summary.results);
81//! break;
82//! }
83//! }
84//! });
85//!
86//! // From a UI button: session.pause() / session.resume() / session.cancel()
87//! let summary = session.join().await?;
88//! println!("visited {} urls", summary.visited_urls.len());
89//! Ok(())
90//! }
91//! ```
92
93#[cfg(not(target_arch = "wasm32"))]
94pub mod builder;
95#[cfg(not(target_arch = "wasm32"))]
96pub mod control;
97pub mod engine;
98#[cfg(not(target_arch = "wasm32"))]
99pub mod exporter;
100#[cfg(not(target_arch = "wasm32"))]
101pub mod mapper;
102#[cfg(not(target_arch = "wasm32"))]
103pub mod output;
104#[cfg(not(target_arch = "wasm32"))]
105pub mod pipeline;
106#[cfg(not(target_arch = "wasm32"))]
107pub mod renderer;
108#[cfg(not(target_arch = "wasm32"))]
109pub mod runner;
110#[cfg(not(target_arch = "wasm32"))]
111pub mod session;
112#[cfg(not(target_arch = "wasm32"))]
113pub mod stealth;
114pub mod transformer;
115pub mod types;
116pub mod utils;
117
118#[cfg(target_arch = "wasm32")]
119pub mod wasm;
120
121#[cfg(not(target_arch = "wasm32"))]
122pub use builder::{Browser, BrowserBuilder};
123#[cfg(not(target_arch = "wasm32"))]
124pub use control::CrawlControl;
125#[cfg(not(target_arch = "wasm32"))]
126pub use engine::common::{Crawler, PageFetch};
127#[cfg(not(target_arch = "wasm32"))]
128pub use exporter::{FileStorageExporter, FileStorageExporterBuilder};
129#[cfg(not(target_arch = "wasm32"))]
130pub use mapper::{SiteMapper, SiteMapperBuilder};
131#[cfg(not(target_arch = "wasm32"))]
132pub use output::{configure_output, StandardWriter};
133#[cfg(not(target_arch = "wasm32"))]
134pub use pipeline::{BrowserPipeline, BrowserPipelineBuilder, CrawlerPipeline};
135#[cfg(not(target_arch = "wasm32"))]
136pub use renderer::{crawl_single_page, PageFetcher};
137#[cfg(not(target_arch = "wasm32"))]
138pub use runner::{Runner, RunnerSummary};
139#[cfg(not(target_arch = "wasm32"))]
140pub use session::{CrawlConfig, CrawlSession, CrawlSessionHandle, SessionPhase, SessionSnapshot};
141#[cfg(not(target_arch = "wasm32"))]
142pub use stealth::{stealth_chrome_args, STEALTH_JS};
143pub use transformer::{extract_links, transform_html_to_ir};
144pub use types::result::Result as CrawlResult;
145pub use types::events::{CrawlerEvent, SessionSummary};
146pub use types::{
147 AnalysisCallback, CrawlSummary, Options, PageIR, RenderMode, RenderOptions, Request, Response,
148 SitemapNode, Strategy, StorageExporter, TimeoutStrategy, WaitUntil,
149};