Skip to main content

celestia_spider/
lib.rs

1//! # celestia-spider
2//! 
3//! A high-performance Rust web browsing library and AI Intermediate Representation (IR) generator.
4//! Built around a streaming builder architecture (`Browser::builder()`) with support for static HTTP fetching,
5//! dynamic JavaScript rendering, anti-bot stealth mode via Headless Chrome, and one-off single page crawling utilities (`crawl_single_page`).
6//! 
7//! ## Interleaved Queue-Based Streaming Architecture
8//! 
9//! Pushes starting landing page onto navigation queue, pops tasks one by one to fetch HTML, extract `PageIR` markdown,
10//! execute analysis callbacks, export payloads to disk/storage sinks, scan HTML for same-domain child links, and push
11//! discovered tasks back onto the queue until draining completes.
12//! 
13//! ## Quick Stealth Dynamic Rendering Example
14//! 
15//! ```rust,no_run
16//! use std::time::Duration;
17//! use celestia_spider::{Browser, RenderMode, WaitUntil, TimeoutStrategy};
18//! 
19//! #[tokio::main]
20//! async fn main() -> Result<(), String> {
21//!     let browser = Browser::builder()
22//!         .start_url("https://example.com")
23//!         .render_mode(RenderMode::Dynamic)
24//!         .stealth(true)
25//!         .wait_until(WaitUntil::NetworkIdle)
26//!         .render_timeout(Duration::from_secs(10))
27//!         .timeout_strategy(TimeoutStrategy::ExtractPartial)
28//!         .on_page(|page_ir| async move {
29//!             println!("Analyzing URL: {}", page_ir.url);
30//!             Ok(())
31//!         })
32//!         .build()?;
33//! 
34//!     let summary = browser.run().await?;
35//!     println!("Pages Processed: {}", summary.pages_processed);
36//!     Ok(())
37//! }
38//! ```
39//!
40//! ## Single Page Crawl Utility Example
41//!
42//! ```rust,no_run
43//! use celestia_spider::{crawl_single_page, RenderOptions, RenderMode};
44//!
45//! #[tokio::main]
46//! async fn main() -> Result<(), String> {
47//!     let options = RenderOptions {
48//!         render_mode: RenderMode::Static,
49//!         ..Default::default()
50//!     };
51//!     let page_ir = crawl_single_page("https://example.com", &options).await?;
52//!     println!("Title: {}", page_ir.title);
53//!     println!("Markdown IR:\n{}", page_ir.markdown_ir);
54//!     Ok(())
55//! }
56//! ```
57//!
58//! ## UI / GUI Integration (Tauri, Electron, egui, ...)
59//!
60//! For UI hosts, [`CrawlSession`] is the integration surface: a serde-serializable
61//! [`CrawlConfig`], a broadcast [`CrawlerEvent`] stream, and a
62//! [`crate::CrawlSessionHandle`] with `cancel` / `pause` / `resume` / `join`.
63//!
64//! ```rust,no_run
65//! use celestia_spider::{CrawlConfig, CrawlSession, CrawlerEvent};
66//!
67//! #[tokio::main]
68//! async fn main() -> Result<(), String> {
69//!     let session = CrawlSession::spawn(CrawlConfig {
70//!         urls: vec!["https://example.com".into()],
71//!         max_depth: 2,
72//!         ..Default::default()
73//!     })?;
74//!     let mut events = session.take_events().unwrap();
75//!
76//!     // In a real UI: forward each event to the window (Tauri emit, IPC, ...).
77//!     tokio::spawn(async move {
78//!         while let Ok(event) = events.recv().await {
79//!             if let CrawlerEvent::Finished { summary } = event {
80//!                 println!("done: {} pages", summary.results);
81//!                 break;
82//!             }
83//!         }
84//!     });
85//!
86//!     // From a UI button: session.pause() / session.resume() / session.cancel()
87//!     let summary = session.join().await?;
88//!     println!("visited {} urls", summary.visited_urls.len());
89//!     Ok(())
90//! }
91//! ```
92
93#[cfg(not(target_arch = "wasm32"))]
94pub mod builder;
95#[cfg(not(target_arch = "wasm32"))]
96pub mod control;
97pub mod engine;
98#[cfg(not(target_arch = "wasm32"))]
99pub mod exporter;
100#[cfg(not(target_arch = "wasm32"))]
101pub mod mapper;
102#[cfg(not(target_arch = "wasm32"))]
103pub mod output;
104#[cfg(not(target_arch = "wasm32"))]
105pub mod pipeline;
106#[cfg(not(target_arch = "wasm32"))]
107pub mod renderer;
108#[cfg(not(target_arch = "wasm32"))]
109pub mod runner;
110#[cfg(not(target_arch = "wasm32"))]
111pub mod session;
112#[cfg(not(target_arch = "wasm32"))]
113pub mod stealth;
114pub mod transformer;
115pub mod types;
116pub mod utils;
117
118#[cfg(target_arch = "wasm32")]
119pub mod wasm;
120
121#[cfg(not(target_arch = "wasm32"))]
122pub use builder::{Browser, BrowserBuilder};
123#[cfg(not(target_arch = "wasm32"))]
124pub use control::CrawlControl;
125#[cfg(not(target_arch = "wasm32"))]
126pub use engine::common::{Crawler, PageFetch};
127#[cfg(not(target_arch = "wasm32"))]
128pub use exporter::{FileStorageExporter, FileStorageExporterBuilder};
129#[cfg(not(target_arch = "wasm32"))]
130pub use mapper::{SiteMapper, SiteMapperBuilder};
131#[cfg(not(target_arch = "wasm32"))]
132pub use output::{configure_output, StandardWriter};
133#[cfg(not(target_arch = "wasm32"))]
134pub use pipeline::{BrowserPipeline, BrowserPipelineBuilder, CrawlerPipeline};
135#[cfg(not(target_arch = "wasm32"))]
136pub use renderer::{crawl_single_page, PageFetcher};
137#[cfg(not(target_arch = "wasm32"))]
138pub use runner::{Runner, RunnerSummary};
139#[cfg(not(target_arch = "wasm32"))]
140pub use session::{CrawlConfig, CrawlSession, CrawlSessionHandle, SessionPhase, SessionSnapshot};
141#[cfg(not(target_arch = "wasm32"))]
142pub use stealth::{stealth_chrome_args, STEALTH_JS};
143pub use transformer::{extract_links, transform_html_to_ir};
144pub use types::result::Result as CrawlResult;
145pub use types::events::{CrawlerEvent, SessionSummary};
146pub use types::{
147    AnalysisCallback, CrawlSummary, Options, PageIR, RenderMode, RenderOptions, Request, Response,
148    SitemapNode, Strategy, StorageExporter, TimeoutStrategy, WaitUntil,
149};