Re-exports§
pub use browser_setup::apply_stealth_measures;pub use browser_setup::download_managed_browser;pub use browser_setup::find_browser_executable;pub use browser_setup::launch_browser;pub use config::CrawlConfig;pub use content_saver::CacheMetadata;pub use content_saver::save_json_data;pub use crawl_engine::ChromiumoxideCrawler;pub use crawl_engine::CrawlError;pub use crawl_engine::CrawlProgress;pub use crawl_engine::CrawlQueue;pub use crawl_engine::CrawlResult;pub use crawl_engine::Crawler;pub use runtime::AsyncJsonSave;pub use runtime::AsyncStream;pub use runtime::BrowserAction;pub use runtime::CrawlRequest;pub use utils::get_mirror_path;pub use utils::get_uri_from_path;pub use web_search::BrowserManager;pub use imurl::ImUrl;pub use crawl_engine::rate_limiter as crawl_rate_limiter;pub use page_extractor::link_rewriter;pub use mcp::ActiveCrawlSession;pub use mcp::ConfigSummary;pub use mcp::CrawlManifest;pub use mcp::CrawlSessionManager;pub use mcp::CrawlStatus;pub use mcp::GetCrawlResultsTool;pub use mcp::ManifestManager;pub use mcp::SearchCrawlResultsTool;pub use mcp::SearchEngineCache;pub use mcp::StartCrawlTool;pub use mcp::WebSearchTool;pub use mcp::url_to_output_dir;pub use page_extractor::schema::*;
Modules§
- browser_
setup - config
- Configuration module for web crawling
- content_
saver - Content saving utilities for web scraping
- crawl_
engine - Crawl Engine Module
- crawl_
events - Event system for tracking and reporting crawl progress
- imurl
- inline_
css - Resource inlining functionality
- kromekover
- mcp
- MCP (Model Context Protocol) Tools for Web Crawling
- page_
extractor - Page data extraction functions.
- runtime
- Zero-allocation, blazing-fast async runtime
- search
- Search functionality using Tantivy for markdown content indexing and retrieval
- utils
- web_
search - Web search functionality using browser automation
Macros§
- log_
search_ operation - Helper macro for logging search operations with performance metrics
- on_
chunk - Macro for handling streaming data chunks with safe unwrapping
- on_
error - Macro for handling errors with safe unwrapping