crawlberg/
lib.rs

1//! crawlberg -- A Rust crawling engine for turning websites into structured data.
2
3#[cfg(feature = "api")]
4pub(crate) mod api;
5mod assets;
6pub(crate) mod bindings;
7#[cfg(feature = "browser")]
8mod browser;
9mod browser_detect;
10#[cfg(feature = "browser")]
11pub mod browser_pool;
12#[cfg(feature = "browser")]
13pub(crate) mod browser_profile;
14#[cfg(feature = "browser")]
15pub mod browser_session_pool;
16pub mod budget;
17pub(crate) mod citations;
18#[cfg(feature = "browser")]
19mod stealth;
20
21pub(crate) mod defaults;
22mod document;
23pub(crate) mod engine;
24mod error;
25#[cfg(not(target_arch = "wasm32"))]
26mod helpers;
27mod html;
28pub mod http;
29pub mod interact;
30mod map;
31mod markdown;
32#[cfg(feature = "mcp")]
33pub(crate) mod mcp;
34#[cfg(feature = "browser-native")]
35mod native_browser;
36pub mod net;
37mod normalize;
38pub mod proxy;
39mod pruning;
40#[cfg(feature = "ai")]
41pub(crate) mod research;
42pub mod robots;
43mod scrape;
44#[cfg(not(target_arch = "wasm32"))]
45pub mod sink;
46pub mod sitemap;
47pub mod telemetry;
48pub(crate) mod tower;
49pub mod traits;
50mod types;
51pub(crate) mod waf;
52#[cfg(feature = "warc")]
53pub(crate) mod warc;
54
55#[cfg(feature = "api")]
56pub use api::serve_with_config as serve_api;
57pub use bindings::{
58    BatchCrawlResult, BatchCrawlResults, BatchScrapeResult, BatchScrapeResults, CrawlEngineHandle, batch_crawl,
59    batch_scrape, crawl, create_engine, interact, map_urls, scrape,
60};
61#[cfg(not(target_arch = "wasm32"))]
62pub use bindings::{batch_crawl_stream, crawl_stream};
63#[cfg(feature = "browser")]
64pub use browser_pool::{BrowserPool, BrowserPoolConfig};
65#[cfg(feature = "browser")]
66pub use browser_session_pool::{BrowserSessionPool, SessionKey};
67pub use budget::{BudgetError, DefaultPageBudget, PageBudget};
68pub use citations::{CitationReference, CitationResult, generate_citations};
69#[cfg(feature = "browser-native")]
70pub use crawlberg_browser::adapter::{NativeBrowserExecutor, NativeBrowserExecutorConfig};
71#[doc(hidden)]
72pub use defaults::compute_backoff_ms;
73pub use defaults::{
74    AdaptiveStrategy, BestFirstStrategy, BfsStrategy, DfsStrategy, EwmaDomainState, EwmaTracker, FixedBudget,
75    InMemoryFrontier, LearningRetryPolicy, NoopCache, NoopEmitter, NoopFilter, NoopStore, PerDomainThrottle,
76    SimpleRetryPolicy, UnlimitedBudget, default_retry_policy, in_memory_domain_state, unlimited_budget,
77};
78pub use engine::{CrawlEngine, CrawlEngineBuilder};
79pub use error::CrawlError;
80pub use interact::{
81    MAX_ACTIONS, MAX_SCRIPT_LEN, MAX_SCROLL_AMOUNT, MAX_SELECTOR_LEN, MAX_SINGLE_WAIT_MS, MAX_TEXT_LEN,
82    MAX_TOTAL_WAIT_SECS, PageAction, ScrollDirection, validate_actions,
83};
84#[cfg(feature = "mcp")]
85pub use mcp::{CrawlbergHttpMcpService, start_mcp_server, start_mcp_server_with_config, streamable_http_service};
86pub use net::ssrf::{HostMatcher, SsrfError, SsrfPolicy, validate_url};
87pub use proxy::{ProxyProvider, StaticProxyProvider};
88#[cfg(not(target_arch = "wasm32"))]
89pub use sink::{EventSink, MultiEventSink, TracingEventSink};
90#[cfg(feature = "telemetry-init")]
91pub use telemetry::{InitError as TelemetryInitError, TelemetryConfig, TelemetryGuard, init_otlp};
92pub use telemetry::{current_traceparent, with_traceparent};
93pub use types::antibot::{AntibotError, AntibotStrategy, Decision, DefaultAntibotStrategy, DynAntibotStrategy};
94pub use types::{
95    ActionResult, ArticleMetadata, AssetCategory, AttemptOutcome, AuthConfig, BrowserBackend, BrowserConfig,
96    BrowserExtras, BrowserMode, BrowserWait, BudgetExhausted, BypassProvider, BypassResponse, CachedPage,
97    ContentConfig, CookieInfo, CrawlConfig, CrawlConfigBuilder, CrawlPageResult, CrawlResult, DispatchProfile,
98    DispatchProfileBuilder, DomainObservation, DomainRecommendation, DomainStatePort, DownloadedAsset,
99    DownloadedDocument, DynBypassProvider, DynDomainStatePort, DynEscalationBudget, DynRetryPolicy, DynWafClassifier,
100    EscalationBudget, EscalationReason, EscalationStrategy, ExtractionMeta, FaviconInfo, FeedInfo, FeedType,
101    HeadingInfo, HreflangEntry, ImageInfo, ImageSource, InteractionResult, JsonLdEntry, LinkInfo, LinkType, MapResult,
102    MarkdownResult, ObservedOutcome, PageMetadata, ProxyConfig, ResponseMeta, RetryDirective, RetryPolicy,
103    ScrapeResult, SitemapUrl, Tier, WafClassifier, WafClassifyError, WafSignal,
104};
105#[cfg(not(target_arch = "wasm32"))]
106pub use types::{BatchCrawlStreamRequest, CrawlEvent, CrawlStreamRequest};
107pub use waf::rules::load_from_path as waf_rules_from_path;
108pub use waf::{
109    Rules as WafRules, RulesError as WafRulesError, TomlClassifier, WatchError as WafWatchError,
110    WatchHandle as WafWatchHandle, load_from_str as waf_rules_from_str,
111};
crawlberg/lib.rs

crawlberg/
lib.rs
