kreuzcrawl 0.1.0

High-performance web crawling engine
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69

//! kreuzcrawl -- A Rust crawling engine for turning websites into structured data.

#[cfg(feature = "api")]
pub mod api;
mod assets;
pub mod bindings;
#[cfg(feature = "browser")]
mod browser;
mod browser_detect;
#[cfg(feature = "browser")]
mod browser_pool;
#[cfg(feature = "browser")]
pub mod browser_profile;
pub mod citations;

pub mod defaults;
pub mod engine;
mod error;
#[cfg(not(target_arch = "wasm32"))]
mod helpers;
mod html;
mod http;
#[cfg(feature = "interact")]
pub mod interact;
mod map;
mod markdown;
#[cfg(feature = "mcp")]
pub mod mcp;
mod normalize;
mod pruning;
#[cfg(feature = "ai")]
pub mod research;
mod robots;
mod scrape;
mod sitemap;
pub mod tower;
pub mod traits;
mod types;
#[cfg(feature = "warc")]
pub mod warc;

pub use bindings::{
    BatchCrawlResult, BatchScrapeResult, CrawlEngineHandle, batch_crawl, batch_scrape, crawl, create_engine, map_urls,
    scrape,
};
#[cfg(feature = "browser")]
pub use browser_pool::{BrowserPool, BrowserPoolConfig, PooledPage};
pub use citations::{CitationReference, CitationResult};
#[cfg(not(target_arch = "wasm32"))]
pub use defaults::DiskCache;
#[cfg(feature = "ai")]
pub use defaults::LlmExtractor;
pub use defaults::{
    AdaptiveStrategy, BestFirstStrategy, BfsStrategy, Bm25Filter, DfsStrategy, InMemoryFrontier, NoopCache,
    NoopEmitter, NoopFilter, NoopRateLimiter, NoopStore, PerDomainThrottle,
};
pub use engine::{CrawlEngine, CrawlEngineBuilder};
pub use error::CrawlError;
pub use traits::{
    CompleteEvent, ContentFilter, CrawlCache, CrawlStats, CrawlStore, CrawlStrategy, ErrorEvent, EventEmitter,
    Frontier, FrontierEntry, PageEvent, RateLimiter,
};
pub use types::{
    ActionResult, ArticleMetadata, AssetCategory, AuthConfig, BrowserConfig, BrowserMode, BrowserWait, CachedPage,
    CookieInfo, CrawlConfig, CrawlEvent, CrawlPageResult, CrawlResult, DownloadedAsset, DownloadedDocument,
    ExtractionMeta, FaviconInfo, FeedInfo, FeedType, HeadingInfo, HreflangEntry, ImageInfo, ImageSource,
    InteractionResult, JsonLdEntry, LinkInfo, LinkType, MapResult, MarkdownResult, PageMetadata, ProxyConfig,
    ResponseMeta, ScrapeResult, SitemapUrl,
};