kreuzcrawl 0.1.1

High-performance web crawling engine
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

//! Default trait implementations for kreuzcrawl.

mod cache;
mod emitter;
mod filter;
mod frontier;
mod llm_extractor;
mod rate_limiter;
mod store;
mod strategy;

#[cfg(not(target_arch = "wasm32"))]
pub use cache::DiskCache;
pub use cache::NoopCache;
pub use emitter::NoopEmitter;
pub use filter::{Bm25Filter, NoopFilter};
pub use frontier::InMemoryFrontier;
#[cfg(feature = "ai")]
pub use llm_extractor::LlmExtractor;
pub use rate_limiter::{NoopRateLimiter, PerDomainThrottle};
pub use store::NoopStore;
pub use strategy::{AdaptiveStrategy, BestFirstStrategy, BfsStrategy, DfsStrategy};