millipede_core/
lib.rs

1#![doc = include_str!("../README.md")]
2
3#[cfg(test)]
4extern crate self as millipede_core;
5
6#[cfg(test)]
7#[allow(dead_code)]
8#[path = "../../millipede-storage-memory/src/dataset.rs"]
9mod dataset;
10#[cfg(test)]
11#[allow(dead_code)]
12#[path = "../../millipede-storage-memory/src/kvs.rs"]
13mod kvs;
14#[cfg(test)]
15#[allow(dead_code)]
16#[path = "../../millipede-storage-memory/src/client.rs"]
17mod memory_client;
18#[cfg(test)]
19#[allow(dead_code)]
20#[path = "../../millipede-storage-memory/src/policy.rs"]
21mod policy;
22#[cfg(test)]
23#[allow(dead_code, unreachable_patterns)]
24#[path = "../../millipede-storage-memory/src/queue.rs"]
25mod queue;
26#[cfg(test)]
27use dataset::MemoryDataset;
28#[cfg(test)]
29use kvs::MemoryKeyValueStore;
30#[cfg(test)]
31use queue::MemoryRequestQueue;
32
33/// Content-based anti-bot and web application firewall detection.
34pub mod antibot;
35/// Autoscaling: dynamic concurrency, load signals, and rate limiting.
36pub mod autoscale;
37/// Crawler configuration and environment resolution.
38pub mod config;
39/// Session cookie storage and persistence.
40pub mod cookies;
41/// Crawler lifecycle kinds, handles, and shared state.
42pub mod crawler;
43/// Link enqueueing from handler contexts.
44pub mod enqueue;
45/// Crawl error taxonomy and retry classification.
46pub mod errors;
47/// Crawler lifecycle events and broadcast support.
48pub mod events;
49/// Request handler and middleware contracts.
50pub mod handler;
51/// Backend-independent HTTP request, response, and client abstractions.
52pub mod http_client;
53/// Link-extraction strategies, URL patterns, and crawl policy.
54pub mod link_extraction;
55/// Proxy configuration and rotation strategies.
56pub mod proxy;
57/// Request data types and construction helpers.
58pub mod request;
59/// Attempt-level retry strategy hooks.
60pub mod retry_strategy;
61/// Label- and method-based request routing.
62pub mod router;
63/// Sessions and reusable session pools.
64pub mod session;
65/// Streaming sitemap ingestion.
66pub mod sitemap;
67/// Failure-artifact capture and reload support.
68pub mod snapshot;
69/// Crawl statistics, rates, and persistence.
70pub mod statistics;
71/// Object-safe storage abstractions and typed convenience wrappers.
72pub mod storage;
73
74mod util;
75
76/// Commonly used items from this crate.
77pub mod prelude {
78    pub use crate::antibot::{AntiBotDetector, AntiBotSignals, DefaultAntiBotDetector};
79    pub use crate::autoscale::{
80        AimdController, AutoscaleMode, AutoscaledPool, AutoscaledPoolOptions, ClientLoadSignal,
81        ClientLoadSignalHandle, CpuLoadSignal, CpuLoadSignalOptions, LoadSignal, LoadSnapshot,
82        MemoryLoadSignal, MemoryLoadSignalOptions, ScaleDecision, Snapshotter, SnapshotterOptions,
83        SystemStatus, SystemStatusOptions, TokioRuntimeLoadSignal, TokioRuntimeLoadSignalOptions,
84    };
85    pub use crate::config::{ConfigError, Configuration, ConfigurationBuilder, LogLevel};
86    pub use crate::cookies::{Cookie, CookieJar, CookieJarError, SameSite};
87    pub use crate::crawler::{
88        AttemptObservation, AutoscalerSnapshot, BasicContext, BasicCrawler, BasicKind, Crawler,
89        CrawlerBuildError, CrawlerBuilder, CrawlerEnv, CrawlerHandle, CrawlerKind,
90        IntoStartRequest, IntoStartRequests, RequestEnv, RequestOutcome, RequestPrep,
91    };
92    pub use crate::enqueue::{
93        EnqueueLinker, EnqueueLinksOptions, EnqueueResult, SkipReason, SkippedUrl,
94    };
95    pub use crate::errors::{AntiBotTech, CrawlError};
96    pub use crate::events::{
97        CrawlerEvent, EventBus, EventStream, HandledRequest, RequestFinalState, ResultStream,
98    };
99    pub use crate::handler::{
100        FailedRequestContext, FailedRequestHandler, Middleware, RequestHandler,
101    };
102    pub use crate::http_client::{
103        HttpClient, HttpClientError, HttpRequest, HttpResponse, HttpStatusError, StreamingResponse,
104    };
105    pub use crate::link_extraction::{
106        CrawlPolicy, EnqueueStrategy, ExtractedLink, GlobPattern, LinkExtractor, LinkPatternError,
107        SkippedHandler, TransformResult, UrlMatch, UrlPattern,
108    };
109    pub use crate::proxy::{
110        ProxyBuckets, ProxyConfiguration, ProxyInfo, ProxyKind, ProxyResolveContext, ProxyResolver,
111        ProxyRouteContext, ProxyStrategy, RotationStrategy,
112    };
113    pub use crate::request::{
114        HeaderMap, IntoUrl, Method, Request, RequestBody, RequestBuildError, RequestBuilder,
115        RequestId, RequestState, UserData,
116    };
117    pub use crate::retry_strategy::{
118        AttemptOutcome, AttemptOverrides, RetryDirective, RetryStrategy, SessionRetryAction,
119    };
120    pub use crate::router::{HasRequest, MethodFilter, Router};
121    pub use crate::session::{
122        SESSION_POOL_PERSIST_KEY, Session, SessionConfig, SessionId, SessionPool,
123        SessionPoolOptions, SessionToken,
124    };
125    pub use crate::sitemap::{
126        RequestQueueWithSitemap, SITEMAP_STATE_KEY, SitemapEntry, SitemapRequestList,
127        SitemapRequestListBuilder,
128    };
129    pub use crate::snapshot::{ErrorSnapshot, ErrorSnapshotter};
130    pub use crate::statistics::{
131        FinalStatistics, STATISTICS_PERSIST_KEY, StatisticsHandle, StatisticsSnapshot,
132    };
133    pub use crate::storage::{
134        AddOptions, AddRequestsBatchedResult, AutoSaved, BatchAddHandle, Dataset, DatasetExt,
135        DatasetInfo, KeyInfo, KeyList, KeyValueStore, KeyValueStoreExt, KvEntry, Lease, LeaseId,
136        ListKeysOptions, ListOptions, Page, ProcessedRequest, QueueOpInfo,
137        RateLimitReportingClient, ReclaimOptions, RequestQueue, RequestSource, StorageClient,
138        StorageError, StorageHandle, StorageResult,
139    };
140}
millipede_core/lib.rs

millipede_core/
lib.rs
