mocra 0.4.0

A distributed, event-driven crawling and data collection framework
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135

//! mocra 的公开 API 契约 —— `use mocra::prelude::*` 一站式引入。
//!
//! 分层:
//! - **核心门面**:`Spider` / `Mocra` / `DataSink` / `on_item` / `Ctx` / `Seeds`
//!   + `Request` / `Response` / `Result` —— 80% 场景只需这些。
//! - **集群**(`cluster-embedded`):`ClusterConfig` / `RaftTuning`。
//! - **后台管理 / 监控**(`dashboard`):`prelude::dashboard`(`EngineStats` / `ClusterStatusView`),
//!   配合门面 `Mocra::builder().dashboard(port)` 暴露 admin + 可观测 HTTP API。
//! - **扩展点**:自定义后端时实现 `queue::MqBackend` / `sync::CoordinationBackend` /
//!   `common::ModuleTrait` / `common::DataMiddleware` 等。
//!
//! 进阶 / 扩展类型在子模块 `prelude::{engine, queue, sync, schedule, common, utils, proxy,
//! cacheable, errors}`;内部实现细节已收敛为 `pub(crate)`,不进公开 API。

// High-level Spider facade (重构 Phase 1)
#[cfg(feature = "cluster-embedded")]
pub use crate::facade::ClusterConfig;
pub use crate::facade::{ChannelSink, Ctx, DataSink, Mocra, MocraBuilder, Seeds, Spider, on_item};
#[cfg(feature = "cluster-embedded")]
pub use mocra_cluster::RaftTuning;

// Common Traits and Structs
pub use crate::common::interface::{
    DataMiddleware, DataStoreMiddleware, DownloadMiddleware, MiddlewareManager, ModuleNodeTrait,
    ModuleTrait, StoreTrait, SyncBoxStream,
};
pub use crate::common::model::data::DataEvent;
pub use crate::common::model::login_info::LoginInfo;
pub use crate::common::model::message::{TaskOutputEvent, TaskParserEvent};
pub use crate::common::model::request::RequestMethod;
pub use crate::common::model::{Cookies, ExecutionMark, Headers, ModuleConfig, Request, Response};

// Errors
pub use crate::errors::{BoxError, Error, ErrorKind, ParserError, RequestError, Result};

// Utils
pub use crate::utils::date_utils::DateUtils;

pub mod common {
    pub use crate::common::interface::DataMiddleware;
    pub use crate::common::interface::DataStoreMiddleware;
    pub use crate::common::interface::DownloadMiddleware;
    pub use crate::common::interface::MiddlewareManager;
    pub use crate::common::interface::ModuleNodeTrait;
    pub use crate::common::interface::ModuleTrait;
    pub use crate::common::interface::StoreTrait;
    pub use crate::common::interface::SyncBoxStream;
    pub use crate::common::interface::module::ToSyncBoxStream;

    pub use crate::common::model::Cookies;
    pub use crate::common::model::ExecutionMark;
    pub use crate::common::model::Headers;
    pub use crate::common::model::ModuleConfig;
    pub use crate::common::model::Request;
    pub use crate::common::model::Response;
}
pub mod downloader {
    /// Needed to implement [`Downloader::set_config`] in a custom downloader.
    pub use crate::common::model::download_config::DownloadConfig;
    pub use crate::downloader::Downloader;
    pub use crate::downloader::DownloaderManager;
    pub use crate::downloader::WebSocketDownloader;
}
pub mod engine {
    pub use crate::engine::engine::Engine;
}
/// 后台管理 / 监控可观测数据类型(需 `dashboard` 特性)。供后台管理页面 / 客户端消费。
#[cfg(feature = "dashboard")]
pub mod dashboard {
    pub use crate::engine::api::observability::{EngineStats, PendingBreakdown};
    pub use crate::engine::monitor::SystemSnapshot;
    pub use crate::sync::ClusterStatusView;
    pub use crate::utils::logger::LogRecord;
}
pub mod queue {
    pub use crate::queue::Channel;
    pub use crate::queue::Compensator;
    pub use crate::queue::Identifiable;
    pub use crate::queue::Message;
    pub use crate::queue::MqBackend;
    pub use crate::queue::QueueManager;
    pub use crate::queue::RedisCompensator;
    pub use crate::queue::RedisQueue;
}
pub mod schedule {
    pub use crate::schedule::Dag;
    pub use crate::schedule::DagError;
    pub use crate::schedule::DagScheduler;
    pub use crate::schedule::TaskPayload;
}
pub mod sync {
    pub use crate::sync::ClusterStatusView;
    pub use crate::sync::CoordinationBackend;
    pub use crate::sync::DistributedSync;
    #[cfg(feature = "queue-kafka")]
    pub use crate::sync::KafkaBackend;
    pub use crate::sync::RedisBackend;
    pub use crate::sync::SyncAble;
    pub use crate::sync::SyncService;
}
pub mod utils {
    pub use crate::utils::date_utils::DateUtils;
    pub use crate::utils::distributed_rate_limit::{
        DistributedSlidingWindowRateLimiter, RateLimitConfig,
    };
    pub use crate::utils::redis_lock::DistributedLockManager;
}
pub mod proxy {
    pub use crate::proxy::*;
}
pub mod errors {
    pub use crate::errors::BoxError;
    pub use crate::errors::Error;
    pub use crate::errors::ErrorKind;
    pub use crate::errors::ParserError;
    pub use crate::errors::RequestError;
    pub use crate::errors::Result;
}
pub mod cacheable {
    pub use crate::cacheable::CacheAble;
    pub use crate::cacheable::CacheService;
}

#[cfg(feature = "polars")]
pub mod polars {
    pub mod polars {
        pub use ::polars::*;
    }
    pub mod polars_lazy {
        pub use ::polars_lazy::*;
    }
    pub mod polars_ops {
        pub use ::polars_ops::*;
    }
}