crawlrs 0.2.0

crawlrs - Enterprise-grade web scraping and crawling platform built with Rust
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
// Copyright (c) 2025-2026 Kirky.X🌠
// SPDX-License-Identifier: Apache-2.0

//! DRL (Deep Reinforcement Learning) 自适应爬取策略
//!
//! 使用预训练的 ONNX 模型根据爬取状态预测最优动作:
//! - URL 优先级调整
//! - 并发度变化
//! - 引擎选择
//! - 重试决策
//!
//! 模型通过 ONNX Runtime 推理。不可用时退化为启发式规则。

use serde::{Deserialize, Serialize};

/// 爬取状态(DRL 模型输入)
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CrawlState {
    /// 当前队列深度(待处理 URL 数)
    pub queue_depth: u32,
    /// 域名平均响应时间(毫秒)
    pub domain_response_time_avg: f32,
    /// 爬取成功率(0.0-1.0)
    pub success_rate: f32,
    /// 内存压力(0.0-1.0,当前使用/总量)
    pub memory_pressure: f32,
    /// 剩余预算比例(0.0-1.0)
    pub budget_remaining: f32,
}

impl CrawlState {
    /// 将状态转为特征向量(用于模型推理)
    pub fn to_feature_vector(&self) -> Vec<f32> {
        vec![
            self.queue_depth as f32 / 1000.0, // 归一化
            self.domain_response_time_avg / 10000.0,
            self.success_rate,
            self.memory_pressure,
            self.budget_remaining,
        ]
    }

    /// 验证状态有效性
    pub fn validate(&self) -> Result<(), String> {
        if self.success_rate < 0.0 || self.success_rate > 1.0 {
            return Err(format!("success_rate out of range: {}", self.success_rate));
        }
        if self.memory_pressure < 0.0 || self.memory_pressure > 1.0 {
            return Err(format!(
                "memory_pressure out of range: {}",
                self.memory_pressure
            ));
        }
        if self.budget_remaining < 0.0 || self.budget_remaining > 1.0 {
            return Err(format!(
                "budget_remaining out of range: {}",
                self.budget_remaining
            ));
        }
        Ok(())
    }
}

impl Default for CrawlState {
    fn default() -> Self {
        Self {
            queue_depth: 0,
            domain_response_time_avg: 0.0,
            success_rate: 1.0,
            memory_pressure: 0.0,
            budget_remaining: 1.0,
        }
    }
}

/// 爬取动作(DRL 模型输出)
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CrawlAction {
    /// URL 优先级调整因子(0.5-2.0,1.0 = 不变)
    pub url_priority_adjustment: f32,
    /// 并发度变化(-3 到 +3,0 = 不变)
    pub concurrency_delta: i32,
    /// 引擎选择偏好(0 = reqwest, 1 = playwright, 2 = tls_fingerprint, 3 = mllm)
    pub engine_selection: u8,
    /// 重试决策(0 = 不重试, 1 = 立即重试, 2 = 延迟重试)
    pub retry_decision: u8,
}

impl CrawlAction {
    /// 默认动作(不调整)
    pub fn noop() -> Self {
        Self {
            url_priority_adjustment: 1.0,
            concurrency_delta: 0,
            engine_selection: 0,
            retry_decision: 0,
        }
    }

    /// 保守动作(降低并发,提高优先级)
    pub fn conservative() -> Self {
        Self {
            url_priority_adjustment: 1.2,
            concurrency_delta: -1,
            engine_selection: 0,
            retry_decision: 2,
        }
    }

    /// 激进动作(增加并发,降低优先级阈值)
    pub fn aggressive() -> Self {
        Self {
            url_priority_adjustment: 0.8,
            concurrency_delta: 2,
            engine_selection: 0,
            retry_decision: 1,
        }
    }
}

impl Default for CrawlAction {
    fn default() -> Self {
        Self::noop()
    }
}

/// ONNX 推理 trait(抽象模型推理,便于测试和替换)
pub trait OnnxInference: Send + Sync {
    /// 运行推理:输入特征向量 → 输出动作
    fn predict(&self, features: &[f32]) -> Result<Vec<f32>, String>;
}

/// 启发式策略(无 ONNX 模型时的退化实现)
pub struct HeuristicPolicy;

impl OnnxInference for HeuristicPolicy {
    fn predict(&self, features: &[f32]) -> Result<Vec<f32>, String> {
        if features.len() < 5 {
            return Err(format!("expected 5 features, got {}", features.len()));
        }

        let _queue_depth = features[0];
        let response_time = features[1];
        let success_rate = features[2];
        let memory_pressure = features[3];
        let budget_remaining = features[4];

        // 启发式规则
        let priority_adj = if success_rate < 0.5 {
            1.5 // 成功率低 → 提高优先级阈值
        } else if success_rate > 0.9 {
            0.8 // 成功率高 → 降低阈值,加速爬取
        } else {
            1.0
        };

        let concurrency = if memory_pressure > 0.8 {
            -2 // 内存压力高 → 减少并发
        } else if success_rate > 0.8 && budget_remaining > 0.5 {
            1 // 状态好 → 增加并发
        } else {
            0
        };

        let engine = if response_time > 0.5 {
            1 // 响应慢 → 换用 playwright
        } else {
            0 // 正常 → reqwest
        };

        let retry = if success_rate < 0.3 {
            2 // 成功率很低 → 延迟重试
        } else if success_rate < 0.6 {
            1 // 成功率中等 → 立即重试
        } else {
            0 // 成功率高 → 不重试
        };

        Ok(vec![
            priority_adj,
            concurrency as f32,
            engine as f32,
            retry as f32,
        ])
    }
}

/// DRL 策略
///
/// 加载预训练 ONNX 模型,根据爬取状态预测最优动作。
/// 模型不可用时退化为启发式规则。
pub struct DrlPolicy {
    inference: Box<dyn OnnxInference>,
    /// 是否启用(默认 false)
    enabled: bool,
}

impl DrlPolicy {
    /// 使用指定推理后端创建策略
    pub fn new(inference: Box<dyn OnnxInference>, enabled: bool) -> Self {
        Self { inference, enabled }
    }

    /// 创建启发式策略(无 ONNX 模型)
    pub fn heuristic(enabled: bool) -> Self {
        Self {
            inference: Box::new(HeuristicPolicy),
            enabled,
        }
    }

    /// 预测最优动作
    pub fn predict(&self, state: &CrawlState) -> CrawlAction {
        if !self.enabled {
            return CrawlAction::noop();
        }

        if let Err(e) = state.validate() {
            log::warn!(
                "Invalid crawl state for DRL prediction: {}, using default",
                e
            );
            return CrawlAction::noop();
        }

        let features = state.to_feature_vector();
        match self.inference.predict(&features) {
            Ok(output) => self.decode_output(&output),
            Err(e) => {
                log::error!("DRL inference failed: {}, using default action", e);
                CrawlAction::noop()
            }
        }
    }

    /// 解码模型输出为动作
    fn decode_output(&self, output: &[f32]) -> CrawlAction {
        if output.len() < 4 {
            log::warn!("DRL output too short ({} < 4), using default", output.len());
            return CrawlAction::noop();
        }

        CrawlAction {
            url_priority_adjustment: output[0].clamp(0.5, 2.0),
            concurrency_delta: output[1].round().clamp(-3.0, 3.0) as i32,
            engine_selection: output[2].round().clamp(0.0, 3.0) as u8,
            retry_decision: output[3].round().clamp(0.0, 2.0) as u8,
        }
    }

    /// 是否启用
    pub fn is_enabled(&self) -> bool {
        self.enabled
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    // Mock ONNX Inference

    struct MockInference {
        output: Vec<f32>,
    }

    impl OnnxInference for MockInference {
        fn predict(&self, _features: &[f32]) -> Result<Vec<f32>, String> {
            Ok(self.output.clone())
        }
    }

    struct FailingInference;

    impl OnnxInference for FailingInference {
        fn predict(&self, _features: &[f32]) -> Result<Vec<f32>, String> {
            Err("Mock inference failure".to_string())
        }
    }

    // 测试

    #[test]
    fn test_crawl_state_to_feature_vector() {
        let state = CrawlState {
            queue_depth: 500,
            domain_response_time_avg: 2000.0,
            success_rate: 0.8,
            memory_pressure: 0.5,
            budget_remaining: 0.7,
        };

        let features = state.to_feature_vector();
        assert_eq!(features.len(), 5);
        assert!((features[0] - 0.5).abs() < 0.01); // 500/1000
        assert!((features[1] - 0.2).abs() < 0.01); // 2000/10000
    }

    #[test]
    fn test_crawl_state_validate_valid() {
        let state = CrawlState::default();
        assert!(state.validate().is_ok());
    }

    #[test]
    fn test_crawl_state_validate_invalid_success_rate() {
        let state = CrawlState {
            success_rate: 1.5,
            ..Default::default()
        };
        assert!(state.validate().is_err());
    }

    #[test]
    fn test_crawl_state_validate_invalid_memory_pressure() {
        let state = CrawlState {
            memory_pressure: -0.1,
            ..Default::default()
        };
        assert!(state.validate().is_err());
    }

    #[test]
    fn test_drl_predict_with_mock() {
        let mock = MockInference {
            output: vec![1.5, 2.0, 1.0, 1.0],
        };
        let policy = DrlPolicy::new(Box::new(mock), true);
        let state = CrawlState::default();

        let action = policy.predict(&state);
        assert_eq!(action.url_priority_adjustment, 1.5);
        assert_eq!(action.concurrency_delta, 2);
        assert_eq!(action.engine_selection, 1);
        assert_eq!(action.retry_decision, 1);
    }

    #[test]
    fn test_drl_predict_disabled_returns_noop() {
        let mock = MockInference {
            output: vec![1.5, 2.0, 1.0, 1.0],
        };
        let policy = DrlPolicy::new(Box::new(mock), false);
        let state = CrawlState::default();

        let action = policy.predict(&state);
        assert_eq!(action.url_priority_adjustment, 1.0);
        assert_eq!(action.concurrency_delta, 0);
    }

    #[test]
    fn test_drl_predict_invalid_state_returns_noop() {
        let mock = MockInference {
            output: vec![1.5, 2.0, 1.0, 1.0],
        };
        let policy = DrlPolicy::new(Box::new(mock), true);
        let state = CrawlState {
            success_rate: 2.0, // invalid
            ..Default::default()
        };

        let action = policy.predict(&state);
        assert_eq!(action.url_priority_adjustment, 1.0);
    }

    #[test]
    fn test_drl_predict_inference_failure_returns_noop() {
        let policy = DrlPolicy::new(Box::new(FailingInference), true);
        let state = CrawlState::default();

        let action = policy.predict(&state);
        assert_eq!(action.url_priority_adjustment, 1.0);
    }

    #[test]
    fn test_drl_predict_output_clamped() {
        let mock = MockInference {
            output: vec![5.0, 10.0, 255.0, 100.0], // 全部超范围
        };
        let policy = DrlPolicy::new(Box::new(mock), true);
        let state = CrawlState::default();

        let action = policy.predict(&state);
        assert_eq!(action.url_priority_adjustment, 2.0); // clamped to max
        assert_eq!(action.concurrency_delta, 3); // clamped to max
        assert_eq!(action.engine_selection, 3); // clamped to max
        assert_eq!(action.retry_decision, 2); // clamped to max
    }

    #[test]
    fn test_heuristic_policy_low_success_rate() {
        let policy = DrlPolicy::heuristic(true);
        let state = CrawlState {
            queue_depth: 100,
            domain_response_time_avg: 500.0,
            success_rate: 0.2, // 低于 0.3 → 延迟重试
            memory_pressure: 0.3,
            budget_remaining: 0.8,
        };

        let action = policy.predict(&state);
        assert!(
            action.url_priority_adjustment > 1.0,
            "low success rate should increase priority"
        );
        assert_eq!(
            action.retry_decision, 2,
            "very low success rate should use delayed retry"
        );
    }

    #[test]
    fn test_heuristic_policy_high_memory_pressure() {
        let policy = DrlPolicy::heuristic(true);
        let state = CrawlState {
            queue_depth: 100,
            domain_response_time_avg: 500.0,
            success_rate: 0.9,
            memory_pressure: 0.9,
            budget_remaining: 0.8,
        };

        let action = policy.predict(&state);
        assert!(
            action.concurrency_delta < 0,
            "high memory should reduce concurrency"
        );
    }

    #[test]
    fn test_crawl_action_noop() {
        let action = CrawlAction::noop();
        assert_eq!(action.url_priority_adjustment, 1.0);
        assert_eq!(action.concurrency_delta, 0);
        assert_eq!(action.engine_selection, 0);
        assert_eq!(action.retry_decision, 0);
    }

    #[test]
    fn test_drl_short_output_returns_noop() {
        let mock = MockInference {
            output: vec![1.0], // too short
        };
        let policy = DrlPolicy::new(Box::new(mock), true);
        let state = CrawlState::default();

        let action = policy.predict(&state);
        assert_eq!(
            action.url_priority_adjustment, 1.0,
            "short output should default to noop"
        );
    }
}