mocra 0.5.0

A distributed, event-driven crawling and data collection framework
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54

//! Offline proxy-pool example: configure two direct proxies, select one, report a simulated
//! failed attempt, then select a different proxy for the retry. No network requests are made.
//!
//! Run: `cargo run --example proxy_pool`
//!
//! This uses `ProxyManager` as a standalone library. In a standalone TOML string passed to
//! `ProxyManager::from_config`, direct proxies use `[[direct]]`. The crawler's `config.toml`
//! uses `[[proxy.direct]]`; its module JSON config also needs `{"enable_proxy": true}`.

use mocra::prelude::proxy::{DirectProxy, PoolConfig, ProxyConfig, ProxyManager};

#[tokio::main]
async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
    let mut pool_config = PoolConfig::default();
    pool_config.health_check_interval_secs = 0; // No background probes in this offline example.

    let config = ProxyConfig {
        direct: Some(vec![
            DirectProxy {
                name: Some("proxy-a".into()),
                url: "http://127.0.0.1:18080".into(),
                rate_limit: Some(0.0),
                expire_time: None,
            },
            DirectProxy {
                name: Some("proxy-b".into()),
                url: "http://127.0.0.1:18081".into(),
                rate_limit: Some(0.0),
                expire_time: None,
            },
        ]),
        tunnel: None,
        ip_provider: None,
        pool_config: Some(pool_config),
    };

    let manager = ProxyManager::from_proxy_config(&config).await?;

    // The engine normally calls begin_proxy_attempt immediately before a real download and
    // reports the actual outcome afterwards. Here we simulate a failure to show the API flow.
    let first = manager.get_proxy_for_attempt(None).await?;
    manager.begin_proxy_attempt(&first).await?;
    manager.report_failure(&first).await?;
    println!("first attempt used {first}; simulated failure");

    // Exclude the failed proxy within the same retry budget.
    let next = manager.get_proxy_for_attempt(Some(&first)).await?;
    manager.begin_proxy_attempt(&next).await?;
    manager.report_success(&next, None).await?;
    println!("retry selected {next}; simulated success");
    println!("pool status: {:?}", manager.get_status().await);

    Ok(())
}