aniscraper 0.1.0

Rust library designed for efficient web scraping and data extraction. It simplifies the process of fetching, parsing, and extracting data from websites.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59

use rand::seq::SliceRandom;
use reqwest::Client;

use crate::{env::EnvVar, error::AniRustError};

// Define a struct to hold proxy data
#[derive(Debug, Clone)]
pub struct Proxy {
    pub address: String,
}

// Function to get a random proxy from the list
pub fn get_random_proxy(proxies: &[Proxy]) -> Option<Proxy> {
    proxies.choose(&mut rand::thread_rng()).cloned()
}

// Fetch proxy list from URL
pub async fn fetch_proxy_list(url: &str) -> Result<Vec<Proxy>, AniRustError> {
    let client = Client::new();
    let response = client.get(url).send().await?.text().await?;
    let proxies = response
        .lines()
        .filter_map(|line| {
            let line = line.trim();
            if !line.is_empty() {
                Some(Proxy {
                    address: line.to_string(),
                })
            } else {
                None
            }
        })
        .collect();
    Ok(proxies)
}

// Load proxies from multiple sources
pub async fn load_proxies() -> Result<Vec<Proxy>, AniRustError> {
    let sock5_url = EnvVar::SOCK5_URL.get_config();
    let sock4_url = EnvVar::SOCK4_URL.get_config();
    let http_url = EnvVar::HTTP_URL.get_config();

    let mut all_proxies = Vec::new();

    if !sock5_url.is_empty() {
        let sock5_proxies = fetch_proxy_list(&sock5_url).await?;
        all_proxies.extend(sock5_proxies);
    }
    if !sock4_url.is_empty() {
        let sock4_proxies = fetch_proxy_list(&sock4_url).await?;
        all_proxies.extend(sock4_proxies);
    }
    if !http_url.is_empty() {
        let http_proxies = fetch_proxy_list(&http_url).await?;
        all_proxies.extend(http_proxies);
    }

    Ok(all_proxies)
}