turboscraper 0.1.1

A high-performance, concurrent web scraping framework for Rust with built-in support for retries, storage backends, and concurrent request handling
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51

use super::{base::StorageBackend, factory::Storage, StorageCategory, StorageConfig};
use crate::ScraperResult;
use std::collections::HashMap;

#[derive(Clone)]
pub struct StorageManager {
    storages: HashMap<StorageCategory, (Storage, Box<dyn StorageConfig>)>,
    default_storage: StorageCategory,
}

impl Default for StorageManager {
    fn default() -> Self {
        Self::new()
    }
}

impl StorageManager {
    pub fn new() -> Self {
        Self {
            storages: HashMap::new(),
            default_storage: StorageCategory::default(),
        }
    }

    pub fn register_storage(
        mut self,
        category: StorageCategory,
        storage: Storage,
        destination: &str,
    ) -> Self {
        let config = storage.create_config(destination);
        self.storages.insert(category.clone(), (storage, config));

        self
    }

    pub fn set_default_storage(mut self, category: StorageCategory) -> ScraperResult<Self> {
        self.default_storage = category;
        Ok(self)
    }

    pub fn get_storage(&self, category: &StorageCategory) -> &(Storage, Box<dyn StorageConfig>) {
        self.storages
            .get(category)
            .unwrap_or_else(|| self.get_default_storage())
    }

    pub fn get_default_storage(&self) -> &(Storage, Box<dyn StorageConfig>) {
        self.storages.get(&self.default_storage).unwrap()
    }
}