crawlex 1.0.4

Stealth crawler with Chrome-perfect TLS/H2 fingerprint, render pool, hooks, persistent queue
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

use bytes::Bytes;
use dashmap::DashMap;
use http::HeaderMap;
use url::Url;

use crate::storage::{
    ArtifactStorage, ChallengeStorage, IntelStorage, PageMetadata, StateStorage, Storage,
    TelemetryStorage,
};
use crate::Result;

#[derive(Default)]
pub struct MemoryStorage {
    pub raw: DashMap<String, Bytes>,
    pub rendered: DashMap<String, String>,
    pub edges: DashMap<(String, String), u32>,
}

impl MemoryStorage {
    pub fn new() -> Self {
        Self::default()
    }
}

#[async_trait::async_trait]
impl ArtifactStorage for MemoryStorage {
    async fn save_raw(&self, url: &Url, _headers: &HeaderMap, body: &Bytes) -> Result<()> {
        self.raw.insert(url.to_string(), body.clone());
        Ok(())
    }
    async fn save_rendered(&self, url: &Url, html: &str, _meta: &PageMetadata) -> Result<()> {
        self.rendered.insert(url.to_string(), html.to_string());
        Ok(())
    }
    async fn save_edge(&self, from: &Url, to: &Url) -> Result<()> {
        *self
            .edges
            .entry((from.to_string(), to.to_string()))
            .or_insert(0) += 1;
        Ok(())
    }
}

impl StateStorage for MemoryStorage {}
impl ChallengeStorage for MemoryStorage {}
impl TelemetryStorage for MemoryStorage {}
impl IntelStorage for MemoryStorage {}

impl Storage for MemoryStorage {
    fn as_any_ref(&self) -> Option<&dyn std::any::Any> {
        Some(self)
    }
}