crawlex 1.0.4

Stealth crawler with Chrome-perfect TLS/H2 fingerprint, render pool, hooks, persistent queue
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

// robots.txt fetch/cache/evaluate per host.
// Uses `texting_robots` for RFC-compliant parsing. TODO: wire into Crawler.

use dashmap::DashMap;
use std::sync::Arc;
use std::time::{Duration, Instant};
use texting_robots::Robot;
use url::Url;

use crate::Result;

pub struct RobotsCache {
    ttl: Duration,
    inner: DashMap<String, (Instant, Arc<Option<Robot>>)>,
}

impl RobotsCache {
    pub fn new(ttl: Duration) -> Self {
        Self {
            ttl,
            inner: DashMap::new(),
        }
    }

    pub fn check(&self, url: &Url, user_agent: &str) -> Option<bool> {
        let host = url.host_str()?;
        let entry = self.inner.get(host)?;
        let (inserted, robot) = entry.value();
        if inserted.elapsed() > self.ttl {
            return None;
        }
        robot
            .as_ref()
            .as_ref()
            .map(|r| r.allowed(url.as_str()))
            .or(Some(true))
            .map(|allowed| {
                let _ = user_agent;
                allowed
            })
    }

    pub fn store(&self, host: &str, txt: Option<&str>, user_agent: &str) -> Result<()> {
        let robot = txt.and_then(|t| Robot::new(user_agent, t.as_bytes()).ok());
        self.inner
            .insert(host.to_string(), (Instant::now(), Arc::new(robot)));
        Ok(())
    }
}