robotparser_fork/service/
fetched_robots_txt.rs

1use crate::model::RequestRate;
2use crate::model::{FetchedRobotsTxt, FetchedRobotsTxtContainer};
3use crate::service::RobotsTxtService;
4use std::time::Duration;
5use url::Url;
6
7impl RobotsTxtService for FetchedRobotsTxt {
8    fn can_fetch(&self, user_agent: &str, url: &Url) -> bool {
9        match *self.get_container() {
10            FetchedRobotsTxtContainer::FetchDenied => false,
11            FetchedRobotsTxtContainer::FetchFailed => true,
12            FetchedRobotsTxtContainer::Fetched(ref robots_txt) => robots_txt.can_fetch(user_agent, url),
13        }
14    }
15
16    fn get_crawl_delay(&self, user_agent: &str) -> Option<Duration> {
17        if let FetchedRobotsTxtContainer::Fetched(ref robots_txt) = *self.get_container() {
18            return robots_txt.get_crawl_delay(user_agent);
19        }
20        None
21    }
22
23    fn normalize_url(&self, url: &mut Url) -> bool {
24        if let FetchedRobotsTxtContainer::Fetched(ref robots_txt) = *self.get_container() {
25            return robots_txt.normalize_url(url);
26        }
27        true
28    }
29
30    fn normalize_url_ignore_origin(&self, url: &mut Url) {
31        if let FetchedRobotsTxtContainer::Fetched(ref robots_txt) = *self.get_container() {
32            robots_txt.normalize_url_ignore_origin(url);
33        }
34    }
35
36    fn get_sitemaps(&self) -> &[Url] {
37        if let FetchedRobotsTxtContainer::Fetched(ref robots_txt) = *self.get_container() {
38            return robots_txt.get_sitemaps();
39        }
40        &[]
41    }
42
43    fn get_req_rate(&self, user_agent: &str) -> Option<RequestRate> {
44        if let FetchedRobotsTxtContainer::Fetched(ref robots_txt) = *self.get_container() {
45            return robots_txt.get_req_rate(user_agent);
46        }
47        None
48    }
49}
robotparser_fork/service/fetched_robots_txt.rs

robotparser_fork/service/
fetched_robots_txt.rs
