robotparser_fork/model/
robots_txt.rs

1use crate::model::clean_params::CleanParams;
2use crate::model::group::Group;
3use url::{Origin, Url};
4
5#[derive(Debug, Clone)]
6/// The robots.txt model that was obtained after parsing the text of the robots.txt file.
7/// To work with this model you should use the trait `robotparser::service::RobotsTxtService`.
8/// To create this structure you should use the `robotparser::parser::parse_robots_txt`.
9pub struct RobotsTxt {
10    origin: Origin,
11    groups: Vec<Group>,
12    sitemaps: Vec<Url>,
13    clean_params: Vec<CleanParams>,
14}
15
16impl RobotsTxt {
17    pub(crate) fn new(origin: Origin) -> RobotsTxt {
18        RobotsTxt {
19            origin,
20            groups: Vec::new(),
21            sitemaps: Vec::new(),
22            clean_params: Vec::new(),
23        }
24    }
25
26    pub(crate) fn add_sitemap(&mut self, url: Url) {
27        self.sitemaps.push(url);
28    }
29
30    pub(crate) fn get_sitemaps_slice(&self) -> &[Url] {
31        self.sitemaps.as_slice()
32    }
33
34    pub(crate) fn add_clean_params(&mut self, clean_params: CleanParams) {
35        self.clean_params.push(clean_params);
36    }
37
38    pub(crate) fn get_clean_params(&self) -> &[CleanParams] {
39        self.clean_params.as_slice()
40    }
41
42    pub(crate) fn add_group(&mut self, group: Group) {
43        self.groups.push(group);
44    }
45
46    pub(crate) fn get_origin(&self) -> &Origin {
47        &self.origin
48    }
49
50    pub(crate) fn find_in_group<'a, T>(
51        &'a self,
52        user_agent: &str,
53        callback: impl Fn(&'a Group) -> Option<T>,
54    ) -> Option<T> {
55        // Search by user agents
56        for group in self.groups.iter() {
57            if group.applies_to(user_agent) {
58                if let Some(output) = (callback)(group) {
59                    return Some(output);
60                }
61            }
62        }
63        if let Some(group) = self.get_default_group() {
64            if let Some(output) = (callback)(group) {
65                return Some(output);
66            }
67        }
68        None
69    }
70
71    pub(crate) fn get_default_group(&self) -> Option<&Group> {
72        for group in self.groups.iter() {
73            if group.is_default() {
74                return Some(group);
75            }
76        }
77        None
78    }
79}
robotparser_fork/model/robots_txt.rs

robotparser_fork/model/
robots_txt.rs
