robotparser-fork 0.11.0

robots.txt parser for Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54

use crate::model::{Group, RobotsTxt};
enum State {
    WaitingForNewGroup,
    WaitingForAdditionalUserAgent,
}

pub struct GroupBuilder {
    state: State,
    active_group: Option<usize>,
    groups: Vec<Group>,
}

impl GroupBuilder {
    pub fn new() -> GroupBuilder {
        GroupBuilder {
            state: State::WaitingForNewGroup,
            active_group: None,
            groups: Vec::new(),
        }
    }

    pub fn handle_user_agent(&mut self, user_agent: &str) {
        match self.state {
            State::WaitingForNewGroup => {
                let mut group = Group::new();
                group.push_useragent(user_agent);
                self.groups.push(group);
                self.active_group = Some(self.groups.len() - 1);
                self.state = State::WaitingForAdditionalUserAgent;
            }
            State::WaitingForAdditionalUserAgent => {
                let active_group = self.active_group.expect("Unable to get active group");
                let group = self.groups.get_mut(active_group).expect("Unable to get group index");
                if !group.contains_user_agent(user_agent) {
                    group.push_useragent(user_agent);
                }
            }
        }
    }

    pub fn get_mut_active_group(&mut self) -> Option<&mut Group> {
        self.state = State::WaitingForNewGroup;
        if let Some(active_group) = self.active_group {
            return self.groups.get_mut(active_group);
        }
        None
    }

    pub fn fill_entries(mut self, robots_txt: &mut RobotsTxt) {
        for group in self.groups.drain(..) {
            robots_txt.add_group(group);
        }
    }
}