robotparser 0.11.0

robots.txt parser for Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136

use super::line::Line;
use super::warning_reason::WarningReason;
use std::error::Error;
use std::fmt;
use std::num::{ParseFloatError, ParseIntError};
use url::ParseError as ParseUrlError;

#[derive(Clone, Debug)]
/// Warning of robots.txt parser about problems when parsing robots.txt file.
pub struct ParseWarning {
    line_no: usize,
    line: String,
    reason: WarningReason,
}

impl Error for ParseWarning {}

impl ParseWarning {
    /// Returns the line number in the text of the robots.txt file.
    pub fn get_line_no(&self) -> usize {
        self.line_no
    }

    /// Returns the text of the robots.txt file string.
    pub fn get_line_text(&self) -> &String {
        &self.line
    }

    /// Returns the reason of warning.
    pub fn get_reason(&self) -> &WarningReason {
        &self.reason
    }

    pub(crate) fn invalid_directive_format(line: &Line) -> ParseWarning {
        ParseWarning {
            line_no: line.get_line_number(),
            line: line.get_line_text().into(),
            reason: WarningReason::InvalidDirectiveFormat,
        }
    }

    pub(crate) fn directive_key_is_empty(line: &Line) -> ParseWarning {
        ParseWarning {
            line_no: line.get_line_number(),
            line: line.get_line_text().into(),
            reason: WarningReason::DirectiveKeyIsEmpty,
        }
    }

    pub(crate) fn unsupported_directive_key(line: &Line, key: String) -> ParseWarning {
        ParseWarning {
            line_no: line.get_line_number(),
            line: line.get_line_text().into(),
            reason: WarningReason::UnsupportedDirectiveKey(key),
        }
    }

    pub(crate) fn user_agent_cannot_be_empty(line: &Line) -> ParseWarning {
        ParseWarning {
            line_no: line.get_line_number(),
            line: line.get_line_text().into(),
            reason: WarningReason::UserAgentCannotBeEmpty,
        }
    }

    pub(crate) fn wrong_path_format(line: &Line) -> ParseWarning {
        ParseWarning {
            line_no: line.get_line_number(),
            line: line.get_line_text().into(),
            reason: WarningReason::WrongPathFormat,
        }
    }

    pub(crate) fn directive_without_user_agent(line: &Line) -> ParseWarning {
        ParseWarning {
            line_no: line.get_line_number(),
            line: line.get_line_text().into(),
            reason: WarningReason::DirectiveWithoutUserAgent,
        }
    }

    pub(crate) fn parse_crawl_delay_error(line: &Line, error: ParseFloatError) -> ParseWarning {
        ParseWarning {
            line_no: line.get_line_number(),
            line: line.get_line_text().into(),
            reason: WarningReason::ParseCrawlDelayError(error),
        }
    }

    pub(crate) fn wrong_request_rate_format(line: &Line) -> ParseWarning {
        ParseWarning {
            line_no: line.get_line_number(),
            line: line.get_line_text().into(),
            reason: WarningReason::WrongRequestRateFormat,
        }
    }

    pub(crate) fn parse_request_rate(line: &Line, error: ParseIntError) -> ParseWarning {
        ParseWarning {
            line_no: line.get_line_number(),
            line: line.get_line_text().into(),
            reason: WarningReason::ParseRequestRate(error),
        }
    }

    pub(crate) fn parse_url(line: &Line, error: ParseUrlError) -> ParseWarning {
        ParseWarning {
            line_no: line.get_line_number(),
            line: line.get_line_text().into(),
            reason: WarningReason::ParseUrl(error),
        }
    }

    pub(crate) fn wrong_clean_param_format(line: &Line) -> ParseWarning {
        ParseWarning {
            line_no: line.get_line_number(),
            line: line.get_line_text().into(),
            reason: WarningReason::WrongCleanParamFormat,
        }
    }

    pub(crate) fn ignored_clean_params(line: &Line, ignored_clean_params: Vec<String>) -> ParseWarning {
        ParseWarning {
            line_no: line.get_line_number(),
            line: line.get_line_text().into(),
            reason: WarningReason::IgnoredCleanParams(ignored_clean_params),
        }
    }
}

/// Displays text of warning.
impl fmt::Display for ParseWarning {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        write!(f, "Line: {}. Text: `{}`. {}", self.line_no, self.line, self.reason)
    }
}