sim_codec_robots/
lib.rs

1//! RFC 9309 robots.txt records and deterministic path matching, without HTTP policy.
2#![forbid(unsafe_code)]
3#![deny(missing_docs)]
4
5mod matching;
6
7use matching::*;
8
9/// Stable runtime codec symbol.
10pub const CODEC_SYMBOL: &str = "codec/robots";
11/// Stable accepted media-type alias.
12pub const MEDIA_TYPES: &[&str] = &["text/plain"];
13/// RFC 9309 requires parsers to accept at least this many octets.
14pub const RFC_MINIMUM_BYTES: usize = 500 * 1024;
15/// Embedded pure recipes.
16pub static RECIPES: sim_cookbook::EmbeddedDir =
17    include!(concat!(env!("OUT_DIR"), "/cookbook_recipes.rs"));
18/// Redirect observations supplied by a transport owner, retained only as metadata.
19#[derive(Clone, Debug, PartialEq, Eq)]
20pub struct RedirectMetadata {
21    /// Redirect count.
22    pub hops: usize,
23    /// Final claimed URL.
24    pub final_url: Option<String>,
25}
26/// Rule disposition.
27#[derive(Clone, Copy, Debug, PartialEq, Eq)]
28pub enum RuleKind {
29    /// Permit matching paths.
30    Allow,
31    /// Deny matching paths.
32    Disallow,
33}
34/// One path rule.
35#[derive(Clone, Debug, PartialEq, Eq)]
36pub struct Rule {
37    /// Kind.
38    pub kind: RuleKind,
39    /// Original pattern.
40    pub pattern: String,
41}
42/// A group of user-agent products and rules.
43#[derive(Clone, Debug, PartialEq, Eq)]
44pub struct Group {
45    /// Case-insensitive product tokens.
46    pub user_agents: Vec<String>,
47    /// Ordered source rules.
48    pub rules: Vec<Rule>,
49}
50/// Parsed robots data.
51#[derive(Clone, Debug, PartialEq, Eq)]
52pub struct RobotsDoc {
53    /// Groups.
54    pub groups: Vec<Group>,
55    /// Sitemap URL claims.
56    pub sitemaps: Vec<String>,
57    /// Optional externally supplied redirect metadata.
58    pub redirect: Option<RedirectMetadata>,
59    /// Decode warnings.
60    pub warnings: Vec<String>,
61}
62/// Parse limits. The default input ceiling honors the RFC floor.
63#[derive(Clone, Debug, PartialEq, Eq)]
64pub struct RobotsLimits {
65    /// Byte ceiling, never less than 500 KiB.
66    pub max_input_bytes: usize,
67    /// Line ceiling.
68    pub max_lines: usize,
69    /// Rule ceiling.
70    pub max_rules: usize,
71}
72impl Default for RobotsLimits {
73    fn default() -> Self {
74        Self {
75            max_input_bytes: RFC_MINIMUM_BYTES,
76            max_lines: 100_000,
77            max_rules: 50_000,
78        }
79    }
80}
81/// Parse failure.
82#[derive(Clone, Debug, PartialEq, Eq)]
83pub struct RobotsError(pub String);
84impl std::fmt::Display for RobotsError {
85    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
86        f.write_str(&self.0)
87    }
88}
89impl std::error::Error for RobotsError {}
90
91/// Parse robots bytes as inert records. HTTP status and redirect policy remain outside this crate.
92pub fn parse_robots(
93    input: &[u8],
94    limits: &RobotsLimits,
95    redirect: Option<RedirectMetadata>,
96) -> Result<RobotsDoc, RobotsError> {
97    let ceiling = limits.max_input_bytes.max(RFC_MINIMUM_BYTES);
98    if input.len() > ceiling {
99        return Err(RobotsError("robots input byte limit exceeded".into()));
100    }
101    let text = String::from_utf8_lossy(input);
102    let mut groups = Vec::new();
103    let mut agents = Vec::new();
104    let mut rules = Vec::new();
105    let mut sitemaps = Vec::new();
106    let mut warnings = Vec::new();
107    let mut count = 0;
108    for (line_no, raw) in text.lines().enumerate() {
109        if line_no >= limits.max_lines {
110            return Err(RobotsError("robots line limit exceeded".into()));
111        }
112        let line = raw.split('#').next().unwrap_or("").trim();
113        if line.is_empty() {
114            continue;
115        }
116        let Some((field, value)) = line.split_once(':') else {
117            warnings.push(format!("line {} has no field separator", line_no + 1));
118            continue;
119        };
120        let field = field.trim().to_ascii_lowercase();
121        let value = value.trim();
122        match field.as_str() {
123            "user-agent" => {
124                if !rules.is_empty() {
125                    groups.push(Group {
126                        user_agents: std::mem::take(&mut agents),
127                        rules: std::mem::take(&mut rules),
128                    });
129                }
130                agents.push(value.to_ascii_lowercase())
131            }
132            "allow" | "disallow" => {
133                if agents.is_empty() {
134                    warnings.push(format!("line {} rule precedes user-agent", line_no + 1));
135                    continue;
136                }
137                if field == "disallow" && value.is_empty() {
138                    continue;
139                }
140                count += 1;
141                if count > limits.max_rules {
142                    return Err(RobotsError("robots rule limit exceeded".into()));
143                }
144                rules.push(Rule {
145                    kind: if field == "allow" {
146                        RuleKind::Allow
147                    } else {
148                        RuleKind::Disallow
149                    },
150                    pattern: value.to_owned(),
151                })
152            }
153            "sitemap" => sitemaps.push(value.to_owned()),
154            _ => warnings.push(format!("unknown robots field {field}")),
155        }
156    }
157    if !agents.is_empty() {
158        groups.push(Group {
159            user_agents: agents,
160            rules,
161        });
162    }
163    if std::str::from_utf8(input).is_err() {
164        warnings.push("invalid UTF-8 replaced during decode".into())
165    }
166    Ok(RobotsDoc {
167        groups,
168        sitemaps,
169        redirect,
170        warnings,
171    })
172}
173impl RobotsDoc {
174    /// Decide access for a product token and URL path. No matching rule means allow.
175    pub fn allows(&self, product: &str, path: &str) -> bool {
176        let product = product.to_ascii_lowercase();
177        let mut best_agent = 0;
178        let mut chosen = Vec::new();
179        for g in &self.groups {
180            let specificity = g
181                .user_agents
182                .iter()
183                .filter_map(|a| {
184                    if a == "*" {
185                        Some(0)
186                    } else if product.contains(a) {
187                        Some(a.len())
188                    } else {
189                        None
190                    }
191                })
192                .max();
193            if let Some(n) = specificity {
194                if n > best_agent {
195                    best_agent = n;
196                    chosen.clear();
197                }
198                if n == best_agent {
199                    chosen.push(g)
200                }
201            }
202        }
203        let normalized = normalize_percent(path);
204        let mut winner: Option<(usize, RuleKind)> = None;
205        for g in chosen {
206            for r in &g.rules {
207                if pattern_matches(&normalize_percent(&r.pattern), &normalized) {
208                    let n = match_len(&r.pattern);
209                    match winner {
210                        None => winner = Some((n, r.kind)),
211                        Some((old, _)) if n > old => winner = Some((n, r.kind)),
212                        Some((old, RuleKind::Disallow))
213                            if n == old && r.kind == RuleKind::Allow =>
214                        {
215                            winner = Some((n, r.kind))
216                        }
217                        _ => {}
218                    }
219                }
220            }
221        }
222        winner.is_none_or(|(_, k)| k == RuleKind::Allow)
223    }
224}
225#[cfg(test)]
226mod tests {
227    // conformance: robots rules apply RFC precedence and matching boundaries.
228    use super::*;
229    #[test]
230    fn precedence_table() {
231        let d=parse_robots(b"User-agent: *\nDisallow: /fish\nAllow: /fish$\nDisallow: /fish*heads\nAllow: /fishheads\n",&Default::default(),None).unwrap();
232        let cases = [
233            ("/", true),
234            ("/fish", true),
235            ("/fish/", false),
236            ("/fishheads", true),
237            ("/fishXYZheads", false),
238        ];
239        for (c, want) in cases {
240            assert_eq!(d.allows("bot", c), want, "{c}")
241        }
242    }
243    #[test]
244    fn groups_case_and_percent() {
245        let d=parse_robots(b"User-agent: Bot\nDisallow: /a%2fb\nUser-agent: *\nAllow: /\nSitemap: https://e/s.xml\n",&Default::default(),None).unwrap();
246        assert!(!d.allows("MyBOT", "/a%2Fb"));
247        assert!(d.allows("other", "/a%2Fb"));
248        assert_eq!(d.sitemaps.len(), 1)
249    }
250}
sim_codec_robots/lib.rs

sim_codec_robots/
lib.rs
