codoseo_diff/
robots.rs

1//! robots.txt comparison: rules matter, comments and `Sitemap:` lines don't.
2
3use codoseo_core::change::{Change, ChangeKind};
4use codoseo_core::check::Severity;
5use codoseo_core::crawl::RobotsFile;
6use xxhash_rust::xxh3::xxh3_64;
7
8/// Hash of the rules in a robots.txt body. Comments, blank lines and `Sitemap:` lines are
9/// dropped and field names are lower-cased, so edits that don't change what bots may do
10/// don't change the fingerprint. Values keep their case, since paths are case-sensitive.
11pub fn robots_fingerprint(body: &str) -> u64 {
12    let mut rules = String::new();
13    for line in body.lines() {
14        let line = line.split('#').next().unwrap_or("").trim();
15        if line.is_empty() {
16            continue;
17        }
18        let (field, value) = match line.split_once(':') {
19            Some((field, value)) => (field.trim().to_ascii_lowercase(), value.trim()),
20            None => (line.to_ascii_lowercase(), ""),
21        };
22        if field == "sitemap" {
23            continue;
24        }
25        if !rules.is_empty() {
26            rules.push('\n');
27        }
28        rules.push_str(&field);
29        rules.push(':');
30        rules.push_str(value);
31    }
32    xxh3_64(rules.as_bytes())
33}
34
35/// A change when the robots.txt status or its rules differ. A missing file on both sides is
36/// no change; missing on one side shows as `none`.
37///
38/// `newly_blocked` (this crawl stopped as `RobotsBlocked` and the previous one didn't)
39/// always gives one critical change, even when the file reads the same.
40pub(crate) fn robots_change(
41    prev: Option<&RobotsFile>,
42    curr: Option<&RobotsFile>,
43    newly_blocked: bool,
44) -> Option<Change> {
45    let status =
46        |f: Option<&RobotsFile>| f.map_or_else(|| "none".to_owned(), |f| f.status.to_string());
47    let (before, after, rules_changed) = match (prev, curr) {
48        (None, None) => (status(prev), status(curr), false),
49        (Some(p), Some(c)) => {
50            let rules_differ = robots_fingerprint(&p.body) != robots_fingerprint(&c.body);
51            (
52                status(prev),
53                status(curr),
54                p.status == c.status && rules_differ,
55            )
56        }
57        _ => (status(prev), status(curr), false),
58    };
59    let differs = before != after || rules_changed;
60    if !differs && !newly_blocked {
61        return None;
62    }
63    let notes: Vec<&str> = [
64        rules_changed.then_some("rules changed"),
65        newly_blocked.then_some("blocks the crawl"),
66    ]
67    .into_iter()
68    .flatten()
69    .collect();
70    let after = if notes.is_empty() {
71        after
72    } else {
73        format!("{after} ({})", notes.join(", "))
74    };
75    Some(Change {
76        kind: ChangeKind::RobotsTxtChanged,
77        severity: if newly_blocked {
78            Severity::Critical
79        } else {
80            Severity::Warning
81        },
82        url: None,
83        before,
84        after,
85    })
86}
codoseo_diff/robots.rs

codoseo_diff/
robots.rs
