scrape-le 0.3.1

//! robots.txt parsing and matching — the port of the extension's
//! `src/detectors/robotstxt.ts`.
//!
//! Flagless behaviour is the extension's, exactly: only the generic
//! (`User-agent: *`) groups are evaluated, and `fixtures/robots/cases.json`
//! pins the results. Passing an agent selects that agent's groups per
//! RFC 9309 instead — the documented divergence, opt-in, recorded in
//! the report so a reader knows which rules answered.

use regex::Regex;

#[derive(Debug, PartialEq)]
pub(crate) struct RobotsTxtInfo {
    pub(crate) exists: bool,
    pub(crate) allows_crawling: bool,
    pub(crate) crawl_delay: Option<f64>,
    pub(crate) disallowed_paths: Vec<String>,
    pub(crate) sitemaps: Vec<String>,
    /// which group answered: `*`, or the agent token that matched
    pub(crate) agent: String,
    /// the rule that decided a refusal, when one did
    pub(crate) matched_rule: Option<String>,
}

struct RobotsRule {
    allow: bool,
    pattern: String,
    /// Compiled once, when the group is parsed.
    ///
    /// Every pattern becomes a regex, and building one is orders of
    /// magnitude dearer than running it: a robots.txt with ten thousand
    /// `Disallow` lines compiled ten thousand regexes **per path
    /// checked**, and a batch is many paths against one host's rules.
    /// `None` is a pattern the engine refused, which matches nothing —
    /// the same answer the per-call version gave.
    matcher: Option<Regex>,
    /// The **encoded** pattern's length — RFC 9309 §2.2.2's "most
    /// octets".
    ///
    /// Longest-match-wins compares these. Measuring the raw pattern made
    /// the unit part of the answer: `/café` is six bytes and five UTF-16
    /// code units, so a rule beside it won the tie on one server and
    /// lost it on the other. Encoding first dissolves that rather than
    /// picking a side — the canonical form is pure ASCII, where octets,
    /// characters and UTF-16 code units are one number, so both
    /// frontends count the same thing whatever their string type is.
    length: i64,
}

struct Group {
    agents: Vec<String>,
    rules: Vec<RobotsRule>,
    crawl_delay: Option<f64>,
}

/// One parsed robots.txt, with every rule's matcher already compiled.
///
/// **Parsing is per document; evaluating is per path.** Splitting them
/// is what lets a batch parse a host's file once and ask it fifty
/// questions: `fetch::RobotsCache` holds one of these per origin for
/// the length of a run. Nothing here is path- or agent-specific, so a
/// held document answers exactly what a fresh parse of the same bytes
/// would — asserted by a test, because that equivalence is the whole
/// licence to reuse it.
pub(crate) struct RobotsDocument {
    groups: Vec<Group>,
    sitemaps: Vec<String>,
    /// The rules exist and could not be read.
    ///
    /// RFC 9309 §2.3.1.4: a robots.txt unreachable through a server
    /// error or a network failure means a crawler "MUST assume complete
    /// disallow". That is not the same as §2.3.1.3's *unavailable* —
    /// a 404, which allows everything — and collapsing the two reported
    /// `exists: false, allows_crawling: true` over a 500, which is
    /// "nothing forbids you" from a blip.
    ///
    /// Carried rather than baked into the rules so the report can say
    /// which of the two happened: the verdict is a complete disallow
    /// either way, but `exists` stays false and the finding says the
    /// file could not be read instead of quoting a rule no server sent.
    unreachable: bool,
}

impl RobotsDocument {
    pub(crate) fn parse(content: &str) -> Self {
        let (groups, sitemaps) = parse_groups(content);
        Self {
            groups,
            sitemaps,
            unreachable: false,
        }
    }

    /// A robots.txt that could not be read: RFC 9309 §2.3.1.4's complete
    /// disallow, carrying the fact that no rule was actually served.
    pub(crate) fn unreachable() -> Self {
        let (groups, sitemaps) = parse_groups("User-agent: *\nDisallow: /\n");
        Self {
            groups,
            sitemaps,
            unreachable: true,
        }
    }

    /// Whether the rules were assumed rather than served.
    pub(crate) fn is_unreachable(&self) -> bool {
        self.unreachable
    }

    /// Evaluates `pathname` against the rules that apply. `agent` is the
    /// caller's product token (`MyBot/1.0` → `mybot`); `None` evaluates
    /// the generic rules only, as the extension does.
    pub(crate) fn evaluate(&self, pathname: &str, agent: Option<&str>) -> RobotsTxtInfo {
        let token = agent.map(product_token);

        // An agent-specific group wins; with no match — or no agent —
        // the generic groups answer, which is RFC 9309 and also the
        // extension's only behaviour.
        let (selected, answering_agent) = match token.as_deref() {
            Some(token)
                if self
                    .groups
                    .iter()
                    .any(|g| g.agents.iter().any(|a| a == token)) =>
            {
                (select(&self.groups, token), token.to_string())
            }
            _ => (select(&self.groups, "*"), "*".to_string()),
        };

        let rules: Vec<&RobotsRule> = selected.iter().flat_map(|g| g.rules.iter()).collect();
        // RFC 9309 §2.2.2: the comparison happens on the encoded form,
        // on both sides of it. The path arrives encoded from
        // `Url::path()` and raw from an MCP caller that typed it, and
        // `canonicalize` is idempotent so either spelling lands here as
        // one string.
        let path = canonicalize(pathname);
        // **The last one wins**, across groups as well as within one.
        // The extension keeps assigning to a single `crawlDelay` as it
        // walks the file, so a document with two applicable groups
        // reports the second; taking the first here had the two servers
        // answer the same file differently.
        let crawl_delay = selected.iter().rev().find_map(|group| group.crawl_delay);
        let decision = decide_path(&path, &rules);

        RobotsTxtInfo {
            // Never claim a file the origin did not serve.
            exists: !self.unreachable,
            allows_crawling: decision.allowed,
            crawl_delay,
            disallowed_paths: rules
                .iter()
                .filter(|r| !r.allow)
                .map(|r| r.pattern.clone())
                .collect(),
            sitemaps: self.sitemaps.clone(),
            agent: answering_agent,
            matched_rule: decision.matched,
        }
    }
}

/// Parses and evaluates in one call — for a caller holding a document it
/// asks exactly one question of, which is every caller that did not
/// fetch it: `analyze_robots_txt` is handed the content by the agent.
pub(crate) fn parse_robots_txt(
    content: &str,
    pathname: &str,
    agent: Option<&str>,
) -> RobotsTxtInfo {
    RobotsDocument::parse(content).evaluate(pathname, agent)
}

fn select<'a>(groups: &'a [Group], token: &str) -> Vec<&'a Group> {
    groups
        .iter()
        .filter(|g| g.agents.iter().any(|a| a == token))
        .collect()
}

/// `MyBot/1.0` → `mybot`. RFC 9309 matches the product token,
/// case-insensitively, ignoring any version suffix.
fn product_token(agent: &str) -> String {
    agent
        .split('/')
        .next()
        .unwrap_or(agent)
        .trim()
        .to_lowercase()
}

fn parse_groups(content: &str) -> (Vec<Group>, Vec<String>) {
    let mut groups: Vec<Group> = Vec::new();
    let mut sitemaps: Vec<String> = Vec::new();
    let mut agents: Vec<String> = Vec::new();
    let mut in_group_header = false;

    for raw_line in content.split('\n') {
        // comments run from '#' to end of line
        let line = match raw_line.find('#') {
            Some(hash_index) => &raw_line[..hash_index],
            None => raw_line,
        }
        .trim();
        if line.is_empty() {
            continue;
        }

        let Some(colon_index) = line.find(':') else {
            continue;
        };
        let directive = line[..colon_index].trim().to_lowercase();
        let value = line[colon_index + 1..].trim();

        if directive == "user-agent" {
            if !in_group_header {
                agents = Vec::new();
                in_group_header = true;
                groups.push(Group {
                    agents: Vec::new(),
                    rules: Vec::new(),
                    crawl_delay: None,
                });
            }
            agents.push(value.to_lowercase());
            if let Some(group) = groups.last_mut() {
                group.agents.clone_from(&agents);
            }
            continue;
        }

        // any non-user-agent directive closes the group header
        in_group_header = false;

        if directive == "sitemap" {
            // sitemap is not group-scoped
            if !value.is_empty() {
                sitemaps.push(value.to_string());
            }
            continue;
        }

        let Some(group) = groups.last_mut() else {
            continue;
        };

        if (directive == "disallow" || directive == "allow") && !value.is_empty() {
            // Matched and measured encoded; reported raw, so the finding
            // quotes the line the file actually carries rather than a
            // canonical form the reader would not find in it.
            let encoded = canonicalize(value);
            group.rules.push(RobotsRule {
                allow: directive == "allow",
                matcher: compile_robots_pattern(&encoded),
                length: i64::try_from(encoded.len()).unwrap_or(i64::MAX),
                pattern: value.to_string(),
            });
            continue;
        }

        if directive == "crawl-delay" {
            let Some(delay) = js_parse_float(value) else {
                continue;
            };
            if delay >= 0.0 {
                group.crawl_delay = Some(delay);
            }
        }
    }

    (groups, sitemaps)
}

/// RFC 9309 §2.2.2's canonical form for comparison: every octet outside
/// ASCII percent-encoded, and a well-formed `%XX` already present left
/// where it is with its hex digits upper-cased.
///
/// **The scope is Google's reference parser's `MaybeEscapePattern`,
/// exactly** — high-bit octets and hex casing, nothing else. Reserved
/// ASCII is deliberately untouched: a pattern's `*` and `$` are the
/// RFC's own special characters (§2.2.3) and `/` is the path separator,
/// so encoding them would turn every wildcard into a literal. A
/// robots.txt that means a literal asterisk writes `%2A` itself.
///
/// Recognising an existing escape is what makes this idempotent, and
/// that is load-bearing rather than tidy: the two entry points disagree
/// about what they hand over. `Url::path()` and `URL.pathname` give an
/// already-encoded path, while an MCP caller gives whatever it typed.
/// Encoding blindly would turn `/caf%C3%A9` into `/caf%25C3%25A9` and
/// match nothing.
///
/// **Not implemented, on purpose:** §2.2.2's second paragraph also has a
/// percent-encoded *unreserved* octet decoded before comparison, so
/// `/foo/%62%61%7A` matches `/foo/baz`. Google's parser does not do it
/// and neither do we — following the reference implementation keeps the
/// two frontends answering what the crawler ecosystem answers, and
/// guessing wider than the reference here would move paths toward
/// "allowed" on nobody else's authority.
fn canonicalize(value: &str) -> String {
    const HEX: &[u8; 16] = b"0123456789ABCDEF";

    let bytes = value.as_bytes();
    let mut out = String::with_capacity(bytes.len());
    let mut index = 0;
    while index < bytes.len() {
        if let Some((high, low)) = existing_escape(bytes, index) {
            out.push('%');
            out.push(high);
            out.push(low);
            index += 3;
            continue;
        }
        let byte = bytes[index];
        index += 1;
        if byte.is_ascii() {
            out.push(char::from(byte));
            continue;
        }
        out.push('%');
        out.push(char::from(HEX[usize::from(byte >> 4)]));
        out.push(char::from(HEX[usize::from(byte & 0x0f)]));
    }
    out
}

/// The upper-cased hex digits of a well-formed `%XX` at `index`.
fn existing_escape(bytes: &[u8], index: usize) -> Option<(char, char)> {
    if bytes[index] != b'%' {
        return None;
    }
    let high = *bytes.get(index + 1)?;
    let low = *bytes.get(index + 2)?;
    (high.is_ascii_hexdigit() && low.is_ascii_hexdigit()).then(|| {
        (
            char::from(high.to_ascii_uppercase()),
            char::from(low.to_ascii_uppercase()),
        )
    })
}

struct Decision {
    allowed: bool,
    matched: Option<String>,
}

/// RFC 9309 matching: longest matching pattern wins, Allow wins ties;
/// no matching rule means allowed. Both `path` and every rule's matcher
/// are already in the canonical encoded form — §2.2.2 compares there,
/// and "longest" counts that form's octets.
fn decide_path(path: &str, rules: &[&RobotsRule]) -> Decision {
    let mut best_length: i64 = -1;
    let mut best_allow = true;
    let mut best_pattern: Option<String> = None;

    for rule in rules {
        if !rule.matcher.as_ref().is_some_and(|re| re.is_match(path)) {
            continue;
        }
        let length = rule.length;
        let wins_tie = length == best_length && rule.allow && !best_allow;
        if length > best_length || wins_tie {
            best_length = length;
            best_allow = rule.allow;
            best_pattern = Some(rule.pattern.clone());
        }
    }

    Decision {
        allowed: best_allow,
        matched: if best_allow { None } else { best_pattern },
    }
}

/// Builds the matcher for one robots.txt pattern, already canonicalized:
/// anchored at the start, `*` matches any character sequence, a trailing
/// `$` anchors the end. Built the same way the extension builds it, so
/// the two cannot disagree about an edge.
///
/// A pattern the engine refuses — one past its size limit, say —
/// becomes `None` and matches nothing.
fn compile_robots_pattern(pattern: &str) -> Option<Regex> {
    let anchored = pattern.ends_with('$');
    let body = if anchored {
        &pattern[..pattern.len() - 1]
    } else {
        pattern
    };

    let escaped = body
        .split('*')
        .map(regex::escape)
        .collect::<Vec<_>>()
        .join("[\\s\\S]*");

    let end = if anchored { "$" } else { "" };
    Regex::new(&format!("^{escaped}{end}")).ok()
}

/// JS `Number.parseFloat` semantics: the longest numeric prefix parses,
/// so `10` and `10s` are both 10 and `not-a-number` is `None`.
///
/// Rust's float parser is the more generous of the two, and the
/// difference is a real answer: it reads `inf`, `infinity` and `nan` in
/// any casing, while `Number.parseFloat` accepts only `Infinity` spelled
/// exactly that way and never a NaN literal. `Crawl-delay: infinity` had
/// this server reporting a delay and the npm server reporting none — the
/// same tool, the same file, two answers.
fn js_parse_float(value: &str) -> Option<f64> {
    for end in (1..=value.len()).rev() {
        let Some(prefix) = value.get(..end) else {
            continue;
        };
        let Ok(parsed) = prefix.parse::<f64>() else {
            continue;
        };
        if parsed.is_nan() {
            return None;
        }
        if !parsed.is_finite() && prefix.strip_prefix(['+', '-']).unwrap_or(prefix) != "Infinity" {
            // A shorter prefix may still be a number JavaScript reads.
            continue;
        }
        return Some(parsed);
    }
    None
}

#[cfg(test)]
mod tests {
    use super::*;

    const CASES: &str = include_str!("../../fixtures/robots/cases.json");

    fn fixture_body(file: &str) -> &'static str {
        match file {
            "simple.txt" => include_str!("../../fixtures/robots/simple.txt"),
            "disallow-all.txt" => include_str!("../../fixtures/robots/disallow-all.txt"),
            "wildcards.txt" => include_str!("../../fixtures/robots/wildcards.txt"),
            "multi-group.txt" => include_str!("../../fixtures/robots/multi-group.txt"),
            "agent-specific.txt" => include_str!("../../fixtures/robots/agent-specific.txt"),
            "encoded.txt" => include_str!("../../fixtures/robots/encoded.txt"),
            other => panic!("fixture body {other} not embedded — add it here"),
        }
    }

    #[test]
    fn every_fixture_case_reproduces() {
        let cases: serde_json::Value = serde_json::from_str(CASES).expect("fixture JSON");
        for case in cases.as_array().expect("array of cases") {
            let name = case["name"].as_str().expect("name");
            let body = fixture_body(case["file"].as_str().expect("file"));
            let path = case["path"].as_str().expect("path");
            let expected = &case["expected"];

            let actual = parse_robots_txt(body, path, None);

            assert_eq!(
                actual.exists,
                expected["exists"].as_bool().expect("exists"),
                "case {name:?}: exists"
            );
            assert_eq!(
                actual.allows_crawling,
                expected["allowsCrawling"]
                    .as_bool()
                    .expect("allowsCrawling"),
                "case {name:?}: allowsCrawling"
            );
            assert_eq!(
                actual.crawl_delay,
                expected["crawlDelay"].as_f64(),
                "case {name:?}: crawlDelay"
            );
            let expected_disallowed: Vec<String> =
                serde_json::from_value(expected["disallowedPaths"].clone())
                    .expect("disallowedPaths");
            assert_eq!(
                actual.disallowed_paths, expected_disallowed,
                "case {name:?}: disallowedPaths"
            );
            let expected_sitemaps: Vec<String> =
                serde_json::from_value(expected["sitemaps"].clone()).expect("sitemaps");
            assert_eq!(
                actual.sitemaps, expected_sitemaps,
                "case {name:?}: sitemaps"
            );
        }
    }

    /// The licence to hold a parsed document across a whole host's
    /// worth of URLs: one kept document must answer every path and
    /// every agent exactly as a fresh parse of the same bytes does.
    /// Were that ever untrue, a batch would answer differently from a
    /// single check of the same URL — the cache turning into a second
    /// implementation of the rules.
    #[test]
    fn a_kept_document_answers_what_a_fresh_parse_answers() {
        let cases: serde_json::Value = serde_json::from_str(CASES).expect("fixture JSON");
        let mut checked = 0;
        for file in [
            "simple.txt",
            "disallow-all.txt",
            "wildcards.txt",
            "multi-group.txt",
            "agent-specific.txt",
            "encoded.txt",
        ] {
            let body = fixture_body(file);
            let kept = RobotsDocument::parse(body);
            // Every path any case names, against every agent any case
            // names — the paths and agents this corpus actually cares
            // about, rather than ones invented here.
            for case in cases.as_array().expect("array of cases") {
                let path = case["path"].as_str().expect("path");
                for agent in [None, Some("googlebot"), Some("PickyBot/1.0"), Some("mybot")] {
                    assert_eq!(
                        kept.evaluate(path, agent),
                        parse_robots_txt(body, path, agent),
                        "{file} {path:?} as {agent:?}"
                    );
                    checked += 1;
                }
            }
        }
        assert!(checked > 0, "expected paths to check");
    }

    /// The divergence annotations are a contract too: where a fixture
    /// records what the CLI answers with `--agent`, the CLI must
    /// actually answer that.
    #[test]
    fn every_divergence_annotation_holds() {
        let cases: serde_json::Value = serde_json::from_str(CASES).expect("fixture JSON");
        let mut checked = 0;
        for case in cases.as_array().expect("array of cases") {
            let Some(divergence) = case.get("divergence") else {
                continue;
            };
            let name = case["name"].as_str().expect("name");
            let body = fixture_body(case["file"].as_str().expect("file"));
            let path = case["path"].as_str().expect("path");
            let agent = divergence["cli"]["agent"].as_str().expect("cli agent");
            let expected = divergence["cli"]["allowsCrawling"]
                .as_bool()
                .expect("cli allowsCrawling");

            let actual = parse_robots_txt(body, path, Some(agent));
            assert_eq!(
                actual.allows_crawling, expected,
                "divergence {name:?} with --agent {agent}"
            );
            assert_eq!(actual.agent, agent.to_lowercase());
            checked += 1;
        }
        assert!(checked >= 2, "expected divergence cases to exist");
    }

    #[test]
    fn unknown_agent_falls_back_to_the_generic_group() {
        let body = fixture_body("agent-specific.txt");
        let info = parse_robots_txt(body, "/members/area", Some("NobodyBot/2.0"));
        assert!(info.allows_crawling);
        assert_eq!(info.agent, "*");
    }

    #[test]
    fn agent_matching_ignores_case_and_version() {
        let body = "User-agent: MyBot\nDisallow: /x\n";
        let info = parse_robots_txt(body, "/x", Some("mybot/9.9"));
        assert!(!info.allows_crawling);
        assert_eq!(info.agent, "mybot");
    }

    #[test]
    fn refusal_names_the_rule_that_decided_it() {
        let body = "User-agent: *\nDisallow: /search\n";
        let info = parse_robots_txt(body, "/search?q=1", None);
        assert!(!info.allows_crawling);
        assert_eq!(info.matched_rule.as_deref(), Some("/search"));
    }

    #[test]
    fn allowed_paths_name_no_rule() {
        let body = "User-agent: *\nDisallow: /search\n";
        let info = parse_robots_txt(body, "/about", None);
        assert!(info.allows_crawling);
        assert_eq!(info.matched_rule, None);
    }

    /// The pattern matcher, reached the way `decide_path` reaches it —
    /// through the canonical form on both sides.
    fn matches_robots_pattern(pattern: &str, pathname: &str) -> bool {
        compile_robots_pattern(&canonicalize(pattern))
            .is_some_and(|re| re.is_match(&canonicalize(pathname)))
    }

    /// Idempotence is what lets one function serve both entry points:
    /// `Url::path()` hands over an encoded path and an MCP caller hands
    /// over whatever it typed, and encoding blindly would turn
    /// `/caf%C3%A9` into `/caf%25C3%25A9` and match nothing.
    #[test]
    fn canonicalizing_twice_changes_nothing() {
        for value in [
            "/caf\u{e9}",
            "/caf%C3%A9",
            "/caf%c3%a9",
            "/*.json$",
            "/a%2Fb",
            "/100%",
            "/%zz",
            "/%",
            "/\u{1f600}",
            "/",
        ] {
            let once = canonicalize(value);
            assert_eq!(canonicalize(&once), once, "{value:?}");
            assert!(once.is_ascii(), "{value:?} canonicalizes to ASCII");
        }
    }

    /// The examples RFC 9309 §2.2.2 prints, and Google's own doc
    /// comment on `MaybeEscapePattern`.
    #[test]
    fn canonicalizing_matches_the_reference_examples() {
        assert_eq!(canonicalize("/foo/bar/\u{30c4}"), "/foo/bar/%E3%83%84");
        assert_eq!(canonicalize("/foo/bar/%E3%83%84"), "/foo/bar/%E3%83%84");
        assert_eq!(canonicalize("/SanJos\u{e9}Sellers"), "/SanJos%C3%A9Sellers");
        assert_eq!(canonicalize("%aa"), "%AA");
        // A truncated escape is not one, and is left as the literal it is.
        assert_eq!(canonicalize("/a%2"), "/a%2");
    }

    /// **Regression.** RFC 9309 §2.2.2: octets outside ASCII "MUST be
    /// percent-encoded ... prior to comparison". `Disallow: /café` and a
    /// request for `/caf%C3%A9` name one resource, and only the
    /// unencoded spelling was refused — the encoded one, which is what
    /// every URL parser hands over, came back allowed. Python's
    /// `RobotFileParser` refuses both.
    #[test]
    fn a_rule_and_a_path_are_compared_percent_encoded() {
        let body = "User-agent: *\nDisallow: /caf\u{e9}\n";
        for path in ["/caf\u{e9}", "/caf%C3%A9", "/caf%c3%a9"] {
            assert!(
                !parse_robots_txt(body, path, None).allows_crawling,
                "path {path:?} names the same resource the rule forbids"
            );
        }
    }

    /// The other spelling of the same file: the rule already encoded,
    /// the path not. Encoding one side alone would leave this half
    /// broken.
    #[test]
    fn an_encoded_rule_matches_the_unencoded_path() {
        let body = "User-agent: *\nDisallow: /caf%C3%A9\n";
        for path in ["/caf\u{e9}", "/caf%C3%A9", "/caf%c3%a9"] {
            assert!(
                !parse_robots_txt(body, path, None).allows_crawling,
                "path {path:?}"
            );
        }
    }

    /// A pattern's `*` and `$` are the RFC's special characters and
    /// survive encoding — only octets outside ASCII move, which is what
    /// Google's reference parser escapes and no more. Encoding `*` would
    /// turn every wildcard into a literal.
    #[test]
    fn encoding_leaves_the_special_characters_alone() {
        let body = "User-agent: *\nDisallow: /caf\u{e9}/*.json$\n";
        assert!(!parse_robots_txt(body, "/caf%C3%A9/a.json", None).allows_crawling);
        assert!(parse_robots_txt(body, "/caf%C3%A9/a.json?x=1", None).allows_crawling);
    }

    /// **Regression.** Longest-match-wins compares pattern lengths, and
    /// the two runtimes counted the raw pattern in different units —
    /// UTF-16 code units in the extension, bytes in `str::len`. Encoding
    /// first settles it: RFC 9309 §2.2.2 measures "the most octets" of
    /// the encoded form, which is pure ASCII, so octets, characters and
    /// UTF-16 code units are one number. `/café` is ten of each once
    /// encoded, and the five-unit `/ca*e` beside it no longer ties.
    #[test]
    fn a_pattern_is_measured_after_it_is_encoded() {
        let body = "User-agent: *\nDisallow: /caf\u{e9}\nAllow: /ca*e\n";
        let info = parse_robots_txt(body, "/caf\u{e9}/page", None);
        assert!(
            !info.allows_crawling,
            "/caf%C3%A9 is ten octets and /ca*e is five, so the longer Disallow wins"
        );

        // And Allow still wins a genuine tie, now measured encoded.
        let tied = "User-agent: *\nDisallow: /caf\u{e9}\nAllow: /ca*%C3%A9\n";
        assert!(parse_robots_txt(tied, "/caf\u{e9}/page", None).allows_crawling);
    }

    /// **Regression.** The extension assigns to one `crawlDelay` as it
    /// walks the file, so the last applicable group wins. Keeping the
    /// first had the two servers report different delays for the same
    /// document.
    #[test]
    fn the_last_crawl_delay_wins_across_groups() {
        let body = "User-agent: *\nCrawl-delay: 5\nUser-agent: *\nCrawl-delay: 10\n";
        assert_eq!(parse_robots_txt(body, "/", None).crawl_delay, Some(10.0));
    }

    /// **Regression.** Rust's float parser reads `inf`, `infinity` and
    /// `nan` in any casing; `Number.parseFloat` reads only `Infinity`,
    /// spelled exactly that way.
    #[test]
    fn a_crawl_delay_parses_the_way_javascript_parses_it() {
        let delay = |value: &str| {
            parse_robots_txt(&format!("User-agent: *\nCrawl-delay: {value}\n"), "/", None)
                .crawl_delay
        };
        assert_eq!(delay("10"), Some(10.0));
        assert_eq!(delay("10s"), Some(10.0));
        assert_eq!(delay("1e3"), Some(1000.0));
        assert_eq!(delay("0x10"), Some(0.0));
        assert_eq!(delay("+5"), Some(5.0));
        assert_eq!(delay(".5"), Some(0.5));
        assert_eq!(delay("infinity"), None, "JavaScript reads no such number");
        assert_eq!(delay("inf"), None);
        assert_eq!(delay("nan"), None);
        assert_eq!(delay("abc"), None);
        assert_eq!(delay("-1"), None, "a negative delay is not a delay");
        assert!(delay("Infinity").is_some_and(f64::is_infinite));
    }

    #[test]
    fn pattern_matching_edges() {
        assert!(matches_robots_pattern("/admin/", "/admin/settings"));
        assert!(!matches_robots_pattern("/admin/", "/admin"));
        assert!(matches_robots_pattern("/*.json$", "/data.json"));
        assert!(!matches_robots_pattern("/*.json$", "/data.json?x=1"));
        assert!(matches_robots_pattern("/private*", "/private/file"));
        assert!(matches_robots_pattern("/", "/anything"));
    }
}

#[cfg(test)]
mod unreachable_tests {
    use super::RobotsDocument;

    /// **RFC 9309 draws a line this used to collapse.** §2.3.1.3: an
    /// *unavailable* robots.txt — 404 — allows crawling, because the
    /// site has no rules. §2.3.1.4: an *unreachable* one — 5xx or a
    /// network failure — means "MUST assume complete disallow", because
    /// the rules exist and could not be read. Both returned "no
    /// robots.txt", so a 500 reported `allows_crawling: true`.
    #[test]
    fn an_unreachable_robots_txt_disallows_everything() {
        let document = RobotsDocument::unreachable();
        let info = document.evaluate("/anything", None);
        assert!(!info.allows_crawling);
        assert!(document.is_unreachable());
    }

    /// And it never claims a file the origin did not serve. Quoting a
    /// rule here would have a reader fetch robots.txt, get a 500, and
    /// find the tool's explanation contradicted by the server.
    #[test]
    fn an_unreachable_robots_txt_does_not_claim_to_exist() {
        let info = RobotsDocument::unreachable().evaluate("/anything", None);
        assert!(!info.exists);

        // A served document still does.
        let served = RobotsDocument::parse("User-agent: *\nDisallow: /admin\n");
        assert!(served.evaluate("/admin", None).exists);
        assert!(!served.is_unreachable());
    }
}