pub fn parse_robots(
input: &[u8],
limits: &RobotsLimits,
redirect: Option<RedirectMetadata>,
) -> Result<RobotsDoc, RobotsError>Expand description
Parse robots bytes as inert records. HTTP status and redirect policy remain outside this crate.