robotparser_fork/parser/
fetched_robots_txt_parser.rs

1use crate::model::{FetchedRobotsTxt, FetchedRobotsTxtContainer};
2use crate::parser::parse_robots_txt;
3use crate::parser::ParseResult;
4use url::Origin;
5
6const UNAUTHORIZED: u16 = 401;
7const FORBIDDEN: u16 = 403;
8const OK: u16 = 200;
9
10/// Parses the text of the robots.txt file located in the specified place of origin,
11/// taking into account the response status code of the HTTP-request.
12/// **IMPORTANT NOTE**: origin must point to robots.txt url **before redirects**.
13pub fn parse(origin: Origin, status_code: u16, input: &str) -> ParseResult<FetchedRobotsTxt> {
14    match status_code {
15        UNAUTHORIZED | FORBIDDEN => ParseResult::new(FetchedRobotsTxt::new(FetchedRobotsTxtContainer::FetchDenied)),
16        OK => parse_robots_txt(origin, input)
17            .map(|result| FetchedRobotsTxt::new(FetchedRobotsTxtContainer::Fetched(result))),
18        _ => ParseResult::new(FetchedRobotsTxt::new(FetchedRobotsTxtContainer::FetchFailed)),
19    }
20}