Trait robotparser::http::RobotsTxtClient

pub trait RobotsTxtClient {
    type Result;
    fn fetch_robots_txt(&self, origin: Origin) -> Self::Result;
}

Trait to fetch and parse the robots.txt file. Must be implemented on http-client.

Associated Types

type Result

Required methods

fn fetch_robots_txt(&self, origin: Origin) -> Self::Result

Implementations on Foreign Types

impl RobotsTxtClient for Client

type Result = Result<ParseResult<FetchedRobotsTxt>, Error>

fn fetch_robots_txt(&self, origin: Origin) -> Self::Result

impl RobotsTxtClient for Client

type Result = Result<RobotsTxtResponse, Error>

fn fetch_robots_txt(&self, origin: Origin) -> Self::Result

Implementors