Struct robotparser::RobotFileParser [−][src]
pub struct RobotFileParser<'a> { /* fields omitted */ }
robots.txt file parser
Methods
impl<'a> RobotFileParser<'a>
[src]
impl<'a> RobotFileParser<'a>
pub fn new<T: AsRef<str>>(url: T) -> RobotFileParser<'a>
[src]
pub fn new<T: AsRef<str>>(url: T) -> RobotFileParser<'a>
pub fn mtime(&self) -> i64
[src]
pub fn mtime(&self) -> i64
Returns the time the robots.txt file was last fetched.
This is useful for long-running web spiders that need to check for new robots.txt files periodically.
pub fn modified(&self)
[src]
pub fn modified(&self)
Sets the time the robots.txt file was last fetched to the current time.
pub fn set_url<T: AsRef<str>>(&mut self, url: T)
[src]
pub fn set_url<T: AsRef<str>>(&mut self, url: T)
Sets the URL referring to a robots.txt file.
pub fn read(&self)
[src]
pub fn read(&self)
Reads the robots.txt URL and feeds it to the parser.
pub fn from_response(&self, response: &mut Response)
[src]
pub fn from_response(&self, response: &mut Response)
Reads the HTTP response and feeds it to the parser.
pub fn parse<T: AsRef<str>>(&self, lines: &[T])
[src]
pub fn parse<T: AsRef<str>>(&self, lines: &[T])
Parse the input lines from a robots.txt file
We allow that a user-agent: line is not preceded by one or more blank lines.
pub fn can_fetch<T: AsRef<str>>(&self, useragent: T, url: T) -> bool
[src]
pub fn can_fetch<T: AsRef<str>>(&self, useragent: T, url: T) -> bool
Using the parsed robots.txt decide if useragent can fetch url
pub fn get_crawl_delay<T: AsRef<str>>(&self, useragent: T) -> Option<Duration>
[src]
pub fn get_crawl_delay<T: AsRef<str>>(&self, useragent: T) -> Option<Duration>
Returns the crawl delay for this user agent as a Duration
, or None if no crawl delay is defined.
pub fn get_sitemaps<T: AsRef<str>>(&self, useragent: T) -> Vec<Url>
[src]
pub fn get_sitemaps<T: AsRef<str>>(&self, useragent: T) -> Vec<Url>
Returns the sitemaps for this user agent as a Vec<Url>
.
pub fn get_req_rate<T: AsRef<str>>(&self, useragent: T) -> Option<RequestRate>
[src]
pub fn get_req_rate<T: AsRef<str>>(&self, useragent: T) -> Option<RequestRate>
Returns the request rate for this user agent as a RequestRate
, or None if not request rate is defined
Trait Implementations
impl<'a> Debug for RobotFileParser<'a>
[src]
impl<'a> Debug for RobotFileParser<'a>
fn fmt(&self, f: &mut Formatter) -> Result
[src]
fn fmt(&self, f: &mut Formatter) -> Result
Formats the value using the given formatter. Read more
impl<'a> Eq for RobotFileParser<'a>
[src]
impl<'a> Eq for RobotFileParser<'a>
impl<'a> PartialEq for RobotFileParser<'a>
[src]
impl<'a> PartialEq for RobotFileParser<'a>
fn eq(&self, other: &RobotFileParser<'a>) -> bool
[src]
fn eq(&self, other: &RobotFileParser<'a>) -> bool
This method tests for self
and other
values to be equal, and is used by ==
. Read more
fn ne(&self, other: &RobotFileParser<'a>) -> bool
[src]
fn ne(&self, other: &RobotFileParser<'a>) -> bool
This method tests for !=
.
impl<'a> Clone for RobotFileParser<'a>
[src]
impl<'a> Clone for RobotFileParser<'a>
fn clone(&self) -> RobotFileParser<'a>
[src]
fn clone(&self) -> RobotFileParser<'a>
Returns a copy of the value. Read more
fn clone_from(&mut self, source: &Self)
1.0.0[src]
fn clone_from(&mut self, source: &Self)
Performs copy-assignment from source
. Read more
Auto Trait Implementations
impl<'a> Send for RobotFileParser<'a>
impl<'a> Send for RobotFileParser<'a>
impl<'a> !Sync for RobotFileParser<'a>
impl<'a> !Sync for RobotFileParser<'a>