Struct robotxt::RobotsBuilder

pub struct RobotsBuilder { /* private fields */ }

Available on crate feature builder only.

The set of formatted user-agent groups that can be written in the robots.txt compliant format.

Implementations

impl RobotsBuilder

pub fn new() -> Self

Creates a new RobotsBuilder with default settings.

pub fn header(self, header: &str) -> Self

Adds a global header, usually used for permissions or legal notices.

use robotxt::RobotsBuilder;

let txt = RobotsBuilder::default()
    .header("Note: Stop right there!")
    .group(["*"], |u| u.disallow("/"))
    .group(["foobot"], |u| u.allow("/"));

pub fn group<'a>( self, group: impl IntoIterator<Item = &'a str>, factory: impl FnOnce(GroupBuilder) -> GroupBuilder ) -> Self

Adds a new user-agent group from the provided list of user-agents.

use robotxt::RobotsBuilder;

let txt = RobotsBuilder::default()
    .group(["*"], |u| u.disallow("/"))
    .group(["foobot"], |u| u.allow("/"));

pub fn sitemap(self, sitemap: Url) -> Self

Adds the Sitemap directive from the URL address.

use url::Url;
use robotxt::RobotsBuilder;

let txt = RobotsBuilder::default()
    .sitemap("https://example.com/sitemap_1.xml".try_into().unwrap())
    .sitemap("https://example.com/sitemap_1.xml".try_into().unwrap());

pub fn footer(self, footer: &str) -> Self

Adds a global footer, usually used for notices.

use robotxt::RobotsBuilder;

let txt = RobotsBuilder::default()
    .group(["*"], |u| u.disallow("/"))
    .group(["foobot"], |u| u.allow("/"))
    .footer("Note: Have a nice day!");

pub fn parse(&self, user_agent: &str) -> Robots

Available on crate feature parser only.

Parses the constructed output. See Robots::from_bytes.

Trait Implementations

impl Clone for RobotsBuilder

fn clone(&self) -> RobotsBuilder

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for RobotsBuilder

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for RobotsBuilder

fn default() -> RobotsBuilder

Returns the “default value” for a type.

impl Display for RobotsBuilder

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.