robotxt 0.6.1

The implementation of the Robots.txt (or URL exclusion) protocol with the support of crawl-delay, sitemap and universal match extensions.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

use crate::{Error, Result};

/// Returns the expected path to the `robots.txt` file
/// as the [`url::Url`].
///
/// ```rust
/// use url::Url;
/// use robotxt::create_url;
///
/// let path = "https://user:pass@example.com/foo/sample.txt";
/// let path = Url::parse(path).unwrap();
/// let robots = create_url(&path).unwrap().to_string();
/// assert_eq!(robots, "https://example.com/robots.txt")
/// ```
pub fn create_url(path: &url::Url) -> Result<url::Url> {
    let mut path = path.clone();

    if path.cannot_be_a_base() {
        return Err(Error::CannotBeBase);
    }

    if path.scheme() != "http" && path.scheme() != "https" {
        return Err(Error::WrongScheme {
            scheme: path.scheme().to_string(),
        });
    }

    if !path.username().is_empty() {
        path.set_username("").unwrap();
    }

    if path.password().is_some() {
        path.set_password(None).unwrap();
    }

    path.join("/robots.txt").map_err(Into::into)
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn from_url() -> Result<()> {
        let path = "https://user:pass@example.com/foo/sample.txt";
        let path = url::Url::parse(path).unwrap();

        let robots = create_url(&path)?.to_string();
        assert_eq!(robots, "https://example.com/robots.txt");

        Ok(())
    }
}