robotxt 0.6.1

The implementation of the Robots.txt (or URL exclusion) protocol with the support of crawl-delay, sitemap and universal match extensions.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20

use std::sync::OnceLock;

use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS};

/// Returns the prefixed & percent-encoded path.
/// NOTE: Expects relative path.
pub(crate) fn normalize_path(path: &str) -> String {
    static FRAGMENT: OnceLock<AsciiSet> = OnceLock::new();
    let fragment = FRAGMENT.get_or_init(|| CONTROLS.add(b' ').add(b'"').add(b'<').add(b'>'));
    let path = utf8_percent_encode(path, fragment).to_string();

    // Url::make_relative strips leading and trailing /
    // https://github.com/servo/rust-url/issues/772
    // https://github.com/servo/rust-url/issues/766
    if !path.starts_with('/') {
        '/'.to_string() + &path
    } else {
        path
    }
}