Skip to main content

yaiko_core/
robots.rs

1//! Deterministic robots.txt generation.
2use std::fmt::Write;
3#[derive(Debug, Clone, PartialEq, Eq)]
4pub enum RobotsError {
5    InvalidValue,
6}
7#[derive(Debug, Clone, PartialEq, Eq)]
8pub struct RobotsTxt {
9    user_agent: String,
10    disallow: Vec<String>,
11    allow: Vec<String>,
12    sitemaps: Vec<String>,
13}
14impl RobotsTxt {
15    pub fn new(user_agent: impl Into<String>) -> Result<Self, RobotsError> {
16        let user_agent = valid(user_agent.into())?;
17        Ok(Self {
18            user_agent,
19            disallow: Vec::new(),
20            allow: Vec::new(),
21            sitemaps: Vec::new(),
22        })
23    }
24    pub fn disallow(mut self, path: impl Into<String>) -> Result<Self, RobotsError> {
25        self.disallow.push(path.into());
26        self.validate_path(self.disallow.last().unwrap())?;
27        Ok(self)
28    }
29    pub fn allow(mut self, path: impl Into<String>) -> Result<Self, RobotsError> {
30        self.allow.push(path.into());
31        self.validate_path(self.allow.last().unwrap())?;
32        Ok(self)
33    }
34    pub fn sitemap(mut self, url: impl Into<String>) -> Result<Self, RobotsError> {
35        let url = url.into();
36        if !url.starts_with("https://") || url.chars().any(|c| c.is_control() || c.is_whitespace())
37        {
38            return Err(RobotsError::InvalidValue);
39        }
40        self.sitemaps.push(url);
41        Ok(self)
42    }
43    pub fn render(&self) -> String {
44        let mut out = String::new();
45        writeln!(out, "User-agent: {}", self.user_agent).unwrap();
46        for p in &self.disallow {
47            writeln!(out, "Disallow: {p}").unwrap()
48        }
49        for p in &self.allow {
50            writeln!(out, "Allow: {p}").unwrap()
51        }
52        for s in &self.sitemaps {
53            writeln!(out, "Sitemap: {s}").unwrap()
54        }
55        out
56    }
57    fn validate_path(&self, path: &str) -> Result<(), RobotsError> {
58        if path.is_empty()
59            || !path.starts_with('/')
60            || path.chars().any(|c| c.is_control() || c.is_whitespace())
61        {
62            Err(RobotsError::InvalidValue)
63        } else {
64            Ok(())
65        }
66    }
67}
68fn valid(v: String) -> Result<String, RobotsError> {
69    if v.is_empty() || v.len() > 128 || v.chars().any(|c| c.is_control() || c.is_whitespace()) {
70        Err(RobotsError::InvalidValue)
71    } else {
72        Ok(v)
73    }
74}
75#[cfg(test)]
76mod tests {
77    use super::*;
78    #[test]
79    fn renders_directives_deterministically() {
80        let r = RobotsTxt::new("*")
81            .unwrap()
82            .disallow("/admin")
83            .unwrap()
84            .allow("/public")
85            .unwrap()
86            .sitemap("https://example.com/sitemap.xml")
87            .unwrap();
88        assert_eq!(r.render(),"User-agent: *\nDisallow: /admin\nAllow: /public\nSitemap: https://example.com/sitemap.xml\n")
89    }
90    #[test]
91    fn rejects_invalid_values() {
92        assert_eq!(RobotsTxt::new("bad agent"), Err(RobotsError::InvalidValue));
93        assert_eq!(
94            RobotsTxt::new("*").unwrap().disallow("admin"),
95            Err(RobotsError::InvalidValue)
96        );
97        assert_eq!(
98            RobotsTxt::new("*").unwrap().sitemap("http://example.com/x"),
99            Err(RobotsError::InvalidValue)
100        )
101    }
102}