1use std::fmt::Write;
3#[derive(Debug, Clone, PartialEq, Eq)]
4pub enum RobotsError {
5 InvalidValue,
6}
7#[derive(Debug, Clone, PartialEq, Eq)]
8pub struct RobotsTxt {
9 user_agent: String,
10 disallow: Vec<String>,
11 allow: Vec<String>,
12 sitemaps: Vec<String>,
13}
14impl RobotsTxt {
15 pub fn new(user_agent: impl Into<String>) -> Result<Self, RobotsError> {
16 let user_agent = valid(user_agent.into())?;
17 Ok(Self {
18 user_agent,
19 disallow: Vec::new(),
20 allow: Vec::new(),
21 sitemaps: Vec::new(),
22 })
23 }
24 pub fn disallow(mut self, path: impl Into<String>) -> Result<Self, RobotsError> {
25 self.disallow.push(path.into());
26 self.validate_path(self.disallow.last().unwrap())?;
27 Ok(self)
28 }
29 pub fn allow(mut self, path: impl Into<String>) -> Result<Self, RobotsError> {
30 self.allow.push(path.into());
31 self.validate_path(self.allow.last().unwrap())?;
32 Ok(self)
33 }
34 pub fn sitemap(mut self, url: impl Into<String>) -> Result<Self, RobotsError> {
35 let url = url.into();
36 if !url.starts_with("https://") || url.chars().any(|c| c.is_control() || c.is_whitespace())
37 {
38 return Err(RobotsError::InvalidValue);
39 }
40 self.sitemaps.push(url);
41 Ok(self)
42 }
43 pub fn render(&self) -> String {
44 let mut out = String::new();
45 writeln!(out, "User-agent: {}", self.user_agent).unwrap();
46 for p in &self.disallow {
47 writeln!(out, "Disallow: {p}").unwrap()
48 }
49 for p in &self.allow {
50 writeln!(out, "Allow: {p}").unwrap()
51 }
52 for s in &self.sitemaps {
53 writeln!(out, "Sitemap: {s}").unwrap()
54 }
55 out
56 }
57 fn validate_path(&self, path: &str) -> Result<(), RobotsError> {
58 if path.is_empty()
59 || !path.starts_with('/')
60 || path.chars().any(|c| c.is_control() || c.is_whitespace())
61 {
62 Err(RobotsError::InvalidValue)
63 } else {
64 Ok(())
65 }
66 }
67}
68fn valid(v: String) -> Result<String, RobotsError> {
69 if v.is_empty() || v.len() > 128 || v.chars().any(|c| c.is_control() || c.is_whitespace()) {
70 Err(RobotsError::InvalidValue)
71 } else {
72 Ok(v)
73 }
74}
75#[cfg(test)]
76mod tests {
77 use super::*;
78 #[test]
79 fn renders_directives_deterministically() {
80 let r = RobotsTxt::new("*")
81 .unwrap()
82 .disallow("/admin")
83 .unwrap()
84 .allow("/public")
85 .unwrap()
86 .sitemap("https://example.com/sitemap.xml")
87 .unwrap();
88 assert_eq!(r.render(),"User-agent: *\nDisallow: /admin\nAllow: /public\nSitemap: https://example.com/sitemap.xml\n")
89 }
90 #[test]
91 fn rejects_invalid_values() {
92 assert_eq!(RobotsTxt::new("bad agent"), Err(RobotsError::InvalidValue));
93 assert_eq!(
94 RobotsTxt::new("*").unwrap().disallow("admin"),
95 Err(RobotsError::InvalidValue)
96 );
97 assert_eq!(
98 RobotsTxt::new("*").unwrap().sitemap("http://example.com/x"),
99 Err(RobotsError::InvalidValue)
100 )
101 }
102}