typstify_generator/
robots.rs

1//! Robots.txt generation.
2//!
3//! Generates the robots.txt file for search engine crawlers.
4
5use std::{fs::File, io::Write, path::Path};
6
7use thiserror::Error;
8use tracing::info;
9use typstify_core::Config;
10
11/// Robots generation errors.
12#[derive(Debug, Error)]
13pub enum RobotsError {
14    /// IO error.
15    #[error("IO error: {0}")]
16    Io(#[from] std::io::Error),
17}
18
19/// Result type for robots generation.
20pub type Result<T> = std::result::Result<T, RobotsError>;
21
22/// Robots.txt generator.
23#[derive(Debug)]
24pub struct RobotsGenerator<'a> {
25    config: &'a Config,
26}
27
28impl<'a> RobotsGenerator<'a> {
29    /// Create a new robots generator.
30    #[must_use]
31    pub fn new(config: &'a Config) -> Self {
32        Self { config }
33    }
34
35    /// Generate robots.txt.
36    pub fn generate(&self, output_dir: &Path) -> Result<()> {
37        if !self.config.robots.enabled {
38            return Ok(());
39        }
40
41        info!("generating robots.txt");
42
43        let path = output_dir.join("robots.txt");
44        let mut file = File::create(path)?;
45
46        writeln!(file, "User-agent: *")?;
47
48        for path in &self.config.robots.disallow {
49            writeln!(file, "Disallow: {path}")?;
50        }
51
52        for path in &self.config.robots.allow {
53            writeln!(file, "Allow: {path}")?;
54        }
55
56        // Add sitemap reference if configured (defaulting to sitemap.xml in root)
57        let sitemap_url = format!("{}/sitemap.xml", self.config.base_url());
58        writeln!(file, "Sitemap: {sitemap_url}")?;
59
60        Ok(())
61    }
62}
63
64#[cfg(test)]
65mod tests {
66    use std::{collections::HashMap, fs};
67
68    use typstify_core::test_fixtures::test_config;
69
70    use super::*;
71
72    #[test]
73    fn test_disabled_produces_no_file() {
74        let mut config = test_config();
75        config.robots.enabled = false;
76
77        let dir = tempfile::tempdir().expect("create temp dir");
78        let generator = RobotsGenerator::new(&config);
79        generator.generate(dir.path()).unwrap();
80
81        assert!(!dir.path().join("robots.txt").exists());
82    }
83
84    #[test]
85    fn test_enabled_with_disallow_allow_paths() {
86        let mut config = test_config();
87        config.robots.disallow = vec!["/private/".to_string(), "/admin/".to_string()];
88        config.robots.allow = vec!["/public/".to_string()];
89
90        let dir = tempfile::tempdir().expect("create temp dir");
91        let generator = RobotsGenerator::new(&config);
92        generator.generate(dir.path()).unwrap();
93
94        let content = fs::read_to_string(dir.path().join("robots.txt")).unwrap();
95
96        assert!(content.starts_with("User-agent: *\n"));
97        assert!(content.contains("Disallow: /private/\n"));
98        assert!(content.contains("Disallow: /admin/\n"));
99        assert!(content.contains("Allow: /public/\n"));
100        assert!(content.contains("Sitemap: https://example.com/sitemap.xml\n"));
101    }
102
103    #[test]
104    fn test_sitemap_includes_base_path() {
105        let config = Config::from_parts(
106            typstify_core::config::SiteConfig {
107                title: "Test Site".to_string(),
108                host: "https://example.com".to_string(),
109                base_path: "/typstify".to_string(),
110                default_language: "en".to_string(),
111                description: None,
112                author: None,
113            },
114            typstify_core::config::BuildConfig::default(),
115            typstify_core::config::SearchConfig::default(),
116            typstify_core::config::RssConfig::default(),
117            typstify_core::config::RobotsConfig::default(),
118            typstify_core::config::TaxonomyConfig::default(),
119            HashMap::new(),
120        );
121
122        let dir = tempfile::tempdir().expect("create temp dir");
123        let generator = RobotsGenerator::new(&config);
124        generator.generate(dir.path()).unwrap();
125
126        let content = fs::read_to_string(dir.path().join("robots.txt")).unwrap();
127
128        assert!(content.contains("Sitemap: https://example.com/typstify/sitemap.xml\n"));
129    }
130}
typstify_generator/robots.rs

typstify_generator/
robots.rs
