krik 0.1.21

A fast static site generator written in Rust with internationalization, theming, and modern web features
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104

use crate::site::SiteConfig;
use std::fs::File;
use std::io::Write;
use std::path::Path;

/// Generate robots.txt for the website
pub fn generate_robots(
    site_config: &SiteConfig,
    output_dir: &Path,
) -> Result<(), Box<dyn std::error::Error>> {
    let robots_content = generate_robots_content(site_config);

    // Write robots.txt file
    let robots_path = output_dir.join("robots.txt");
    let mut file = File::create(&robots_path)?;
    file.write_all(robots_content.as_bytes())?;

    Ok(())
}

/// Generate robots.txt content following best practices
fn generate_robots_content(site_config: &SiteConfig) -> String {
    let mut robots = String::new();

    // User-agent directive - allow all bots by default
    robots.push_str("User-agent: *\n");
    
    // Allow all content by default (good for most static sites)
    robots.push_str("Allow: /\n");
    
    // Common disallows for static sites
    robots.push_str("\n# Disallow common non-content directories\n");
    robots.push_str("Disallow: /.*\n");  // Hidden files/directories (like .git, .htaccess)
    robots.push_str("Disallow: /_*\n");  // Build directories (like _site, _cache)
    
    // Disallow common temp/system files
    robots.push_str("\n# Disallow temporary and system files\n");
    robots.push_str("Disallow: /*.tmp$\n");
    robots.push_str("Disallow: /*.bak$\n");
    robots.push_str("Disallow: /*.log$\n");
    
    // Add sitemap reference
    robots.push_str("\n# Sitemap location\n");
    if let Some(ref base_url) = site_config.base_url {
        robots.push_str(&format!("Sitemap: {}/sitemap.xml\n", base_url.trim_end_matches('/')));
    } else {
        robots.push_str("Sitemap: /sitemap.xml\n");
    }
    
    // Add crawl delay for politeness (optional but good practice)
    robots.push_str("\n# Crawl delay (optional - be nice to servers)\n");
    robots.push_str("Crawl-delay: 1\n");
    
    // Add a comment about the generator
    robots.push_str("\n# Generated by Krik Static Site Generator\n");
    robots.push_str("# https://github.com/mcaserta/krik\n");

    robots
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_generate_robots_content_with_base_url() {
        let mut site_config = SiteConfig::default();
        site_config.title = Some("Test Site".to_string());
        site_config.base_url = Some("https://example.com".to_string());
        
        let content = generate_robots_content(&site_config);
        
        assert!(content.contains("User-agent: *"));
        assert!(content.contains("Allow: /"));
        assert!(content.contains("Sitemap: https://example.com/sitemap.xml"));
        assert!(content.contains("Disallow: /.*"));
        assert!(content.contains("Crawl-delay: 1"));
        assert!(content.contains("Generated by Krik"));
    }

    #[test]
    fn test_generate_robots_content_without_base_url() {
        let site_config = SiteConfig::default();
        
        let content = generate_robots_content(&site_config);
        
        assert!(content.contains("User-agent: *"));
        assert!(content.contains("Allow: /"));
        assert!(content.contains("Sitemap: /sitemap.xml"));
        assert!(!content.contains("Sitemap: https://"));
    }

    #[test]
    fn test_robots_contains_common_disallows() {
        let site_config = SiteConfig::default();
        let content = generate_robots_content(&site_config);
        
        assert!(content.contains("Disallow: /.*"));
        assert!(content.contains("Disallow: /_*"));
        assert!(content.contains("Disallow: /*.tmp$"));
        assert!(content.contains("Disallow: /*.bak$"));
        assert!(content.contains("Disallow: /*.log$"));
    }
}