krik 0.1.27

A fast static site generator written in Rust with internationalization, theming, and modern web features
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62

use crate::site::SiteConfig;
use std::fs::File;
use std::io::Write;
use std::path::Path;

/// Generate robots.txt for the website
pub fn generate_robots(
    site_config: &SiteConfig,
    output_dir: &Path,
) -> Result<(), Box<dyn std::error::Error>> {
    let robots_content = generate_robots_content(site_config);

    // Write robots.txt file
    let robots_path = output_dir.join("robots.txt");
    let mut file = File::create(&robots_path)?;
    file.write_all(robots_content.as_bytes())?;

    Ok(())
}

/// Generate robots.txt content following best practices
pub fn generate_robots_content(site_config: &SiteConfig) -> String {
    let mut robots = String::new();

    // User-agent directive - allow all bots by default
    robots.push_str("User-agent: *\n");

    // Allow all content by default (good for most static sites)
    robots.push_str("Allow: /\n");

    // Common disallows for static sites
    robots.push_str("\n# Disallow common non-content directories\n");
    robots.push_str("Disallow: /.*\n"); // Hidden files/directories (like .git, .htaccess)
    robots.push_str("Disallow: /_*\n"); // Build directories (like _site, _cache)

    // Disallow common temp/system files
    robots.push_str("\n# Disallow temporary and system files\n");
    robots.push_str("Disallow: /*.tmp$\n");
    robots.push_str("Disallow: /*.bak$\n");
    robots.push_str("Disallow: /*.log$\n");

    // Add sitemap reference
    robots.push_str("\n# Sitemap location\n");
    if let Some(ref base_url) = site_config.base_url {
        robots.push_str(&format!(
            "Sitemap: {}/sitemap.xml\n",
            base_url.trim_end_matches('/')
        ));
    } else {
        robots.push_str("Sitemap: /sitemap.xml\n");
    }

    // Add crawl delay for politeness (optional but good practice)
    robots.push_str("\n# Crawl delay (optional - be nice to servers)\n");
    robots.push_str("Crawl-delay: 1\n");

    // Add a comment about the generator
    robots.push_str("\n# Generated by Krik Static Site Generator\n");
    robots.push_str("# https://github.com/mcaserta/krik\n");

    robots
}