sova_meta/
robots.rs

1//! robots.txt generation.
2
3use crate::defaults::MetaDefaults;
4use crate::page::MetaPage;
5use crate::sitemap::path_is_dynamic;
6use crate::sitemap_config::SitemapConfig;
7use http::Method;
8use sova_core::extend::{RouteEntry, RouteTable};
9
10#[derive(Debug, Clone)]
11pub enum RobotsDirective {
12    Allow(String),
13    Disallow(String),
14    CrawlDelay(f32),
15    Raw(String),
16}
17
18#[derive(Debug, Clone)]
19pub struct RobotsGroup {
20    pub user_agent: String,
21    pub directives: Vec<RobotsDirective>,
22}
23
24/// Builder state for robots.txt.
25#[derive(Debug, Clone)]
26pub struct RobotsConfig {
27    pub path: String,
28    pub block_all: bool,
29    pub from_noindex: bool,
30    pub sitemap_url: Option<String>,
31    pub sitemap_from_plugin: bool,
32    pub crawl_delay: Option<f32>,
33    pub groups: Vec<RobotsGroup>,
34    pub trailing_raw: Vec<String>,
35}
36
37impl Default for RobotsConfig {
38    fn default() -> Self {
39        Self {
40            path: "/robots.txt".into(),
41            block_all: false,
42            from_noindex: true,
43            sitemap_url: None,
44            sitemap_from_plugin: true,
45            crawl_delay: None,
46            groups: vec![RobotsGroup {
47                user_agent: "*".into(),
48                directives: vec![RobotsDirective::Allow("/".into())],
49            }],
50            trailing_raw: Vec::new(),
51        }
52    }
53}
54
55pub fn render_robots(
56    cfg: &RobotsConfig,
57    table: Option<&RouteTable>,
58    sitemap_cfg: Option<&SitemapConfig>,
59    meta: Option<&MetaDefaults>,
60) -> String {
61    if cfg.block_all {
62        return "User-agent: *\nDisallow: /\n".into();
63    }
64
65    let mut out = String::new();
66    for group in &cfg.groups {
67        out.push_str(&format!("User-agent: {}\n", group.user_agent));
68        let mut wrote = false;
69        for d in &group.directives {
70            match d {
71                RobotsDirective::Allow(p) => {
72                    out.push_str(&format!("Allow: {p}\n"));
73                    wrote = true;
74                }
75                RobotsDirective::Disallow(p) => {
76                    out.push_str(&format!("Disallow: {p}\n"));
77                    wrote = true;
78                }
79                RobotsDirective::CrawlDelay(n) => {
80                    out.push_str(&format!("Crawl-delay: {n}\n"));
81                    wrote = true;
82                }
83                RobotsDirective::Raw(line) => {
84                    out.push_str(line);
85                    if !line.ends_with('\n') {
86                        out.push('\n');
87                    }
88                    wrote = true;
89                }
90            }
91        }
92        if group.user_agent == "*" && cfg.from_noindex {
93            if let Some(table) = table {
94                for entry in &table.0 {
95                    if let RouteEntry::Http {
96                        method,
97                        path,
98                        meta,
99                    } = entry
100                    {
101                        if *method != Method::GET || path_is_dynamic(path) {
102                            continue;
103                        }
104                        if meta.get::<MetaPage>().is_some_and(|p| p.noindex) {
105                            out.push_str(&format!("Disallow: {path}\n"));
106                            wrote = true;
107                        }
108                    }
109                }
110            }
111        }
112        if !wrote {
113            out.push_str("Allow: /\n");
114        }
115        if let Some(delay) = cfg.crawl_delay {
116            if group.user_agent == "*"
117                && !group
118                    .directives
119                    .iter()
120                    .any(|d| matches!(d, RobotsDirective::CrawlDelay(_)))
121            {
122                out.push_str(&format!("Crawl-delay: {delay}\n"));
123            }
124        }
125        out.push('\n');
126    }
127
128    for line in &cfg.trailing_raw {
129        out.push_str(line);
130        if !line.ends_with('\n') {
131            out.push('\n');
132        }
133    }
134
135    let sitemap = cfg
136        .sitemap_url
137        .clone()
138        .or_else(|| {
139            if cfg.sitemap_from_plugin {
140                sitemap_cfg.and_then(|c| c.sitemap_url()).or_else(|| {
141                    let base = meta.and_then(|m| m.public_url.as_ref())?;
142                    let path = sitemap_cfg
143                        .map(|c| c.path.as_str())
144                        .unwrap_or("/sitemap.xml");
145                    Some(format!("{}{}", base.trim_end_matches('/'), path))
146                })
147            } else {
148                None
149            }
150        });
151    if let Some(url) = sitemap {
152        out.push_str(&format!("Sitemap: {url}\n"));
153    }
154    out
155}
sova_meta/robots.rs

sova_meta/
robots.rs
