sova_meta/
robots_plugin.rs

1//! Plugin `Robots` — `/robots.txt`.
2
3use crate::defaults::MetaDefaults;
4use crate::robots::{render_robots, RobotsConfig, RobotsDirective, RobotsGroup};
5use crate::sitemap_config::SitemapConfig;
6use sova_core::extend::RouteTable;
7use sova_core::{App, Plugin, Request, Response};
8
9/// Full-control robots.txt plugin.
10pub struct Robots {
11    cfg: RobotsConfig,
12}
13
14impl Robots {
15    pub fn new() -> Self {
16        Self {
17            cfg: RobotsConfig::default(),
18        }
19    }
20
21    pub fn path(mut self, p: impl Into<String>) -> Self {
22        self.cfg.path = p.into();
23        self
24    }
25
26    pub fn block_all(mut self) -> Self {
27        self.cfg.block_all = true;
28        self
29    }
30
31    pub fn allow(mut self, path: impl Into<String>) -> Self {
32        ensure_star_group(&mut self.cfg)
33            .directives
34            .push(RobotsDirective::Allow(path.into()));
35        self
36    }
37
38    pub fn disallow(mut self, path: impl Into<String>) -> Self {
39        ensure_star_group(&mut self.cfg)
40            .directives
41            .push(RobotsDirective::Disallow(path.into()));
42        self
43    }
44
45    pub fn user_agent<F>(mut self, name: impl Into<String>, f: F) -> Self
46    where
47        F: FnOnce(RobotsUaBuilder) -> RobotsUaBuilder,
48    {
49        let built = f(RobotsUaBuilder {
50            group: RobotsGroup {
51                user_agent: name.into(),
52                directives: Vec::new(),
53            },
54        });
55        self.cfg.groups.push(built.group);
56        self
57    }
58
59    pub fn sitemap(mut self, url: impl Into<String>) -> Self {
60        self.cfg.sitemap_url = Some(url.into());
61        self.cfg.sitemap_from_plugin = false;
62        self
63    }
64
65    pub fn sitemap_from_plugin(mut self, yes: bool) -> Self {
66        self.cfg.sitemap_from_plugin = yes;
67        if !yes {
68            self.cfg.sitemap_url = None;
69        }
70        self
71    }
72
73    pub fn from_noindex(mut self, yes: bool) -> Self {
74        self.cfg.from_noindex = yes;
75        self
76    }
77
78    pub fn crawl_delay(mut self, seconds: f32) -> Self {
79        self.cfg.crawl_delay = Some(seconds);
80        self
81    }
82
83    pub fn raw(mut self, line: impl Into<String>) -> Self {
84        self.cfg.trailing_raw.push(line.into());
85        self
86    }
87}
88
89impl Default for Robots {
90    fn default() -> Self {
91        Self::new()
92    }
93}
94
95/// Fluent builder for a User-agent group.
96pub struct RobotsUaBuilder {
97    group: RobotsGroup,
98}
99
100impl RobotsUaBuilder {
101    pub fn allow(mut self, path: impl Into<String>) -> Self {
102        self.group
103            .directives
104            .push(RobotsDirective::Allow(path.into()));
105        self
106    }
107
108    pub fn disallow(mut self, path: impl Into<String>) -> Self {
109        self.group
110            .directives
111            .push(RobotsDirective::Disallow(path.into()));
112        self
113    }
114
115    pub fn crawl_delay(mut self, seconds: f32) -> Self {
116        self.group
117            .directives
118            .push(RobotsDirective::CrawlDelay(seconds));
119        self
120    }
121
122    pub fn raw(mut self, line: impl Into<String>) -> Self {
123        self.group
124            .directives
125            .push(RobotsDirective::Raw(line.into()));
126        self
127    }
128}
129
130fn ensure_star_group(cfg: &mut RobotsConfig) -> &mut RobotsGroup {
131    if let Some(i) = cfg.groups.iter().position(|g| g.user_agent == "*") {
132        &mut cfg.groups[i]
133    } else {
134        cfg.groups.insert(
135            0,
136            RobotsGroup {
137                user_agent: "*".into(),
138                directives: Vec::new(),
139            },
140        );
141        &mut cfg.groups[0]
142    }
143}
144
145impl Plugin for Robots {
146    fn id(&self) -> &'static str {
147        "robots"
148    }
149
150    fn meta(&self) -> sova_core::PluginMeta {
151        sova_core::PluginMeta::new("Robots")
152            .description("Serve robots.txt with allow/disallow and Sitemap line")
153            .version(env!("CARGO_PKG_VERSION"))
154    }
155
156    fn install(mut self, app: &mut App) {
157        if let Some(doc) = app.config_doc() {
158            if let Some(section) = doc.section("robots") {
159                if section
160                    .get("block_all")
161                    .and_then(|v| v.as_bool())
162                    .unwrap_or(false)
163                    || section.get("block_all").and_then(|v| v.as_str()) == Some("true")
164                {
165                    self.cfg.block_all = true;
166                }
167                if let Some(v) = section.get("path").and_then(|v| v.as_str()) {
168                    self.cfg.path = v.to_string();
169                }
170                if let Some(v) = section.get("sitemap").and_then(|v| v.as_str()) {
171                    self.cfg.sitemap_url = Some(v.to_string());
172                    self.cfg.sitemap_from_plugin = false;
173                }
174                if let Some(v) = section.get("from_noindex").and_then(|v| v.as_bool()) {
175                    self.cfg.from_noindex = v;
176                }
177                if let Some(arr) = section.get("allow").and_then(|v| v.as_array()) {
178                    for item in arr {
179                        if let Some(s) = item.as_str() {
180                            ensure_star_group(&mut self.cfg)
181                                .directives
182                                .push(RobotsDirective::Allow(s.to_string()));
183                        }
184                    }
185                }
186                if let Some(arr) = section.get("disallow").and_then(|v| v.as_array()) {
187                    for item in arr {
188                        if let Some(s) = item.as_str() {
189                            ensure_star_group(&mut self.cfg)
190                                .directives
191                                .push(RobotsDirective::Disallow(s.to_string()));
192                        }
193                    }
194                }
195            }
196            // legacy [meta] robots = "block-all"
197            if let Some(section) = doc.section("meta") {
198                if section.get("robots").and_then(|v| v.as_str()) == Some("block-all") {
199                    self.cfg.block_all = true;
200                }
201            }
202        }
203
204        let mount = self.cfg.path.clone();
205        app.state(self.cfg.clone());
206
207        app.get(mount.as_str(), |req: Request| async move {
208            let cfg = req
209                .try_state::<RobotsConfig>()
210                .map(|c| (*c).clone())
211                .unwrap_or_default();
212            let table = req.try_state::<RouteTable>();
213            let sitemap = req.try_state::<SitemapConfig>();
214            let meta = req.try_state::<MetaDefaults>();
215            let body = render_robots(
216                &cfg,
217                table.as_deref(),
218                sitemap.as_deref(),
219                meta.as_deref(),
220            );
221            Response::text(body).header("content-type", "text/plain; charset=utf-8")
222        });
223    }
224}
sova_meta/robots_plugin.rs

sova_meta/
robots_plugin.rs
