Skip to main content

rustango_cms/
robots.rs

1//! `/robots.txt` for the public CMS.
2//!
3//! The CMS has served [`/sitemap.xml`](crate::sitemap) for a long time,
4//! and a sitemap no crawler is told about is a sitemap nobody reads: the
5//! conventional way to advertise one is a `Sitemap:` line in robots.txt.
6//! Until this module every rustango-cms site answered **404** there,
7//! which is also why crawlers were free to spend their budget on
8//! `/cms-admin/` and the login form.
9//!
10//! ## What is served
11//!
12//! A default that lets crawlers have the public site and keeps them out
13//! of the admin and the auth endpoints, plus an absolute `Sitemap:` line
14//! built from the request's own scheme and host — the same
15//! [`base_url`](crate::sitemap) the sitemap's `<loc>` entries use, so the
16//! two can never disagree about which host they are on.
17//!
18//! ## Overriding it
19//!
20//! A tenant that needs something else — a staging site that should
21//! disallow everything, say — stores a replacement body in the
22//! `seo` site-setting scope under `robots_txt`, and that is served
23//! verbatim. Nothing sensitive can leak through this: robots.txt is a
24//! public document by definition, and the value is only ever echoed
25//! back as `text/plain`.
26//!
27//! ## Wire-up
28//!
29//! The CMS public [`router`](crate::router()) mounts `GET /robots.txt`
30//! automatically, exactly as it does the sitemap. No registration needed.
31
32use axum::http::{header, HeaderMap, StatusCode};
33use axum::response::{IntoResponse, Response};
34use rustango::extractors::Tenant;
35
36/// Site-setting scope and key a tenant can override the body with.
37const SETTING_SCOPE: &str = "seo";
38const SETTING_KEY: &str = "robots_txt";
39
40/// Paths kept out of the crawl. These are the admin and auth mounts,
41/// which are hardcoded here the same way they are in
42/// [`crate::admin`] — there is no shared constant to reference yet, so
43/// a change to the admin prefix has to be made in both places.
44const DISALLOWED: &[&str] = &["/cms-admin/", "/login", "/logout"];
45
46/// `GET /robots.txt`.
47///
48/// Answers `text/plain`, always 200 — a crawler that gets an error here
49/// may treat the whole site as disallowed, so a failure to read the
50/// override falls back to the default rather than surfacing a 5xx.
51pub async fn handle_robots(t: Tenant, headers: HeaderMap) -> Response {
52    let body = match crate::site_setting::get(t.pool(), SETTING_SCOPE).await {
53        Ok(Some(setting)) => custom_body(&setting.value_json),
54        // No `seo` scope stored: the common case, not a problem.
55        Ok(None) => None,
56        Err(e) => {
57            tracing::warn!(
58                target: "rustango_cms::robots",
59                error = %e,
60                "could not read the seo site setting; serving the default robots.txt"
61            );
62            None
63        }
64    };
65
66    let body = body.unwrap_or_else(|| default_body(&crate::sitemap::base_url(&headers)));
67
68    (
69        StatusCode::OK,
70        [(header::CONTENT_TYPE, "text/plain; charset=utf-8")],
71        body,
72    )
73        .into_response()
74}
75
76/// The stored override, when the scope holds a non-empty `robots_txt`
77/// string. A blank value is treated as absent so clearing the field in
78/// an admin form restores the default instead of serving an empty file,
79/// which a crawler reads as "everything is allowed" — including the
80/// admin.
81fn custom_body(value: &serde_json::Value) -> Option<String> {
82    let text = value.get(SETTING_KEY)?.as_str()?.trim();
83    if text.is_empty() {
84        return None;
85    }
86    Some(format!("{text}\n"))
87}
88
89/// The default policy: the public site is crawlable, the admin and auth
90/// endpoints are not, and the sitemap is advertised absolutely.
91fn default_body(base: &str) -> String {
92    let mut out = String::from("User-agent: *\n");
93    for path in DISALLOWED {
94        out.push_str("Disallow: ");
95        out.push_str(path);
96        out.push('\n');
97    }
98    out.push_str("\nSitemap: ");
99    out.push_str(base);
100    out.push_str("/sitemap.xml\n");
101    out
102}
103
104#[cfg(test)]
105mod tests {
106    use super::*;
107
108    #[test]
109    fn the_default_advertises_the_sitemap_on_the_requested_host() {
110        let body = default_body("https://rustango.com");
111
112        assert!(body.contains("\nSitemap: https://rustango.com/sitemap.xml\n"));
113        // Same host the sitemap's own <loc> entries use — the point of
114        // sharing `base_url` is that these cannot drift apart.
115        assert!(!body.contains("localhost"));
116    }
117
118    #[test]
119    fn the_default_keeps_crawlers_out_of_the_admin_and_auth() {
120        let body = default_body("https://example.org");
121
122        assert!(body.starts_with("User-agent: *\n"));
123        assert!(body.contains("\nDisallow: /cms-admin/\n"));
124        assert!(body.contains("\nDisallow: /login\n"));
125        assert!(body.contains("\nDisallow: /logout\n"));
126    }
127
128    #[test]
129    fn a_stored_override_is_served_verbatim() {
130        let value = serde_json::json!({ "robots_txt": "User-agent: *\nDisallow: /" });
131
132        assert_eq!(
133            custom_body(&value).as_deref(),
134            Some("User-agent: *\nDisallow: /\n"),
135        );
136    }
137
138    /// A blank or missing override must fall back, never serve an empty
139    /// file — an empty robots.txt allows everything, including the admin
140    /// paths the default is there to exclude.
141    #[test]
142    fn a_blank_or_absent_override_falls_back_to_the_default() {
143        assert_eq!(custom_body(&serde_json::json!({ "robots_txt": "" })), None);
144        assert_eq!(custom_body(&serde_json::json!({ "robots_txt": "   \n" })), None);
145        assert_eq!(custom_body(&serde_json::json!({ "other": "x" })), None);
146        assert_eq!(custom_body(&serde_json::json!({})), None);
147        // Wrong type, e.g. someone stored a bool through the JSON column.
148        assert_eq!(custom_body(&serde_json::json!({ "robots_txt": true })), None);
149    }
150}