dioxus_docs_kit/
server.rs

1//! Server-side (Axum) routes for crawler-facing endpoints.
2//!
3//! Available behind the `server` feature. These are plain Axum routes, **not**
4//! server functions: a server function JSON-encodes its `String` return value
5//! (quoted body, `application/json` content type), which robots.txt / sitemap /
6//! llms.txt consumers can't parse.
7//!
8//! # Example
9//!
10//! ```rust,ignore
11//! dioxus::server::serve(|| async {
12//!     let seo = SeoRouter::new("https://example.com", "My Docs", "Documentation for My Product")
13//!         .with_docs(&DOCS, "/docs")
14//!         .with_blog(&BLOG, "/blog")
15//!         .into_router();
16//!     Ok(dioxus::server::router(App).merge(seo))
17//! });
18//! ```
19
20use dioxus::server::axum::{Router, http::header, routing::get};
21
22use crate::blog::BlogRegistry;
23use crate::components::seo::xml_escape;
24use crate::registry::DocsRegistry;
25
26const TEXT: &str = "text/plain; charset=utf-8";
27const XML: &str = "application/xml; charset=utf-8";
28const MD: &str = "text/markdown; charset=utf-8";
29const RSS: &str = "application/rss+xml; charset=utf-8";
30
31/// Builder for the crawler-facing routes of a docs/blog site.
32///
33/// Generated routes:
34///
35/// | Route | Condition | Content |
36/// |---|---|---|
37/// | `<docs_path>/<page>.md` | docs | raw Markdown per doc page |
38/// | `/llms.txt`, `/llms-full.txt` | docs | LLM-friendly docs index / full corpus |
39/// | `/sitemap-docs.xml` | docs | docs sitemap |
40/// | `<blog_path>/<slug>.md` | blog | raw Markdown per post |
41/// | `<blog_path>/rss.xml` | blog | RSS feed |
42/// | `/sitemap-blog.xml` | blog | blog sitemap |
43/// | `/sitemap.xml` | always | sitemap index over the above |
44/// | `/robots.txt` | always | allow-all incl. explicit AI-crawler entries |
45pub struct SeoRouter {
46    site_url: String,
47    site_title: String,
48    site_description: String,
49    docs: Option<(&'static DocsRegistry, String)>,
50    blog: Option<(&'static BlogRegistry, String)>,
51}
52
53/// Normalize a base path to the form `/docs` (leading slash, no trailing slash).
54fn normalize_base(base_path: &str) -> String {
55    let trimmed = base_path.trim_matches('/');
56    if trimmed.is_empty() {
57        String::new()
58    } else {
59        format!("/{trimmed}")
60    }
61}
62
63impl SeoRouter {
64    /// Create a builder.
65    ///
66    /// - `site_url`: public origin, e.g. `"https://example.com"` (no trailing slash needed).
67    /// - `site_title` / `site_description`: used in `llms.txt` headers and the RSS channel.
68    pub fn new(site_url: &str, site_title: &str, site_description: &str) -> Self {
69        Self {
70            site_url: site_url.trim_end_matches('/').to_string(),
71            site_title: site_title.to_string(),
72            site_description: site_description.to_string(),
73            docs: None,
74            blog: None,
75        }
76    }
77
78    /// Serve docs endpoints for `registry`, mounted under `base_path` (e.g. `"/docs"`).
79    pub fn with_docs(mut self, registry: &'static DocsRegistry, base_path: &str) -> Self {
80        self.docs = Some((registry, normalize_base(base_path)));
81        self
82    }
83
84    /// Serve blog endpoints for `registry`, mounted under `base_path` (e.g. `"/blog"`).
85    pub fn with_blog(mut self, registry: &'static BlogRegistry, base_path: &str) -> Self {
86        self.blog = Some((registry, normalize_base(base_path)));
87        self
88    }
89
90    /// Build the Axum router. Merge it into your app router with
91    /// [`Router::merge`].
92    pub fn into_router(self) -> Router {
93        let mut router = Router::new();
94        let site_url = &self.site_url;
95
96        let mut sitemap_index_entries: Vec<String> = Vec::new();
97
98        if let Some((docs, base)) = &self.docs {
99            // Raw Markdown for each doc page at `<base>/<page>.md`. Registered as
100            // literal routes, one per known doc path, so they take priority over
101            // the SSR fallback without shadowing the HTML pages. OpenAPI endpoint
102            // pages have no Markdown source.
103            for path in docs.get_all_paths() {
104                if let Some(markdown) = docs.get_doc_content(path) {
105                    router = router.route(
106                        &format!("{base}/{path}.md"),
107                        get(move || async move { ([(header::CONTENT_TYPE, MD)], markdown) }),
108                    );
109                }
110            }
111
112            let docs_base_url = format!("{site_url}{base}");
113
114            let llms =
115                docs.generate_llms_txt(&self.site_title, &self.site_description, &docs_base_url);
116            router = router.route(
117                "/llms.txt",
118                get(move || async move { ([(header::CONTENT_TYPE, TEXT)], llms) }),
119            );
120
121            let llms_full = docs.generate_llms_full_txt(
122                &self.site_title,
123                &self.site_description,
124                &docs_base_url,
125            );
126            router = router.route(
127                "/llms-full.txt",
128                get(move || async move { ([(header::CONTENT_TYPE, TEXT)], llms_full) }),
129            );
130
131            let sitemap = docs.generate_sitemap(site_url, base);
132            router = router.route(
133                "/sitemap-docs.xml",
134                get(move || async move { ([(header::CONTENT_TYPE, XML)], sitemap) }),
135            );
136            sitemap_index_entries.push(format!("{site_url}/sitemap-docs.xml"));
137        }
138
139        if let Some((blog, base)) = &self.blog {
140            // Raw Markdown for each post at `<base>/<slug>.md`.
141            for post in blog.all_posts() {
142                let markdown = post.raw_markdown.as_str();
143                router = router.route(
144                    &format!("{base}/{}.md", post.slug),
145                    get(move || async move { ([(header::CONTENT_TYPE, MD)], markdown) }),
146                );
147            }
148
149            let rss = blog.generate_rss(&self.site_title, site_url, base);
150            router = router.route(
151                &format!("{base}/rss.xml"),
152                get(move || async move { ([(header::CONTENT_TYPE, RSS)], rss) }),
153            );
154
155            let sitemap = blog.generate_sitemap(site_url, base);
156            router = router.route(
157                "/sitemap-blog.xml",
158                get(move || async move { ([(header::CONTENT_TYPE, XML)], sitemap) }),
159            );
160            sitemap_index_entries.push(format!("{site_url}/sitemap-blog.xml"));
161        }
162
163        let sitemap_index = format!(
164            "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n\
165             <sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n{}\
166             </sitemapindex>\n",
167            sitemap_index_entries
168                .iter()
169                .map(|loc| format!("<sitemap><loc>{}</loc></sitemap>\n", xml_escape(loc)))
170                .collect::<String>()
171        );
172        router = router.route(
173            "/sitemap.xml",
174            get(move || async move { ([(header::CONTENT_TYPE, XML)], sitemap_index) }),
175        );
176
177        let robots = robots_txt(site_url);
178        router.route(
179            "/robots.txt",
180            get(move || async move { ([(header::CONTENT_TYPE, TEXT)], robots) }),
181        )
182    }
183}
184
185/// Allow-all robots.txt with explicit per-AI-crawler sections.
186///
187/// AI crawlers are listed explicitly so each can be controlled with a single
188/// line: flip its `Allow: /` to `Disallow: /` to block that bot. Covers
189/// training crawlers (GPTBot, ClaudeBot, anthropic-ai, Google-Extended, CCBot)
190/// and live-retrieval/search agents (ChatGPT-User, OAI-SearchBot,
191/// PerplexityBot).
192fn robots_txt(site_url: &str) -> String {
193    const AI_CRAWLERS: &[&str] = &[
194        "GPTBot",
195        "ChatGPT-User",
196        "OAI-SearchBot",
197        "ClaudeBot",
198        "anthropic-ai",
199        "Claude-Web",
200        "Google-Extended",
201        "PerplexityBot",
202        "CCBot",
203    ];
204
205    let mut out = String::from("User-agent: *\nAllow: /\n");
206    for bot in AI_CRAWLERS {
207        out.push_str(&format!("\nUser-agent: {bot}\nAllow: /\n"));
208    }
209    out.push_str(&format!("\nSitemap: {site_url}/sitemap.xml\n"));
210    out
211}
212
213#[cfg(test)]
214mod tests {
215    use super::{normalize_base, robots_txt};
216
217    #[test]
218    fn normalizes_base_paths() {
219        assert_eq!(normalize_base("/docs"), "/docs");
220        assert_eq!(normalize_base("docs/"), "/docs");
221        assert_eq!(normalize_base("/"), "");
222        assert_eq!(normalize_base(""), "");
223    }
224
225    #[test]
226    fn robots_txt_lists_ai_crawlers_and_sitemap() {
227        let out = robots_txt("https://example.com");
228        assert!(out.starts_with("User-agent: *\nAllow: /\n"));
229        assert!(out.contains("User-agent: GPTBot\nAllow: /\n"));
230        assert!(out.contains("User-agent: ClaudeBot\nAllow: /\n"));
231        assert!(out.ends_with("Sitemap: https://example.com/sitemap.xml\n"));
232    }
233}
dioxus_docs_kit/server.rs

dioxus_docs_kit/
server.rs
