Skip to main content

rustango_cms/
sitemap.rs

1//! `/sitemap.xml` for the public CMS — adopts [`rustango::sitemaps`].
2//!
3//! Walks every published, indexable [`Page`] in the current tenant,
4//! turns each row into a [`SitemapEntry`] (using the row's
5//! `updated_at` for `<lastmod>` and `sitemap_priority` for
6//! `<priority>`), and renders the sitemaps.org `<urlset>` XML.
7//!
8//! Pages excluded:
9//!
10//! - `status != "published"` — drafts / scheduled / archived rows.
11//! - `robots_index = false` — the per-page noindex toggle on
12//!   [`Page::robots_index`].
13//! - Locale variants (`locale_variant_of IS NOT NULL`) — the
14//!   canonical row covers the URL; including both produces
15//!   duplicate `<loc>` for the same path.
16//!
17//! ## Wire-up
18//!
19//! The CMS public [`router`] mounts `GET /sitemap.xml` automatically.
20//! No extra handler registration is required.
21//!
22//! [`router`]: crate::router()
23//! [`Page`]: crate::Page
24
25use axum::http::{header, HeaderMap, StatusCode};
26use axum::response::{IntoResponse, Response};
27use rustango::core::Column as _;
28use rustango::extractors::Tenant;
29use rustango::sql::FetcherPool as _;
30
31use crate::locale_mode::LocaleMode;
32use crate::page::Page;
33
34/// Sitemap shard size — sitemaps.org caps each `<urlset>` at 50 000
35/// URLs / 50 MB. Below the cap we emit one flat sitemap.xml; above,
36/// we emit a `<sitemapindex>` pointing at numbered shards.
37pub const SHARD_SIZE: usize = 50_000;
38
39// ---- #441 — custom (non-page) sitemap sources --------------------
40
41/// One non-page URL contributed to the sitemap by a registered
42/// [`SitemapSource`] — e.g. a document, a routable sub-URL, or a
43/// host-specific landing page. Appended after the page URLs in the
44/// same sharded / indexed `/sitemap.xml`.
45#[derive(Debug, Clone)]
46pub struct SitemapEntry {
47    /// Absolute URL (`https://host/path`). Sources build it from the
48    /// `base` they're handed so scheme/host stay consistent with the
49    /// page locs.
50    pub loc: String,
51    pub lastmod: Option<chrono::DateTime<chrono::Utc>>,
52    /// `<changefreq>` hint (`daily`, `weekly`, …); emitted verbatim.
53    pub changefreq: Option<String>,
54    /// `<priority>` 0.0–1.0 (clamped on render).
55    pub priority: Option<f32>,
56}
57
58impl SitemapEntry {
59    #[must_use]
60    pub fn new(loc: impl Into<String>) -> Self {
61        Self {
62            loc: loc.into(),
63            lastmod: None,
64            changefreq: None,
65            priority: None,
66        }
67    }
68
69    #[must_use]
70    pub fn with_lastmod(mut self, ts: chrono::DateTime<chrono::Utc>) -> Self {
71        self.lastmod = Some(ts);
72        self
73    }
74
75    #[must_use]
76    pub fn with_changefreq(mut self, cf: impl Into<String>) -> Self {
77        self.changefreq = Some(cf.into());
78        self
79    }
80
81    #[must_use]
82    pub fn with_priority(mut self, p: f32) -> Self {
83        self.priority = Some(p);
84        self
85    }
86
87    /// Emit this entry as a `<url>…</url>` block (2-space indent).
88    fn push_url(&self, xml: &mut String) {
89        xml.push_str("  <url>\n");
90        push_lp(xml, "loc", &self.loc, 4);
91        if let Some(ts) = self.lastmod {
92            push_lp(xml, "lastmod", &format_iso8601(ts), 4);
93        }
94        if let Some(cf) = &self.changefreq {
95            push_lp(xml, "changefreq", cf, 4);
96        }
97        if let Some(p) = self.priority {
98            push_lp(xml, "priority", &format!("{:.1}", p.clamp(0.0, 1.0)), 4);
99        }
100        xml.push_str("  </url>\n");
101    }
102}
103
104/// A registered source of non-page sitemap URLs. Implement + register
105/// with [`crate::register_sitemap_source!`]; the public `/sitemap.xml`
106/// appends every source's entries after the page URLs, inside the same
107/// shard / index accounting.
108#[async_trait::async_trait]
109pub trait SitemapSource: Send + Sync {
110    /// The URLs this source contributes. `base` is the scheme+host
111    /// prefix (no trailing slash) so entries match the page locs.
112    async fn entries(&self, pool: &rustango::sql::Pool, base: &str) -> Vec<SitemapEntry>;
113}
114
115/// Inventory registration for a [`SitemapSource`]. Mirrors
116/// [`crate::page_type::PageTypeHandlerRegistration`].
117pub struct SitemapSourceRegistration {
118    pub factory: fn() -> Box<dyn SitemapSource>,
119}
120inventory::collect!(SitemapSourceRegistration);
121
122/// Gather every registered source's entries, in registration order.
123pub async fn collect_custom_entries(pool: &rustango::sql::Pool, base: &str) -> Vec<SitemapEntry> {
124    let mut out = Vec::new();
125    for reg in inventory::iter::<SitemapSourceRegistration> {
126        out.extend((reg.factory)().entries(pool, base).await);
127    }
128    out
129}
130
131/// Register a [`SitemapSource`]. Pair with a `Default` impl.
132///
133/// ```ignore
134/// #[derive(Default)]
135/// struct DocsSource;
136/// #[async_trait::async_trait]
137/// impl rustango_cms::sitemap::SitemapSource for DocsSource {
138///     async fn entries(&self, _pool: &rustango::sql::Pool, base: &str)
139///         -> Vec<rustango_cms::sitemap::SitemapEntry> {
140///         vec![rustango_cms::sitemap::SitemapEntry::new(format!("{base}/docs"))
141///             .with_changefreq("weekly")]
142///     }
143/// }
144/// rustango_cms::register_sitemap_source!(DocsSource);
145/// ```
146#[macro_export]
147macro_rules! register_sitemap_source {
148    ($ty:ty) => {
149        $crate::inventory::submit! {
150            $crate::sitemap::SitemapSourceRegistration {
151                factory: || ::std::boxed::Box::new(<$ty>::default()),
152            }
153        }
154    };
155}
156
157/// `<lastmod>` source for a page: `last_published_at` (set only on
158/// public-visible changes) preferred over `updated_at`.
159fn page_lastmod(p: &Page) -> Option<chrono::DateTime<chrono::Utc>> {
160    if let Some(ts) = p.last_published_at {
161        Some(ts)
162    } else if let rustango::sql::Auto::Set(ts) = p.updated_at {
163        Some(ts)
164    } else {
165        None
166    }
167}
168
169/// Ids per `IN (…)` list. SQLite refuses a statement with more than
170/// 32,766 binds, and a shard holds up to [`SHARD_SIZE`] pages.
171const IN_CHUNK: usize = 10_000;
172
173/// Page-extension loads [`collect_images_for_pages`] runs at once.
174const HERO_LOADS: usize = 8;
175
176/// Unified shard window `[start, end)` over `total` URLs for a 1-based
177/// shard number, or `None` when the shard is out of range (`0`
178/// included).
179fn shard_window(total: usize, shard_one_based: usize) -> Option<(usize, usize)> {
180    if shard_one_based == 0 {
181        return None;
182    }
183    let start = (shard_one_based - 1) * SHARD_SIZE;
184    if start >= total {
185        return None;
186    }
187    Some((start, (start + SHARD_SIZE).min(total)))
188}
189
190/// The part of the tree a host's sitemap covers.
191///
192/// A host mapped to a page (Sites screen) serves only that page's
193/// subtree, at paths relative to it, so its sitemap lists the same. A
194/// host on the default root lists everything, as before.
195#[derive(Default)]
196struct SiteScope {
197    /// The root's `url_path` without a trailing slash, `""` for the default root.
198    prefix: String,
199    /// The root's materialized tree path, `None` for the default root.
200    tree_path: Option<String>,
201}
202
203impl SiteScope {
204    async fn for_host(pool: &rustango::sql::Pool, headers: &HeaderMap) -> Self {
205        // A failed lookup lists the whole tenant, like the page router's fallback.
206        match crate::site::root_for_host(pool, crate::router::host_header(headers)).await {
207            Ok(Some(root)) => {
208                let prefix = root.url_path.trim_end_matches('/').to_owned();
209                if prefix.is_empty() {
210                    Self::default()
211                } else {
212                    Self { prefix, tree_path: Some(root.path) }
213                }
214            }
215            _ => Self::default(),
216        }
217    }
218
219    /// Narrow a page query to this site's subtree.
220    fn narrow(
221        &self,
222        qs: rustango::query::QuerySet<Page>,
223    ) -> rustango::query::QuerySet<Page> {
224        match &self.tree_path {
225            Some(tree) => qs.where_(Page::path.startswith(tree)),
226            None => qs,
227        }
228    }
229
230    /// Rewrite fetched rows to the paths this host serves them at.
231    fn to_public(&self, rows: &mut [Page]) {
232        if self.prefix.is_empty() {
233            return;
234        }
235        for p in rows {
236            p.url_path = crate::site::to_public_path(&self.prefix, &p.url_path);
237        }
238    }
239}
240
241/// `GET /sitemap.xml` — emits the sitemaps.org XML for the current
242/// tenant's published, indexable pages.
243///
244/// Below [`SHARD_SIZE`] URLs, returns a flat `<urlset>` (the old
245/// behavior). At/above, returns a `<sitemapindex>` pointing at
246/// `/sitemap/1`, `/sitemap/2`, … (handled by
247/// [`handle_sitemap_shard`]).
248///
249/// Includes `xhtml:link rel="alternate" hreflang=…` clustering for
250/// locale variants + an `<image:image>` extension when the page-type
251/// handler surfaces a hero media id.
252// The public router mounts this automatically and supplies `PublicState`
253// (a crate-internal type); consumers never call it directly, so exposing
254// the private state in the signature is intentional.
255#[allow(private_interfaces)]
256pub async fn handle_sitemap(
257    axum::extract::State(state): axum::extract::State<crate::router::PublicState>,
258    t: Tenant,
259    headers: HeaderMap,
260) -> Response {
261    let base = base_url(&headers);
262    let mode = state.locale_mode();
263
264    // Fetch every public+indexable row — both canonicals AND locale
265    // variants — so we can emit one `<url>` per URL plus hreflang
266    // annotations grouping the variants. #556 — archived pages stay in
267    // the sitemap (no link rot) at a capped priority; `expired` (takedown)
268    // is excluded.
269    let scope = SiteScope::for_host(t.pool(), &headers).await;
270    let listed = scope.narrow(
271        Page::objects()
272            .where_(Page::status.is_in(crate::page::PageStatus::public_strings()))
273            .where_(Page::robots_index.eq(true)),
274    );
275    // #644 — a view-restricted page is not listed: crawlers are anonymous.
276    let mut rows: Vec<Page> = match crate::view_restriction::only_anonymous_visible(t.pool(), listed)
277        .await
278        .order_by(&[("path", false)])
279        .fetch(t.pool())
280        .await
281    {
282        Ok(rows) => rows,
283        Err(e) => {
284            tracing::error!(error = %e, "sitemap: page fetch failed");
285            return (StatusCode::INTERNAL_SERVER_ERROR, "sitemap unavailable").into_response();
286        }
287    };
288    // Error pages (404/500 …) serve as substitutes for failures, not
289    // destinations — keep them out of the sitemap even when published.
290    if let Some(tid) = crate::error_pages::error_page_type_id(t.pool()).await {
291        rows.retain(|p| p.page_type_id != tid);
292    }
293    scope.to_public(&mut rows);
294
295    // #441 — non-page URLs from registered sources, appended after the
296    // pages in the same (sharded, indexed) sitemap.
297    let custom = collect_custom_entries(t.pool(), &base).await;
298
299    // #187 — split when total URLs exceed the shard cap. Return an
300    // index pointing at numbered shards instead of one giant urlset.
301    if rows.len() + custom.len() > SHARD_SIZE {
302        let mut lastmods: Vec<_> = rows.iter().map(page_lastmod).collect();
303        lastmods.extend(custom.iter().map(|e| e.lastmod));
304        return build_sitemap_index(&base, &lastmods);
305    }
306
307    let xml = build_urlset(t.pool(), &base, &rows, &custom, mode).await;
308    (
309        StatusCode::OK,
310        [(header::CONTENT_TYPE, "application/xml; charset=utf-8")],
311        xml,
312    )
313        .into_response()
314}
315
316/// `GET /sitemap/{n}` — one shard of [`SHARD_SIZE`] URLs from the
317/// full page list, in `path` order. Returns 404 when `n` is past the
318/// last shard.
319#[allow(private_interfaces)]
320pub async fn handle_sitemap_shard(
321    axum::extract::State(state): axum::extract::State<crate::router::PublicState>,
322    t: Tenant,
323    headers: HeaderMap,
324    axum::extract::Path(shard_one_based): axum::extract::Path<usize>,
325) -> Response {
326    let base = base_url(&headers);
327    let mode = state.locale_mode();
328    // Same filter as `handle_sitemap`, so shard windows line up with the
329    // index. Counted, then only this shard's window fetched (#691): a
330    // crawl of K shards used to fetch every public page K times.
331    use rustango::sql::CounterPool as _;
332    let error_tid = crate::error_pages::error_page_type_id(t.pool()).await;
333    let scope = SiteScope::for_host(t.pool(), &headers).await;
334    let listed = || async {
335        let qs = scope.narrow(
336            Page::objects()
337                .where_(Page::status.is_in(crate::page::PageStatus::public_strings()))
338                .where_(Page::robots_index.eq(true)),
339        );
340        let qs = match error_tid {
341            Some(tid) => qs.where_(Page::page_type_id.ne(tid)),
342            None => qs,
343        };
344        crate::view_restriction::only_anonymous_visible(t.pool(), qs).await
345    };
346    let p = match listed().await.count(t.pool()).await {
347        Ok(n) => usize::try_from(n).unwrap_or(0),
348        Err(e) => {
349            tracing::error!(error = %e, "sitemap shard: page count failed");
350            return (StatusCode::INTERNAL_SERVER_ERROR, "sitemap unavailable").into_response();
351        }
352    };
353    // #441 — pages occupy the first `p` slots of the unified URL list,
354    // custom-source entries the tail; shard windows span both.
355    let custom = collect_custom_entries(t.pool(), &base).await;
356    let total = p + custom.len();
357    let Some((start, end)) = shard_window(total, shard_one_based) else {
358        return (StatusCode::NOT_FOUND, "shard out of range").into_response();
359    };
360    let mut page_slice: Vec<Page> = if start < p {
361        match listed()
362            .await
363            .order_by(&[("path", false), ("id", false)])
364            .offset(start as i64)
365            .limit((end.min(p) - start) as i64)
366            .fetch(t.pool())
367            .await
368        {
369            Ok(rows) => rows,
370            Err(e) => {
371                tracing::error!(error = %e, "sitemap shard: page fetch failed");
372                return (StatusCode::INTERNAL_SERVER_ERROR, "sitemap unavailable").into_response();
373            }
374        }
375    } else {
376        Vec::new()
377    };
378    scope.to_public(&mut page_slice);
379    let custom_slice: Vec<SitemapEntry> =
380        custom[start.saturating_sub(p)..end.saturating_sub(p)].to_vec();
381    // Re-use the same renderer the flat path uses. hreflang grouping is
382    // scoped to this shard's rows (so a translation in a different
383    // shard won't emit a cross-shard link — acceptable, since the
384    // sitemaps.org spec doesn't require cross-shard
385    // grouping).
386    let xml = build_urlset(t.pool(), &base, &page_slice, &custom_slice, mode).await;
387    (
388        StatusCode::OK,
389        [(header::CONTENT_TYPE, "application/xml; charset=utf-8")],
390        xml,
391    )
392        .into_response()
393}
394
395/// Build a `<sitemapindex>` over the unified URL list (pages then
396/// custom-source entries), pointing at /sitemap/1, /sitemap/2, …
397/// `lastmods[i]` is the `<lastmod>` of the i-th URL; each shard's
398/// `<lastmod>` is the max across its window.
399fn build_sitemap_index(base: &str, lastmods: &[Option<chrono::DateTime<chrono::Utc>>]) -> Response {
400    let total = lastmods.len();
401    let shard_count = total.div_ceil(SHARD_SIZE);
402    let mut xml = String::with_capacity(256 + shard_count * 128);
403    xml.push_str("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
404    xml.push_str("<sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n");
405    for shard in 0..shard_count {
406        let start = shard * SHARD_SIZE;
407        let end = (start + SHARD_SIZE).min(total);
408        let max_updated = lastmods[start..end].iter().flatten().copied().max();
409        xml.push_str("  <sitemap>\n");
410        push_lp(&mut xml, "loc", &format!("{base}/sitemap/{}", shard + 1), 4);
411        if let Some(ts) = max_updated {
412            push_lp(&mut xml, "lastmod", &format_iso8601(ts), 4);
413        }
414        xml.push_str("  </sitemap>\n");
415    }
416    xml.push_str("</sitemapindex>\n");
417    (
418        StatusCode::OK,
419        [(header::CONTENT_TYPE, "application/xml; charset=utf-8")],
420        xml,
421    )
422        .into_response()
423}
424
425/// Render one `<urlset>` for the given pages. Extracted so both the
426/// flat `/sitemap.xml` path AND the per-shard `/sitemap/{n}`
427/// handler can call into the same code.
428async fn build_urlset(
429    pool: &rustango::sql::Pool,
430    base: &str,
431    rows: &[Page],
432    custom: &[SitemapEntry],
433    mode: LocaleMode,
434) -> String {
435    let mut group_key: std::collections::HashMap<i64, i64> = std::collections::HashMap::new();
436    for p in rows {
437        let Some(pid) = p.id.get().copied() else {
438            continue;
439        };
440        let key = p.locale_variant_of.or(p.alias_of).unwrap_or(pid);
441        group_key.insert(pid, key);
442    }
443    let mut groups: std::collections::HashMap<i64, Vec<&Page>> = std::collections::HashMap::new();
444    for p in rows {
445        let Some(pid) = p.id.get().copied() else {
446            continue;
447        };
448        let key = *group_key.get(&pid).unwrap_or(&pid);
449        groups.entry(key).or_default().push(p);
450    }
451    let locales: Vec<crate::locale::Locale> = crate::locale::Locale::objects()
452        .where_(crate::locale::Locale::active.eq(true))
453        .fetch(pool)
454        .await
455        .unwrap_or_default();
456    let default_locale_code = locales
457        .iter()
458        .find(|l| l.is_default)
459        .map(|l| l.code.clone())
460        .unwrap_or_else(|| "en".to_owned());
461    // Resolve every page's `<image:image>` rows up front in a bounded
462    // number of queries (#521). This replaced a per-page N+1 (one
463    // PageType + ReferenceIndex + Media lookup *per page*) that, under
464    // concurrency, pinned the tenant connection pool long enough to
465    // exhaust it. The batched map feeds `render_images` per page below.
466    let images_by_page = collect_images_for_pages(pool, rows, base).await;
467    // #i18n — override-model localization (one canonical page + per-field
468    // `cms_translation` overrides, the current model) has no per-locale
469    // sibling pages, so a page can still be served in every active locale
470    // via its locale-prefixed URL. When ≥1 active non-default locale
471    // exists ("languages set"), emit one `<url>` per locale for each page
472    // with a full bidirectional hreflang cluster. Legacy `locale_variant_of`
473    // pages keep their existing sibling-grouped behavior below.
474    let non_default: Vec<&crate::locale::Locale> =
475        locales.iter().filter(|l| !l.is_default).collect();
476    let langs_set = !non_default.is_empty();
477    let mut xml = String::with_capacity(512 + rows.len() * 256);
478    xml.push_str("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
479    xml.push_str("<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\"\n");
480    xml.push_str("        xmlns:xhtml=\"http://www.w3.org/1999/xhtml\"\n");
481    xml.push_str("        xmlns:image=\"http://www.google.com/schemas/sitemap-image/1.1\">\n");
482    for p in rows {
483        let Some(pid) = p.id.get().copied() else {
484            continue;
485        };
486        // #251 — prefer `last_published_at` (set only when public content
487        // changed) over `updated_at`; see [`page_lastmod`]. Computed once
488        // and shared across a page's per-locale `<url>` entries.
489        let lastmod = page_lastmod(p).map(format_iso8601);
490        // #556 — archived pages stay indexed but rank below current
491        // content: cap their priority at 0.3 (crawlers keep them, prefer
492        // the live pages). Non-archived pages keep their configured value.
493        let priority_f = f64::from(p.sitemap_priority).clamp(0.0, 1.0);
494        let priority_f = if p.status == crate::page::PageStatus::Archived.as_str() {
495            priority_f.min(0.3)
496        } else {
497            priority_f
498        };
499        let priority = format!("{priority_f:.1}");
500        let images_xml = render_images(images_by_page.get(&pid).map(Vec::as_slice).unwrap_or(&[]));
501        let group = group_key.get(&pid).copied().unwrap_or(pid);
502        let has_variants = groups.get(&group).is_some_and(|g| g.len() > 1);
503
504        if langs_set && !has_variants && p.locale_variant_of.is_none() && p.alias_of.is_none() {
505            // Override-model i18n: emit the default-locale URL plus one URL
506            // per active non-default locale, each carrying the same full
507            // hreflang cluster (every locale + x-default → the default URL),
508            // so search engines discover + bidirectionally link all variants.
509            let default_url = url_for(base, p);
510            let mut links = hreflang_link(&default_locale_code, &default_url);
511            for l in &non_default {
512                links.push_str(&hreflang_link(&l.code, &locale_url(base, p, &l.code, mode)));
513            }
514            links.push_str(&hreflang_link("x-default", &default_url));
515            emit_url(
516                &mut xml,
517                &default_url,
518                lastmod.as_deref(),
519                &priority,
520                &links,
521                &images_xml,
522            );
523            for l in &non_default {
524                let loc = locale_url(base, p, &l.code, mode);
525                emit_url(
526                    &mut xml,
527                    &loc,
528                    lastmod.as_deref(),
529                    &priority,
530                    &links,
531                    &images_xml,
532                );
533            }
534        } else {
535            // Single `<url>` — with legacy `locale_variant_of` sibling
536            // alternates when the page belongs to a variant group.
537            let url = url_for(base, p);
538            let mut links = String::new();
539            if let Some(group_pages) = groups.get(&group) {
540                if group_pages.len() > 1 {
541                    for member in group_pages {
542                        // Canonical (no variant parent) → default locale.
543                        // A locale-variant page carries its language as the
544                        // first URL segment (`/de/…`, `/fr/…`) for standalone
545                        // per-locale trees, so derive the hreflang from that
546                        // rather than emitting an unhelpful `und`.
547                        let code = if member.locale_variant_of.is_some() {
548                            variant_hreflang(&member.url_path)
549                        } else {
550                            default_locale_code.clone()
551                        };
552                        links.push_str(&hreflang_link(&code, &url_for(base, member)));
553                    }
554                    if let Some(canonical) =
555                        group_pages.iter().find(|m| m.locale_variant_of.is_none())
556                    {
557                        links.push_str(&hreflang_link("x-default", &url_for(base, canonical)));
558                    }
559                }
560            }
561            emit_url(
562                &mut xml,
563                &url,
564                lastmod.as_deref(),
565                &priority,
566                &links,
567                &images_xml,
568            );
569        }
570        // (images + `</url>` are emitted inside `emit_url` above, using the
571        // batched `images_by_page` map.)
572    }
573    // #441 — non-page entries from registered sources.
574    for entry in custom {
575        entry.push_url(&mut xml);
576    }
577    xml.push_str("</urlset>\n");
578    xml
579}
580
581/// Build the canonical URL for a page (`base + url_path`).
582fn url_for(base: &str, p: &Page) -> String {
583    let path = if p.url_path.is_empty() {
584        "/".to_owned()
585    } else {
586        p.url_path.clone()
587    };
588    format!("{base}{path}")
589}
590
591/// Build the URL that serves page `p` in locale `code`, per the router's
592/// [`LocaleMode`]: path modes prefix `/{code}` (`…/fr/blog`), query mode
593/// appends `?lang={code}` (`…/blog?lang=fr`). Mirrors
594/// [`crate::language_switcher::build`] so sitemap links match the switcher.
595fn locale_url(base: &str, p: &Page, code: &str, mode: LocaleMode) -> String {
596    let path = if p.url_path.is_empty() {
597        "/"
598    } else {
599        p.url_path.as_str()
600    };
601    if mode.reads_path() {
602        format!("{base}/{code}{path}")
603    } else {
604        format!("{base}{path}?lang={code}")
605    }
606}
607
608/// Derive the hreflang code for a standalone locale-variant page from the
609/// first segment of its `url_path` (`/de/guides/x` → `de`). Accepts a bare
610/// two-letter language code or a `ll-RR` language-region tag; anything else
611/// (e.g. a version token like `0.51`) falls back to `und`. Used for per-locale
612/// page trees whose language is expressed in the URL rather than a `cms_locale`
613/// association.
614fn variant_hreflang(url_path: &str) -> String {
615    let seg = url_path
616        .trim_start_matches('/')
617        .split('/')
618        .next()
619        .unwrap_or("");
620    let two_alpha = |s: &str| s.len() == 2 && s.bytes().all(|b| b.is_ascii_lowercase());
621    let is_code = two_alpha(seg)
622        || (seg.len() == 5
623            && seg.as_bytes()[2] == b'-'
624            && two_alpha(&seg[..2])
625            && two_alpha(&seg[3..]));
626    if is_code {
627        seg.to_owned()
628    } else {
629        "und".to_owned()
630    }
631}
632
633/// One `<xhtml:link rel="alternate" hreflang=…>` line (4-space indent).
634fn hreflang_link(code: &str, href: &str) -> String {
635    format!(
636        "    <xhtml:link rel=\"alternate\" hreflang=\"{}\" href=\"{}\"/>\n",
637        xml_escape(code),
638        xml_escape(href)
639    )
640}
641
642/// Emit one `<url>` block: `<loc>`, optional `<lastmod>`, `<priority>`,
643/// then the pre-rendered hreflang `links` + `images_xml` (both already
644/// indented). Shared by the per-locale and single-URL paths.
645fn emit_url(
646    xml: &mut String,
647    loc: &str,
648    lastmod: Option<&str>,
649    priority: &str,
650    links: &str,
651    images_xml: &str,
652) {
653    xml.push_str("  <url>\n");
654    push_lp(xml, "loc", loc, 4);
655    if let Some(ts) = lastmod {
656        push_lp(xml, "lastmod", ts, 4);
657    }
658    push_lp(xml, "priority", priority, 4);
659    xml.push_str(links);
660    xml.push_str(images_xml);
661    xml.push_str("  </url>\n");
662}
663
664/// Pre-render a page's `<image:image>` blocks (6-space indent) so they can
665/// be reused across per-locale `<url>` entries without re-querying.
666fn render_images(images: &[ImageRow]) -> String {
667    let mut s = String::new();
668    for img in images {
669        s.push_str("    <image:image>\n");
670        push_lp(&mut s, "image:loc", &img.loc, 6);
671        if let Some(title) = &img.title {
672            push_lp(&mut s, "image:title", title, 6);
673        }
674        if let Some(caption) = &img.caption {
675            push_lp(&mut s, "image:caption", caption, 6);
676        }
677        s.push_str("    </image:image>\n");
678    }
679    s
680}
681
682/// One `<image:image>` row.
683struct ImageRow {
684    loc: String,
685    title: Option<String>,
686    caption: Option<String>,
687}
688
689/// Resolve every page's `<image:image>` rows in a bounded number of
690/// queries — one `PageType` fetch, one `ReferenceIndex` fetch and one
691/// `Media` fetch for the whole page set — instead of the former
692/// per-page N+1 (which emitted those three queries *per page*, so a
693/// large sitemap fired thousands and, under concurrency, held the
694/// tenant pool long enough to deadlock it).
695///
696/// Hero media still come from the page-type handler's `load_extension`
697/// (v0 convention: a `hero_media_id` key in the extension JSON), which
698/// has no batch form: one query per page of a type with an extension
699/// table. Those run [`HERO_LOADS`] at a time rather than one after
700/// another. Id lists are sent in [`IN_CHUNK`]-sized pieces, under
701/// SQLite's bind limit. Returns a `page_id -> rows` map; pages with no
702/// images are absent.
703async fn collect_images_for_pages(
704    pool: &rustango::sql::Pool,
705    pages: &[Page],
706    base: &str,
707) -> std::collections::HashMap<i64, Vec<ImageRow>> {
708    use crate::page_type::find_handler;
709    use crate::page_type_model::PageType;
710    use crate::reference_index::ReferenceIndex;
711
712    let mut out: std::collections::HashMap<i64, Vec<ImageRow>> = std::collections::HashMap::new();
713    let page_ids: Vec<i64> = pages
714        .iter()
715        .filter_map(|p| p.id.get().copied())
716        .filter(|&id| id != 0)
717        .collect();
718    if page_ids.is_empty() {
719        return out;
720    }
721
722    // 1 query — page types keyed by id, so each page resolves its
723    // handler from memory rather than via a per-page lookup.
724    let mut type_ids: Vec<i64> = pages.iter().map(|p| p.page_type_id).collect();
725    type_ids.sort_unstable();
726    type_ids.dedup();
727    let type_name_by_id: std::collections::HashMap<i64, String> = PageType::objects()
728        .where_(PageType::id.is_in(type_ids))
729        .fetch(pool)
730        .await
731        .unwrap_or_default()
732        .into_iter()
733        .filter_map(|pt| pt.id.get().copied().map(|id| (id, pt.type_name)))
734        .collect();
735
736    // 1 query — every page->media reference, grouped by page. Ordered
737    // by `to_id` to preserve the former per-page query's ordering.
738    let mut indexed_by_page: std::collections::HashMap<i64, Vec<i64>> =
739        std::collections::HashMap::new();
740    for chunk in page_ids.chunks(IN_CHUNK) {
741        let refs = ReferenceIndex::objects()
742            .where_(ReferenceIndex::from_kind.eq(crate::reference_index::KIND_PAGE))
743            .where_(ReferenceIndex::from_id.is_in(chunk.iter().copied()))
744            .where_(ReferenceIndex::to_kind.eq(crate::reference_index::KIND_MEDIA))
745            .order_by(&[("to_id", false)])
746            .fetch(pool)
747            .await
748            .unwrap_or_default();
749        for r in refs {
750            indexed_by_page.entry(r.from_id).or_default().push(r.to_id);
751        }
752    }
753
754    // Per page: hero id from the handler's extension (no-op for types
755    // without one), then the indexed references. Accumulate the union
756    // of media ids for a single bulk fetch below.
757    let mut heroes: std::collections::HashMap<i64, i64> = std::collections::HashMap::new();
758    {
759        let permits = std::sync::Arc::new(rustango::__private_runtime::tokio::sync::Semaphore::new(HERO_LOADS));
760        let mut loads = rustango::__private_runtime::tokio::task::JoinSet::new();
761        for p in pages {
762            let Some(pid) = p.id.get().copied().filter(|&id| id != 0) else {
763                continue;
764            };
765            let Some(handler) = type_name_by_id
766                .get(&p.page_type_id)
767                .and_then(|name| find_handler(name))
768            else {
769                continue;
770            };
771            let (pool, permits) = (pool.clone(), std::sync::Arc::clone(&permits));
772            loads.spawn(async move {
773                let _permit = permits.acquire_owned().await.ok()?;
774                let hero = handler
775                    .load_extension(&pool, pid)
776                    .await
777                    .ok()?
778                    .get("hero_media_id")
779                    .and_then(serde_json::Value::as_i64)?;
780                Some((pid, hero))
781            });
782        }
783        while let Some(done) = loads.join_next().await {
784            if let Ok(Some((pid, hero))) = done {
785                heroes.insert(pid, hero);
786            }
787        }
788    }
789
790    let mut ids_by_page: Vec<(i64, Vec<i64>)> = Vec::with_capacity(pages.len());
791    let mut all_media: Vec<i64> = Vec::new();
792    for p in pages {
793        let Some(pid) = p.id.get().copied().filter(|&id| id != 0) else {
794            continue;
795        };
796        let hero = heroes.get(&pid).copied();
797        let indexed = indexed_by_page.remove(&pid).unwrap_or_default();
798        let ids = collect_media_ids(hero, indexed);
799        if !ids.is_empty() {
800            all_media.extend(ids.iter().copied());
801            ids_by_page.push((pid, ids));
802        }
803    }
804    if all_media.is_empty() {
805        return out;
806    }
807
808    // 1 query — all referenced media at once.
809    all_media.sort_unstable();
810    all_media.dedup();
811    let mut by_id: std::collections::HashMap<i64, crate::media::Media> =
812        std::collections::HashMap::new();
813    for chunk in all_media.chunks(IN_CHUNK) {
814        let found = crate::media::Media::objects()
815            .where_(crate::media::Media::id.is_in(chunk.iter().copied()))
816            .fetch(pool)
817            .await
818            .unwrap_or_default();
819        by_id.extend(found.into_iter().filter_map(|m| m.id.get().copied().map(|id| (id, m))));
820    }
821
822    // Emit per page in id-collection order (hero first), images only.
823    for (pid, ids) in ids_by_page {
824        let rows: Vec<ImageRow> = ids
825            .iter()
826            .filter_map(|id| {
827                let m = by_id.get(id)?;
828                if m.kind != "image" {
829                    return None;
830                }
831                Some(ImageRow {
832                    // Signed like every CMS-rendered image URL (#643).
833                    loc: format!(
834                        "{base}{}",
835                        crate::rendition::rendition_url_for(*id, "max-2000x2000", None).unwrap_or_default()
836                    ),
837                    title: Some(m.title.clone()).filter(|s| !s.is_empty()),
838                    caption: Some(m.alt_text.clone()).filter(|s| !s.is_empty()),
839                })
840            })
841            .collect();
842        if !rows.is_empty() {
843            out.insert(pid, rows);
844        }
845    }
846    out
847}
848
849/// Order-preserving dedup of a page's media ids — hero first, then the
850/// indexed stream/body references; drops non-positive ids.
851fn collect_media_ids(hero: Option<i64>, indexed: Vec<i64>) -> Vec<i64> {
852    let mut seen = std::collections::HashSet::new();
853    let mut out = Vec::new();
854    for id in hero.into_iter().chain(indexed) {
855        if id > 0 && seen.insert(id) {
856            out.push(id);
857        }
858    }
859    out
860}
861
862#[cfg(test)]
863mod image_id_tests {
864    use super::collect_media_ids;
865
866    #[test]
867    fn hero_first_then_indexed_deduped() {
868        assert_eq!(collect_media_ids(Some(5), vec![3, 5, 7, 3]), vec![5, 3, 7]);
869    }
870
871    #[test]
872    fn no_hero_keeps_indexed_order() {
873        assert_eq!(collect_media_ids(None, vec![9, 2, 9]), vec![9, 2]);
874    }
875
876    #[test]
877    fn drops_nonpositive_and_empty() {
878        assert_eq!(collect_media_ids(Some(0), vec![-1, 4]), vec![4]);
879        assert!(collect_media_ids(None, vec![]).is_empty());
880    }
881}
882
883#[cfg(test)]
884mod custom_source_tests {
885    use super::{shard_window, SitemapEntry, SitemapSource, SitemapSourceRegistration, SHARD_SIZE};
886
887    #[test]
888    fn entry_emits_all_fields_clamped_and_escaped() {
889        let ts = chrono::DateTime::parse_from_rfc3339("2026-06-04T12:00:00Z")
890            .unwrap()
891            .with_timezone(&chrono::Utc);
892        let e = SitemapEntry::new("https://x.test/a?b=1&c=2")
893            .with_lastmod(ts)
894            .with_changefreq("weekly")
895            .with_priority(1.5); // out of range → clamps to 1.0
896        let mut xml = String::new();
897        e.push_url(&mut xml);
898        assert!(
899            xml.contains("<loc>https://x.test/a?b=1&amp;c=2</loc>"),
900            "{xml}"
901        );
902        assert!(
903            xml.contains("<lastmod>2026-06-04T12:00:00Z</lastmod>"),
904            "{xml}"
905        );
906        assert!(xml.contains("<changefreq>weekly</changefreq>"), "{xml}");
907        assert!(xml.contains("<priority>1.0</priority>"), "{xml}");
908    }
909
910    #[test]
911    fn minimal_entry_emits_only_loc() {
912        let mut xml = String::new();
913        SitemapEntry::new("https://x.test/").push_url(&mut xml);
914        assert!(xml.contains("<loc>https://x.test/</loc>"), "{xml}");
915        assert!(!xml.contains("<lastmod>"), "{xml}");
916        assert!(!xml.contains("<changefreq>"), "{xml}");
917        assert!(!xml.contains("<priority>"), "{xml}");
918    }
919
920    #[test]
921    fn shard_window_spans_pages_and_custom() {
922        // Out-of-range shards (0 and past the end) → None.
923        assert_eq!(shard_window(10, 0), None);
924        assert_eq!(shard_window(10, 2), None);
925        // Single small shard covers everything.
926        assert_eq!(shard_window(10, 1), Some((0, 10)));
927        // Cross-shard windows at the cap boundary.
928        assert_eq!(shard_window(SHARD_SIZE + 5, 1), Some((0, SHARD_SIZE)));
929        assert_eq!(
930            shard_window(SHARD_SIZE + 5, 2),
931            Some((SHARD_SIZE, SHARD_SIZE + 5))
932        );
933        assert_eq!(shard_window(SHARD_SIZE + 5, 3), None);
934        assert_eq!(shard_window(0, 1), None);
935    }
936
937    // A registered test source proves the inventory + macro wiring.
938    #[derive(Default)]
939    struct TestSource;
940    #[async_trait::async_trait]
941    impl SitemapSource for TestSource {
942        async fn entries(&self, _pool: &rustango::sql::Pool, base: &str) -> Vec<SitemapEntry> {
943            vec![SitemapEntry::new(format!("{base}/__test_custom__"))]
944        }
945    }
946    crate::register_sitemap_source!(TestSource);
947
948    #[test]
949    fn registry_collects_registered_sources() {
950        let count = inventory::iter::<SitemapSourceRegistration>
951            .into_iter()
952            .count();
953        assert!(count >= 1, "expected the test source to be registered");
954    }
955}
956
957/// Indent-prefixed `<tag>text</tag>\n` writer with XML escaping on
958/// the value.
959fn push_lp(buf: &mut String, tag: &str, value: &str, indent: usize) {
960    use std::fmt::Write as _;
961    let pad = " ".repeat(indent);
962    let _ = writeln!(buf, "{pad}<{tag}>{}</{tag}>", xml_escape(value));
963}
964
965fn xml_escape(s: &str) -> String {
966    s.replace('&', "&amp;")
967        .replace('<', "&lt;")
968        .replace('>', "&gt;")
969        .replace('"', "&quot;")
970        .replace('\'', "&apos;")
971}
972
973fn format_iso8601(ts: chrono::DateTime<chrono::Utc>) -> String {
974    ts.to_rfc3339_opts(chrono::SecondsFormat::Secs, true)
975}
976
977/// Build the scheme+host prefix (no trailing slash) for the absolute
978/// URLs in the sitemap, the feeds and robots.txt. Scheme defaults to
979/// `https`, downgrades to `http` only when the request was clearly
980/// plaintext (`X-Forwarded-Proto: http`, or no proxy header AND a
981/// localhost-shape host).
982///
983/// The host is the `Host` header — the one the tenant was resolved from.
984/// `X-Forwarded-Host` is not read: nothing here knows whether a
985/// trusted proxy set it, so a client could get tenant A's sitemap with
986/// every URL on a host of its choosing, and a shared cache would keep it.
987pub(crate) fn base_url(headers: &HeaderMap) -> String {
988    let host = headers
989        .get(header::HOST)
990        .and_then(|v| v.to_str().ok())
991        .map(|s| s.split(',').next().unwrap_or(s).trim().to_owned())
992        .unwrap_or_else(|| "localhost".to_owned());
993    let scheme = headers
994        .get("x-forwarded-proto")
995        .and_then(|v| v.to_str().ok())
996        .map(|s| s.split(',').next().unwrap_or(s).trim().to_ascii_lowercase())
997        .filter(|s| s == "http" || s == "https")
998        .unwrap_or_else(|| {
999            if is_localhost_shape(&host) {
1000                "http".to_owned()
1001            } else {
1002                "https".to_owned()
1003            }
1004        });
1005    // A site served under a path on the main domain lives at host + prefix.
1006    let prefix = crate::path_prefix::of(headers).unwrap_or("");
1007    format!("{scheme}://{host}{prefix}")
1008}
1009
1010fn is_localhost_shape(host: &str) -> bool {
1011    let bare = host.split(':').next().unwrap_or(host);
1012    bare == "localhost"
1013        || bare.ends_with(".localtest.me")
1014        || bare.ends_with(".localhost")
1015        || bare == "127.0.0.1"
1016        || bare == "::1"
1017}
1018
1019#[cfg(test)]
1020mod base_url_tests {
1021    use axum::http::{HeaderMap, HeaderValue};
1022
1023    fn headers(pairs: &[(&'static str, &'static str)]) -> HeaderMap {
1024        let mut h = HeaderMap::new();
1025        for (k, v) in pairs {
1026            h.insert(*k, HeaderValue::from_static(v));
1027        }
1028        h
1029    }
1030
1031    /// The host is the one the tenant was resolved from.
1032    #[test]
1033    fn a_forwarded_host_cannot_move_the_urls() {
1034        let h = headers(&[("host", "victim.example"), ("x-forwarded-host", "evil.example")]);
1035        assert_eq!(super::base_url(&h), "https://victim.example");
1036    }
1037
1038    #[test]
1039    fn only_a_real_scheme_is_taken_from_the_proxy() {
1040        let h = headers(&[("host", "site.example"), ("x-forwarded-proto", "http")]);
1041        assert_eq!(super::base_url(&h), "http://site.example");
1042        let h = headers(&[("host", "site.example"), ("x-forwarded-proto", "javascript")]);
1043        assert_eq!(super::base_url(&h), "https://site.example");
1044        let h = headers(&[("host", "demo.localhost:8080")]);
1045        assert_eq!(super::base_url(&h), "http://demo.localhost:8080");
1046    }
1047}
1048
1049#[cfg(all(test, feature = "sqlite"))]
1050mod site_scope_tests {
1051    use super::*;
1052    use rustango::core::Model as _;
1053
1054    async fn pool_with_tree() -> rustango::sql::Pool {
1055        let pool = rustango::sql::Pool::connect("sqlite::memory:").await.expect("pool");
1056        for schema in [
1057            &crate::page_type_model::PageType::SCHEMA,
1058            &crate::media::Media::SCHEMA,
1059            &crate::theme::Theme::SCHEMA,
1060            &Page::SCHEMA,
1061            &crate::site::Site::SCHEMA,
1062        ] {
1063            let sql = rustango::migrate::ddl::create_table_if_not_exists_sql_with_dialect(
1064                pool.dialect(),
1065                schema,
1066            );
1067            for stmt in sql.split(';').map(str::trim).filter(|s| !s.is_empty()) {
1068                rustango::sql::raw_execute_pool(&pool, stmt, Vec::new()).await.expect("ddl");
1069            }
1070        }
1071        for sql in [
1072            "INSERT INTO cms_page_type (id, app_label, type_name, verbose_name, default_template, is_creatable, allowed_parent_types) \
1073             VALUES (1, 'cms', 'P', 'P', 'p.html', 1, '[]')",
1074            "INSERT INTO cms_page (id, page_type_id, title, slug, path, url_path, depth, sort_order, status, seo_title, seo_description) VALUES \
1075             (1, 1, 'Shop', 'shop', '0001/', '/shop', 1, 0, 'published', '', ''), \
1076             (2, 1, 'Catalog', 'catalog', '0001/0001/', '/shop/catalog', 2, 0, 'published', '', ''), \
1077             (3, 1, 'Blog', 'blog', '0002/', '/blog', 1, 0, 'published', '', '')",
1078            "INSERT INTO cms_site (hostname, root_page_id) VALUES ('shop.example', 1)",
1079        ] {
1080            rustango::sql::raw_execute_pool(&pool, sql, Vec::new()).await.expect("row");
1081        }
1082        pool
1083    }
1084
1085    async fn listed(pool: &rustango::sql::Pool, host: &'static str) -> Vec<String> {
1086        let mut headers = HeaderMap::new();
1087        headers.insert(header::HOST, host.parse().unwrap());
1088        let scope = SiteScope::for_host(pool, &headers).await;
1089        let mut rows: Vec<Page> = scope
1090            .narrow(Page::objects())
1091            .order_by(&[("path", false)])
1092            .fetch(pool)
1093            .await
1094            .expect("fetch");
1095        scope.to_public(&mut rows);
1096        rows.into_iter().map(|p| p.url_path).collect()
1097    }
1098
1099    /// A mapped host lists only its subtree, at the paths it serves.
1100    #[tokio::test]
1101    async fn a_mapped_host_lists_its_own_subtree_relative_to_its_root() {
1102        let pool = pool_with_tree().await;
1103        assert_eq!(listed(&pool, "shop.example:8000").await, ["/", "/catalog"]);
1104        // An unmapped host still lists the whole tenant, unchanged.
1105        assert_eq!(listed(&pool, "other.example").await, ["/shop", "/shop/catalog", "/blog"]);
1106    }
1107}