Skip to main content

autumn_web/
app.rs

1//! Application builder -- the entry point for configuring and running
2//! an Autumn server.
3//!
4//! Every Autumn application follows the same pattern:
5//!
6//! 1. Call [`app()`] to create an [`AppBuilder`].
7//! 2. Register routes with [`.routes()`](AppBuilder::routes).
8//! 3. Call [`.run()`](AppBuilder::run) to start serving.
9//!
10//! # Example
11//!
12//! ```rust,no_run
13//! use autumn_web::prelude::*;
14//!
15//! #[get("/hello")]
16//! async fn hello() -> &'static str { "Hello!" }
17//!
18//! #[autumn_web::main]
19//! async fn main() {
20//!     autumn_web::app()
21//!         .routes(routes![hello])
22//!         .run()
23//!         .await;
24//! }
25//! ```
26
27// autumn-determinism-gate: production code in this module must read time and
28// mint identifiers through the framework's injected seams (ClockSource /
29// Entropy), never `Instant::now()` / `Utc::now()` / `SystemTime::now()` /
30// `Uuid::new_v4()` directly. See CONTRIBUTING.md "Determinism seam gate"
31// (issue #1797). Justify exceptions with
32// #[allow(clippy::disallowed_methods, reason = "…")] at the narrowest scope.
33#![cfg_attr(not(test), deny(clippy::disallowed_methods))]
34
35use std::any::{Any, TypeId};
36use std::collections::{BTreeSet, HashMap, HashSet};
37use std::future::Future;
38use std::pin::Pin;
39use std::sync::Arc;
40
41use futures::FutureExt as _;
42use tracing::Instrument as _;
43
44use crate::config::{AutumnConfig, ConfigLoader};
45#[cfg(feature = "maud")]
46use crate::error_pages::{ErrorPageRenderer, SharedRenderer};
47use crate::middleware::exception_filter::ExceptionFilter;
48#[cfg(feature = "db")]
49use crate::migrate;
50use crate::route::Route;
51use crate::state::AppState;
52
53/// Create a new [`AppBuilder`].
54///
55/// This is the primary entry point for constructing an Autumn application.
56/// Chain [`.routes()`](AppBuilder::routes) calls to register handlers, then
57/// call [`.run()`](AppBuilder::run) to start the server.
58///
59/// # Examples
60///
61/// ```rust,no_run
62/// use autumn_web::prelude::*;
63///
64/// #[get("/")]
65/// async fn index() -> &'static str { "hi" }
66///
67/// #[autumn_web::main]
68/// async fn main() {
69///     autumn_web::app()
70///         .routes(routes![index])
71///         .run()
72///         .await;
73/// }
74/// ```
75#[must_use]
76pub fn app() -> AppBuilder {
77    let builder = AppBuilder {
78        routes: Vec::new(),
79        api_versions: Vec::new(),
80        route_sources: Vec::new(),
81        current_plugin: None,
82        tasks: Vec::new(),
83        one_off_tasks: Vec::new(),
84        jobs: Vec::new(),
85        listeners: Vec::new(),
86        #[cfg(feature = "db")]
87        outbox_handlers: crate::outbox::OutboxHandlers::default(),
88        static_metas: Vec::new(),
89        exception_filters: Vec::new(),
90        scoped_groups: Vec::new(),
91        merge_routers: Vec::new(),
92        nest_routers: Vec::new(),
93        custom_layers: Vec::new(),
94        static_gate_layers: Vec::new(),
95        startup_hooks: Vec::new(),
96        state_initializers: Vec::new(),
97        shutdown_hooks: Vec::new(),
98        extensions: HashMap::new(),
99        registered_plugins: HashSet::new(),
100        plugin_contracts: Vec::new(),
101        plugin_config_roots: BTreeSet::new(),
102        #[cfg(feature = "maud")]
103        error_page_renderer: None,
104        #[cfg(feature = "db")]
105        migrations: Vec::new(),
106        config_loader_factory: None,
107        #[cfg(feature = "db")]
108        pool_provider_factory: None,
109        #[cfg(feature = "db")]
110        shard_provider_factory: None,
111        #[cfg(feature = "db")]
112        shard_router: None,
113        #[cfg(feature = "db")]
114        directory_shard_router: false,
115        telemetry_provider: None,
116        session_store: None,
117        #[cfg(feature = "ws")]
118        channels_backend: None,
119        #[cfg(feature = "storage")]
120        blob_store: None,
121        cache_backend: None,
122        #[cfg(feature = "reporting")]
123        error_reporters: Vec::new(),
124        alert_channels: Vec::new(),
125        #[cfg(feature = "openapi")]
126        openapi: None,
127        #[cfg(feature = "mcp")]
128        mcp: None,
129        #[cfg(feature = "webmcp")]
130        webmcp: None,
131        audit_logger: None,
132        #[cfg(feature = "i18n")]
133        i18n_bundle: None,
134        #[cfg(feature = "i18n")]
135        i18n_auto_load: false,
136        #[cfg(feature = "embed-assets")]
137        embedded_static: None,
138        #[cfg(all(feature = "embed-assets", feature = "i18n"))]
139        embedded_locales: None,
140        policy_registrations: Vec::new(),
141        #[cfg(feature = "mail")]
142        mail_delivery_queue_factory: None,
143        #[cfg(feature = "mail")]
144        suppression_store: None,
145        #[cfg(feature = "mail")]
146        mail_suppression_store: None,
147        #[cfg(feature = "mail")]
148        mount_unsubscribe_endpoint: false,
149        #[cfg(feature = "mail")]
150        mail_previews: Vec::new(),
151        #[cfg(feature = "maud")]
152        story_gallery: None,
153        declared_routes: Vec::new(),
154        plugin_asset_bundles: Vec::new(),
155        idempotency_enabled: false,
156        #[cfg(feature = "mail")]
157        mail_interceptor: None,
158        job_interceptor: None,
159        #[cfg(feature = "db")]
160        db_interceptor: None,
161        #[cfg(feature = "ws")]
162        channels_interceptor: None,
163        #[cfg(feature = "oauth2")]
164        http_interceptor: None,
165        seo_sources: Vec::new(),
166        agent_skills: Vec::new(),
167        commerce_docs: crate::aeo::RegisteredCommerceDocs::default(),
168        metrics_sources: Vec::new(),
169        health_indicators: Vec::new(),
170        #[cfg(feature = "inbound-mail")]
171        inbound_mail_router: None,
172    };
173    // Strip the edge lane's internal fallthrough-sentinel header from every
174    // outbound response, for every app — not only apps that call
175    // `with_edge_kv`. `EdgeCacheUnavailable` (autumn-edge's `extract.rs`) sets
176    // this header on its 500 so the EDGE CAPSULE runtime knows to fall
177    // through to the origin; the same handler code also runs at the origin,
178    // and a `#[edge(needs(kv))]` route with no `with_edge_kv` call — a wiring
179    // bug — hits that same 500 at the origin. Without this layer the internal
180    // header would leak straight to a real HTTP client. See
181    // `StripEdgeFallthroughSentinelLayer` below.
182    #[cfg(feature = "edge")]
183    let builder = builder.layer(StripEdgeFallthroughSentinelLayer);
184    builder
185}
186
187/// Removes [`autumn_edge::FALLTHROUGH_SENTINEL`] from an outbound response.
188///
189/// `autumn-edge` is substrate-agnostic on purpose: `extract.rs` cannot tell
190/// whether it is running at the edge or at the origin, so it always sets the
191/// sentinel on an `EdgeCacheUnavailable` response. Only the origin knows it is
192/// the origin, so only the origin strips the header before a real client ever
193/// sees it. The response body's actionable message is left untouched — only
194/// the internal signaling header is removed.
195///
196/// A bespoke `tower::Layer`, not `axum::middleware::from_fn`: this type's
197/// `TypeId` is what `router::is_idempotency_transparent_app_layer` matches
198/// on to recognize this one framework-owned registration without forcing
199/// fail-closed idempotency on every app built with the `edge` feature. A
200/// name (even a function's) is not unique enough for that — a user's own
201/// `from_fn` middleware could share it by coincidence; a crate-private type
202/// cannot.
203#[cfg(feature = "edge")]
204#[derive(Clone, Copy, Default)]
205pub(crate) struct StripEdgeFallthroughSentinelLayer;
206
207#[cfg(feature = "edge")]
208impl<S> tower::Layer<S> for StripEdgeFallthroughSentinelLayer {
209    type Service = StripEdgeFallthroughSentinelService<S>;
210
211    fn layer(&self, inner: S) -> Self::Service {
212        StripEdgeFallthroughSentinelService { inner }
213    }
214}
215
216/// `true` for a `custom_layers` registration that [`app()`] installs itself
217/// rather than a user calling [`AppBuilder::layer`] — [`get_layer_types`](AppBuilder::get_layer_types)
218/// filters these out to keep its documented "user-installed only" contract,
219/// even though they share the same underlying `custom_layers` vector as a
220/// real user layer (needed so the router-build step applies them the same
221/// way, in the same registration-order pass).
222#[cfg(feature = "edge")]
223fn is_framework_owned_layer(type_id: TypeId) -> bool {
224    type_id == TypeId::of::<StripEdgeFallthroughSentinelLayer>()
225}
226
227#[cfg(not(feature = "edge"))]
228const fn is_framework_owned_layer(_type_id: TypeId) -> bool {
229    false
230}
231
232/// Tower [`Service`](tower::Service) produced by
233/// [`StripEdgeFallthroughSentinelLayer`].
234#[cfg(feature = "edge")]
235#[derive(Clone, Debug)]
236pub(crate) struct StripEdgeFallthroughSentinelService<S> {
237    inner: S,
238}
239
240#[cfg(feature = "edge")]
241impl<S, ReqBody> tower::Service<axum::http::Request<ReqBody>>
242    for StripEdgeFallthroughSentinelService<S>
243where
244    S: tower::Service<axum::http::Request<ReqBody>, Response = axum::response::Response>
245        + Send
246        + 'static,
247    S::Future: Send + 'static,
248{
249    type Response = S::Response;
250    type Error = S::Error;
251    type Future = std::pin::Pin<
252        Box<dyn std::future::Future<Output = Result<Self::Response, Self::Error>> + Send>,
253    >;
254
255    fn poll_ready(
256        &mut self,
257        cx: &mut std::task::Context<'_>,
258    ) -> std::task::Poll<Result<(), Self::Error>> {
259        self.inner.poll_ready(cx)
260    }
261
262    fn call(&mut self, req: axum::http::Request<ReqBody>) -> Self::Future {
263        let response = self.inner.call(req);
264        Box::pin(async move {
265            let mut response = response.await?;
266            response
267                .headers_mut()
268                .remove(autumn_edge::FALLTHROUGH_SENTINEL);
269            Ok(response)
270        })
271    }
272}
273
274/// Count the raw routers omitted from `autumn routes` output because their
275/// endpoints can't be enumerated — the value `autumn routes audit` treats as a
276/// hard failure (an unprovable route defeats the coverage guarantee).
277///
278/// Every `.merge()` router is rootless — it has no mount prefix to match
279/// declarations against — so it is always opaque and always counts. A `.nest()`
280/// router carries a mount prefix, so it is treated as **covered** (enumerable,
281/// not omitted) when at least one declared route (from
282/// [`declare_plugin_routes`](AppBuilder::declare_plugin_routes)) has a path that
283/// falls under that prefix. This makes the documented
284/// `app.nest(prefix, router).declare_plugin_routes(routes)` pattern audit-clean
285/// without any dedicated bookkeeping: the declared routes prove the mount.
286///
287/// Soundness (fail-closed) is preserved: a bare `nest(prefix, raw_router)` with
288/// no declared route under `prefix` stays uncovered and counts, and every
289/// `merge()` counts unconditionally.
290fn omitted_router_count<'a>(
291    merge_routers: usize,
292    nest_prefixes: impl IntoIterator<Item = &'a str>,
293    declared_routes: &[crate::route_listing::RouteInfo],
294) -> usize {
295    let uncovered_nests = nest_prefixes
296        .into_iter()
297        .filter(|prefix| !nest_prefix_is_covered(prefix, declared_routes))
298        .count();
299    merge_routers + uncovered_nests
300}
301
302/// A nested mount at `prefix` is "covered" when at least one declared route's
303/// path falls under that prefix, proving the nested router's endpoints are
304/// enumerable in the `autumn routes` listing.
305fn nest_prefix_is_covered(
306    prefix: &str,
307    declared_routes: &[crate::route_listing::RouteInfo],
308) -> bool {
309    declared_routes
310        .iter()
311        .any(|route| path_is_under_prefix(&route.path, prefix))
312}
313
314/// Whether `path` is mounted under `prefix` — i.e. equal to the prefix or a
315/// descendant of it at a path-segment boundary (`/admin` covers `/admin` and
316/// `/admin/users`, but not `/administrators`). A root prefix (`/` or empty)
317/// covers everything.
318fn path_is_under_prefix(path: &str, prefix: &str) -> bool {
319    let prefix = prefix.trim_end_matches('/');
320    if prefix.is_empty() {
321        return true;
322    }
323    path == prefix
324        || path
325            .strip_prefix(prefix)
326            .is_some_and(|rest| rest.starts_with('/'))
327}
328
329type StartupHookFuture = Pin<Box<dyn Future<Output = crate::AutumnResult<()>> + Send>>;
330type StartupHook = Box<dyn Fn(AppState) -> StartupHookFuture + Send + Sync>;
331type StateInitializer = Box<dyn FnOnce(&AppState) + Send>;
332type ShutdownHookFuture = Pin<Box<dyn Future<Output = ()> + Send>>;
333type ShutdownHook = Box<dyn Fn() -> ShutdownHookFuture + Send + Sync>;
334
335// ── Tier-1 subsystem factories ────────────────────────────────
336//
337// `ConfigLoader` and `DatabasePoolProvider` use RPIT (`-> impl Future + Send`)
338// in their trait methods, so `Box<dyn Trait>` is not dyn-compatible. We store
339// boxed factory closures that capture the concrete impl at the call site and
340// erase its future type via `Pin<Box<dyn Future>>`. `TelemetryProvider`'s
341// `init` is sync, so it's stored as a normal `Box<dyn>`.
342type ConfigLoaderFactory = Box<
343    dyn FnOnce() -> Pin<
344            Box<dyn Future<Output = Result<AutumnConfig, crate::config::ConfigError>> + Send>,
345        > + Send,
346>;
347#[cfg(feature = "db")]
348type PoolProviderFactory = Box<
349    dyn FnOnce(
350            crate::config::DatabaseConfig,
351        ) -> Pin<
352            Box<
353                dyn Future<
354                        Output = Result<Option<crate::db::DatabaseTopology>, crate::db::PoolError>,
355                    > + Send,
356            >,
357        > + Send,
358>;
359/// Captured [`DatabasePoolProvider::create_shard_topology`] calls: builds
360/// one topology per configured shard, in declaration order.
361#[cfg(feature = "db")]
362type ShardProviderFactory = Box<
363    dyn FnOnce(
364            crate::config::DatabaseConfig,
365        ) -> Pin<
366            Box<
367                dyn Future<Output = Result<Vec<crate::db::DatabaseTopology>, crate::db::PoolError>>
368                    + Send,
369            >,
370        > + Send,
371>;
372
373/// Closure that registers a policy or scope on the runtime
374/// [`PolicyRegistry`](crate::authorization::PolicyRegistry).
375type PolicyRegistration = Box<dyn FnOnce(&crate::authorization::PolicyRegistry) + Send>;
376
377/// Represents an API version registration.
378#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)]
379pub struct ApiVersion {
380    /// The version name (e.g. "v1", "v2").
381    pub version: String,
382    /// When this version was deprecated.
383    pub deprecated_at: Option<chrono::DateTime<chrono::Utc>>,
384    /// When this version was sunsetted.
385    pub sunset_at: Option<chrono::DateTime<chrono::Utc>>,
386}
387
388/// A wrapper for registered API versions in the app state.
389#[derive(Clone, Debug)]
390pub struct RegisteredApiVersions(pub Vec<ApiVersion>);
391
392/// Builder for configuring and launching an Autumn application.
393///
394/// Created by [`app()`]. Collect routes with [`.routes()`](Self::routes),
395/// then call [`.run()`](Self::run) to start the HTTP server.
396///
397/// The builder follows the **builder pattern**: each method consumes `self`
398/// and returns a new `AppBuilder`, allowing chained calls.
399///
400/// # Examples
401///
402/// ```rust,no_run
403/// use autumn_web::prelude::*;
404///
405/// #[get("/a")]
406/// async fn route_a() -> &'static str { "a" }
407///
408/// #[get("/b")]
409/// async fn route_b() -> &'static str { "b" }
410///
411/// #[autumn_web::main]
412/// async fn main() {
413///     autumn_web::app()
414///         .routes(routes![route_a])
415///         .routes(routes![route_b])
416///         .run()
417///         .await;
418/// }
419/// ```
420#[allow(clippy::struct_excessive_bools)]
421pub struct AppBuilder {
422    pub(crate) routes: Vec<Route>,
423    /// Registered API versions.
424    pub api_versions: Vec<ApiVersion>,
425    /// Parallel to `routes`: registration origin for each route.
426    route_sources: Vec<crate::route_listing::RouteSource>,
427    /// Non-None while a plugin's `build()` is executing; routes and scoped
428    /// groups added during that window are attributed to this plugin.
429    current_plugin: Option<String>,
430    pub(crate) tasks: Vec<crate::task::TaskInfo>,
431    one_off_tasks: Vec<crate::task::OneOffTaskInfo>,
432    pub(crate) jobs: Vec<crate::job::JobInfo>,
433    /// Registered event listeners; durable ones are synthesized into jobs at
434    /// build time and the rest dispatch synchronously via the event registry.
435    pub(crate) listeners: Vec<crate::events::ListenerInfo>,
436    /// Outbox topic handlers (issue #3062).
437    #[cfg(feature = "db")]
438    pub(crate) outbox_handlers: crate::outbox::OutboxHandlers,
439    pub(crate) static_metas: Vec<crate::static_gen::StaticRouteMeta>,
440    pub(crate) exception_filters: Vec<Arc<dyn ExceptionFilter>>,
441    pub(crate) scoped_groups: Vec<ScopedGroup>,
442    pub(crate) merge_routers: Vec<axum::Router<AppState>>,
443    pub(crate) nest_routers: Vec<(String, axum::Router<AppState>)>,
444    /// Custom Tower layers registered via [`AppBuilder::layer`], applied
445    /// inside `RequestIdLayer` on ingress so they observe the request ID.
446    pub(crate) custom_layers: Vec<CustomLayerRegistration>,
447    /// Pre-static gate layers registered via [`AppBuilder::static_gate`],
448    /// applied outermost (outside session and before the static cache lookup)
449    /// so they can auth-gate / redirect requests before a cached SSG/ISG page
450    /// is served.
451    pub(crate) static_gate_layers: Vec<CustomLayerRegistration>,
452    pub(crate) startup_hooks: Vec<StartupHook>,
453    pub(crate) state_initializers: Vec<StateInitializer>,
454    pub(crate) shutdown_hooks: Vec<ShutdownHook>,
455    pub(crate) extensions: HashMap<TypeId, Box<dyn Any + Send>>,
456    /// Plugin names that have already been applied, for duplicate detection.
457    pub(crate) registered_plugins: HashSet<String>,
458    /// Compatibility contracts declared by the plugins applied to this builder
459    /// (issue #1601), in registration order. Emitted after
460    /// [`PLUGIN_CONTRACT_MARKER`](crate::plugin_contract::PLUGIN_CONTRACT_MARKER)
461    /// by the route dump so `autumn plugin-check` can report experimental
462    /// surface use without linking the plugin itself.
463    pub(crate) plugin_contracts: Vec<crate::plugin_contract::PluginContract>,
464    /// Top-level config roots plugins have declared as their own opaque config
465    /// sections via [`config_section`](AppBuilder::config_section). Threaded into
466    /// the default config loader so `server.strict_config` treats them as
467    /// known-and-opaque instead of unknown-key hard errors.
468    pub(crate) plugin_config_roots: BTreeSet<String>,
469    /// Custom error page renderer (overrides built-in pages).
470    #[cfg(feature = "maud")]
471    error_page_renderer: Option<SharedRenderer>,
472    /// Embedded Diesel migrations, registered via `.migrations()` (tagged
473    /// `"app"`) or [`Self::plugin_migrations`] (tagged with the caller's
474    /// `name`) — see [`Self::plugin_migrations`] for why the name matters.
475    #[cfg(feature = "db")]
476    migrations: Vec<(&'static str, migrate::EmbeddedMigrations)>,
477    /// Custom config loader (tier-1 subsystem replacement). When `None`, the
478    /// default [`TomlEnvConfigLoader`](crate::config::TomlEnvConfigLoader) runs.
479    config_loader_factory: Option<ConfigLoaderFactory>,
480    /// Custom DB pool provider (tier-1 subsystem replacement). When `None`,
481    /// the default [`DieselDeadpoolPoolProvider`](crate::db::DieselDeadpoolPoolProvider) runs.
482    #[cfg(feature = "db")]
483    pool_provider_factory: Option<PoolProviderFactory>,
484    /// Companion to `pool_provider_factory` for `[[database.shards]]`
485    /// topologies; captured from the same provider in `with_pool_provider`.
486    #[cfg(feature = "db")]
487    shard_provider_factory: Option<ShardProviderFactory>,
488    /// Custom shard routing strategy. When `None` and shards are
489    /// configured, the default [`HashShardRouter`](crate::sharding::HashShardRouter)
490    /// is used.
491    #[cfg(feature = "db")]
492    shard_router: Option<Arc<dyn crate::sharding::ShardRouter>>,
493    /// Builder opt-in for the control-DB [`DirectoryShardRouter`](crate::sharding::DirectoryShardRouter),
494    /// applied to `config.database.directory_shard_router` at build time.
495    #[cfg(feature = "db")]
496    directory_shard_router: bool,
497    /// Custom telemetry provider (tier-1 subsystem replacement). When `None`,
498    /// the default [`TracingOtlpTelemetryProvider`](crate::telemetry::TracingOtlpTelemetryProvider) runs.
499    telemetry_provider: Option<Box<dyn crate::telemetry::TelemetryProvider>>,
500    /// Custom session store (tier-1 subsystem replacement). When `Some`,
501    /// `build_session_layer` skips the config-driven `memory`/`redis` selection
502    /// and uses this store directly.
503    session_store: Option<Arc<dyn crate::session::BoxedSessionStore>>,
504    /// Custom channel backend (tier-1 subsystem replacement). When `Some`,
505    /// `AppState` skips config-driven `in_process`/`redis` channel selection.
506    #[cfg(feature = "ws")]
507    channels_backend: Option<Arc<dyn crate::channels::ChannelsBackend>>,
508    /// Custom blob store installed via
509    /// [`AppBuilder::with_blob_store`]. When `Some`, `preflight_storage`
510    /// is skipped and this store is installed directly onto `AppState`.
511    #[cfg(feature = "storage")]
512    blob_store: Option<crate::storage::SharedBlobStore>,
513    /// Shared cache backend installed via [`AppBuilder::with_cache_backend`].
514    /// When `Some`, installed onto `AppState` as `shared_cache` before startup
515    /// hooks run.
516    cache_backend: Option<Arc<dyn crate::cache::Cache>>,
517    /// Error reporters registered via [`AppBuilder::with_error_reporter`].
518    /// Installed onto `AppState` so the
519    /// [`ReportingLayer`](crate::reporting::ReportingLayer) delivers panic and
520    /// 5xx [`ErrorEvent`](crate::reporting::ErrorEvent)s to each. Empty means
521    /// the built-in [`LogReporter`](crate::reporting::LogReporter) is used.
522    #[cfg(feature = "reporting")]
523    pub(crate) error_reporters: Vec<Arc<dyn crate::reporting::ErrorReporter>>,
524    /// Operator-alert channels registered via [`AppBuilder::with_alert_channel`].
525    /// Combined with the built-in mail/webhook channels derived from
526    /// `[alerts]` config and installed onto `AppState` so the built-in
527    /// condition hooks can fan out to each. Empty means only config-derived
528    /// destinations are used. See [`crate::alerts`].
529    pub(crate) alert_channels: Vec<Arc<dyn crate::alerts::AlertChannel>>,
530    /// `OpenAPI` generation configuration. When `Some`, the router mounts
531    /// `/openapi.json` (serving the generated spec) and `/swagger-ui` (if the
532    /// Swagger UI path is set). When `None`, no docs endpoints are mounted.
533    ///
534    /// Gated behind the `openapi` feature: apps that don't need a
535    /// served `OpenAPI` document shouldn't pay for the spec types or the
536    /// runtime collision-check machinery.
537    #[cfg(feature = "openapi")]
538    openapi: Option<crate::openapi::OpenApiConfig>,
539    /// MCP (Model Context Protocol) runtime config. `Some` once
540    /// [`AppBuilder::mount_mcp`] is called; the contained `expose_all` flag is
541    /// flipped by [`AppBuilder::expose_all_as_mcp`]. Gated behind the `mcp`
542    /// feature (which implies `openapi`).
543    #[cfg(feature = "mcp")]
544    mcp: Option<crate::mcp::McpRuntime>,
545    /// `WebMCP` runtime config. `Some` once [`AppBuilder::mount_webmcp`] or
546    /// [`AppBuilder::expose_all_as_webmcp`] is called.
547    #[cfg(feature = "webmcp")]
548    webmcp: Option<crate::webmcp::WebMcpRuntime>,
549    /// Shared audit logger used for append-only compliance events.
550    audit_logger: Option<Arc<crate::audit::AuditLogger>>,
551    /// Loaded i18n translation bundle. When `Some`, an `axum::Extension`
552    /// layer publishing this bundle is added at `run()` time so the
553    /// [`Locale`](crate::i18n::Locale) extractor can resolve translations.
554    #[cfg(feature = "i18n")]
555    i18n_bundle: Option<Arc<crate::i18n::Bundle>>,
556    /// Whether to load the i18n bundle after the active config loader resolves
557    /// [`AutumnConfig`]. This keeps `.i18n_auto()` aligned with
558    /// `.with_config_loader(...)`.
559    #[cfg(feature = "i18n")]
560    i18n_auto_load: bool,
561    /// Embedded `static/` tree (incl. the fingerprint manifest) registered via
562    /// [`embedded_static`](AppBuilder::embedded_static). When set, `/static/*`
563    /// is served from the binary and `asset_url()` resolves against the embedded
564    /// manifest — no `static/` sidecar directory is read at runtime.
565    #[cfg(feature = "embed-assets")]
566    embedded_static: Option<crate::assets::EmbeddedStaticDir>,
567    /// Embedded i18n locale bundles registered via
568    /// [`embedded_locales`](AppBuilder::embedded_locales). When set (and no
569    /// explicit bundle was provided), the bundle is loaded from the binary
570    /// instead of the `i18n/` directory on disk.
571    #[cfg(all(feature = "embed-assets", feature = "i18n"))]
572    embedded_locales: Option<&'static include_dir::Dir<'static>>,
573    /// Deferred [`Policy`](crate::authorization::Policy) and
574    /// [`Scope`](crate::authorization::Scope) registrations applied
575    /// to [`AppState::policy_registry`] just before the router is
576    /// built. Stored as boxed closures so we can carry the
577    /// generic type parameters across the builder boundary.
578    policy_registrations: Vec<PolicyRegistration>,
579    /// Durable mail delivery queue factory registered at builder time. Invoked
580    /// with the freshly-built [`AppState`] before `install_mailer` runs so it
581    /// can capture framework-managed resources (DB pool, channels, etc.).
582    #[cfg(feature = "mail")]
583    mail_delivery_queue_factory: Option<MailDeliveryQueueFactory>,
584    #[cfg(feature = "mail")]
585    pub(crate) suppression_store: Option<crate::mail::SuppressionStoreHandle>,
586    #[cfg(feature = "mail")]
587    pub(crate) mail_suppression_store: Option<crate::mail::suppression::SuppressionStoreHandle>,
588    #[cfg(feature = "mail")]
589    pub(crate) mount_unsubscribe_endpoint: bool,
590    /// Mail template previews registered for the dev preview UI.
591    #[cfg(feature = "mail")]
592    mail_previews: Vec<crate::mail::MailPreview>,
593    /// Widget story gallery registered for the `/_stories` UI (#1526).
594    #[cfg(feature = "maud")]
595    story_gallery: Option<crate::stories::StoryGallery>,
596    /// Routes explicitly declared by plugins for listing purposes, to complement
597    /// opaque `nest_routers`. Included in `autumn routes` output even though
598    /// the underlying Axum router is not enumerable, and handed to the router
599    /// build so the duplicate-route preflight can see inside those otherwise
600    /// opaque mounts.
601    ///
602    /// `pub(crate)` so [`TestApp`](crate::test::TestApp) can carry them too — a
603    /// harness that dropped them would mount a colliding plugin cleanly in
604    /// tests and panic at boot in production.
605    pub(crate) declared_routes: Vec<crate::route_listing::RouteInfo>,
606    /// Asset bundles installed via [`plugin_assets`](Self::plugin_assets),
607    /// kept so a second bundle claiming a taken namespace is refused.
608    /// `pub(crate)` so [`TestApp`](crate::test::TestApp) can carry them.
609    pub(crate) plugin_asset_bundles: Vec<&'static crate::assets::PluginAssets>,
610    /// Whether `.idempotent()` was called on this builder. Applied to the
611    /// loaded `AutumnConfig` before router assembly so that startup validation
612    /// and `apply_middleware` both see `config.idempotency.enabled = true`.
613    idempotency_enabled: bool,
614    #[cfg(feature = "mail")]
615    mail_interceptor: Option<Arc<dyn crate::interceptor::MailInterceptor>>,
616    job_interceptor: Option<Arc<dyn crate::interceptor::JobInterceptor>>,
617    #[cfg(feature = "db")]
618    db_interceptor: Option<Arc<dyn crate::interceptor::DbConnectionInterceptor>>,
619    #[cfg(feature = "ws")]
620    channels_interceptor: Option<Arc<dyn crate::interceptor::ChannelsInterceptor>>,
621    #[cfg(feature = "oauth2")]
622    http_interceptor: Option<Arc<dyn crate::interceptor::HttpInterceptor>>,
623    /// Sitemap sources registered via [`AppBuilder::seo_source`].
624    /// Each source provides dynamic URL entries for `/sitemap.xml`.
625    seo_sources: Vec<Arc<dyn crate::seo::SitemapSource>>,
626    /// Agent skills registered via [`AppBuilder::agent_skill`].
627    agent_skills: Vec<crate::aeo::AgentSkill>,
628    /// UCP/ACP documents registered via [`AppBuilder::ucp_profile`] and
629    /// [`AppBuilder::acp_discovery`].
630    commerce_docs: crate::aeo::RegisteredCommerceDocs,
631
632    /// Plugin-contributed metrics sources registered via [`AppBuilder::metrics_source`].
633    pub(crate) metrics_sources: Vec<(String, Arc<dyn crate::actuator::MetricsSource>)>,
634    /// Custom health indicators registered via [`AppBuilder::health_indicator`].
635    pub(crate) health_indicators: Vec<(
636        String,
637        crate::actuator::IndicatorGroup,
638        Arc<dyn crate::actuator::HealthIndicator>,
639    )>,
640    /// Inbound mail router registered via [`AppBuilder::inbound_mail_router`].
641    /// HTTP webhook routes are derived from the router's endpoint configs and
642    /// merged into the Axum router at startup.
643    #[cfg(feature = "inbound-mail")]
644    pub(crate) inbound_mail_router: Option<Arc<crate::inbound_mail::InboundMailRouter>>,
645}
646
647/// Boxed builder closure that constructs a durable
648/// [`MailDeliveryQueue`](crate::mail::MailDeliveryQueue) from the live
649/// [`AppState`].
650#[cfg(feature = "mail")]
651pub(crate) type MailDeliveryQueueFactory = Box<
652    dyn FnOnce(&AppState) -> crate::AutumnResult<Arc<dyn crate::mail::MailDeliveryQueue>> + Send,
653>;
654
655/// A group of routes sharing a common path prefix and middleware layer.
656///
657/// Created by [`AppBuilder::scoped`]. The routes are mounted under the
658/// prefix with the middleware applied only to this group.
659pub struct ScopedGroup {
660    pub prefix: String,
661    pub routes: Vec<Route>,
662    /// Registration origin: user application or a named plugin.
663    pub source: crate::route_listing::RouteSource,
664    /// The app declared that the layer of this group authenticates its routes
665    /// (`AppBuilder::scoped_guarded`). This is a claim, not a proof.
666    pub layer_guarded: bool,
667    /// Closure that applies the layer to a sub-router.
668    pub apply_layer: Box<dyn FnOnce(axum::Router<AppState>) -> axum::Router<AppState> + Send>,
669}
670
671/// The one service type every user-registered layer is composed against.
672///
673/// Erasing to a single concrete service type is what lets an arbitrary number
674/// of operator layers be folded into ONE `Router::layer` call: a `tower-layer`
675/// tuple needs its members' types known at compile time, and a `Vec` of
676/// registrations does not have that (#2198).
677///
678/// # When you need to name this type
679///
680/// Almost never. A layer that is generic over the service it wraps — the shape
681/// of every layer in tower, tower-http, and this repo — satisfies
682/// [`AppBuilder::layer`]'s bounds without mentioning it. The exception is a
683/// layer deliberately written against ONE concrete inner service type. Before
684/// #2198 that target was `axum::routing::Route`; it is now this alias:
685///
686/// ```rust,ignore
687/// impl tower::Layer<ErasedAppService> for MyRouteSpecificLayer { /* … */ }
688/// ```
689///
690/// Prefer the generic form (`impl<S> tower::Layer<S> for MyLayer`) unless the
691/// layer genuinely cannot be written that way.
692pub type ErasedAppService = tower::util::BoxCloneSyncService<
693    axum::extract::Request,
694    axum::response::Response,
695    std::convert::Infallible,
696>;
697
698/// A user-registered layer with its type erased at registration time, so a
699/// heterogeneous `Vec` of them can be composed into a single application.
700pub(crate) type ErasedAppLayer = tower::util::BoxCloneSyncServiceLayer<
701    ErasedAppService,
702    axum::extract::Request,
703    axum::response::Response,
704    std::convert::Infallible,
705>;
706
707/// Metadata and the type-erased layer for a user-registered middleware.
708///
709/// `Clone` (the erased `layer` is a `BoxCloneSyncServiceLayer`, which is
710/// itself `Clone`) so a registration set can be applied to more than one
711/// router — see `try_build_router_with_static_inner`'s `mcp_dispatch_extra_layers`,
712/// which clones the SSG/ISG path's drained `custom_layers` onto the MCP
713/// dispatch clone without disturbing how the original set wraps the
714/// live-serving router.
715#[derive(Clone)]
716pub(crate) struct CustomLayerRegistration {
717    /// Concrete type for the registered layer.
718    pub(crate) type_id: TypeId,
719    /// Concrete type name for generic layer families that need router-time
720    /// classification without unstable specialization.
721    pub(crate) type_name: &'static str,
722    /// The registered layer, erased so registrations of unrelated types can
723    /// share one `Vec` and one `Router::layer` application.
724    pub(crate) layer: ErasedAppLayer,
725}
726
727mod sealed {
728    pub trait Sealed {}
729}
730
731/// Marker trait for types that can be registered with
732/// [`AppBuilder::layer`] as an app-wide Tower middleware.
733///
734/// Any [`tower::Layer`] whose produced service is a compatible axum
735/// service (i.e. `Service<Request, Response = Response, Error = Infallible>`,
736/// plus the usual `Clone + Send + Sync + 'static` bounds and a `Send`
737/// future) implements this trait automatically via a blanket impl.
738///
739/// The trait is **sealed**: it exists only to surface a clean
740/// `IntoAppLayer is not implemented for YourType` error message when a
741/// candidate layer fails to meet axum's service bounds, instead of a
742/// 40-line associated-type wall. You cannot implement it manually, and
743/// you should not need to — just bring your own `tower::Layer`.
744///
745/// The layer is applied to Autumn's own erased ingress service rather than to
746/// `axum::routing::Route` directly, so registrations of unrelated types can be
747/// composed into a single `Router::layer` call (#2198). In practice this is
748/// invisible: a real-world layer is generic over its inner service and
749/// satisfies both.
750#[diagnostic::on_unimplemented(
751    message = "`{Self}` is not a usable Autumn app-wide Tower layer",
752    label = "this type is not a `tower::Layer` over Autumn's ingress service with the required service bounds",
753    note = "`AppBuilder::layer(..)` requires a layer that is generic over the service it wraps (or written against `autumn_web::app::ErasedAppService`), producing:\n    L::Service: Service<axum::extract::Request, Response = axum::response::Response, Error = Infallible> + Clone + Send + Sync + 'static,\n    <L::Service as Service<axum::extract::Request>>::Future: Send + 'static\nand the layer itself must be Clone + Send + Sync + 'static.\nSee docs/guide/middleware.md for common patterns and how to wrap raw-error layers (e.g. TimeoutLayer) with HandleErrorLayer."
754)]
755pub trait IntoAppLayer: sealed::Sealed + Send + Sync + 'static {
756    /// Erase this layer's type so it can be stored alongside registrations of
757    /// other types and composed into one application. Not intended for direct
758    /// use.
759    #[doc(hidden)]
760    fn erase(self) -> ErasedAppLayer;
761}
762
763impl<L> sealed::Sealed for L
764where
765    L: tower::Layer<ErasedAppService> + Clone + Send + Sync + 'static,
766    L::Service: tower::Service<
767            axum::extract::Request,
768            Response = axum::response::Response,
769            Error = std::convert::Infallible,
770        > + Clone
771        + Send
772        + Sync
773        + 'static,
774    <L::Service as tower::Service<axum::extract::Request>>::Future: Send + 'static,
775{
776}
777
778impl<L> IntoAppLayer for L
779where
780    L: tower::Layer<ErasedAppService> + Clone + Send + Sync + 'static,
781    L::Service: tower::Service<
782            axum::extract::Request,
783            Response = axum::response::Response,
784            Error = std::convert::Infallible,
785        > + Clone
786        + Send
787        + Sync
788        + 'static,
789    <L::Service as tower::Service<axum::extract::Request>>::Future: Send + 'static,
790{
791    fn erase(self) -> ErasedAppLayer {
792        tower::util::BoxCloneSyncServiceLayer::new(self)
793    }
794}
795
796impl AppBuilder {
797    /// Register a collection of routes with the application.
798    ///
799    /// Can be called multiple times -- routes are combined additively.
800    /// Use the [`routes!`](crate::routes) macro to collect annotated
801    /// handlers into the expected `Vec<Route>`.
802    ///
803    /// # Examples
804    ///
805    /// ```rust,no_run
806    /// # use autumn_web::prelude::*;
807    /// # #[get("/users")] async fn list_users() -> &'static str { "" }
808    /// # #[get("/posts")] async fn list_posts() -> &'static str { "" }
809    /// # #[autumn_web::main]
810    /// # async fn main() {
811    /// autumn_web::app()
812    ///     .routes(routes![list_users])
813    ///     .routes(routes![list_posts])
814    ///     .run()
815    ///     .await;
816    /// # }
817    /// ```
818    #[must_use]
819    pub fn routes(mut self, routes: Vec<Route>) -> Self {
820        let source = self
821            .current_plugin
822            .as_ref()
823            .map_or(crate::route_listing::RouteSource::User, |name| {
824                crate::route_listing::RouteSource::Plugin(name.clone())
825            });
826        for _ in &routes {
827            self.route_sources.push(source.clone());
828        }
829        self.routes.extend(routes);
830        self
831    }
832
833    /// Register scheduled background tasks with the application.
834    ///
835    /// Tasks run alongside the HTTP server and are stopped during
836    /// graceful shutdown. Use the [`tasks!`](crate::tasks) macro
837    /// to collect `#[scheduled]` handlers.
838    #[must_use]
839    pub fn tasks(mut self, tasks: Vec<crate::task::TaskInfo>) -> Self {
840        self.tasks.extend(tasks);
841        self
842    }
843
844    /// Register one-off operational tasks runnable with `autumn task <name>`.
845    ///
846    /// Use the [`one_off_tasks!`](crate::one_off_tasks) macro to collect
847    /// `#[task]` handlers.
848    #[must_use]
849    pub fn one_off_tasks(mut self, tasks: Vec<crate::task::OneOffTaskInfo>) -> Self {
850        self.one_off_tasks.extend(tasks);
851        self
852    }
853
854    /// Register ad-hoc background jobs with the application.
855    #[must_use]
856    pub fn jobs(mut self, jobs: Vec<crate::job::JobInfo>) -> Self {
857        self.jobs.extend(jobs);
858        self
859    }
860
861    /// Register event listeners with the application.
862    ///
863    /// Collect them with `listeners![..]`. Durable listeners are wired onto the
864    /// job runtime automatically (no separate `jobs![..]` entry needed); sync
865    /// listeners run in-request when their event is published. Decoupled from
866    /// emitters: adding a listener never touches the code that publishes.
867    #[must_use]
868    pub fn listeners(mut self, listeners: Vec<crate::events::ListenerInfo>) -> Self {
869        self.listeners.extend(listeners);
870        self
871    }
872
873    /// Set the outbox handler of `topic` (issue #3062).
874    ///
875    /// The relay calls it for each message written with
876    /// [`Outbox::write`](crate::outbox::Outbox::write) on `topic`. It runs
877    /// only when `outbox.enabled = true`. A message can arrive more than
878    /// once; drop copies with [`Inbox::seen`](crate::outbox::Inbox::seen).
879    ///
880    /// # Panics
881    ///
882    /// Panics when `topic` is empty or starts with `autumn.` (reserved).
883    #[cfg(feature = "db")]
884    #[must_use]
885    pub fn outbox_handler<F, Fut>(mut self, topic: impl Into<String>, handler: F) -> Self
886    where
887        F: Fn(AppState, crate::outbox::OutboxMessage) -> Fut + Send + Sync + 'static,
888        Fut: std::future::Future<Output = crate::AutumnResult<()>> + Send + 'static,
889    {
890        self.outbox_handlers.insert(topic, handler);
891        self
892    }
893
894    /// Register static route metadata for build-time rendering.
895    ///
896    /// Use the [`static_routes!`](crate::static_routes) macro to collect
897    /// `#[static_get]` handlers' metadata.
898    #[must_use]
899    pub fn static_routes(mut self, metas: Vec<crate::static_gen::StaticRouteMeta>) -> Self {
900        self.static_metas.extend(metas);
901        self
902    }
903
904    /// Register a [`SitemapSource`](crate::seo::SitemapSource) for dynamic sitemap entries.
905    ///
906    /// When called at least once, the framework automatically serves `/sitemap.xml` and
907    /// `/robots.txt`. Dynamic sources (e.g. blog posts from a database) produce entries
908    /// collected at request time.
909    ///
910    /// Combine with `[seo] base_url` in `autumn.toml` to auto-inject the `Sitemap:`
911    /// directive in `robots.txt` and compute canonical URLs.
912    ///
913    /// # Example
914    ///
915    /// ```rust,no_run
916    /// use autumn_web::prelude::*;
917    /// use autumn_web::seo::{SitemapEntry, SitemapSource};
918    /// use std::pin::Pin;
919    /// use std::future::Future;
920    ///
921    /// struct PostsSitemap;
922    ///
923    /// impl SitemapSource for PostsSitemap {
924    ///     fn entries(&self) -> Pin<Box<dyn Future<Output = Vec<SitemapEntry>> + Send>> {
925    ///         Box::pin(async {
926    ///             vec![SitemapEntry::new("https://example.com/posts/hello")]
927    ///         })
928    ///     }
929    /// }
930    ///
931    /// # #[autumn_web::main]
932    /// # async fn main() {
933    /// # #[get("/")] async fn index() -> &'static str { "" }
934    /// autumn_web::app()
935    ///     .routes(routes![index])
936    ///     .seo_source(PostsSitemap)
937    ///     .run()
938    ///     .await;
939    /// # }
940    /// ```
941    #[must_use]
942    pub fn seo_source<S: crate::seo::SitemapSource + 'static>(mut self, source: S) -> Self {
943        self.seo_sources.push(Arc::new(source));
944        self
945    }
946
947    /// Publish an agent skill at `/.well-known/agent-skills/<name>/SKILL.md`
948    /// and list it in `/.well-known/agent-skills/index.json`.
949    ///
950    /// A skill with the name `site-guide` replaces the generated one. A second
951    /// skill with a name already registered is ignored.
952    ///
953    /// ```rust,no_run
954    /// use autumn_web::aeo::AgentSkill;
955    ///
956    /// # #[autumn_web::main]
957    /// # async fn main() {
958    /// let skill = AgentSkill::new("refunds", "Draft a refund for an order.", "# Refunds\n")
959    ///     .expect("valid skill");
960    /// autumn_web::app().agent_skill(skill).run().await;
961    /// # }
962    /// ```
963    #[must_use]
964    pub fn agent_skill(mut self, skill: crate::aeo::AgentSkill) -> Self {
965        self.agent_skills.push(skill);
966        self
967    }
968
969    /// Serve a UCP business profile at `/.well-known/ucp`.
970    #[must_use]
971    pub fn ucp_profile(mut self, profile: crate::aeo::commerce::UcpProfile) -> Self {
972        self.commerce_docs.ucp = Some(profile);
973        self
974    }
975
976    /// Serve an ACP discovery document at `/.well-known/acp.json`.
977    #[must_use]
978    pub fn acp_discovery(mut self, document: crate::aeo::commerce::AcpDiscovery) -> Self {
979        self.commerce_docs.acp = Some(document);
980        self
981    }
982
983    /// Enable `OpenAPI` (Swagger) spec auto-generation.
984    ///
985    /// When called, the framework inspects every registered route's
986    /// [`ApiDoc`](crate::openapi::ApiDoc) metadata — inferred at compile
987    /// time from the route path, HTTP method, extractor types, and any
988    /// [`#[api_doc(...)]`](crate::api_doc) overrides — and serves an
989    /// `OpenAPI` 3.1 JSON document at `OpenApiConfig::openapi_json_path`
990    /// (default `/openapi.json`). If
991    /// `OpenApiConfig::swagger_ui_path` is set (default `/swagger-ui`),
992    /// a Swagger UI HTML page is served there too.
993    ///
994    /// Routes marked `#[api_doc(hidden)]` are excluded.
995    ///
996    /// Narrative guide: `docs/guide/openapi.md`.
997    ///
998    /// **Gated behind the `openapi` Cargo feature.** Add
999    /// `features = ["openapi"]` to your `autumn-web` dependency to
1000    /// enable it; the default build excludes the runtime spec types
1001    /// and endpoints to keep the binary small.
1002    ///
1003    /// # Examples
1004    ///
1005    /// Zero-config:
1006    ///
1007    /// ```rust,ignore
1008    /// use autumn_web::prelude::*;
1009    /// use autumn_web::openapi::OpenApiConfig;
1010    ///
1011    /// # #[get("/hello")] async fn hello() -> &'static str { "hi" }
1012    /// # #[autumn_web::main]
1013    /// # async fn main() {
1014    /// autumn_web::app()
1015    ///     .routes(routes![hello])
1016    ///     .openapi(OpenApiConfig::new("My API", "1.0.0"))
1017    ///     .run()
1018    ///     .await;
1019    /// # }
1020    /// ```
1021    ///
1022    /// With custom paths:
1023    ///
1024    /// ```rust,ignore
1025    /// use autumn_web::openapi::OpenApiConfig;
1026    ///
1027    /// let config = OpenApiConfig::new("My API", "1.0.0")
1028    ///     .description("Full product API")
1029    ///     .openapi_json_path("/openapi.json")
1030    ///     .swagger_ui_path(Some("/docs".to_owned()));
1031    /// ```
1032    #[cfg(feature = "openapi")]
1033    #[must_use]
1034    pub fn openapi(mut self, config: crate::openapi::OpenApiConfig) -> Self {
1035        self.openapi = Some(config);
1036        self
1037    }
1038
1039    /// Mount a Model Context Protocol (MCP) endpoint at `path` (e.g. `/mcp`).
1040    ///
1041    /// Projects opted-in routes — those tagged `#[api_doc(mcp)]` — as
1042    /// agent-callable MCP tools over Streamable HTTP, handling `initialize`,
1043    /// `tools/list`, and `tools/call`. A tool's `name`, `description`, and
1044    /// `inputSchema` are derived from the handler's existing
1045    /// [`ApiDoc`](crate::openapi::ApiDoc), so the tool catalog cannot drift
1046    /// from the handler's typed contract. `tools/call` dispatches through the
1047    /// real handler pipeline, so `#[secured]`, authorization, rate limits, and
1048    /// validation apply identically to agent and HTTP calls.
1049    ///
1050    /// Opt-in is per-endpoint; nothing is exposed implicitly. Use
1051    /// [`expose_all_as_mcp`](Self::expose_all_as_mcp) for the whole-API hatch.
1052    ///
1053    /// Only **JSON** endpoints are projected: a route is eligible when it
1054    /// returns `Json<T>` (the structural signal for a JSON response). The
1055    /// generated tool's `body` input is derived solely from a `Json<T>`
1056    /// request extractor, so a handler that returns `Json<T>` but reads its
1057    /// body via `Form<T>`, `Multipart`, `Bytes`, or `String` should **not** be
1058    /// opted in — the tool would carry no body input and replay an empty
1059    /// request. Use JSON request bodies for endpoints exposed as MCP tools.
1060    ///
1061    /// `tools/call` replays through the same pipeline as a direct HTTP request,
1062    /// so `#[secured]`, route guards, rate limits, and validation apply
1063    /// identically. One caveat applies only in **static/ISR mode** (an app with
1064    /// a `dist` manifest): a global [`layer`](Self::layer) is applied outside
1065    /// the static-first middleware and is therefore *not* traversed by MCP
1066    /// `tools/call` replays. Prefer `#[secured]` or route-level guards (which do
1067    /// apply) for MCP-exposed handlers in that mode.
1068    ///
1069    /// Requires the `mcp` Cargo feature.
1070    ///
1071    /// ```rust,ignore
1072    /// autumn_web::app()
1073    ///     .routes(routes![list_todos, create_todo])
1074    ///     .mount_mcp("/mcp")
1075    ///     .run()
1076    ///     .await;
1077    /// ```
1078    #[cfg(feature = "mcp")]
1079    #[must_use]
1080    pub fn mount_mcp(mut self, path: impl Into<String>) -> Self {
1081        let path = path.into();
1082        if let Some(rt) = self.mcp.as_mut() {
1083            rt.mount_path = path;
1084        } else {
1085            self.mcp = Some(crate::mcp::McpRuntime::new(path));
1086        }
1087        self
1088    }
1089
1090    /// Whole-API escape hatch: expose **every** eligible read (`GET`) endpoint
1091    /// as an MCP tool without per-endpoint tags.
1092    ///
1093    /// This is an explicit, separate opt-in — never the default. It still
1094    /// honors per-endpoint exclusions (`#[api_doc(mcp = false)]`) and the
1095    /// JSON-only rule, and **mutating verbs (`POST`/`PUT`/`PATCH`/`DELETE`)
1096    /// still require an explicit `#[api_doc(mcp)]` opt-in** even under the
1097    /// hatch.
1098    ///
1099    /// On its own this mounts the endpoint at the default `/mcp`; chain
1100    /// [`mount_mcp`](Self::mount_mcp) to serve it at a different path.
1101    ///
1102    /// Requires the `mcp` Cargo feature.
1103    #[cfg(feature = "mcp")]
1104    #[must_use]
1105    pub fn expose_all_as_mcp(mut self) -> Self {
1106        if let Some(rt) = self.mcp.as_mut() {
1107            rt.expose_all = true;
1108        } else {
1109            let mut rt = crate::mcp::McpRuntime::new("/mcp");
1110            rt.expose_all = true;
1111            self.mcp = Some(rt);
1112        }
1113        self
1114    }
1115
1116    /// Set the limits for MCP task tools (`#[api_doc(mcp, task)]`): TTL,
1117    /// poll interval, store size and per-caller quota. See
1118    /// [`McpTaskConfig`](crate::mcp::McpTaskConfig).
1119    ///
1120    /// If no MCP endpoint is set, this call mounts it at `/mcp`. Call
1121    /// [`mount_mcp`](Self::mount_mcp) to use a different path.
1122    ///
1123    /// Requires the `mcp` Cargo feature.
1124    #[cfg(feature = "mcp")]
1125    #[must_use]
1126    pub fn mcp_tasks(mut self, config: crate::mcp::McpTaskConfig) -> Self {
1127        self.mcp
1128            .get_or_insert_with(|| crate::mcp::McpRuntime::new("/mcp"))
1129            .tasks = config;
1130        self
1131    }
1132
1133    /// Set the MCP server instructions.
1134    ///
1135    /// `initialize` and `server/discover` return the text. A client can put
1136    /// it in the model's system prompt. Say what the server does and when to
1137    /// search for its tools. Do not repeat the tool descriptions.
1138    ///
1139    /// If no MCP endpoint is set, this call mounts it at `/mcp`. Call
1140    /// [`mount_mcp`](Self::mount_mcp) to use a different path.
1141    ///
1142    /// Requires the `mcp` Cargo feature.
1143    #[cfg(feature = "mcp")]
1144    #[must_use]
1145    pub fn mcp_instructions(mut self, text: impl Into<String>) -> Self {
1146        self.mcp
1147            .get_or_insert_with(|| crate::mcp::McpRuntime::new("/mcp"))
1148            .instructions = Some(text.into());
1149        self
1150    }
1151
1152    /// Select how `tools/list` presents the MCP tool catalog.
1153    ///
1154    /// [`McpCatalog::Full`](crate::mcp::McpCatalog::Full), the default,
1155    /// lists every tool.
1156    /// [`McpCatalog::Progressive`](crate::mcp::McpCatalog::Progressive)
1157    /// lists the core tools (`#[api_doc(mcp, core)]`) and two meta-tools:
1158    /// `search_tools` finds a tool, and `call_tool` runs it. Use it when the
1159    /// catalog is too large to send in full.
1160    ///
1161    /// If no MCP endpoint is set, this call mounts it at `/mcp`. Call
1162    /// [`mount_mcp`](Self::mount_mcp) to use a different path.
1163    ///
1164    /// Requires the `mcp` Cargo feature.
1165    #[cfg(feature = "mcp")]
1166    #[must_use]
1167    pub fn mcp_catalog(mut self, catalog: crate::mcp::McpCatalog) -> Self {
1168        self.mcp
1169            .get_or_insert_with(|| crate::mcp::McpRuntime::new("/mcp"))
1170            .catalog = catalog;
1171        self
1172    }
1173
1174    /// Gate the **entire** MCP endpoint — the catalog (`initialize`/
1175    /// `tools/list`) as well as tool dispatch — behind a tower `layer`.
1176    ///
1177    /// The `/mcp` envelope is otherwise reachable without the app's global
1178    /// middleware. Pass an auth layer (e.g.
1179    /// [`RequireApiToken`](crate::auth::RequireApiToken)) here to require a
1180    /// credential for the whole endpoint, the way you'd protect a normal
1181    /// route group. Combine with [`mount_mcp`](Self::mount_mcp); the MCP
1182    /// transport's spec-required `Origin` validation (sourced from your CORS
1183    /// `allowed_origins`) always applies regardless of this layer.
1184    ///
1185    /// Requires the `mcp` Cargo feature.
1186    #[cfg(feature = "mcp")]
1187    #[must_use]
1188    pub fn secure_mcp<L>(mut self, layer: L) -> Self
1189    where
1190        L: tower::Layer<axum::routing::Route> + Clone + Send + Sync + 'static,
1191        L::Service: tower::Service<
1192                axum::http::Request<axum::body::Body>,
1193                Response = axum::http::Response<axum::body::Body>,
1194                Error = std::convert::Infallible,
1195            > + Clone
1196            + Send
1197            + Sync
1198            + 'static,
1199        <L::Service as tower::Service<axum::http::Request<axum::body::Body>>>::Future:
1200            Send + 'static,
1201    {
1202        let applier: crate::mcp::McpEndpointLayer = Box::new(move |router| router.layer(layer));
1203        if let Some(rt) = self.mcp.as_mut() {
1204            rt.endpoint_layer = Some(applier);
1205        } else {
1206            let mut rt = crate::mcp::McpRuntime::new("/mcp");
1207            rt.endpoint_layer = Some(applier);
1208            self.mcp = Some(rt);
1209        }
1210        self
1211    }
1212
1213    /// Serve the `WebMCP` tool list, so the page script
1214    /// (`asset_url("js/autumn-webmcp.js")`) can register the routes tagged
1215    /// `#[api_doc(webmcp)]` with the browser agent.
1216    ///
1217    /// Each tool comes from the same derivation as `/mcp`: the same
1218    /// eligibility rule, names and `inputSchema`. A tool call is a
1219    /// same-origin `fetch` with the user's session cookie and CSRF token, so
1220    /// the full route pipeline runs. `/mcp` does not have to be mounted.
1221    ///
1222    /// `#[api_doc(mcp)]` does not imply `WebMCP`. See `docs/guide/webmcp.md`.
1223    ///
1224    /// Requires the `webmcp` Cargo feature.
1225    ///
1226    /// ```rust,ignore
1227    /// autumn_web::app()
1228    ///     .routes(routes![list_todos, add_todo])
1229    ///     .mount_webmcp()
1230    ///     .run()
1231    ///     .await;
1232    /// ```
1233    #[cfg(feature = "webmcp")]
1234    #[must_use]
1235    pub fn mount_webmcp(mut self) -> Self {
1236        self.webmcp.get_or_insert_with(Default::default);
1237        self
1238    }
1239
1240    /// Whole-API hatch for `WebMCP`: each eligible `GET` route is a tool
1241    /// with no `#[api_doc(webmcp)]` tag. A write still needs the tag, and
1242    /// `#[api_doc(webmcp = false)]` still wins. Also mounts the tool list.
1243    ///
1244    /// Requires the `webmcp` Cargo feature.
1245    #[cfg(feature = "webmcp")]
1246    #[must_use]
1247    pub fn expose_all_as_webmcp(mut self) -> Self {
1248        self.webmcp.get_or_insert_with(Default::default).expose_all = true;
1249        self
1250    }
1251
1252    /// Register a global exception filter.
1253    ///
1254    /// Exception filters intercept error responses produced by
1255    /// [`AutumnError`](crate::AutumnError) before they are sent to the
1256    /// client. Filters run in registration order.
1257    ///
1258    /// # Examples
1259    ///
1260    /// ```rust,no_run
1261    /// use autumn_web::middleware::{ExceptionFilter, AutumnErrorInfo};
1262    /// use axum::response::Response;
1263    ///
1264    /// struct LogFilter;
1265    /// impl ExceptionFilter for LogFilter {
1266    ///     fn filter(&self, error: &AutumnErrorInfo, response: Response) -> Response {
1267    ///         eprintln!("Error: {}", error.message);
1268    ///         response
1269    ///     }
1270    /// }
1271    ///
1272    /// # use autumn_web::prelude::*;
1273    /// # #[get("/")] async fn index() -> &'static str { "" }
1274    /// # #[autumn_web::main]
1275    /// # async fn main() {
1276    /// autumn_web::app()
1277    ///     .exception_filter(LogFilter)
1278    ///     .routes(routes![index])
1279    ///     .run()
1280    ///     .await;
1281    /// # }
1282    /// ```
1283    #[must_use]
1284    pub fn exception_filter(mut self, filter: impl ExceptionFilter) -> Self {
1285        self.exception_filters.push(Arc::new(filter));
1286        self
1287    }
1288
1289    /// Register a custom error page renderer.
1290    ///
1291    /// The renderer replaces the built-in default error pages (404, 422, 500,
1292    /// and generic errors). Implement [`ErrorPageRenderer`] to provide your
1293    /// own branded error pages.
1294    ///
1295    /// Only one renderer can be active. Calling this method multiple times
1296    /// replaces the previous renderer.
1297    ///
1298    /// Requires the `maud` feature.
1299    ///
1300    /// # Examples
1301    ///
1302    /// ```rust,no_run
1303    /// use autumn_web::error_pages::{ErrorPageRenderer, ErrorContext};
1304    /// use maud::{Markup, html};
1305    ///
1306    /// struct MyErrors;
1307    ///
1308    /// impl ErrorPageRenderer for MyErrors {
1309    ///     fn render_error(&self, ctx: &ErrorContext) -> Markup {
1310    ///         html! {
1311    ///             h1 { (ctx.status.as_u16()) " - Custom error page" }
1312    ///         }
1313    ///     }
1314    /// }
1315    ///
1316    /// # use autumn_web::prelude::*;
1317    /// # #[get("/")] async fn index() -> &'static str { "" }
1318    /// # #[autumn_web::main]
1319    /// # async fn main() {
1320    /// autumn_web::app()
1321    ///     .error_pages(MyErrors)
1322    ///     .routes(routes![index])
1323    ///     .run()
1324    ///     .await;
1325    /// # }
1326    /// ```
1327    #[must_use]
1328    #[cfg(feature = "maud")]
1329    pub fn error_pages(mut self, renderer: impl ErrorPageRenderer) -> Self {
1330        self.error_page_renderer = Some(Arc::new(renderer));
1331        self
1332    }
1333
1334    /// Register a group of routes with a shared path prefix and middleware.
1335    ///
1336    /// The `layer` is applied only to routes within this group, not to the
1337    /// rest of the application. The routes are mounted under `prefix`.
1338    ///
1339    /// # Examples
1340    ///
1341    /// ```rust,no_run
1342    /// use autumn_web::prelude::*;
1343    /// use autumn_web::middleware::RequestIdLayer; // any Tower Layer
1344    ///
1345    /// # #[get("/")]  async fn index() -> &'static str { "" }
1346    /// # #[get("/users")] async fn list_users() -> &'static str { "" }
1347    /// # #[autumn_web::main]
1348    /// # async fn main() {
1349    /// autumn_web::app()
1350    ///     .routes(routes![index])
1351    ///     .scoped("/api", RequestIdLayer::default(), routes![list_users])
1352    ///     .run()
1353    ///     .await;
1354    /// # }
1355    /// ```
1356    #[must_use]
1357    pub fn scoped<L>(self, prefix: &str, layer: L, routes: Vec<Route>) -> Self
1358    where
1359        L: tower::Layer<axum::routing::Route> + Clone + Send + Sync + 'static,
1360        L::Service: tower::Service<
1361                axum::http::Request<axum::body::Body>,
1362                Response = axum::http::Response<axum::body::Body>,
1363                Error = std::convert::Infallible,
1364            > + Clone
1365            + Send
1366            + Sync
1367            + 'static,
1368        <L::Service as tower::Service<axum::http::Request<axum::body::Body>>>::Future:
1369            Send + 'static,
1370    {
1371        self.push_scoped_group(prefix, layer, routes, false)
1372    }
1373
1374    /// Like [`scoped`](Self::scoped). Also declares that `layer` authenticates
1375    /// every route in the group.
1376    ///
1377    /// Use it when a layer is the guard, such as `RequireApiToken`. Do not use
1378    /// it with a layer that does not authenticate. That hides routes from the
1379    /// default-deny audit.
1380    /// `autumn routes audit` then lists each otherwise unclassified route in
1381    /// the group as `gated`, with `provenance: "declared"`. A route that has
1382    /// its own `#[public]` or `#[secured]` keeps that classification.
1383    ///
1384    /// The audit cannot check the claim. If the layer does not authenticate,
1385    /// the manifest is wrong.
1386    #[must_use]
1387    pub fn scoped_guarded<L>(self, prefix: &str, layer: L, routes: Vec<Route>) -> Self
1388    where
1389        L: tower::Layer<axum::routing::Route> + Clone + Send + Sync + 'static,
1390        L::Service: tower::Service<
1391                axum::http::Request<axum::body::Body>,
1392                Response = axum::http::Response<axum::body::Body>,
1393                Error = std::convert::Infallible,
1394            > + Clone
1395            + Send
1396            + Sync
1397            + 'static,
1398        <L::Service as tower::Service<axum::http::Request<axum::body::Body>>>::Future:
1399            Send + 'static,
1400    {
1401        self.push_scoped_group(prefix, layer, routes, true)
1402    }
1403
1404    fn push_scoped_group<L>(
1405        mut self,
1406        prefix: &str,
1407        layer: L,
1408        routes: Vec<Route>,
1409        layer_guarded: bool,
1410    ) -> Self
1411    where
1412        L: tower::Layer<axum::routing::Route> + Clone + Send + Sync + 'static,
1413        L::Service: tower::Service<
1414                axum::http::Request<axum::body::Body>,
1415                Response = axum::http::Response<axum::body::Body>,
1416                Error = std::convert::Infallible,
1417            > + Clone
1418            + Send
1419            + Sync
1420            + 'static,
1421        <L::Service as tower::Service<axum::http::Request<axum::body::Body>>>::Future:
1422            Send + 'static,
1423    {
1424        let source = self
1425            .current_plugin
1426            .as_ref()
1427            .map_or(crate::route_listing::RouteSource::User, |name| {
1428                crate::route_listing::RouteSource::Plugin(name.clone())
1429            });
1430        self.scoped_groups.push(ScopedGroup {
1431            prefix: prefix.to_owned(),
1432            routes,
1433            source,
1434            layer_guarded,
1435            apply_layer: Box::new(move |router| router.layer(layer)),
1436        });
1437        self
1438    }
1439
1440    /// Apply a custom [`tower::Layer`] to the entire application.
1441    ///
1442    /// This is the escape hatch for integrating any middleware from the
1443    /// Tower / Tower-HTTP ecosystem (timeouts, rate limiting, bespoke
1444    /// tracing, request signing, etc.) without forking the framework.
1445    ///
1446    /// The generic bound is [`IntoAppLayer`], a sealed trait with a blanket
1447    /// impl for every `tower::Layer` that meets axum's service requirements
1448    /// — in practice this means any standard Tower layer whose service
1449    /// produces `Infallible` errors. If your layer produces real errors
1450    /// (like `TimeoutLayer`'s `BoxError`), wrap it with
1451    /// [`axum::error_handling::HandleErrorLayer`] before passing it here.
1452    ///
1453    /// # Ordering
1454    ///
1455    /// User layers are applied **inside** Autumn's request-ID layer on the
1456    /// ingress path, which means your middleware always sees the generated
1457    /// `RequestId` in the request extensions. The full stack (outermost to
1458    /// innermost on ingress) is:
1459    ///
1460    /// `Metrics -> ExceptionFilter -> ErrorPageContext -> Session ->`
1461    /// `SecurityHeaders -> RequestId -> [user layers, registration order]`
1462    /// `-> CSRF -> CORS -> route handler`
1463    ///
1464    /// When `.layer()` is called multiple times, the **first** call becomes
1465    /// the outermost user layer on ingress (matching `tower::ServiceBuilder`
1466    /// semantics): the layer from the first `.layer(...)` call sees the
1467    /// request first on the way in and the response last on the way out.
1468    ///
1469    /// # Scope
1470    ///
1471    /// This layer applies **globally** to every route in the app, including
1472    /// routes added later by plugins, routes mounted via `.merge` / `.nest`,
1473    /// and the built-in `404` fallback. Use [`AppBuilder::scoped`] when you
1474    /// need middleware scoped to a group of routes.
1475    ///
1476    /// Shared state (pools, metrics registries, rate-limit stores, etc.)
1477    /// should be wrapped in `Arc` so the layer can satisfy the
1478    /// `Clone + Send + Sync + 'static` bounds without moving the state.
1479    ///
1480    /// See [the middleware guide](https://github.com/autumn-foundation/autumn/blob/trunk/docs/guide/middleware.md)
1481    /// for ready-made recipes.
1482    ///
1483    /// # Examples
1484    ///
1485    /// Adding a Tower timeout layer in one line (Tower's `TimeoutLayer`
1486    /// returns `BoxError`, so it must be paired with `HandleErrorLayer` to
1487    /// satisfy axum's `Infallible` error requirement):
1488    ///
1489    /// ```rust,no_run
1490    /// use std::time::Duration;
1491    /// use autumn_web::prelude::*;
1492    /// use axum::{error_handling::HandleErrorLayer, http::StatusCode};
1493    /// use tower::{ServiceBuilder, timeout::TimeoutLayer};
1494    ///
1495    /// # #[get("/")] async fn index() -> &'static str { "ok" }
1496    /// # #[autumn_web::main]
1497    /// # async fn main() {
1498    /// autumn_web::app()
1499    ///     .routes(routes![index])
1500    ///     .layer(
1501    ///         ServiceBuilder::new()
1502    ///             .layer(HandleErrorLayer::new(|_| async {
1503    ///                 StatusCode::REQUEST_TIMEOUT
1504    ///             }))
1505    ///             .layer(TimeoutLayer::new(Duration::from_secs(5))),
1506    ///     )
1507    ///     .run()
1508    ///     .await;
1509    /// # }
1510    /// ```
1511    #[must_use]
1512    pub fn layer<L: IntoAppLayer>(mut self, layer: L) -> Self {
1513        self.custom_layers.push(CustomLayerRegistration {
1514            type_id: TypeId::of::<L>(),
1515            type_name: std::any::type_name::<L>(),
1516            layer: layer.erase(),
1517        });
1518        self
1519    }
1520
1521    /// Returns `true` when a custom layer of type `L` has already been
1522    /// registered via [`AppBuilder::layer`].
1523    ///
1524    /// Intended for plugin pre-flight validation before the app is started.
1525    #[must_use]
1526    pub fn has_layer<L: 'static>(&self) -> bool {
1527        let layer_type = TypeId::of::<L>();
1528        self.custom_layers
1529            .iter()
1530            .any(|registered| registered.type_id == layer_type)
1531    }
1532
1533    /// Enable the HTTP idempotency-key middleware for this application.
1534    ///
1535    /// Mutating requests (`POST`, `PUT`, `PATCH`, `DELETE`) that carry an
1536    /// `Idempotency-Key` header are deduplicated: the first response is cached
1537    /// and replayed byte-for-byte on subsequent identical requests.
1538    /// Session-mutating responses are cached after the outer session middleware
1539    /// has finalized `Set-Cookie`, so retries can observe the successful
1540    /// mutation without re-entering the handler.
1541    ///
1542    /// Raw Axum routers registered with [`merge`](Self::merge) or
1543    /// [`nest`](Self::nest) are opaque to Autumn. They are protected from
1544    /// duplicate mutating retries by failing closed on cache hits; install
1545    /// idempotency and replay-stop layers inside those routers when raw routes
1546    /// need successful cached-response replay after their own route-local
1547    /// checks.
1548    ///
1549    /// The cache is scoped to the caller: the cookie-backed session, the
1550    /// resolved tenant, and for a token client the principal
1551    /// [`RequireApiToken`](crate::auth::RequireApiToken) verified (or, with no
1552    /// verified principal and no session, a digest of the raw `Authorization`
1553    /// header). A route that dedupes by its own key can opt out with
1554    /// `#[post("/x", idempotent = false)]`.
1555    ///
1556    /// The storage backend and TTL are taken from the `[idempotency]` block in
1557    /// `autumn.toml` (defaulting to in-process memory with a 24 h TTL).
1558    /// For multi-replica deployments set `backend = "redis"` and configure
1559    /// `[idempotency.redis]`.
1560    ///
1561    /// # Startup validation
1562    ///
1563    /// In production (`AUTUMN_PROFILE=production`) the memory backend is
1564    /// rejected unless `allow_memory_in_production = true` is set explicitly.
1565    #[must_use]
1566    pub const fn idempotent(mut self) -> Self {
1567        self.idempotency_enabled = true;
1568        self
1569    }
1570
1571    /// Returns the registered custom layer types in registration order.
1572    ///
1573    /// This includes only user-installed layers from
1574    /// [`AppBuilder::layer`], not framework-managed middleware — even one
1575    /// installed through this same `custom_layers` vector internally, such
1576    /// as the `edge` feature's own sentinel-strip layer, which this filters
1577    /// back out.
1578    #[must_use]
1579    pub fn get_layer_types(&self) -> Vec<TypeId> {
1580        self.custom_layers
1581            .iter()
1582            .map(|registered| registered.type_id)
1583            .filter(|type_id| !is_framework_owned_layer(*type_id))
1584            .collect()
1585    }
1586
1587    /// Register a Tower layer that runs **before** the static file middleware
1588    /// and the static cache lookup — Autumn's equivalent of Next.js *Edge
1589    /// Middleware*.
1590    ///
1591    /// Cached SSG/ISG pages are served by the static-first middleware before
1592    /// the inner router (session, auth) is ever reached, so framework auth
1593    /// layers cannot gate pre-rendered responses. A `static_gate` layer runs
1594    /// outermost — outside the session layer and ahead of the static cache —
1595    /// so it can redirect or reject a request before a cached page is served.
1596    ///
1597    /// This is the right place for auth gating that protects pre-rendered
1598    /// routes: redirect unauthenticated visitors to a login page while leaving
1599    /// the cached HTML free of user-specific content. Personalised content
1600    /// still requires a fully dynamic route or client-side fetching.
1601    ///
1602    /// # Position and limitations
1603    ///
1604    /// * Runs as the **outermost** user middleware in *both* SSG/ISG and
1605    ///   fully-dynamic modes, so the same gate behaves identically regardless
1606    ///   of whether static generation is active.
1607    /// * Has access to request **headers and cookies**, but **NOT** the
1608    ///   session [`Extension`](axum::Extension) — the session layer runs inside
1609    ///   it. Verify a signed/JWT session cookie directly (e.g. with the same
1610    ///   signing key configured for the session) rather than relying on
1611    ///   session-populated extensions.
1612    /// * Like [`layer`](Self::layer), it applies globally to every route.
1613    /// * **Page-cache gate, not API auth.** The gate guards GET/HEAD page
1614    ///   serving and acts by issuing a browser redirect/reject. It is **not**
1615    ///   applied to MCP `tools/call` dispatch (a JSON-RPC call, where a redirect
1616    ///   is meaningless) in *either* mode: the gate is applied after the MCP
1617    ///   dispatch clone is taken. Gate MCP tools and JSON APIs with route-level
1618    ///   guards / `#[secured]` / session auth, which always traverse the
1619    ///   dispatch path. A well-behaved gate should therefore no-op on non-GET
1620    ///   requests (such as the `/mcp` JSON-RPC POST transport).
1621    /// * Short-circuit responses (the redirect/reject) are wrapped by the
1622    ///   framework's security-header layer, so they still carry HSTS/CSP, etc.
1623    /// * Because the gate runs **outside** the request stack (it must run before
1624    ///   session and the static cache), a gate short-circuit does **not** pass
1625    ///   through trusted-host validation or the per-request timeout — same as any
1626    ///   middleware registered with [`layer`](Self::layer) that runs before
1627    ///   those framework layers. Keep gate work bounded (prefer local
1628    ///   cookie/JWT checks over unbounded remote calls), and rely on the
1629    ///   framework's trusted-host policy for the routes the gate forwards to.
1630    ///
1631    /// Layers are wrapped in registration order with the first-registered gate
1632    /// outermost, matching [`tower::ServiceBuilder`] semantics.
1633    ///
1634    /// # Examples
1635    ///
1636    /// ```rust,no_run
1637    /// use autumn_web::prelude::*;
1638    /// use axum::{
1639    ///     extract::Request,
1640    ///     http::{header, Method, StatusCode},
1641    ///     middleware::Next,
1642    ///     response::Response,
1643    /// };
1644    ///
1645    /// async fn require_auth(req: Request, next: Next) -> Response {
1646    ///     // Only gate page navigation. Pass non-GET/HEAD requests (JSON APIs,
1647    ///     // form POSTs, the `/mcp` JSON-RPC transport, CORS preflights) straight
1648    ///     // through so a browser redirect never turns them into a 302.
1649    ///     let is_page = matches!(req.method(), &Method::GET | &Method::HEAD);
1650    ///     // Inspect a signed session cookie directly — no session Extension
1651    ///     // is available this far out in the stack.
1652    ///     if !is_page || req.headers().contains_key("x-authed") {
1653    ///         next.run(req).await
1654    ///     } else {
1655    ///         Response::builder()
1656    ///             .status(StatusCode::FOUND)
1657    ///             .header(header::LOCATION, "/login")
1658    ///             .body(axum::body::Body::empty())
1659    ///             .unwrap()
1660    ///     }
1661    /// }
1662    ///
1663    /// # #[get("/")] async fn index() -> &'static str { "ok" }
1664    /// # #[autumn_web::main]
1665    /// # async fn main() {
1666    /// autumn_web::app()
1667    ///     .routes(routes![index])
1668    ///     .static_gate(axum::middleware::from_fn(require_auth))
1669    ///     .run()
1670    ///     .await;
1671    /// # }
1672    /// ```
1673    #[must_use]
1674    pub fn static_gate<L: IntoAppLayer>(mut self, layer: L) -> Self {
1675        self.static_gate_layers.push(CustomLayerRegistration {
1676            type_id: TypeId::of::<L>(),
1677            type_name: std::any::type_name::<L>(),
1678            layer: layer.erase(),
1679        });
1680        self
1681    }
1682
1683    /// Returns `true` when a pre-static gate layer of type `L` has already
1684    /// been registered via [`AppBuilder::static_gate`].
1685    ///
1686    /// Intended for plugin pre-flight validation before the app is started.
1687    #[must_use]
1688    pub fn has_static_gate<L: 'static>(&self) -> bool {
1689        let layer_type = TypeId::of::<L>();
1690        self.static_gate_layers
1691            .iter()
1692            .any(|registered| registered.type_id == layer_type)
1693    }
1694
1695    /// Returns the registered pre-static gate layer types in registration
1696    /// order.
1697    ///
1698    /// This includes only user-installed gates from
1699    /// [`AppBuilder::static_gate`], not regular layers or framework
1700    /// middleware.
1701    #[must_use]
1702    pub fn get_static_gate_types(&self) -> Vec<TypeId> {
1703        self.static_gate_layers
1704            .iter()
1705            .map(|registered| registered.type_id)
1706            .collect()
1707    }
1708
1709    /// Merge a raw Axum router into the application.
1710    ///
1711    /// This is an escape hatch for when Autumn's route macros are not
1712    /// sufficient -- for example, when integrating a third-party Axum
1713    /// middleware crate or mounting a hand-built WebSocket handler.
1714    ///
1715    /// The merged router shares the same [`AppState`] (database pool,
1716    /// config, etc.) and Autumn's global middleware (request IDs,
1717    /// security headers, session management) applies to its routes.
1718    /// When `.idempotent()` is enabled, retries that hit an existing raw-route
1719    /// idempotency record fail closed instead of rerunning the raw handler or
1720    /// replaying around opaque route-local checks. Install idempotency and
1721    /// replay-stop layers inside the raw router when successful replay is
1722    /// required.
1723    ///
1724    /// Merged routes are added **after** Autumn's annotated routes.
1725    /// If both define the same method+path pair, Axum treats that as an
1726    /// overlap and router construction will fail.
1727    ///
1728    /// Can be called multiple times -- routers are accumulated.
1729    ///
1730    /// # Examples
1731    ///
1732    /// ```rust,no_run
1733    /// use autumn_web::prelude::*;
1734    /// use autumn_web::AppState;
1735    ///
1736    /// #[get("/")]
1737    /// async fn index() -> &'static str { "hi" }
1738    ///
1739    /// #[autumn_web::main]
1740    /// async fn main() {
1741    ///     let raw = axum::Router::<AppState>::new()
1742    ///         .route("/ws", axum::routing::get(|| async { "websocket" }));
1743    ///
1744    ///     autumn_web::app()
1745    ///         .routes(routes![index])
1746    ///         .merge(raw)
1747    ///         .run()
1748    ///         .await;
1749    /// }
1750    /// ```
1751    #[must_use]
1752    pub fn merge(mut self, router: axum::Router<AppState>) -> Self {
1753        self.merge_routers.push(router);
1754        self
1755    }
1756
1757    /// Mount a raw Axum router under a path prefix.
1758    ///
1759    /// This is an escape hatch similar to [`merge`](Self::merge), but the
1760    /// router's routes are nested under the given `path` prefix. Useful
1761    /// for mounting a self-contained API version or third-party router.
1762    ///
1763    /// The nested router shares the same [`AppState`] and Autumn's global
1764    /// middleware applies to its routes. When `.idempotent()` is enabled,
1765    /// retries that hit an existing raw-route idempotency record fail closed
1766    /// instead of rerunning the raw handler or replaying around opaque
1767    /// route-local checks. Install idempotency and replay-stop layers inside
1768    /// the raw router when successful replay is required.
1769    ///
1770    /// Can be called multiple times with different prefixes.
1771    ///
1772    /// # Examples
1773    ///
1774    /// ```rust,no_run
1775    /// use autumn_web::prelude::*;
1776    /// use autumn_web::AppState;
1777    ///
1778    /// #[get("/")]
1779    /// async fn index() -> &'static str { "hi" }
1780    ///
1781    /// #[autumn_web::main]
1782    /// async fn main() {
1783    ///     let v2 = axum::Router::<AppState>::new()
1784    ///         .route("/users", axum::routing::get(|| async { "v2 users" }));
1785    ///
1786    ///     autumn_web::app()
1787    ///         .routes(routes![index])
1788    ///         .nest("/api/v2", v2)
1789    ///         .run()
1790    ///         .await;
1791    /// }
1792    /// ```
1793    #[must_use]
1794    pub fn nest(mut self, path: &str, router: axum::Router<AppState>) -> Self {
1795        self.nest_routers.push((path.to_owned(), router));
1796        self
1797    }
1798
1799    /// Explicitly register route metadata for listing via `autumn routes`.
1800    ///
1801    /// Plugins that mount routes via [`AppBuilder::nest`] (which is opaque to
1802    /// the route listing) can call this method so that `autumn routes --format json`
1803    /// shows their routes with the correct plugin attribution.
1804    ///
1805    /// Routes are automatically attributed to the current plugin when called from
1806    /// within a plugin's `build()` method. The `source` field of each supplied
1807    /// `RouteInfo` is overwritten with that attribution.
1808    ///
1809    /// Declaring routes also makes a [`nest`](Self::nest) mount *coverage-clean*
1810    /// for `autumn routes audit`: a nested router is normally opaque and counts
1811    /// as an omitted, unprovable router that hard-fails the gate, but when at
1812    /// least one declared route's path falls under the nest's prefix, the mount
1813    /// is treated as enumerable and no longer counts. So the documented
1814    /// `app.nest(prefix, router).declare_plugin_routes(routes)` pattern — with
1815    /// `routes` covering everything the raw router serves under `prefix` — passes
1816    /// the audit. A bare `nest`/`merge` with no covering declaration stays
1817    /// opaque and still fails closed.
1818    #[must_use]
1819    pub fn declare_plugin_routes(
1820        mut self,
1821        routes: impl IntoIterator<Item = crate::route_listing::RouteInfo>,
1822    ) -> Self {
1823        let source = self.current_route_source();
1824        for mut route in routes {
1825            route.source = source.clone();
1826            // The asset-bundle marker is the framework's to set: only routes
1827            // `plugin_assets` generated may carry it, because the conformance
1828            // checks exempt a route that does.
1829            route
1830                .middleware
1831                .retain(|label| label != crate::assets::PLUGIN_ASSETS_ROUTE_MARKER);
1832            self.declared_routes.push(route);
1833        }
1834        self
1835    }
1836
1837    /// The `RouteSource` a route declared right now is attributed to: the
1838    /// plugin whose `build` is running, or the user.
1839    fn current_route_source(&self) -> crate::route_listing::RouteSource {
1840        self.current_plugin
1841            .as_deref()
1842            .map_or(crate::route_listing::RouteSource::User, |name| {
1843                crate::route_listing::RouteSource::Plugin(name.to_owned())
1844            })
1845    }
1846
1847    /// Serve a [`PluginAssets`](crate::assets::PluginAssets) bundle: files
1848    /// compiled into a crate, with content-hashed URLs.
1849    ///
1850    /// Call this from [`Plugin::build`](crate::plugin::Plugin::build). The
1851    /// bundle is mounted under `/static/_plugins/<namespace>/`, each file at
1852    /// its plain URL (`must-revalidate`) and its fingerprinted URL (`immutable`
1853    /// for a year), and its routes are declared for `autumn routes` as public
1854    /// static files. After this,
1855    /// [`asset_url("_plugins/<namespace>/<file>")`](crate::assets::asset_url)
1856    /// returns the fingerprinted URL.
1857    ///
1858    /// ```rust,ignore
1859    /// use autumn_web::assets::PluginAssets;
1860    ///
1861    /// pub static ASSETS: PluginAssets = autumn_web::plugin_assets!("motion");
1862    ///
1863    /// impl Plugin for MotionPlugin {
1864    ///     fn build(self, app: AppBuilder) -> AppBuilder {
1865    ///         app.plugin_assets(&ASSETS)
1866    ///     }
1867    /// }
1868    /// ```
1869    ///
1870    /// Installing the same bundle twice is a no-op.
1871    ///
1872    /// # Panics
1873    ///
1874    /// Panics if a *different* bundle with the same namespace is already
1875    /// installed. The two would fight over the same URLs.
1876    #[must_use]
1877    pub fn plugin_assets(mut self, assets: &'static crate::assets::PluginAssets) -> Self {
1878        if let Some(existing) = self
1879            .plugin_asset_bundles
1880            .iter()
1881            .find(|installed| installed.namespace() == assets.namespace())
1882        {
1883            assert!(
1884                std::ptr::eq(*existing, assets),
1885                "two different PluginAssets bundles use the namespace `{}`; each bundle needs \
1886                 its own namespace because it becomes the URL path {}",
1887                assets.namespace(),
1888                assets.mount_path(),
1889            );
1890            return self;
1891        }
1892        self.plugin_asset_bundles.push(assets);
1893        crate::assets::plugin::register(assets);
1894        if assets.iter().next().is_none() {
1895            tracing::warn!(
1896                namespace = assets.namespace(),
1897                "PluginAssets bundle has no files; nothing to serve"
1898            );
1899            return self;
1900        }
1901        // Declared directly rather than through `declare_plugin_routes`, which
1902        // strips the asset-bundle marker these routes carry.
1903        let source = self.current_route_source();
1904        for mut route in assets.route_infos() {
1905            route.source = source.clone();
1906            self.declared_routes.push(route);
1907        }
1908        let mount = assets.mount_path();
1909        self.nest(&mount, assets.nested_router())
1910    }
1911
1912    /// The route manifest this builder would dump for `autumn routes` —
1913    /// enumerable routes plus everything declared via
1914    /// [`declare_plugin_routes`](Self::declare_plugin_routes).
1915    ///
1916    /// This is the seam a plugin author's conformance test needs: it runs the
1917    /// same collection the `AUTUMN_DUMP_ROUTES` path runs, in-process, so
1918    /// `autumn_web::plugin_conformance::run_conformance` can be pointed at a
1919    /// host app built in a test without compiling and executing a binary.
1920    ///
1921    /// Framework-owned routes (probes, actuator, docs) are **not** included:
1922    /// they depend on the loaded configuration, and a conformance run is about
1923    /// what the plugin contributes.
1924    ///
1925    /// # Errors
1926    ///
1927    /// Returns [`RouterBuildError::UnregisteredApiVersion`](crate::RouterBuildError::UnregisteredApiVersion)
1928    /// if a route names an API version this builder has not registered — the
1929    /// same refusal `autumn routes` reports.
1930    pub fn plugin_route_infos(
1931        &self,
1932    ) -> Result<Vec<crate::route_listing::RouteInfo>, crate::RouterBuildError> {
1933        let mut infos = crate::route_listing::collect_route_infos(
1934            &self.routes,
1935            &self.route_sources,
1936            &self.scoped_groups,
1937            &self.api_versions,
1938        )?;
1939        infos.extend(self.declared_routes.iter().cloned());
1940        Ok(infos)
1941    }
1942
1943    /// Register an async startup hook that runs after [`AppState`] exists and
1944    /// before the server begins accepting requests.
1945    ///
1946    /// This is intended for background runtimes that need the fully built app
1947    /// state, such as workers or pollers that share the database pool.
1948    #[must_use]
1949    pub fn on_startup<F, Fut>(mut self, hook: F) -> Self
1950    where
1951        F: Fn(AppState) -> Fut + Send + Sync + 'static,
1952        Fut: Future<Output = crate::AutumnResult<()>> + Send + 'static,
1953    {
1954        self.startup_hooks
1955            .push(Box::new(move |state| Box::pin(hook(state))));
1956        self
1957    }
1958
1959    /// Register a synchronous initializer that mutates [`AppState`] after
1960    /// framework-managed extensions are installed and before job workers start.
1961    #[must_use]
1962    pub fn state_initializer<F>(mut self, initializer: F) -> Self
1963    where
1964        F: FnOnce(&AppState) + Send + 'static,
1965    {
1966        self.state_initializers.push(Box::new(initializer));
1967        self
1968    }
1969
1970    /// Serve plugins that `mounts` mounts while the app runs (issue #2433).
1971    ///
1972    /// Keep a clone of `mounts`. Call `mount` and `unmount` on it at any time,
1973    /// with no restart. The configured actuator and probe paths are reserved.
1974    /// See [`PluginMounts`](crate::plugin_sandbox::PluginMounts).
1975    #[cfg(feature = "plugin-sandbox")]
1976    #[must_use]
1977    pub fn plugin_mounts(self, mounts: crate::plugin_sandbox::PluginMounts) -> Self {
1978        self.state_initializer(move |state| {
1979            mounts.reserve(crate::plugin_sandbox::framework_prefixes(&state.config()));
1980            state.insert_extension(mounts);
1981        })
1982    }
1983
1984    /// Designate a block of typed in-memory state to survive an in-place
1985    /// upgrade (issue #1674).
1986    ///
1987    /// On `SIGUSR2` the block is snapshotted, frozen against further writes,
1988    /// and handed to the successor build along with the listening socket; the
1989    /// successor installs it before it serves its first request. On an ordinary
1990    /// cold start `initial` is used.
1991    ///
1992    /// Reach the block from a handler with
1993    /// [`AppState::live_state`](crate::AppState::live_state). Use
1994    /// [`with_live_state_from`](Self::with_live_state_from) when the new build
1995    /// changed the shape.
1996    ///
1997    /// One block per app: designating a second is a startup error, because
1998    /// silently carrying only one of them is exactly the data loss this feature
1999    /// exists to prevent.
2000    ///
2001    /// # Examples
2002    ///
2003    /// ```rust,no_run
2004    /// use autumn_web::prelude::*;
2005    /// use autumn_web::upgrade::LiveState;
2006    /// use serde::{Deserialize, Serialize};
2007    ///
2008    /// #[derive(Default, Serialize, Deserialize)]
2009    /// struct Stats { hits: u64 }
2010    /// impl LiveState for Stats { const VERSION: u32 = 1; }
2011    ///
2012    /// # #[get("/")] async fn index() -> &'static str { "ok" }
2013    /// # #[autumn_web::main]
2014    /// # async fn main() {
2015    /// autumn_web::app()
2016    ///     .routes(routes![index])
2017    ///     .with_live_state(Stats::default())
2018    ///     .run()
2019    ///     .await;
2020    /// # }
2021    /// ```
2022    #[must_use]
2023    pub fn with_live_state<T>(self, initial: T) -> Self
2024    where
2025        T: crate::upgrade::LiveState,
2026    {
2027        self.state_initializer(move |state| {
2028            install_live_state(state, initial, crate::upgrade::decode::<T>);
2029        })
2030    }
2031
2032    /// Designate a live-state block whose shape changed since the previous
2033    /// build, carrying an `Old` snapshot across through the
2034    /// [`state_migration!`](crate::state_migration) declared for it.
2035    ///
2036    /// A snapshot at `T`'s own version is adopted directly; one at `Old`'s
2037    /// version is migrated; anything else refuses to start, which aborts the
2038    /// upgrade and leaves the previous build serving.
2039    ///
2040    /// # Examples
2041    ///
2042    /// ```rust,no_run
2043    /// use autumn_web::prelude::*;
2044    /// use autumn_web::state_migration;
2045    /// use autumn_web::upgrade::LiveState;
2046    /// use serde::{Deserialize, Serialize};
2047    ///
2048    /// #[derive(Serialize, Deserialize)]
2049    /// struct StatsV1 { hits: u64 }
2050    /// #[derive(Default, Serialize, Deserialize)]
2051    /// struct Stats { hits: u64, upgrades: u64 }
2052    /// impl LiveState for StatsV1 { const VERSION: u32 = 1; }
2053    /// impl LiveState for Stats { const VERSION: u32 = 2; }
2054    ///
2055    /// state_migration! {
2056    ///     from StatsV1 as old => Stats {
2057    ///         hits: old.hits,
2058    ///         upgrades: 1,
2059    ///     }
2060    /// }
2061    ///
2062    /// # #[get("/")] async fn index() -> &'static str { "ok" }
2063    /// # #[autumn_web::main]
2064    /// # async fn main() {
2065    /// autumn_web::app()
2066    ///     .routes(routes![index])
2067    ///     .with_live_state_from::<StatsV1, _>(Stats::default())
2068    ///     .run()
2069    ///     .await;
2070    /// # }
2071    /// ```
2072    #[must_use]
2073    pub fn with_live_state_from<Old, T>(self, initial: T) -> Self
2074    where
2075        Old: crate::upgrade::LiveState,
2076        T: crate::upgrade::MigrateFrom<Old>,
2077    {
2078        self.state_initializer(move |state| {
2079            install_live_state(state, initial, crate::upgrade::decode_migrating::<Old, T>);
2080        })
2081    }
2082
2083    /// Register an async shutdown hook that runs during graceful shutdown.
2084    ///
2085    /// Hooks execute in reverse registration order so later-added runtimes
2086    /// shut down before earlier infrastructure they might depend on.
2087    #[must_use]
2088    pub fn on_shutdown<F, Fut>(mut self, hook: F) -> Self
2089    where
2090        F: Fn() -> Fut + Send + Sync + 'static,
2091        Fut: Future<Output = ()> + Send + 'static,
2092    {
2093        self.shutdown_hooks.push(Box::new(move || Box::pin(hook())));
2094        self
2095    }
2096
2097    /// Register a single API version. If a version with the same name already exists, it is updated.
2098    #[must_use]
2099    pub fn api_version(mut self, version: ApiVersion) -> Self {
2100        if let Some(pos) = self
2101            .api_versions
2102            .iter()
2103            .position(|v| v.version == version.version)
2104        {
2105            self.api_versions[pos] = version;
2106        } else {
2107            self.api_versions.push(version);
2108        }
2109        self
2110    }
2111
2112    /// Register multiple API versions, replacing duplicates.
2113    #[must_use]
2114    pub fn api_versions(mut self, versions: impl IntoIterator<Item = ApiVersion>) -> Self {
2115        for version in versions {
2116            if let Some(pos) = self
2117                .api_versions
2118                .iter()
2119                .position(|v| v.version == version.version)
2120            {
2121                self.api_versions[pos] = version;
2122            } else {
2123                self.api_versions.push(version);
2124            }
2125        }
2126        self
2127    }
2128
2129    /// Enable **user impersonation** for this app, gated by `gate`.
2130    ///
2131    /// Impersonation is default-deny: without this call (or
2132    /// `AdminPlugin::with_impersonation`, which does it for you)
2133    /// [`begin_impersonation`](crate::auth::impersonation::begin_impersonation)
2134    /// refuses every attempt with `403`. It also requires an audit sink — see
2135    /// [`with_audit_sink`](Self::with_audit_sink).
2136    ///
2137    /// ```rust,no_run
2138    /// use autumn_web::auth::impersonation::ImpersonationGate;
2139    ///
2140    /// # fn wire(app: autumn_web::app::AppBuilder) -> autumn_web::app::AppBuilder {
2141    /// app.impersonation_gate(ImpersonationGate::allow_roles(["admin"]))
2142    /// # }
2143    /// ```
2144    #[must_use]
2145    pub fn impersonation_gate(self, gate: crate::auth::impersonation::ImpersonationGate) -> Self {
2146        self.state_initializer(move |state| {
2147            // Surface a self-destructive `[auth].session_key` at boot rather
2148            // than at the first impersonation attempt, which refuses outright.
2149            let auth_key = state.auth_session_key();
2150            if crate::auth::impersonation::is_reserved_session_key(auth_key) {
2151                tracing::error!(
2152                    auth_session_key = %auth_key,
2153                    "impersonation is enabled but `auth.session_key` collides with a key the \
2154                     impersonation record reserves; every attempt will be refused"
2155                );
2156            }
2157            state.insert_extension(gate);
2158        })
2159    }
2160
2161    /// Run `hook` after each stored session-id rotation.
2162    ///
2163    /// Use it to move state keyed by the session id (such as a tracked-session
2164    /// row) to the new id. Hooks run after the new session is saved and are
2165    /// skipped for a destroyed session. See
2166    /// [`SessionRotationHooks`](crate::session::SessionRotationHooks).
2167    #[must_use]
2168    pub fn on_session_rotation<F, Fut>(self, hook: F) -> Self
2169    where
2170        F: Fn(crate::session::SessionRotation) -> Fut + Send + Sync + 'static,
2171        Fut: Future<Output = Result<(), crate::session::SessionRotationError>> + Send + 'static,
2172    {
2173        self.state_initializer(move |state| state.session_rotation_hooks().register(hook))
2174    }
2175
2176    /// Store or replace a typed builder extension.
2177    ///
2178    /// External crates use this to accumulate configuration across fluent
2179    /// extension-trait calls without Autumn needing to know the concrete type.
2180    #[must_use]
2181    pub fn with_extension<T>(mut self, value: T) -> Self
2182    where
2183        T: Any + Send + 'static,
2184    {
2185        self.extensions.insert(TypeId::of::<T>(), Box::new(value));
2186        self
2187    }
2188
2189    /// Mutate a typed builder extension, inserting a default value first when
2190    /// the extension has not been registered yet.
2191    ///
2192    /// # Panics
2193    ///
2194    /// Panics if the internal extension type map is corrupted and the value
2195    /// stored under `T`'s [`TypeId`] cannot be downcast back to `T`.
2196    #[must_use]
2197    pub fn update_extension<T, Init, Update>(mut self, init: Init, update: Update) -> Self
2198    where
2199        T: Any + Send + 'static,
2200        Init: FnOnce() -> T,
2201        Update: FnOnce(&mut T),
2202    {
2203        let type_id = TypeId::of::<T>();
2204        let entry = self
2205            .extensions
2206            .entry(type_id)
2207            .or_insert_with(|| Box::new(init()));
2208        let typed = entry
2209            .downcast_mut::<T>()
2210            .expect("extension type map corrupted");
2211        update(typed);
2212        self
2213    }
2214
2215    /// Borrow a typed builder extension if it has been registered.
2216    #[must_use]
2217    pub fn extension<T>(&self) -> Option<&T>
2218    where
2219        T: Any + Send + 'static,
2220    {
2221        self.extensions.get(&TypeId::of::<T>())?.downcast_ref::<T>()
2222    }
2223
2224    #[cfg(feature = "mail")]
2225    #[must_use]
2226    pub fn with_mail_interceptor(
2227        mut self,
2228        interceptor: impl crate::interceptor::MailInterceptor,
2229    ) -> Self {
2230        self.mail_interceptor = Some(Arc::new(interceptor));
2231        self
2232    }
2233
2234    #[must_use]
2235    pub fn with_job_interceptor(
2236        mut self,
2237        interceptor: impl crate::interceptor::JobInterceptor,
2238    ) -> Self {
2239        self.job_interceptor = Some(Arc::new(interceptor));
2240        self
2241    }
2242
2243    #[cfg(feature = "db")]
2244    #[must_use]
2245    pub fn with_db_interceptor(
2246        mut self,
2247        interceptor: impl crate::interceptor::DbConnectionInterceptor,
2248    ) -> Self {
2249        self.db_interceptor = Some(Arc::new(interceptor));
2250        self
2251    }
2252
2253    #[cfg(feature = "ws")]
2254    #[must_use]
2255    pub fn with_channels_interceptor(
2256        mut self,
2257        interceptor: impl crate::interceptor::ChannelsInterceptor,
2258    ) -> Self {
2259        self.channels_interceptor = Some(Arc::new(interceptor));
2260        self
2261    }
2262
2263    #[cfg(feature = "oauth2")]
2264    #[must_use]
2265    pub fn with_http_interceptor(
2266        mut self,
2267        interceptor: impl crate::interceptor::HttpInterceptor,
2268    ) -> Self {
2269        self.http_interceptor = Some(Arc::new(interceptor));
2270        self
2271    }
2272
2273    /// Register a pre-loaded i18n translation bundle.
2274    ///
2275    /// Most apps prefer [`Self::i18n_auto`] which loads from the
2276    /// `i18n/` directory using the configured `[i18n]` block. Use this
2277    /// directly when you need to construct a [`Bundle`](crate::i18n::Bundle)
2278    /// from non-filesystem sources (in-memory tests, embedded `.ftl` files,
2279    /// translation-management-system clients, etc.).
2280    #[cfg(feature = "i18n")]
2281    #[must_use]
2282    pub fn i18n(mut self, bundle: crate::i18n::Bundle) -> Self {
2283        self.i18n_bundle = Some(Arc::new(bundle));
2284        self.i18n_auto_load = false;
2285        self
2286    }
2287
2288    /// Auto-load the i18n translation bundle from the configured directory
2289    /// (`i18n/` by default), reading the `[i18n]` block from the active
2290    /// [`AutumnConfig`].
2291    ///
2292    /// Fails fast during [`Self::run`] if the configured default locale's file is
2293    /// missing — the spec calls out this as the desired behaviour: a
2294    /// half-localized app is worse than a clearly-broken one. The error
2295    /// path here panics with the typed [`LoadError`](crate::i18n::LoadError)
2296    /// formatted as a string so it surfaces in the same banner as other
2297    /// fatal startup errors.
2298    ///
2299    /// # Panics
2300    ///
2301    /// Panics when configuration cannot be loaded, the configured i18n
2302    /// directory is unreadable, or the default locale bundle is missing or
2303    /// invalid.
2304    ///
2305    /// # Examples
2306    ///
2307    /// ```rust,no_run
2308    /// use autumn_web::prelude::*;
2309    ///
2310    /// #[get("/")]
2311    /// async fn index() -> &'static str { "ok" }
2312    ///
2313    /// #[autumn_web::main]
2314    /// async fn main() {
2315    ///     # #[cfg(feature = "i18n")]
2316    ///     autumn_web::app()
2317    ///         .i18n_auto()
2318    ///         .routes(routes![index])
2319    ///         .run()
2320    ///         .await;
2321    /// }
2322    /// ```
2323    #[cfg(feature = "i18n")]
2324    #[must_use]
2325    pub fn i18n_auto(mut self) -> Self {
2326        self.i18n_bundle = None;
2327        self.i18n_auto_load = true;
2328        self
2329    }
2330
2331    // ── Tier-1 subsystem replacement hooks ─────────────────────
2332    //
2333    // Each `with_*` method swaps a framework-default subsystem for a
2334    // user-provided trait impl. The defaults preserve current behaviour, so an
2335    // app that does not customize sees no change. Plugins chain these in
2336    // `build()` to ship a subsystem — an `AwsSecretsConfigPlugin` calling
2337    // `app.with_config_loader(...)`. See `docs/guides/extensibility.md`.
2338
2339    /// Install a custom [`ConfigLoader`],
2340    /// replacing the default TOML + env loader.
2341    ///
2342    /// Useful when your config lives somewhere other than `autumn.toml` —
2343    /// AWS Secrets Manager, Vault, a JSON file, an HTTP fetch, etc. Emits a
2344    /// `tracing::warn!` if a loader was already installed.
2345    #[must_use]
2346    pub fn with_config_loader<L>(mut self, loader: L) -> Self
2347    where
2348        L: crate::config::ConfigLoader,
2349    {
2350        if self.config_loader_factory.is_some() {
2351            tracing::warn!(
2352                "config loader replaced; the previously-installed loader was overwritten"
2353            );
2354        }
2355        self.config_loader_factory = Some(Box::new(move || {
2356            Box::pin(async move { loader.load().await })
2357        }));
2358        self
2359    }
2360
2361    /// Install a custom [`crate::db::DatabasePoolProvider`],
2362    /// replacing the default `deadpool + diesel-async` pool factory.
2363    ///
2364    /// Useful for adding metrics/circuit-breaker wrappers, switching to a
2365    /// per-shard pool, or driving a non-default backend at the same
2366    /// `Pool<AsyncPgConnection>` interface. Emits a `tracing::warn!` if a
2367    /// provider was already installed.
2368    #[cfg(feature = "db")]
2369    #[must_use]
2370    pub fn with_pool_provider<P>(mut self, provider: P) -> Self
2371    where
2372        P: crate::db::DatabasePoolProvider,
2373    {
2374        if self.pool_provider_factory.is_some() {
2375            tracing::warn!(
2376                "database pool provider replaced; the previously-installed provider was overwritten"
2377            );
2378        }
2379        // The provider serves both the control topology and any configured
2380        // shard topologies; share it between the two captured closures.
2381        let provider = Arc::new(provider);
2382        let shard_provider = Arc::clone(&provider);
2383        self.pool_provider_factory =
2384            Some(Box::new(move |config: crate::config::DatabaseConfig| {
2385                Box::pin(async move { provider.create_topology(&config).await })
2386            }));
2387        self.shard_provider_factory =
2388            Some(Box::new(move |config: crate::config::DatabaseConfig| {
2389                Box::pin(async move {
2390                    let mut topologies = Vec::with_capacity(config.shards.len());
2391                    for shard in &config.shards {
2392                        topologies
2393                            .push(shard_provider.create_shard_topology(shard, &config).await?);
2394                    }
2395                    Ok(topologies)
2396                })
2397            }));
2398        self
2399    }
2400
2401    /// Install a custom [`ShardRouter`](crate::sharding::ShardRouter),
2402    /// replacing the default slot-hash router for `[[database.shards]]`
2403    /// routing.
2404    ///
2405    /// Useful for directory/lookup routing — e.g. a control-plane table
2406    /// that pins hot tenants to dedicated shards. Custom routers can
2407    /// still compose with the deterministic hash via
2408    /// [`ShardSet::slot_for_key`](crate::sharding::ShardSet::slot_for_key)
2409    /// and
2410    /// [`ShardSet::shard_for_slot`](crate::sharding::ShardSet::shard_for_slot).
2411    #[cfg(feature = "db")]
2412    #[must_use]
2413    pub fn with_shard_router<R>(mut self, router: R) -> Self
2414    where
2415        R: crate::sharding::ShardRouter,
2416    {
2417        if self.shard_router.is_some() {
2418            tracing::warn!(
2419                "shard router replaced; the previously-installed router was overwritten"
2420            );
2421        }
2422        self.shard_router = Some(Arc::new(router));
2423        self
2424    }
2425
2426    /// Route tenants through the control-plane `_autumn_shard_directory` table
2427    /// via a [`DirectoryShardRouter`](crate::sharding::DirectoryShardRouter).
2428    ///
2429    /// The router is bound to the control primary pool at build time. Tenants
2430    /// with a directory row are pinned to the named shard; everyone else falls
2431    /// back to the slot-hash router. Apply the framework migrations to the
2432    /// control database (`autumn migrate`) so `_autumn_shard_directory` exists.
2433    ///
2434    /// An explicit [`with_shard_router`](Self::with_shard_router) takes
2435    /// precedence over this flag.
2436    #[cfg(feature = "db")]
2437    #[must_use]
2438    pub const fn with_directory_shard_router(mut self) -> Self {
2439        self.directory_shard_router = true;
2440        self
2441    }
2442
2443    /// Install a custom [`TelemetryProvider`](crate::telemetry::TelemetryProvider),
2444    /// replacing the default `tracing-subscriber + OTLP` initializer.
2445    ///
2446    /// Useful for shipping a Datadog tracer, Honeycomb beeline, Sentry
2447    /// integration, or any other observability backend. Emits a
2448    /// `tracing::warn!` if a provider was already installed.
2449    #[must_use]
2450    pub fn with_telemetry_provider<T>(mut self, provider: T) -> Self
2451    where
2452        T: crate::telemetry::TelemetryProvider,
2453    {
2454        if self.telemetry_provider.is_some() {
2455            tracing::warn!(
2456                "telemetry provider replaced; the previously-installed provider was overwritten"
2457            );
2458        }
2459        self.telemetry_provider = Some(Box::new(provider));
2460        self
2461    }
2462
2463    /// Install a custom [`SessionStore`](crate::session::SessionStore),
2464    /// bypassing the config-driven `memory`/`redis` backend selection.
2465    ///
2466    /// Useful for backing sessions with a database, encrypted cookie store,
2467    /// or enterprise SSO bridge. Emits a `tracing::warn!` if a store was
2468    /// already installed.
2469    #[must_use]
2470    pub fn with_session_store<S>(mut self, store: S) -> Self
2471    where
2472        S: crate::session::SessionStore,
2473    {
2474        if self.session_store.is_some() {
2475            tracing::warn!(
2476                "session store replaced; the previously-installed store was overwritten"
2477            );
2478        }
2479        self.session_store = Some(Arc::new(store));
2480        self
2481    }
2482
2483    /// Install a custom [`ChannelsBackend`](crate::channels::ChannelsBackend),
2484    /// bypassing the config-driven `in_process`/`redis` backend selection.
2485    ///
2486    /// Useful for NATS, Postgres `LISTEN/NOTIFY`, test harnesses, or a
2487    /// sharded pub/sub fabric. Emits a `tracing::warn!` if a backend was
2488    /// already installed.
2489    #[cfg(feature = "ws")]
2490    #[must_use]
2491    pub fn with_channels_backend<B>(mut self, backend: B) -> Self
2492    where
2493        B: crate::channels::ChannelsBackend,
2494    {
2495        if self.channels_backend.is_some() {
2496            tracing::warn!(
2497                "channels backend replaced; the previously-installed backend was overwritten"
2498            );
2499        }
2500        self.channels_backend = Some(Arc::new(backend));
2501        self
2502    }
2503
2504    /// Install a custom [`BlobStore`](crate::storage::BlobStore),
2505    /// bypassing the config-driven `local`/`s3` backend selection.
2506    ///
2507    /// The typical use case is the `autumn-storage-s3` plugin:
2508    ///
2509    /// ```rust,ignore
2510    /// use autumn_storage_s3::S3BlobStore;
2511    ///
2512    /// # async fn example() {
2513    /// let config = autumn_web::config::TomlEnvConfigLoader::new()
2514    ///     .load().await.unwrap();
2515    /// let store = S3BlobStore::from_config(&config.storage.s3)
2516    ///     .await.unwrap();
2517    /// autumn_web::app()
2518    ///     .with_blob_store(store)
2519    ///     .run()
2520    ///     .await;
2521    /// # }
2522    /// ```
2523    ///
2524    /// Emits a `tracing::warn!` if a store was already installed (last
2525    /// call wins).
2526    ///
2527    /// # Note on `LocalBlobStore`
2528    ///
2529    /// **Do not** pass a [`LocalBlobStore`](crate::storage::LocalBlobStore)
2530    /// here. The local backend requires the framework to mount a `/_blobs`
2531    /// serving route (for HMAC-signed presigned URLs); that route is only
2532    /// wired up when the store is provisioned through the config-driven path
2533    /// (`backend = "local"` in `autumn.toml`). Calling
2534    /// `.with_blob_store(LocalBlobStore::new(...))` will silently succeed but
2535    /// presigned URLs will return 404. Use the `[storage]` config section for
2536    /// local storage.
2537    #[cfg(feature = "storage")]
2538    #[must_use]
2539    pub fn with_blob_store<B>(mut self, store: B) -> Self
2540    where
2541        B: crate::storage::BlobStore,
2542    {
2543        if self.blob_store.is_some() {
2544            tracing::warn!("blob store replaced; the previously-installed store was overwritten");
2545        }
2546        self.blob_store = Some(std::sync::Arc::new(store));
2547        self
2548    }
2549
2550    /// Register a shared cache backend for the application.
2551    ///
2552    /// Once registered, `#[cached]` functions will use this backend as their
2553    /// primary store (falling back to their per-function Moka cache only if the
2554    /// global backend is absent). `CacheResponseLayer::from_app` returns a layer
2555    /// wired to this same backend.
2556    ///
2557    /// # Example
2558    ///
2559    /// ```rust,ignore
2560    /// use autumn_cache_redis::RedisCache;
2561    ///
2562    /// let cache = RedisCache::connect("redis://redis:6379", "myapp:cache").await?;
2563    /// autumn_web::app()
2564    ///     .with_cache_backend(cache)
2565    ///     .run()
2566    ///     .await;
2567    /// ```
2568    #[must_use]
2569    pub fn with_cache_backend<C: crate::cache::Cache>(mut self, cache: C) -> Self {
2570        if self.cache_backend.is_some() {
2571            tracing::warn!(
2572                "cache backend replaced; the previously-installed backend was overwritten"
2573            );
2574        }
2575        self.cache_backend = Some(Arc::new(cache) as Arc<dyn crate::cache::Cache>);
2576        self
2577    }
2578
2579    /// Provide the key/value seam an `#[edge]` handler reads at the origin
2580    /// (issue #1790).
2581    ///
2582    /// An `#[edge(needs(kv))]` handler takes an
2583    /// [`EdgeCache`](autumn_edge::EdgeCache) extractor. At the edge the capsule
2584    /// runtime injects that handle per request; at the origin this method does,
2585    /// installing it as an app-wide extension layer so the *same handler
2586    /// source* serves from both substrates. Without it the extractor declines
2587    /// with an actionable `500` naming this call.
2588    ///
2589    /// Most apps pass [`CacheEdgeKv`](crate::CacheEdgeKv) over the cache they
2590    /// already run, so anything the origin publishes with
2591    /// [`cache::insert_cached`](crate::cache::insert_cached) is visible to the
2592    /// edge lane; any other [`EdgeKv`](autumn_edge::EdgeKv) works just as well.
2593    ///
2594    /// # Not a database
2595    ///
2596    /// The seam is a non-authoritative, read-only replica (ADR-0004 category
2597    /// 2): a miss is always a legal answer and staleness is expected. Routes
2598    /// whose correctness depends on a fresh, authoritative read belong on the
2599    /// origin — see [`edge_support`](crate::edge_support).
2600    ///
2601    /// # Examples
2602    ///
2603    /// ```rust,no_run
2604    /// use std::sync::Arc;
2605    ///
2606    /// use autumn_web::CacheEdgeKv;
2607    /// use autumn_web::cache::{Cache, MokaCache};
2608    /// use autumn_web::edge::EdgeKv;
2609    /// use autumn_web::prelude::*;
2610    ///
2611    /// # #[get("/health")] async fn health() -> &'static str { "ok" }
2612    /// # #[autumn_web::main]
2613    /// # async fn main() {
2614    /// let cache = Arc::new(MokaCache::new(1_024, None)) as Arc<dyn Cache>;
2615    ///
2616    /// autumn_web::app()
2617    ///     .routes(routes![health])
2618    ///     .with_edge_kv(Arc::new(CacheEdgeKv::new(cache)) as Arc<dyn EdgeKv>)
2619    ///     .run()
2620    ///     .await;
2621    /// # }
2622    /// ```
2623    #[cfg(feature = "edge")]
2624    #[must_use]
2625    pub fn with_edge_kv(self, kv: Arc<dyn autumn_edge::EdgeKv>) -> Self {
2626        // `EdgeCache::layer` hands back a ready-made `axum::Extension`, which is
2627        // itself a `tower::Layer` and therefore an `IntoAppLayer` — the same
2628        // plumbing `install_i18n_bundle_layer` uses. No adapter needed, and the
2629        // injection type stays an implementation detail of `autumn-edge`.
2630        self.layer(autumn_edge::EdgeCache::layer(kv))
2631    }
2632
2633    /// Install an arbitrary host-side edge identity source.
2634    ///
2635    /// The provider remains in the native application graph; capsules receive
2636    /// only the resulting [`EdgeIdentity`](crate::edge_support::EdgeIdentity).
2637    #[cfg(feature = "edge")]
2638    #[must_use]
2639    pub fn with_edge_identity_provider<P>(self, provider: P) -> Self
2640    where
2641        P: crate::edge_support::EdgeIdentityProvider,
2642    {
2643        self.layer(crate::edge_support::EdgeIdentityLayer::new(provider))
2644    }
2645
2646    /// Register an [`ErrorReporter`](crate::reporting::ErrorReporter) for
2647    /// unhandled panics and 5xx responses.
2648    ///
2649    /// Reporters receive a structured
2650    /// [`ErrorEvent`](crate::reporting::ErrorEvent) for every caught handler
2651    /// panic and every server-error response, carrying request context (route,
2652    /// method, request id, status) and — for panics — the panic payload and a
2653    /// backtrace (when `RUST_BACKTRACE` is set). Call this multiple times to
2654    /// chain reporters; each receives every event. When none are registered,
2655    /// the built-in [`LogReporter`](crate::reporting::LogReporter) is used.
2656    ///
2657    /// Mirrors [`with_blob_store`](Self::with_blob_store) /
2658    /// [`with_cache_backend`](Self::with_cache_backend).
2659    ///
2660    /// # Examples
2661    ///
2662    /// ```rust,no_run
2663    /// use autumn_web::reporting::{ErrorEvent, ErrorReporter, ReportFuture};
2664    ///
2665    /// struct MyReporter;
2666    /// impl ErrorReporter for MyReporter {
2667    ///     fn report<'a>(&'a self, event: &'a ErrorEvent) -> ReportFuture<'a> {
2668    ///         Box::pin(async move { eprintln!("error: {} {}", event.status, event.message); })
2669    ///     }
2670    /// }
2671    ///
2672    /// # #[autumn_web::main]
2673    /// # async fn main() {
2674    /// autumn_web::app()
2675    ///     .with_error_reporter(MyReporter)
2676    /// #   .routes(vec![])
2677    /// #   ;
2678    /// # }
2679    /// ```
2680    #[cfg(feature = "reporting")]
2681    #[must_use]
2682    pub fn with_error_reporter<R: crate::reporting::ErrorReporter>(mut self, reporter: R) -> Self {
2683        self.error_reporters
2684            .push(Arc::new(reporter) as Arc<dyn crate::reporting::ErrorReporter>);
2685        self
2686    }
2687
2688    /// Register an operator-alert delivery channel.
2689    ///
2690    /// Alerts for the built-in conditions (dead-lettered jobs, Down health
2691    /// indicators, 5xx-rate spikes, scheduled-task failures) are delivered to
2692    /// every registered [`AlertChannel`](crate::alerts::AlertChannel) **plus**
2693    /// the built-in mail/webhook channels derived from `[alerts]` config.
2694    ///
2695    /// This is the extension seam for additional transports (`PagerDuty`, Slack,
2696    /// Discord — follow-up #1630): implement
2697    /// [`AlertChannel`](crate::alerts::AlertChannel) and register it here. The
2698    /// framework core never changes. Most apps need no code at all — configuring
2699    /// an `email` and/or `webhook_url` under `[alerts]` is sufficient.
2700    ///
2701    /// ```rust,no_run
2702    /// use autumn_web::alerts::{Alert, AlertChannel, AlertDeliveryError, AlertDeliveryFuture};
2703    ///
2704    /// struct PagerDuty;
2705    /// impl AlertChannel for PagerDuty {
2706    ///     fn name(&self) -> &'static str { "pagerduty" }
2707    ///     fn deliver<'a>(&'a self, alert: &'a Alert) -> AlertDeliveryFuture<'a> {
2708    ///         Box::pin(async move {
2709    ///             let _ = (&alert.dedup_key, alert.severity);
2710    ///             Ok::<(), AlertDeliveryError>(())
2711    ///         })
2712    ///     }
2713    /// }
2714    ///
2715    /// # #[autumn_web::main]
2716    /// # async fn main() {
2717    /// autumn_web::app()
2718    ///     .with_alert_channel(PagerDuty)
2719    /// #   .routes(vec![])
2720    /// #   ;
2721    /// # }
2722    /// ```
2723    #[must_use]
2724    pub fn with_alert_channel<C: crate::alerts::AlertChannel>(mut self, channel: C) -> Self {
2725        self.alert_channels
2726            .push(Arc::new(channel) as Arc<dyn crate::alerts::AlertChannel>);
2727        self
2728    }
2729
2730    /// Register a [`FlagStore`](crate::feature_flags::FlagStore) backend for
2731    /// feature-flag evaluation.
2732    ///
2733    /// After registration, the [`Flags`](crate::feature_flags::Flags) extractor
2734    /// and `#[feature_flag]` macro are available in route handlers. Without a
2735    /// registered store, both return `500 Internal Server Error`.
2736    ///
2737    /// For tests use [`InMemoryFlagStore`](crate::feature_flags::InMemoryFlagStore);
2738    /// in production use the Postgres-backed
2739    /// `autumn_web::feature_flags::pg::PgFlagStore`.
2740    ///
2741    /// When the app installs the service, it calls
2742    /// [`FlagStore::preload`](crate::feature_flags::FlagStore::preload) once and
2743    /// waits up to 10 seconds for it, before job workers, startup hooks and
2744    /// requests. A failed preload logs a warning; it does not stop startup.
2745    ///
2746    /// # Sharing the store with the poll listener
2747    ///
2748    /// When using `PgFlagStore` in a multi-replica deployment, pass an `Arc`
2749    /// clone so the app service and the poll listener share the **same** snapshot:
2750    ///
2751    /// ```rust,ignore
2752    /// use std::sync::Arc;
2753    /// use std::time::Duration;
2754    /// use autumn_web::feature_flags::pg::PgFlagStore;
2755    ///
2756    /// // `None` when no primary target is set, or it is not Postgres.
2757    /// let store = Arc::new(
2758    ///     PgFlagStore::from_database_config(&config.database).expect("a Postgres target"),
2759    /// );
2760    /// PgFlagStore::spawn_poll_listener(Arc::clone(&store), Duration::from_secs(1));
2761    /// autumn_web::app()
2762    ///     .with_flag_store(Arc::clone(&store))
2763    ///     .run()
2764    ///     .await;
2765    /// ```
2766    ///
2767    /// `Arc<PgFlagStore>` implements `FlagStore`, so the same `Arc` is
2768    /// accepted directly without creating a separate snapshot.
2769    ///
2770    /// # Basic example
2771    ///
2772    /// ```rust,ignore
2773    /// use autumn_web::feature_flags::InMemoryFlagStore;
2774    /// use std::sync::Arc;
2775    ///
2776    /// autumn_web::app()
2777    ///     .with_flag_store(InMemoryFlagStore::new())
2778    ///     .run()
2779    ///     .await;
2780    /// ```
2781    #[must_use]
2782    pub fn with_flag_store<S>(self, store: S) -> Self
2783    where
2784        S: crate::feature_flags::FlagStore,
2785    {
2786        let service = crate::feature_flags::FeatureFlagService::new(Arc::new(store) as Arc<_>);
2787        self.with_flag_service(service)
2788    }
2789
2790    /// Register a configured [`FeatureFlagService`](crate::feature_flags::FeatureFlagService),
2791    /// for example one with declared defaults.
2792    ///
2793    /// Like [`with_flag_store`](Self::with_flag_store), it preloads the store
2794    /// at startup.
2795    ///
2796    /// ```rust,ignore
2797    /// use autumn_web::feature_flags::{FeatureFlagService, InMemoryFlagStore};
2798    /// use std::sync::Arc;
2799    ///
2800    /// let flags = FeatureFlagService::new(Arc::new(InMemoryFlagStore::new()))
2801    ///     .with_default("checkout_v2", true);
2802    /// autumn_web::app().with_flag_service(flags).run().await;
2803    /// ```
2804    #[must_use]
2805    pub fn with_flag_service(self, service: crate::feature_flags::FeatureFlagService) -> Self {
2806        self.state_initializer(move |state| {
2807            // Load flags before anything that reads them runs (#3063).
2808            crate::feature_flags::preload_blocking(&service, crate::feature_flags::PRELOAD_WAIT);
2809            state.insert_extension(service);
2810        })
2811    }
2812
2813    /// Register a feature-flag store with a group-membership resolver.
2814    ///
2815    /// The resolver is called during flag evaluation to check whether an actor
2816    /// belongs to a named group listed in a flag's `group_allowlist`. Without
2817    /// registering a resolver, group gates are silently ignored.
2818    ///
2819    /// # Example
2820    ///
2821    /// ```rust,ignore
2822    /// use autumn_web::feature_flags::{InMemoryFlagStore, GroupResolver};
2823    /// use std::sync::Arc;
2824    ///
2825    /// let resolver: GroupResolver = Arc::new(|actor_id, group| {
2826    ///     group == "staff" && actor_id.starts_with("staff:")
2827    /// });
2828    ///
2829    /// autumn_web::app()
2830    ///     .with_flag_store_and_resolver(InMemoryFlagStore::new(), resolver)
2831    ///     .run()
2832    ///     .await;
2833    /// ```
2834    #[must_use]
2835    pub fn with_flag_store_and_resolver<S>(
2836        self,
2837        store: S,
2838        resolver: crate::feature_flags::GroupResolver,
2839    ) -> Self
2840    where
2841        S: crate::feature_flags::FlagStore,
2842    {
2843        let service = crate::feature_flags::FeatureFlagService::new(Arc::new(store) as Arc<_>)
2844            .with_group_resolver(resolver);
2845        self.with_flag_service(service)
2846    }
2847
2848    /// Register an experiment store, enabling the [`Experiments`] extractor.
2849    ///
2850    /// Wrap any [`ExperimentStore`] implementation. Use [`InMemoryExperimentStore`]
2851    /// for development and tests; use
2852    /// [`pg::PgExperimentStore`](crate::experiments::pg::PgExperimentStore)
2853    /// for production against the `autumn_experiments` tables.
2854    ///
2855    /// # Production example (Postgres-backed)
2856    ///
2857    /// ```rust,ignore
2858    /// use std::sync::Arc;
2859    /// use std::time::Duration;
2860    /// use autumn_web::experiments::pg::PgExperimentStore;
2861    ///
2862    /// // `None` when no primary target is set, or it is not Postgres.
2863    /// let store = Arc::new(
2864    ///     PgExperimentStore::from_database_config(&config.database).expect("a Postgres target"),
2865    /// );
2866    /// PgExperimentStore::spawn_poll_listener(Arc::clone(&store), Duration::from_secs(5));
2867    /// autumn_web::app()
2868    ///     .with_experiment_store(Arc::clone(&store))
2869    ///     .run()
2870    ///     .await;
2871    /// ```
2872    ///
2873    /// # Development / test example
2874    ///
2875    /// ```rust,ignore
2876    /// use autumn_web::experiments::InMemoryExperimentStore;
2877    ///
2878    /// autumn_web::app()
2879    ///     .with_experiment_store(InMemoryExperimentStore::new())
2880    ///     .run()
2881    ///     .await;
2882    /// ```
2883    ///
2884    /// [`Experiments`]: crate::experiments::Experiments
2885    /// [`ExperimentStore`]: crate::experiments::ExperimentStore
2886    /// [`InMemoryExperimentStore`]: crate::experiments::InMemoryExperimentStore
2887    ///
2888    /// When the app installs the service, it calls
2889    /// [`ExperimentStore::preload`](crate::experiments::ExperimentStore::preload) once and waits up to 10 seconds for it,
2890    /// before job workers, startup hooks and requests. A failed preload logs a
2891    /// warning; it does not stop startup.
2892    #[must_use]
2893    pub fn with_experiment_store<S>(self, store: S) -> Self
2894    where
2895        S: crate::experiments::ExperimentStore,
2896    {
2897        let service = crate::experiments::ExperimentService::new(Arc::new(store) as Arc<_>);
2898        self.state_initializer(move |state| {
2899            crate::experiments::preload_blocking(&service, crate::feature_flags::PRELOAD_WAIT);
2900            state.insert_extension(service);
2901        })
2902    }
2903
2904    /// Register a notification store, overriding the default resolution used
2905    /// by the [`Notifications`] extractor.
2906    ///
2907    /// Without this call the extractor resolves its store automatically: the
2908    /// database-backed [`DbNotificationStore`] when a pool is configured (the
2909    /// `notifications` table is scaffolded by `autumn generate
2910    /// notifications`), the process-local
2911    /// [`MemoryNotificationStore`](crate::notifications::MemoryNotificationStore)
2912    /// otherwise. Register a store explicitly to plug in a custom backend.
2913    ///
2914    /// # Example
2915    ///
2916    /// ```rust,ignore
2917    /// use autumn_web::notifications::MemoryNotificationStore;
2918    ///
2919    /// autumn_web::app()
2920    ///     .with_notification_store(MemoryNotificationStore::new())
2921    ///     .run()
2922    ///     .await;
2923    /// ```
2924    ///
2925    /// [`Notifications`]: crate::notifications::Notifications
2926    /// [`DbNotificationStore`]: crate::notifications::DbNotificationStore
2927    #[must_use]
2928    pub fn with_notification_store<S>(self, store: S) -> Self
2929    where
2930        S: crate::notifications::NotificationStore,
2931    {
2932        let service = crate::notifications::Notifications::new(store);
2933        self.state_initializer(move |state| {
2934            state.insert_extension(service);
2935        })
2936    }
2937
2938    /// Register a push subscription store, overriding the default resolution
2939    /// used by the [`WebPush`] extractor.
2940    ///
2941    /// Without this call the extractor resolves its store automatically: the
2942    /// database-backed
2943    /// [`DbPushSubscriptionStore`](crate::push::DbPushSubscriptionStore) when a
2944    /// pool is configured (the `push_subscriptions` table is scaffolded by
2945    /// `autumn generate pwa`), the process-local
2946    /// [`MemoryPushSubscriptionStore`](crate::push::MemoryPushSubscriptionStore)
2947    /// otherwise.
2948    ///
2949    /// # Example
2950    ///
2951    /// ```rust,ignore
2952    /// use autumn_web::push::MemoryPushSubscriptionStore;
2953    ///
2954    /// autumn_web::app()
2955    ///     .with_push_subscription_store(MemoryPushSubscriptionStore::new())
2956    ///     .merge(autumn_web::push::router())
2957    ///     .run()
2958    ///     .await;
2959    /// ```
2960    ///
2961    /// [`WebPush`]: crate::push::WebPush
2962    #[must_use]
2963    pub fn with_push_subscription_store<S>(self, store: S) -> Self
2964    where
2965        S: crate::push::PushSubscriptionStore,
2966    {
2967        self.state_initializer(move |state| {
2968            state.insert_extension(crate::push::WebPush::from_state_with_store(state, store));
2969        })
2970    }
2971
2972    /// Register a fully-built [`WebPush`] service, overriding key, store,
2973    /// transport, TTL and clock at once.
2974    ///
2975    /// This is the registration path for a **custom
2976    /// [`PushTransport`]** — routing push traffic through your own HTTP stack
2977    /// or a queue rather than the built-in one — and for supplying a VAPID key
2978    /// from somewhere `[push] private_key` cannot reach, such as a secrets
2979    /// manager fetched at boot.
2980    ///
2981    /// # Example
2982    ///
2983    /// ```rust,ignore
2984    /// use autumn_web::push::{MemoryPushSubscriptionStore, VapidKey, WebPush};
2985    ///
2986    /// let push = WebPush::new(
2987    ///     MemoryPushSubscriptionStore::new(),
2988    ///     VapidKey::from_base64url(&fetch_key_from_vault().await?)?,
2989    ///     "mailto:ops@example.com",
2990    ///     MyQueueTransport::new(),
2991    /// );
2992    ///
2993    /// autumn_web::app()
2994    ///     .with_web_push(push)
2995    ///     .merge(autumn_web::push::router())
2996    ///     .run()
2997    ///     .await;
2998    /// ```
2999    ///
3000    /// [`WebPush`]: crate::push::WebPush
3001    /// [`PushTransport`]: crate::push::PushTransport
3002    #[must_use]
3003    pub fn with_web_push(self, push: crate::push::WebPush) -> Self {
3004        self.state_initializer(move |state| {
3005            state.insert_extension(push);
3006        })
3007    }
3008
3009    /// Register an experiment store with a custom [`ExposureSink`].
3010    ///
3011    /// Use when you want to forward exposure events to an analytics pipeline
3012    /// rather than the default `tracing` log.
3013    ///
3014    /// # Example
3015    ///
3016    /// ```rust,ignore
3017    /// use autumn_web::experiments::{InMemoryExperimentStore, NoOpExposureSink};
3018    /// use std::sync::Arc;
3019    ///
3020    /// autumn_web::app()
3021    ///     .with_experiment_store_and_sink(
3022    ///         InMemoryExperimentStore::new(),
3023    ///         Arc::new(NoOpExposureSink),
3024    ///     )
3025    ///     .run()
3026    ///     .await;
3027    /// ```
3028    ///
3029    /// [`ExposureSink`]: crate::experiments::ExposureSink
3030    #[must_use]
3031    pub fn with_experiment_store_and_sink<S>(
3032        self,
3033        store: S,
3034        sink: Arc<dyn crate::experiments::ExposureSink>,
3035    ) -> Self
3036    where
3037        S: crate::experiments::ExperimentStore,
3038    {
3039        let service = crate::experiments::ExperimentService::new(Arc::new(store) as Arc<_>)
3040            .with_exposure_sink(sink);
3041        self.state_initializer(move |state| {
3042            crate::experiments::preload_blocking(&service, crate::feature_flags::PRELOAD_WAIT);
3043            state.insert_extension(service);
3044        })
3045    }
3046
3047    /// Register a durable [`MailDeliveryQueue`](crate::mail::MailDeliveryQueue) for
3048    /// [`Mailer::deliver_later`](crate::mail::Mailer::deliver_later).
3049    ///
3050    /// Must be called before [`run`](Self::run). Plugins call this inside their
3051    /// `apply` implementation to satisfy the production delivery guard without
3052    /// requiring `mail.allow_in_process_deliver_later_in_production`.
3053    ///
3054    /// Use [`Self::with_mail_delivery_queue_factory`] when the queue needs
3055    /// framework-managed resources (the DB pool, channels, etc.) that only
3056    /// exist after the [`AppState`] is constructed.
3057    #[cfg(feature = "mail")]
3058    #[must_use]
3059    pub fn with_mail_delivery_queue(
3060        mut self,
3061        queue: impl crate::mail::MailDeliveryQueue + 'static,
3062    ) -> Self {
3063        let arc: Arc<dyn crate::mail::MailDeliveryQueue> = Arc::new(queue);
3064        self.mail_delivery_queue_factory = Some(Box::new(move |_state| Ok(arc)));
3065        self
3066    }
3067
3068    /// Register a factory that builds the durable
3069    /// [`MailDeliveryQueue`](crate::mail::MailDeliveryQueue) from the
3070    /// fully-built [`AppState`].
3071    ///
3072    /// Use this when the queue captures framework-managed resources — for
3073    /// example a DB-outbox queue that needs the connection pool returned by
3074    /// [`AppState::pool`]. The factory runs once, immediately before
3075    /// `install_mailer`, with the live `AppState`. Returning `Err` aborts
3076    /// startup with the propagated error.
3077    #[cfg(feature = "mail")]
3078    #[must_use]
3079    pub fn with_mail_delivery_queue_factory<F, Q>(mut self, factory: F) -> Self
3080    where
3081        F: FnOnce(&AppState) -> crate::AutumnResult<Q> + Send + 'static,
3082        Q: crate::mail::MailDeliveryQueue + 'static,
3083    {
3084        self.mail_delivery_queue_factory = Some(Box::new(move |state| {
3085            factory(state).map(|q| Arc::new(q) as Arc<dyn crate::mail::MailDeliveryQueue>)
3086        }));
3087        self
3088    }
3089
3090    /// Register a [`SuppressionStore`](crate::mail::SuppressionStore) used by
3091    /// List-Unsubscribe sends to skip opted-out recipients and by the default
3092    /// unsubscribe endpoint to record opt-outs.
3093    ///
3094    /// When the `db` feature is enabled and a connection pool is configured, a
3095    /// Diesel-backed store is auto-wired, so most apps never call this — use it
3096    /// to plug a custom backend. Mirrors
3097    /// [`Self::with_mail_delivery_queue`].
3098    #[cfg(feature = "mail")]
3099    #[must_use]
3100    pub fn with_suppression_store(
3101        mut self,
3102        store: impl crate::mail::SuppressionStore + 'static,
3103    ) -> Self {
3104        self.suppression_store = Some(crate::mail::SuppressionStoreHandle::new(store));
3105        self
3106    }
3107
3108    /// Register a bounce/complaint
3109    /// [`SuppressionStore`](crate::mail::suppression::SuppressionStore) so
3110    /// [`Mailer::send`](crate::mail::Mailer::send) skips addresses that have
3111    /// hard-bounced or complained (issue #1247).
3112    ///
3113    /// Zero-config apps need not call this: the framework wires an in-memory
3114    /// default store automatically. Use this to plug the durable
3115    /// [`PgSuppressionStore`](crate::mail::suppression::PgSuppressionStore) (or
3116    /// a custom backend) for multi-instance deploys that must share suppression
3117    /// across replicas. Mirrors [`Self::with_suppression_store`].
3118    #[cfg(feature = "mail")]
3119    #[must_use]
3120    pub fn with_mail_suppression_store(
3121        mut self,
3122        store: impl crate::mail::suppression::SuppressionStore + 'static,
3123    ) -> Self {
3124        self.mail_suppression_store =
3125            Some(crate::mail::suppression::SuppressionStoreHandle::new(store));
3126        self
3127    }
3128
3129    /// Mount the framework's default RFC 8058 one-click unsubscribe endpoint at
3130    /// `/_autumn/unsubscribe` (`GET` confirmation page + `POST` one-click).
3131    ///
3132    /// Opt-in: a plain JSON API never gets an HTML endpoint it didn't ask for.
3133    /// Requires `mail.unsubscribe_base_url` to be configured. When mounted, the
3134    /// path is automatically exempted from CSRF and CAPTCHA (mailbox-provider
3135    /// POSTs carry neither token). To serve a custom unsubscribe page instead,
3136    /// skip this and register your own route at the path.
3137    #[cfg(feature = "mail")]
3138    #[must_use]
3139    pub const fn mount_unsubscribe_endpoint(mut self) -> Self {
3140        self.mount_unsubscribe_endpoint = true;
3141        self
3142    }
3143
3144    /// Register an inbound mail router that creates webhook HTTP endpoints and
3145    /// dispatches parsed [`InboundEmail`](crate::inbound_mail::InboundEmail)
3146    /// values to registered handlers.
3147    ///
3148    /// Calling this method twice replaces the previously registered router.
3149    ///
3150    /// # Example
3151    ///
3152    /// ```rust,ignore
3153    /// use autumn_web::inbound_mail::{
3154    ///     InboundMailRouter, InboundMailEndpointConfig,
3155    ///     InboundMailHandlerInfo, ProcessingMode, RecipientPattern,
3156    /// };
3157    ///
3158    /// autumn_web::app()
3159    ///     .inbound_mail_router(
3160    ///         InboundMailRouter::new()
3161    ///             .endpoint(InboundMailEndpointConfig::mailgun("/inbound/mailgun", "key"))
3162    ///             .handler(InboundMailHandlerInfo {
3163    ///                 name: "support",
3164    ///                 pattern: RecipientPattern::Exact("support@company.com".to_string()),
3165    ///                 processing: ProcessingMode::Background,
3166    ///                 handler: handle_support,
3167    ///             })
3168    ///     )
3169    ///     .routes(routes![...])
3170    ///     .run()
3171    ///     .await;
3172    /// ```
3173    #[cfg(feature = "inbound-mail")]
3174    #[must_use]
3175    pub fn inbound_mail_router(mut self, router: crate::inbound_mail::InboundMailRouter) -> Self {
3176        self.inbound_mail_router = Some(Arc::new(router));
3177        self
3178    }
3179
3180    /// Register mail template previews for the dev mail preview UI.
3181    ///
3182    /// Pair this with `#[mailer_preview]` and `mail_previews![...]`.
3183    #[cfg(feature = "mail")]
3184    #[must_use]
3185    pub fn mail_previews(
3186        mut self,
3187        previews: impl IntoIterator<Item = crate::mail::MailPreview>,
3188    ) -> Self {
3189        self.mail_previews.extend(previews);
3190        self
3191    }
3192
3193    /// Register the widget story gallery served at `/_stories` (#1526).
3194    ///
3195    /// Routes mount only when the resolved config has `[stories] enabled =
3196    /// true` (off by default, opt-in per profile). Start from
3197    /// [`StoryGallery::builtin`](crate::stories::StoryGallery::builtin) for
3198    /// the framework widget set and
3199    /// [`extend`](crate::stories::StoryGallery::extend) it with your app's
3200    /// own `story!{...}` entries. See `docs/guide/stories.md`.
3201    #[cfg(feature = "maud")]
3202    #[must_use]
3203    pub fn with_story_gallery(mut self, gallery: crate::stories::StoryGallery) -> Self {
3204        self.story_gallery = Some(gallery);
3205        self
3206    }
3207
3208    /// Register an additional audit sink for structured audit events.
3209    ///
3210    /// Multiple calls accumulate sinks. Logged events are fanned out to all
3211    /// configured sinks.
3212    #[must_use]
3213    pub fn with_audit_sink<S>(mut self, sink: S) -> Self
3214    where
3215        S: crate::audit::AuditSink,
3216    {
3217        let logger = self
3218            .audit_logger
3219            .take()
3220            .map_or_else(crate::audit::AuditLogger::new, |logger| (*logger).clone())
3221            .with_sink(Arc::new(sink));
3222        self.audit_logger = Some(Arc::new(logger));
3223        self
3224    }
3225
3226    /// Register a [`Policy`](crate::authorization::Policy)
3227    /// implementation for resource type `R`.
3228    ///
3229    /// Multiple policies per resource are not supported: registering
3230    /// `R` twice causes a startup-time panic with a clear error
3231    /// message.
3232    ///
3233    /// # Examples
3234    ///
3235    /// ```rust,ignore
3236    /// use autumn_web::authorization::{Policy, PolicyContext};
3237    ///
3238    /// #[derive(Default)]
3239    /// struct PostPolicy;
3240    /// impl Policy<Post> for PostPolicy { /* ... */ }
3241    ///
3242    /// autumn_web::app()
3243    ///     .routes(routes![...])
3244    ///     .policy::<Post, _>(PostPolicy)
3245    ///     .run()
3246    ///     .await;
3247    /// ```
3248    #[must_use]
3249    pub fn policy<R, P>(mut self, policy: P) -> Self
3250    where
3251        R: Send + Sync + 'static,
3252        P: crate::authorization::Policy<R>,
3253    {
3254        self.policy_registrations.push(Box::new(move |registry| {
3255            registry.register_policy::<R, _>(policy);
3256        }));
3257        self
3258    }
3259
3260    /// Register a [`Scope`](crate::authorization::Scope) implementation
3261    /// for resource type `R`. The scope filters list endpoints
3262    /// (`GET /<api>` for `#[repository(api = "...", scope = ...)]`)
3263    /// to records the current user is allowed to read.
3264    ///
3265    /// Default impls return an empty list so a missing scope opt-in
3266    /// fails closed.
3267    #[must_use]
3268    pub fn scope<R, S>(mut self, scope: S) -> Self
3269    where
3270        R: Send + Sync + 'static,
3271        S: crate::authorization::Scope<R>,
3272    {
3273        self.policy_registrations.push(Box::new(move |registry| {
3274            registry.register_scope::<R, _>(scope);
3275        }));
3276        self
3277    }
3278
3279    /// Apply a [`Plugin`](crate::plugin::Plugin) to the builder.
3280    ///
3281    /// The plugin's [`build`](crate::plugin::Plugin::build) runs exactly once
3282    /// per [`AppBuilder`]. Registering two plugins that share a
3283    /// [`name`](crate::plugin::Plugin::name) is a no-op after the first: the
3284    /// duplicate emits a `tracing::warn!` and the builder is returned
3285    /// unchanged.
3286    #[must_use]
3287    #[track_caller]
3288    pub fn plugin<P>(mut self, plugin: P) -> Self
3289    where
3290        P: crate::plugin::Plugin,
3291    {
3292        let name = plugin.name();
3293        if self.registered_plugins.contains(name.as_ref()) {
3294            tracing::warn!(
3295                plugin = name.as_ref(),
3296                "plugin already registered; skipping duplicate"
3297            );
3298            return self;
3299        }
3300        if let Some(mut contract) = plugin.contract() {
3301            Self::enforce_plugin_contract(&contract);
3302            // Route attribution keys on `Plugin::name()` while a contract names
3303            // the plugin's CRATE; the default `name()` is `type_name`, so a
3304            // plugin that declares `env!("CARGO_PKG_NAME")` without overriding
3305            // `name()` has two identities. Carry both so
3306            // `autumn plugin-check --plugin-name` finds it under either.
3307            contract.registered_as = Some(name.as_ref().to_owned());
3308            self.plugin_contracts.push(contract);
3309        }
3310        let name_str = name.into_owned();
3311        self.registered_plugins.insert(name_str.clone());
3312        // Save outer plugin context so nested plugin() calls don't permanently
3313        // clear it; restore it after this plugin's build() returns.
3314        let outer_plugin = self.current_plugin.replace(name_str);
3315        let mut result = plugin.build(self);
3316        result.current_plugin = outer_plugin;
3317        result
3318    }
3319
3320    /// Apply a [`Plugins`](crate::plugin::Plugins) bundle (a plugin or tuple
3321    /// of plugins) to the builder, in declaration order.
3322    #[must_use]
3323    pub fn plugins<P>(self, plugins: P) -> Self
3324    where
3325        P: crate::plugin::Plugins,
3326    {
3327        plugins.apply(self)
3328    }
3329
3330    /// Return `true` if a plugin with the given [`Plugin::name`](crate::plugin::Plugin::name)
3331    /// has already been applied to this builder.
3332    #[must_use]
3333    pub fn has_plugin(&self, name: &str) -> bool {
3334        self.registered_plugins.contains(name)
3335    }
3336
3337    /// The compatibility contracts declared by the plugins applied so far
3338    /// (issue #1601), in registration order.
3339    ///
3340    /// A plugin that returns `None` from
3341    /// [`Plugin::contract`](crate::plugin::Plugin::contract) contributes
3342    /// nothing here, and a duplicate registration contributes once — the
3343    /// duplicate is skipped before its contract is read.
3344    #[must_use]
3345    pub fn plugin_contracts(&self) -> &[crate::plugin_contract::PluginContract] {
3346        &self.plugin_contracts
3347    }
3348
3349    /// Check one plugin's declared `autumn-web` range against the framework it
3350    /// is actually compiled into.
3351    ///
3352    /// An incompatible pairing **panics** at registration: the plugin is about
3353    /// to wire itself into an application built on a framework it does not
3354    /// claim to support, and the whole point of the contract is that this stops
3355    /// being a silent surprise. The message names both versions and both
3356    /// remedies.
3357    ///
3358    /// A requirement that cannot be parsed only warns. It is the *plugin
3359    /// author's* typo, and `autumn plugin-check` fails on it in their CI —
3360    /// hard-failing here would punish an application author for a mistake they
3361    /// cannot fix.
3362    ///
3363    /// # The escape hatch
3364    ///
3365    /// The one thing an application author *cannot* fix is a plugin whose
3366    /// declared range is merely stale — cargo has already proven the two link
3367    /// one `autumn-web`, so an over-tight literal in somebody else's crate
3368    /// should not be able to strand a working deployment. Setting
3369    /// `AUTUMN_PLUGIN_CONTRACT=warn` downgrades the panic to a `tracing::warn!`
3370    /// carrying the same message. It is named in the panic text itself, so the
3371    /// person who hits it does not have to find this doc first. Loud-by-default
3372    /// is the point; unbootable-with-no-recourse is not.
3373    ///
3374    /// Note that a **duplicate** registration is skipped before its contract is
3375    /// read, so enforcement applies to the first plugin registered under a
3376    /// given name.
3377    #[track_caller]
3378    fn enforce_plugin_contract(contract: &crate::plugin_contract::PluginContract) {
3379        use crate::plugin_contract::{AUTUMN_WEB_VERSION, ContractVerdict, evaluate};
3380
3381        match evaluate(contract, AUTUMN_WEB_VERSION) {
3382            ContractVerdict::Compatible | ContractVerdict::Undeclared => {}
3383            ContractVerdict::Incompatible(err) => {
3384                if std::env::var("AUTUMN_PLUGIN_CONTRACT").as_deref() == Ok("warn") {
3385                    tracing::warn!(
3386                        plugin = contract.plugin.as_str(),
3387                        "{err}\n  (demoted to a warning by AUTUMN_PLUGIN_CONTRACT=warn)"
3388                    );
3389                } else {
3390                    panic!(
3391                        "{err}\n  \u{2192} or, to boot anyway while you sort it out, set \
3392                         AUTUMN_PLUGIN_CONTRACT=warn"
3393                    );
3394                }
3395            }
3396            ContractVerdict::Unparseable {
3397                requirement,
3398                reason,
3399            } => {
3400                tracing::warn!(
3401                    plugin = contract.plugin.as_str(),
3402                    requirement = requirement.as_str(),
3403                    reason = reason.as_str(),
3404                    "plugin declares an autumn-web requirement that cannot be evaluated; \
3405                     compatibility was NOT checked (run `autumn plugin-check` on the plugin)"
3406                );
3407            }
3408        }
3409    }
3410
3411    /// Declare a plugin-owned top-level config section so it coexists with
3412    /// `server.strict_config = true`.
3413    ///
3414    /// Core's [`AutumnConfig`] schema is closed: any
3415    /// top-level `[root]` table it does not know about is an unknown key. Under
3416    /// `strict_config`, an unknown root is a **hard** boot error. A plugin that
3417    /// reads its own top-level table — for example `autumn-media-plugin` reading
3418    /// `[media]` via raw TOML — would therefore make a `strict_config` app fail
3419    /// at boot with `unknown key "media"`.
3420    ///
3421    /// Calling `config_section("media")` registers `[media]` as a **known,
3422    /// opaque** section: the strict unknown-key check accepts the root and does
3423    /// **not** validate its contents (the plugin owns that — core has no schema
3424    /// for it). The seam is **fail-closed**: only the roots a plugin explicitly
3425    /// declares are exempt; every other unknown top-level root still hard-fails,
3426    /// so a typo like `[medai]` is still caught.
3427    ///
3428    /// Call this from your [`Plugin::build`](crate::plugin::Plugin::build)
3429    /// implementation, where the plugin is applied to the builder:
3430    ///
3431    /// ```ignore
3432    /// impl Plugin for MediaPlugin {
3433    ///     fn build(self, app: AppBuilder) -> AppBuilder {
3434    ///         app.config_section("media") // `[media]` is now strict-config-safe
3435    ///         // … register routes, jobs, startup hooks, …
3436    ///     }
3437    /// }
3438    /// ```
3439    ///
3440    /// The declared roots are threaded into the default config loader
3441    /// ([`TomlEnvConfigLoader`](crate::config::TomlEnvConfigLoader)); a fully
3442    /// custom loader installed via
3443    /// [`with_config_loader`](AppBuilder::with_config_loader) owns its own
3444    /// strict-config handling and is unaffected.
3445    ///
3446    /// Future: an optional eager per-section validation hook (handing each
3447    /// plugin its raw `[root]` table at boot to validate uniformly) could be
3448    /// layered on top of this registry later. It is deliberately deferred — the
3449    /// media plugin already fail-fast-validates its own `[media]` config in its
3450    /// startup hook, and an eager hook adds callback-storage and error-surface
3451    /// plumbing this declarative seam does not need.
3452    #[must_use]
3453    pub fn config_section(mut self, root: impl Into<String>) -> Self {
3454        self.plugin_config_roots.insert(root.into());
3455        self
3456    }
3457
3458    /// Return `true` if the given top-level config root has been declared as a
3459    /// plugin config section via [`config_section`](AppBuilder::config_section).
3460    ///
3461    /// Mirrors [`has_plugin`](AppBuilder::has_plugin); useful for tests and
3462    /// builder introspection.
3463    #[must_use]
3464    pub fn has_config_section(&self, root: &str) -> bool {
3465        self.plugin_config_roots.contains(root)
3466    }
3467
3468    /// Register a named [`MetricsSource`](crate::actuator::MetricsSource) that contributes
3469    /// metric families to `/actuator/prometheus` and `/actuator/metrics`.
3470    ///
3471    /// The `name` is a stable identifier used for:
3472    /// - Duplicate-registration detection (same behaviour as duplicate plugins: a
3473    ///   `tracing::warn!` is emitted and the second registration is skipped).
3474    /// - The `source` label in the `autumn_metrics_source_errors_total` counter
3475    ///   that increments when a source panics during a scrape.
3476    ///
3477    /// `Plugin::build` implementations can call this to wire a source with no
3478    /// extra application-level glue code.
3479    ///
3480    /// # Examples
3481    ///
3482    /// ```rust,no_run
3483    /// use autumn_web::actuator::{MetricsSource, MetricFamily, MetricKind, MetricSample};
3484    /// use autumn_web::app::AppBuilder;
3485    /// use std::sync::Arc;
3486    ///
3487    /// struct QueueMetrics;
3488    ///
3489    /// impl MetricsSource for QueueMetrics {
3490    ///     fn collect(&self) -> Vec<MetricFamily> {
3491    ///         vec![MetricFamily {
3492    ///             name: "myapp_queue_depth".to_string(),
3493    ///             help: "Current queue depth".to_string(),
3494    ///             kind: MetricKind::Gauge,
3495    ///             samples: vec![MetricSample { labels: vec![], value: 42.0 }],
3496    ///         }]
3497    ///     }
3498    /// }
3499    ///
3500    /// autumn_web::app()
3501    ///     .metrics_source("myapp_queue", Arc::new(QueueMetrics));
3502    /// ```
3503    #[must_use]
3504    pub fn metrics_source(
3505        mut self,
3506        name: impl Into<String>,
3507        source: Arc<dyn crate::actuator::MetricsSource>,
3508    ) -> Self {
3509        let name = name.into();
3510        if self.metrics_sources.iter().any(|(n, _)| n == &name) {
3511            tracing::warn!(
3512                source_name = %name,
3513                "MetricsSource '{}' is already registered; skipping duplicate",
3514                name
3515            );
3516            return self;
3517        }
3518        self.metrics_sources.push((name, source));
3519        self
3520    }
3521
3522    /// Register a custom [`HealthIndicator`](crate::actuator::HealthIndicator) with the application.
3523    ///
3524    /// The indicator's [`check`](crate::actuator::HealthIndicator::check) method is called on every
3525    /// `/actuator/health` request (and on `/ready` for `Readiness`-group indicators).
3526    ///
3527    /// Duplicate registration names are silently ignored (a warning is logged).
3528    ///
3529    /// # Examples
3530    ///
3531    /// ```rust,no_run
3532    /// use std::sync::Arc;
3533    /// use autumn_web::actuator::{HealthCheckOutput, HealthIndicator};
3534    ///
3535    /// struct StripeIndicator;
3536    /// impl HealthIndicator for StripeIndicator {
3537    ///     fn check(&self) -> futures::future::BoxFuture<'_, HealthCheckOutput> {
3538    ///         Box::pin(async move { HealthCheckOutput::up() })
3539    ///     }
3540    /// }
3541    ///
3542    /// autumn_web::app()
3543    ///     .health_indicator("stripe", Arc::new(StripeIndicator));
3544    /// ```
3545    #[must_use]
3546    pub fn health_indicator(
3547        mut self,
3548        name: impl Into<String>,
3549        indicator: Arc<dyn crate::actuator::HealthIndicator>,
3550    ) -> Self {
3551        let name = name.into();
3552        // "db" is a reserved built-in component name. Allowing a custom indicator
3553        // under this name would produce an inconsistent response: the custom result
3554        // would still gate the aggregate status while the built-in primary ping owns
3555        // the components.db / checks.database display. The "db:shard:" prefix and
3556        // "db:fleet" are reserved for the framework's per-shard and fleet
3557        // indicators for the same reason.
3558        #[cfg(feature = "db")]
3559        if name == "db" || name == "db:fleet" || name.starts_with("db:shard:") {
3560            tracing::warn!(
3561                indicator_name = %name,
3562                "\"db\", \"db:fleet\" and \"db:shard:*\" are reserved built-in health indicator \
3563                 names; registration skipped. Use a different name for your custom indicator."
3564            );
3565            return self;
3566        }
3567        if self.health_indicators.iter().any(|(n, _, _)| n == &name) {
3568            tracing::warn!(
3569                indicator_name = %name,
3570                "HealthIndicator '{}' is already registered; skipping duplicate",
3571                name
3572            );
3573            return self;
3574        }
3575        let group = indicator.group();
3576        self.health_indicators.push((name, group, indicator));
3577        self
3578    }
3579
3580    /// Register embedded Diesel migrations with the application.
3581    ///
3582    /// When migrations are registered:
3583    /// - They always target the primary/write database role
3584    ///   (`database.primary_url`, falling back to legacy `database.url`).
3585    /// - In **dev** mode, pending migrations run automatically on startup.
3586    /// - In **prod** mode, pending migrations are logged as warnings but
3587    ///   not applied -- use a one-shot `autumn migrate` job before rolling web
3588    ///   replicas.
3589    ///
3590    /// # Examples
3591    ///
3592    /// ```rust,ignore
3593    /// use autumn_web::migrate::{EmbeddedMigrations, embed_migrations};
3594    ///
3595    /// const MIGRATIONS: EmbeddedMigrations = embed_migrations!();
3596    ///
3597    /// #[autumn_web::main]
3598    /// async fn main() {
3599    ///     autumn_web::app()
3600    ///         .routes(routes![...])
3601    ///         .migrations(MIGRATIONS)
3602    ///         .run()
3603    ///         .await;
3604    /// }
3605    /// ```
3606    ///
3607    #[cfg(feature = "db")]
3608    #[must_use]
3609    pub fn migrations(mut self, migrations: migrate::EmbeddedMigrations) -> Self {
3610        self.migrations.push(("app", migrations));
3611        self
3612    }
3613
3614    /// Register embedded Diesel migrations owned by a plugin or other
3615    /// third-party integration, distinct from the app's own
3616    /// [`Self::migrations`].
3617    ///
3618    /// Functionally identical to [`Self::migrations`] — the set is applied at
3619    /// the same startup / one-shot points, subject to the same dev/prod
3620    /// auto-apply policy — but tagged with `name` (e.g.
3621    /// `"autumn-admin-plugin"`) rather than the generic `"app"` label
3622    /// [`Self::migrations`] uses.
3623    ///
3624    /// # Examples
3625    ///
3626    /// ```rust,ignore
3627    /// autumn_web::app()
3628    ///     .plugin_migrations("autumn-admin-plugin", autumn_admin_plugin::MIGRATIONS)
3629    ///     .migrations(MIGRATIONS)
3630    ///     .run()
3631    ///     .await;
3632    /// ```
3633    ///
3634    /// # Version collisions are resolved automatically, never rejected
3635    ///
3636    /// Diesel's `__diesel_schema_migrations` table is keyed by **version
3637    /// alone** — it has no notion of which registered source (the
3638    /// framework, a plugin, the app's own `migrations/`) recorded a
3639    /// version. Nothing coordinates timestamps across a plugin, the
3640    /// framework, and an app, so it is entirely normal for two
3641    /// independently authored migrations to reuse the same version by
3642    /// coincidence — e.g. both picking an all-zero placeholder for their
3643    /// first migration. Applied naively, whichever set's apply runs first
3644    /// would "win" the version, and every other set's same-versioned
3645    /// migration would be skipped forever as "already applied" even though
3646    /// its `up.sql` never actually ran.
3647    ///
3648    /// Rather than reject this — which would leave an app unable to use a
3649    /// plugin at all until someone renames a migration in a dependency they
3650    /// may not control — the framework detects the collision at apply time
3651    /// (across every registered set, including ones the framework itself
3652    /// folds in) and transparently tracks one of the colliding migrations
3653    /// under a distinguishing substitute version, so **both** migrations
3654    /// still apply. This is logged at `INFO` so it is visible, not silent.
3655    /// A version reused under the exact same full migration name (e.g. a
3656    /// shard-required set folded verbatim into another bundle too) is the
3657    /// separate, intentional, harmless case and is left untouched.
3658    ///
3659    /// Which of two colliding migrations keeps the plain version is decided
3660    /// by a fixed rule — the lexicographically-first full migration name —
3661    /// derived purely from the migrations' own content, **not** from
3662    /// registration order. So reordering `.migrations()`/`.plugin_migrations()`
3663    /// calls, or adding a new plugin, never changes an already-settled
3664    /// assignment for a collision that existed before.
3665    ///
3666    /// One case this cannot make safe on its own: introducing a **new**
3667    /// colliding source against a database that has **already** applied one
3668    /// side of the collision under its plain version from an *earlier*
3669    /// deploy (before the new source ever existed). Diesel's tracking table
3670    /// records only the bare version string, not which migration produced
3671    /// it, so there is no way to recover that history after the fact — the
3672    /// fixed rule above has no way to know a version it would assign to the
3673    /// new source is actually already claimed, in the real database, by the
3674    /// older one. If you introduce a plugin whose migrations might collide
3675    /// with an already-deployed app, verify manually before rolling out
3676    /// (e.g. confirm the plugin's expected tables don't already exist under
3677    /// a different name) rather than relying on this to resolve it for you.
3678    ///
3679    /// A second, narrower residual gap: `autumn migrate status` / `autumn
3680    /// migrate down` (the CLI's user-migration status and rollback commands)
3681    /// and the migration checksum/drift-detection system (`autumn migrate
3682    /// record-checksums`'s baseline and its later validation) all resolve
3683    /// applied versions against the app's own `migrations/` directory only —
3684    /// none of them have visibility into which plugins were registered at
3685    /// runtime. If your own app's migration is the one that loses a
3686    /// collision and gets tracked under a substitute version: `migrate
3687    /// status`/`migrate down` cannot currently resolve or revert it by name
3688    /// (reverting it needs manual intervention — inspect
3689    /// `__diesel_schema_migrations` directly); and checksum baselining/drift
3690    /// detection cannot see it either, so a later edit to that migration's
3691    /// `up.sql` will not trigger the drift guard. A plugin's own migrations
3692    /// are unaffected by either gap, since neither system touches plugin
3693    /// migrations either way. Fixing this needs the full registered
3694    /// migration set (not just the app's own directory) threaded through to
3695    /// these CLI-facing functions — out of scope for the auto-resolution
3696    /// added here.
3697    ///
3698    /// A third, even narrower edge case: an already-applied migration's
3699    /// substitute version is recomputed fresh on every startup from the
3700    /// CURRENTLY registered sources, reserved only against versions those
3701    /// sources currently claim. If a later release adds an entirely new
3702    /// migration whose own raw version happens to exactly equal an
3703    /// already-applied substitute (astronomically unlikely in practice,
3704    /// since a substitute is a source-name hash suffix no ordinary migration
3705    /// timestamp would organically collide with), the already-applied
3706    /// migration's substitute would be reassigned to free up that version —
3707    /// changing a previously-settled collision's tracked identity. This is
3708    /// accepted as a residual risk rather than solved by, say, persisting
3709    /// substitute assignments to the database, which the framework does not
3710    /// otherwise need to do.
3711    #[cfg(feature = "db")]
3712    #[must_use]
3713    pub fn plugin_migrations(
3714        mut self,
3715        name: &'static str,
3716        migrations: migrate::EmbeddedMigrations,
3717    ) -> Self {
3718        self.migrations.push((name, migrations));
3719        self
3720    }
3721
3722    /// Embed the app's `static/` tree into the binary for single-binary deploys.
3723    ///
3724    /// Pass the directory produced by [`embed_static!`](crate::embed_static)
3725    /// (requires the `embed-assets` feature). When set, `/static/*` is served
3726    /// from the binary and `asset_url()` resolves against the embedded
3727    /// fingerprint manifest — copying only the release binary into an empty
3728    /// directory serves every referenced asset with no `static/` sidecar.
3729    /// Because the manifest and the files are baked from the same build,
3730    /// fingerprint-vs-manifest drift is impossible.
3731    ///
3732    /// This is a release-time concern: leave it unset in development so CSS/JS
3733    /// hot-reload keeps serving from disk.
3734    ///
3735    /// ```rust,ignore
3736    /// static STATIC: autumn_web::include_dir::Dir = autumn_web::embed_static!();
3737    ///
3738    /// #[autumn_web::main]
3739    /// async fn main() {
3740    ///     autumn_web::app().embedded_static(&STATIC).run().await;
3741    /// }
3742    /// ```
3743    #[cfg(feature = "embed-assets")]
3744    #[must_use]
3745    pub const fn embedded_static(mut self, dir: &'static include_dir::Dir<'static>) -> Self {
3746        self.embedded_static = Some(crate::assets::EmbeddedStaticDir(dir));
3747        self
3748    }
3749
3750    /// Embed the app's i18n locale bundles into the binary.
3751    ///
3752    /// Pass the directory produced by [`embed_locales!`](crate::embed_locales)
3753    /// (requires the `embed-assets` and `i18n` features). When set (and no
3754    /// explicit [`i18n`](AppBuilder::i18n) bundle was provided), all configured
3755    /// locales render from the binary with no `i18n/` sidecar directory.
3756    ///
3757    /// ```rust,ignore
3758    /// static LOCALES: autumn_web::include_dir::Dir = autumn_web::embed_locales!();
3759    ///
3760    /// #[autumn_web::main]
3761    /// async fn main() {
3762    ///     autumn_web::app().embedded_locales(&LOCALES).run().await;
3763    /// }
3764    /// ```
3765    #[cfg(all(feature = "embed-assets", feature = "i18n"))]
3766    #[must_use]
3767    pub const fn embedded_locales(mut self, dir: &'static include_dir::Dir<'static>) -> Self {
3768        self.embedded_locales = Some(dir);
3769        self
3770    }
3771
3772    /// Start the HTTP server.
3773    ///
3774    /// This method performs the full application lifecycle:
3775    ///
3776    /// 1. Loads configuration from `autumn.toml` (or defaults).
3777    /// 2. Initializes the tracing subscriber.
3778    /// 3. Validates that at least one route is registered.
3779    /// 4. Creates the database connection pool (if configured).
3780    /// 5. Builds the Axum router from collected routes.
3781    /// 6. Mounts built-in routes (health check, htmx JS, static files).
3782    /// 7. Binds to the configured address and port.
3783    /// 8. Serves requests with graceful shutdown on Ctrl+C (or `SIGTERM`
3784    ///    on Unix).
3785    ///
3786    /// # Panics
3787    ///
3788    /// Panics if no routes have been registered via [`.routes()`](Self::routes).
3789    /// This is intentional -- an application with no routes is always a
3790    /// developer error.
3791    #[allow(clippy::too_many_lines)]
3792    #[allow(clippy::cognitive_complexity)]
3793    // Large frame is expected: the caller boxes this long-lived future.
3794    #[allow(clippy::large_stack_frames)]
3795    pub async fn run(self) {
3796        // Remember the binary this process was started from, before a deploy
3797        // can replace the file underneath it: an in-place upgrade (#1674) execs
3798        // that path, and `/proc/self/exe` reports "… (deleted)" once the file
3799        // has been swapped.
3800        crate::upgrade::record_startup_exe();
3801
3802        // ── Build mode ─────────────────────────────────────────────────
3803        // When AUTUMN_BUILD_STATIC=1, render static routes to dist/ and exit
3804        // instead of starting the HTTP server. This is triggered by `autumn build`.
3805        if is_static_build_mode() {
3806            Box::pin(self.run_build_mode()).await;
3807            return;
3808        }
3809
3810        // ── Route dump mode ────────────────────────────────────────────
3811        // When AUTUMN_DUMP_ROUTES=1, print the route listing JSON and exit.
3812        // This is triggered by `autumn routes` to introspect the app's
3813        // route table without booting the server or connecting to a database.
3814        if is_dump_routes_mode() {
3815            self.run_dump_routes_mode().await;
3816            return;
3817        }
3818
3819        // ── OpenAPI spec dump mode ─────────────────────────────────────
3820        // When AUTUMN_DUMP_OPENAPI=1, print the generated OpenAPI document
3821        // and exit. Triggered by `autumn openapi export`, which needs the
3822        // contract without booting the server or connecting to a database.
3823        // The guard is deliberately outside the feature gate: a binary built
3824        // without `openapi` must report that on the dump protocol rather than
3825        // ignore the request and start serving.
3826        if is_dump_openapi_mode() {
3827            #[cfg(feature = "openapi")]
3828            {
3829                self.run_dump_openapi_mode().await;
3830                return;
3831            }
3832            #[cfg(not(feature = "openapi"))]
3833            {
3834                eprintln!(
3835                    "{marker}{reason}",
3836                    marker = crate::openapi::OPENAPI_UNAVAILABLE_MARKER,
3837                    reason = crate::openapi::OPENAPI_UNAVAILABLE_FEATURE,
3838                );
3839                std::process::exit(2);
3840            }
3841        }
3842
3843        // ── Cache-coherence manifest dump mode ─────────────────────────
3844        // When AUTUMN_DUMP_CACHE_COHERENCE=1, print the cache-coherence
3845        // manifest (#1716) and exit. Triggered by `autumn cache audit`, which
3846        // needs the whole binary's registrations — every `#[cached]` read and
3847        // every `#[repository]` write, across the app AND its plugins — and
3848        // link-time `inventory` collection is the only place they all exist
3849        // together. Runs before any database or port is touched.
3850        if crate::cache::coherence::is_dump_mode() {
3851            crate::cache::coherence::print_manifest_dump(&crate::cache::coherence::audit());
3852            return;
3853        }
3854
3855        // ── Data-flow manifest dump mode ───────────────────────────────
3856        // When AUTUMN_DUMP_DATA_FLOW=1, print the classified-data flow manifest
3857        // (#1654) and exit. Triggered by `autumn data-flow`, which needs the
3858        // whole binary's registrations -- every `#[classified]` column and every
3859        // declared declassification boundary, across the app AND its plugins --
3860        // and link-time `inventory` collection is the only place they all exist
3861        // together. Runs before any database or port is touched.
3862        if crate::classify::manifest::is_dump_mode() {
3863            crate::classify::manifest::print_manifest_dump(&crate::classify::manifest::audit());
3864            return;
3865        }
3866
3867        // ── Agent-authority manifest dump mode ─────────────────────────
3868        // With AUTUMN_DUMP_AGENT_AUTHORITY=1, print the agent-authority manifest
3869        // (#1691) and exit. `autumn agents manifest` triggers this because it
3870        // needs the whole binary's registrations — every `#[agent_operable]`
3871        // action and every `authority_grant!`, across the app and its plugins —
3872        // joined against this app's route table: which actions an agent can reach
3873        // is a fact about the mounted routes, not the annotations alone. Runs
3874        // before any database or port is touched.
3875        if crate::agent_authority::manifest::is_dump_mode() {
3876            self.run_dump_agent_authority_mode();
3877            return;
3878        }
3879
3880        // ── Architecture-graph dump mode ───────────────────────────────
3881        // When AUTUMN_DUMP_GRAPH=1, print the application architecture graph
3882        // (#1747) and exit. Triggered by `autumn graph`, which needs the whole
3883        // binary's registrations -- every `#[route]`, `#[model]`,
3884        // `#[repository]` and `#[job]`, across the app AND its plugins --
3885        // joined against this app's mounted route table, because a route's
3886        // served path and resolved auth posture are facts about the mount and
3887        // not about the annotation alone. Runs before any database or port is
3888        // touched.
3889        if crate::graph::manifest::is_dump_mode() {
3890            self.run_dump_graph_mode().await;
3891            return;
3892        }
3893
3894        // ── Jobs manifest dump mode ────────────────────────────────────
3895        // When AUTUMN_DUMP_JOBS=1, print the effective drained-queue manifest
3896        // (TOML `queues = [...]`) and exit. Triggered by `autumn jobs manifest`
3897        // so a topology-aware `autumn doctor` sees exactly what the runtime
3898        // drains without booting the server or connecting to a database.
3899        if is_dump_jobs_mode() {
3900            self.run_dump_jobs_mode().await;
3901            return;
3902        }
3903
3904        if is_list_one_off_tasks_mode() {
3905            self.run_list_one_off_tasks_mode();
3906            return;
3907        }
3908
3909        if let Some(task_name) = one_off_task_name_from_env() {
3910            self.run_one_off_task_mode(task_name).await;
3911            return;
3912        }
3913
3914        // ── Migrate one-shot mode ──────────────────────────────────────
3915        // With AUTUMN_MIGRATE=1, apply pending embedded migrations to the
3916        // configured databases and exit; never start the HTTP server or bind a
3917        // port. `autumn deploy`'s redeploy cutover runs migrations before
3918        // flipping traffic (#1607), so a non-zero exit aborts the deploy with the
3919        // old release still serving (AC-3). Unlike startup auto-migration this
3920        // applies on every profile, because the deploy invokes it explicitly.
3921        if is_migrate_only_mode() {
3922            self.run_migrate_only_mode().await;
3923            return;
3924        }
3925
3926        // ── Retention dry-run mode ──────────────────────────────────────
3927        // When AUTUMN_RETENTION_DRY_RUN=1, count (never delete) the rows every
3928        // registered `#[repository(..., retention(...))]` policy would sweep,
3929        // print the report as JSON, and exit — never starting the HTTP server.
3930        // Triggered by `autumn retention --dry-run` (issue #1342).
3931        if is_retention_dry_run_mode() {
3932            self.run_retention_dry_run_mode().await;
3933            return;
3934        }
3935
3936        // ── Framework data-retention mode ───────────────────────────────
3937        // With AUTUMN_DB_RETENTION=report|purge, report or enforce the unified
3938        // `[retention]` policy over every framework-owned dataset and exit,
3939        // never starting the HTTP server. Triggered by `autumn db retention`
3940        // (#1605). It runs inside the app rather than the standalone CLI, so the
3941        // report reflects the app's own resolved config, GDPR legal-hold
3942        // registrations, and audit sinks — the inputs the scheduled sweep uses.
3943        if let Some(mode) = framework_retention_mode_from_env() {
3944            self.run_framework_retention_mode(mode).await;
3945            return;
3946        }
3947
3948        // ── Data-capsule mode ───────────────────────────────────────────
3949        // With AUTUMN_DATA_CAPSULE=export|import|verify, export one subject
3950        // to a signed capsule, import a capsule, or verify one, and exit.
3951        // Triggered by `autumn data capsule` (issue #1811). It runs inside the
3952        // app, so it uses the app's own registry, database, and secret.
3953        if let Some(mode) = data_capsule_mode_from_env() {
3954            self.run_data_capsule_mode(mode).await;
3955            return;
3956        }
3957
3958        // ── Capsule replay mode ────────────────────────────────────────
3959        // When AUTUMN_REPLAY_CAPSULE=<path> is set, rebuild this application
3960        // offline, drive the request the capsule recorded through it, print the
3961        // verdict and EXIT — never start the HTTP server, never open a socket
3962        // (issue #1598). Triggered by `autumn replay <capsule>`.
3963        if is_replay_mode()
3964            && let Some(capsule_path) = replay_capsule_from_env()
3965        {
3966            #[cfg(feature = "reporting")]
3967            {
3968                self.run_replay_mode(capsule_path).await;
3969                return;
3970            }
3971            // Capsules are a `reporting`-feature surface; without it there is
3972            // nothing to load the document with. Refuse (exit 2) rather than
3973            // silently booting a server the operator did not ask for.
3974            #[cfg(not(feature = "reporting"))]
3975            {
3976                eprintln!(
3977                    "REFUSED  {capsule_path}\n  this binary was built without the `reporting` \
3978                     feature, which failure capsules require"
3979                );
3980                std::process::exit(2);
3981            }
3982        }
3983
3984        // Register the in-place upgrade signal (#1674) before the long boot —
3985        // config, database, migrations — rather than when the watcher task
3986        // first runs. Until a handler is installed, `SIGUSR2`'s default
3987        // disposition is to *terminate* the process, so a deploy script that
3988        // signals a process that is still booting would kill it.
3989        #[cfg(unix)]
3990        let upgrade_signal =
3991            match tokio::signal::unix::signal(tokio::signal::unix::SignalKind::user_defined2()) {
3992                Ok(signal) => Some(signal),
3993                Err(e) => {
3994                    tracing::warn!(
3995                        error = %e,
3996                        "could not install the SIGUSR2 handler; in-place upgrade is unavailable"
3997                    );
3998                    None
3999                }
4000            };
4001
4002        let Self {
4003            routes,
4004            api_versions,
4005            route_sources: _,
4006            current_plugin: _,
4007            mut tasks,
4008            one_off_tasks: _,
4009            mut jobs,
4010            listeners,
4011            #[cfg(feature = "db")]
4012            outbox_handlers,
4013            static_metas,
4014            exception_filters,
4015            scoped_groups,
4016            merge_routers,
4017            nest_routers,
4018            custom_layers,
4019            static_gate_layers,
4020            startup_hooks,
4021            state_initializers,
4022            shutdown_hooks,
4023            extensions: _,
4024            registered_plugins: _,
4025            plugin_contracts: _,
4026            plugin_config_roots,
4027            #[cfg(feature = "maud")]
4028            error_page_renderer,
4029            #[cfg(feature = "db")]
4030            migrations,
4031            config_loader_factory,
4032            #[cfg(feature = "db")]
4033            pool_provider_factory,
4034            #[cfg(feature = "db")]
4035            shard_provider_factory,
4036            #[cfg(feature = "db")]
4037            shard_router,
4038            #[cfg(feature = "db")]
4039            directory_shard_router,
4040            telemetry_provider,
4041            session_store,
4042            #[cfg(feature = "ws")]
4043            channels_backend,
4044            #[cfg(feature = "storage")]
4045            blob_store,
4046            cache_backend,
4047            #[cfg(feature = "reporting")]
4048            error_reporters,
4049            alert_channels,
4050            #[cfg(feature = "openapi")]
4051            openapi,
4052            #[cfg(feature = "mcp")]
4053            mcp,
4054            #[cfg(feature = "webmcp")]
4055            webmcp,
4056            audit_logger,
4057            #[cfg(feature = "i18n")]
4058            i18n_bundle,
4059            #[cfg(feature = "i18n")]
4060            i18n_auto_load,
4061            #[cfg(feature = "embed-assets")]
4062            embedded_static,
4063            #[cfg(all(feature = "embed-assets", feature = "i18n"))]
4064            embedded_locales,
4065            policy_registrations,
4066            #[cfg(feature = "mail")]
4067            mail_delivery_queue_factory,
4068            #[cfg(feature = "mail")]
4069            suppression_store,
4070            #[cfg(feature = "mail")]
4071            mail_suppression_store,
4072            #[cfg(feature = "mail")]
4073            mount_unsubscribe_endpoint,
4074            #[cfg(feature = "mail")]
4075            mail_previews,
4076            #[cfg(feature = "maud")]
4077            story_gallery,
4078            declared_routes,
4079            plugin_asset_bundles: _,
4080            idempotency_enabled,
4081            #[cfg(feature = "mail")]
4082            mail_interceptor,
4083            job_interceptor,
4084            #[cfg(feature = "db")]
4085            db_interceptor,
4086            #[cfg(feature = "ws")]
4087            channels_interceptor,
4088            #[cfg(feature = "oauth2")]
4089            http_interceptor,
4090            seo_sources,
4091            agent_skills,
4092            commerce_docs,
4093            metrics_sources,
4094            health_indicators,
4095            #[cfg(feature = "inbound-mail")]
4096            inbound_mail_router,
4097        } = self;
4098
4099        // #1342: every `#[repository(..., retention(...))]` policy compiled
4100        // into this binary auto-registers here — no `tasks![...]` entry
4101        // required. `inventory`-collected, so this is a no-op allocation
4102        // when no model declares a policy.
4103        #[cfg(feature = "db")]
4104        tasks.extend(crate::retention::collect_retention_tasks());
4105
4106        // `collect_retention_tasks()` catches collisions only among
4107        // retention-generated names; it cannot see hand-declared `tasks![...]`
4108        // entries merged in above. An operator's `#[scheduled]` task sharing a
4109        // name with a generated `retention-sweep-<table>` task, or with another
4110        // hand-declared task, would silently spawn two competing scheduler loops.
4111        // Validate the fully merged list now that every name is visible, rather
4112        // than asking operators to avoid the generated namespace by convention.
4113        if let Err(error) = crate::task::validate_unique_scheduled_task_names(&tasks) {
4114            panic!("{error}");
4115        }
4116
4117        let all_routes = routes;
4118
4119        // 1 & 2. Load configuration and initialize logging/telemetry
4120        let (mut config, telemetry_guard) = load_config_and_telemetry(
4121            config_loader_factory,
4122            telemetry_provider,
4123            plugin_config_roots,
4124        )
4125        .await;
4126
4127        // #1605: the unified framework-owned data-retention sweep. Registered
4128        // here rather than with the `#[repository(..., retention(...))]` policies
4129        // above, because it is config-driven and the config loads only now.
4130        // `framework_retention_task` returns `None` unless at least one
4131        // `[retention]` window is set, so an app that never mentions the section
4132        // gets no extra scheduler loop.
4133        let retention_tasks = crate::data_retention::framework_retention_tasks(&config.retention);
4134        if !retention_tasks.is_empty() {
4135            tasks.extend(retention_tasks);
4136            // Re-validate: the merged-name check above ran before this task
4137            // existed, and an app is free to declare a `#[scheduled]` fn
4138            // named `autumn-retention-sweep`, which would otherwise spawn two
4139            // loops competing for one coordination lock.
4140            if let Err(error) = crate::task::validate_unique_scheduled_task_names(&tasks) {
4141                panic!("{error}");
4142            }
4143        }
4144
4145        // The process role selects which slice of the runtime this replica runs.
4146        // A split role (web/worker) needs a durable jobs backend both processes
4147        // share. Anything else — the in-process `local` queue, a typo, a blank
4148        // value — falls through to the per-process local runtime: the web replica
4149        // enqueues into an in-memory queue no worker can drain, and a worker
4150        // replica's queue starts empty. Reject it here, before any boot work,
4151        // rather than in `validate()`, so the doctor can still load the config.
4152        // A combined role is always fine.
4153        let role = config.role;
4154        // Both config-only preconditions, through the same helper the no-boot
4155        // export calls — the split-role/durable-backend rule and the merged
4156        // scheduled-task-name check. The helper returns the message rather than
4157        // exiting, because this path has a database pool to stop on the way out
4158        // and the export has none.
4159        if let Err(message) = validate_config_preconditions(&config, &tasks) {
4160            tracing::error!("{message}");
4161            #[cfg(feature = "managed-pg")]
4162            crate::managed_pg::emergency_stop_async().await;
4163            std::process::exit(1);
4164        }
4165
4166        // The sqlite queue backs a split role only because both processes open
4167        // the same file. Against an in-memory target each gets its own private
4168        // database, so the web replica would enqueue where no worker can ever
4169        // look — the same silent stranding, one step further in (issue #1907).
4170        if crate::config::split_role_requires_file_backed_sqlite(
4171            role,
4172            &config.jobs.backend,
4173            config.database.effective_primary_url(),
4174        ) {
4175            tracing::error!(
4176                role = role.as_str(),
4177                "process role '{}' with jobs.backend = \"sqlite\" requires a FILE-backed \
4178                 database: an in-memory SQLite target is private to each process, so the web \
4179                 replica would enqueue into a queue no worker process can see. Point \
4180                 database.url at a sqlite:// file, or run the combined role.",
4181                role.as_str(),
4182            );
4183            #[cfg(feature = "managed-pg")]
4184            crate::managed_pg::emergency_stop_async().await;
4185            std::process::exit(1);
4186        }
4187
4188        #[cfg(feature = "mail")]
4189        apply_mail_builder_overrides(&mut config, mount_unsubscribe_endpoint);
4190
4191        // Apply builder-level flag: `.idempotent()` enables the middleware when
4192        // neither `autumn.toml` nor the environment explicitly disable it.
4193        // The env var `AUTUMN_IDEMPOTENCY__ENABLED` is re-checked here so
4194        // operators can disable idempotency at runtime (e.g. during a Redis
4195        // incident) without code changes, even when `.idempotent()` is called.
4196        if idempotency_enabled {
4197            let env_disabled = std::env::var("AUTUMN_IDEMPOTENCY__ENABLED")
4198                .is_ok_and(|v| matches!(v.to_lowercase().as_str(), "false" | "0" | "no" | "off"));
4199            // Only apply the builder default when neither the env var nor the
4200            // loaded config file explicitly sets enabled = false.
4201            if !env_disabled && config.idempotency.enabled != Some(false) {
4202                config.idempotency.enabled = Some(true);
4203            }
4204        }
4205
4206        // Register the embedded `static/` tree (if any) before the router is
4207        // built so `/static/*` serves from the binary and `asset_url()` resolves
4208        // against the embedded manifest, then prefer embedded locales over disk
4209        // auto-loading when no explicit bundle was provided.
4210        #[cfg(feature = "embed-assets")]
4211        register_embedded_static_dir(embedded_static);
4212
4213        #[cfg(all(feature = "embed-assets", feature = "i18n"))]
4214        let i18n_bundle = embedded_i18n_bundle(i18n_bundle, embedded_locales, &config);
4215
4216        #[cfg(feature = "i18n")]
4217        let i18n_bundle =
4218            resolve_i18n_bundle(i18n_bundle, i18n_auto_load, &config, &crate::config::OsEnv);
4219
4220        // 3. Validate routes.
4221        //
4222        // Both of this function's own pre-router checks now run here, through
4223        // the same helper the no-boot export calls. The repository-policy audit
4224        // used to sit ~140 lines further down; running it here only moves it
4225        // ahead of the startup banner, and a refusal is better reported before
4226        // announcing a start than after.
4227        if let Err(message) =
4228            validate_pre_router_preconditions(&all_routes, &scoped_groups, &config)
4229        {
4230            panic!("{message}");
4231        }
4232
4233        // 4. Log banner with profile info
4234        let profile_display = config.profile.as_deref().unwrap_or("none");
4235        tracing::info!(
4236            version = env!("CARGO_PKG_VERSION"),
4237            profile = profile_display,
4238            "Autumn starting"
4239        );
4240
4241        // 4b. Startup transparency log (AUTUMN_SHOW_CONFIG=1 or log level <= DEBUG)
4242        let show_config = std::env::var("AUTUMN_SHOW_CONFIG").as_deref() == Ok("1");
4243        if show_config {
4244            log_startup_transparency(&all_routes, &tasks, &scoped_groups, &config);
4245        }
4246
4247        // 4c. Fail-fast on invalid session config — but only when no custom
4248        // SessionStore was installed via with_session_store(...). Done before
4249        // setup_database so a doomed boot doesn't run migrations first.
4250        fail_fast_on_invalid_session_config(&config, session_store.is_some());
4251
4252        // 4d. Validate signing secret — production must have a stable, private,
4253        // entropy-meeting secret before the server binds. Dev/test are exempt.
4254        fail_fast_on_invalid_signing_secret(&config);
4255        fail_fast_on_missing_encryption_keys(&config);
4256        fail_fast_on_invalid_trusted_hosts(&config);
4257
4258        // 4e. Signed webhook configs must resolve to usable key material
4259        // before the app binds. Missing secrets should fail before a real
4260        // provider retry loop starts hammering a broken endpoint.
4261        fail_fast_on_invalid_webhook_config(&config);
4262
4263        // 4f. Idempotency backend must be production-ready when enabled.
4264        fail_fast_on_invalid_idempotency_config(&config);
4265
4266        // 4f. Provision the configured BlobStore before `setup_database`.
4267        // `LocalBlobStore::new` does real IO (it creates and canonicalizes the
4268        // root) and the storage code may `process::exit(1)` on failure — an
4269        // unwritable root, or `storage.backend = "s3"` with no plugin. Running it
4270        // before migrations keeps a doomed boot from mutating the DB schema
4271        // first. A store installed via `.with_blob_store(...)` bypasses
4272        // config-driven instantiation entirely: no IO, no fail-fast.
4273        #[cfg(feature = "storage")]
4274        let storage_bootstrap = blob_store.map_or_else(
4275            || preflight_storage(&config),
4276            |store| {
4277                Some(StorageBootstrap {
4278                    store,
4279                    serving: None,
4280                })
4281            },
4282        );
4283
4284        // 5. Create database pool and run migrations (if configured)
4285        //
4286        // With `[failure_capture] enabled = true` the pool is built through the
4287        // recording factory, so a failing request's database traffic is captured
4288        // at the wire (#1598). An app with its own `DatabasePoolProvider` keeps
4289        // it, and DB capture stands down.
4290        //
4291        // This wraps the control topology only. `[[database.shards]]` pools are
4292        // built by `create_shard_set` below, outside `setup_database`, and are
4293        // not recorded in this slice. So `Db::checkout` notes the gap and marks
4294        // the capsule truncated for any request that checks out a shard
4295        // connection (`capsule::record_db::note_shard_capture_gap`), and
4296        // `maybe_capture_pool_provider` warns at boot when both are configured.
4297        #[cfg(all(feature = "db", feature = "reporting", not(feature = "sqlite")))]
4298        let pool_provider_factory =
4299            crate::capsule::record_db::maybe_capture_pool_provider(pool_provider_factory, &config);
4300        #[cfg(feature = "db")]
4301        let database = setup_database(
4302            &config,
4303            migrations,
4304            pool_provider_factory,
4305            shard_provider_factory,
4306            shard_router,
4307            directory_shard_router,
4308            RepositoryCommitHookQueueMigrationMode::Runtime,
4309        )
4310        .await
4311        .unwrap_or_else(|e| {
4312            tracing::error!("{e}");
4313            std::process::exit(1);
4314        });
4315        #[cfg(feature = "db")]
4316        let pool = database.topology;
4317        #[cfg(feature = "db")]
4318        let shards = database.shards;
4319        #[cfg(feature = "db")]
4320        let replica_readiness = database.replica_readiness;
4321        #[cfg(feature = "db")]
4322        let replica_migration_check = database.replica_migration_check;
4323
4324        #[cfg(feature = "db")]
4325        if pool.is_some() || shards.is_some() {
4326            // Pool sizes multiply across shards: surface the total so
4327            // N-shard deployments notice the aggregate connection count.
4328            let shard_max_connections = shards
4329                .as_ref()
4330                .map_or(0, crate::sharding::ShardSet::total_max_connections);
4331            let control_max_connections = pool.as_ref().map_or(0, |topology| {
4332                topology.primary().status().max_size
4333                    + topology.replica().map_or(0, |p| p.status().max_size)
4334            });
4335            let total_max_connections = control_max_connections + shard_max_connections;
4336            tracing::info!(
4337                primary_max_connections = config.database.effective_primary_pool_size(),
4338                replica_configured = config.database.replica_url.is_some(),
4339                replica_max_connections = config.database.effective_replica_pool_size(),
4340                shard_count = shards.as_ref().map_or(0, crate::sharding::ShardSet::len),
4341                total_max_connections,
4342                "Database topology configured"
4343            );
4344            // Pool sizes multiply across shards; warn before the aggregate
4345            // silently exhausts Postgres's server-side `max_connections`.
4346            let warn_threshold = config.database.max_connections_warn_threshold;
4347            if crate::config::should_warn_total_connections(total_max_connections, warn_threshold) {
4348                tracing::warn!(
4349                    total_max_connections,
4350                    warn_threshold,
4351                    "Aggregate database connection count is high: the control \
4352                     topology and all shard pools together may open \
4353                     {total_max_connections} connections (warn threshold \
4354                     {warn_threshold}). Ensure each Postgres server's \
4355                     max_connections (plus headroom for migrations and \
4356                     psql) exceeds the pools that target it, or lower \
4357                     database.pool_size. Set \
4358                     database.max_connections_warn_threshold = 0 to silence."
4359                );
4360            }
4361            // A transaction-mode pooler breaks session state (issue #3065).
4362            for warning in crate::db::pooler::pooler_warnings(&config.database) {
4363                tracing::warn!("{warning}");
4364            }
4365            if config.database.replica_max_lag_ms.is_some_and(|ms| ms > 0)
4366                && config.database.replica_url.is_none()
4367            {
4368                tracing::warn!(
4369                    "database.replica_max_lag_ms has no effect: database.replica_url is not set"
4370                );
4371            }
4372        } else {
4373            tracing::info!("Database not configured");
4374        }
4375
4376        // 5b. Fail-fast on `#[repository(api = ...)]` endpoints that
4377        // were mounted without a paired `policy = ...` argument when
4378        // running in `prod` profile and the explicit escape hatch is
4379        // off. Hides exactly the footgun called out in the issue:
4380        // "a developer who flips the `api =` switch on a
4381        // `#[repository]` exposes mutate endpoints that any
4382        // authenticated user can call against any record."
4383        // (The audit itself now runs above, with the other pre-router
4384        // precondition, so the exporter shares both — see
4385        // `validate_pre_router_preconditions`.)
4386
4387        // Subsystems whose backend the builder installed do not use the
4388        // configured Redis, so they get no Redis indicator (#3059).
4389        #[cfg(feature = "redis")]
4390        let mut unused_redis_subsystems: Vec<&'static str> = Vec::new();
4391        #[cfg(feature = "redis")]
4392        if session_store.is_some() {
4393            unused_redis_subsystems.push("sessions");
4394        }
4395        #[cfg(all(feature = "redis", feature = "ws"))]
4396        if channels_backend.is_some() {
4397            unused_redis_subsystems.push("channels");
4398        }
4399
4400        // 6. Build the router (with optional static-file layer)
4401        let mut state = build_state(
4402            &config,
4403            #[cfg(feature = "db")]
4404            pool.as_ref(),
4405            #[cfg(feature = "db")]
4406            shards,
4407            #[cfg(feature = "ws")]
4408            channels_backend,
4409        );
4410
4411        // Tee clock reads into the capsule of whatever request took them, so a
4412        // replayed handler sees the same `now()` sequence the failure did.
4413        // Mirrored by `TestApp::build` so test apps exercise the same wiring.
4414        #[cfg(feature = "reporting")]
4415        if config.failure_capture.enabled {
4416            let recording =
4417                std::sync::Arc::new(crate::capsule::RecordingClock::new(state.clock_arc()))
4418                    as std::sync::Arc<dyn crate::time::ClockSource>;
4419            state = state.with_clock(recording);
4420        }
4421        // Same for the entropy source (#1634): a handler that mints a session
4422        // id, a token or a job id must mint the *recorded* one on replay, or
4423        // the identifier in the capsule's SQL binds will not be the one the
4424        // replayed code produced.
4425        #[cfg(feature = "reporting")]
4426        if config.failure_capture.enabled {
4427            let recording =
4428                std::sync::Arc::new(crate::capsule::RecordingEntropy::new(state.entropy_arc()))
4429                    as std::sync::Arc<dyn crate::entropy::Entropy>;
4430            state = state.with_entropy(recording);
4431        }
4432
4433        // Wire the in-memory log capture buffer from the telemetry guard into the
4434        // app state so the `/actuator/logfile` endpoint can serve it.
4435        if let Some(buf) = telemetry_guard.log_buffer.clone() {
4436            state.insert_extension(buf);
4437        }
4438        // Wire the live-subscriber reload handle into the loggers actuator so
4439        // `PUT /actuator/loggers/{name}` affects the running subscriber, not
4440        // just an in-memory map (issue #1044).
4441        if let Some(handle) = telemetry_guard.filter_reload.clone() {
4442            state.log_levels().attach_reload_handle(handle);
4443        }
4444
4445        // Build MaintenanceState, load the flag synchronously, insert it as an
4446        // extension, and start the background poller.
4447        //
4448        // #1621: the flag path comes from `maintenance::resolve_flag_file_path()`,
4449        // not the bare cwd-relative const. A deploy-managed slot unit runs with
4450        // `WorkingDirectory={release_dir}`, a fresh dir every release, so the
4451        // legacy path made a cutover orphan the flag and silently un-maintain the
4452        // host. The resolver honours `AUTUMN_MAINTENANCE_FLAG_FILE`, stamped by
4453        // `autumn deploy` at the per-app `shared/` dir that survives cutovers, and
4454        // falls back to the legacy path when unset, so a non-deploy-managed app is
4455        // unaffected. This boot load and the 500 ms poller below use the same
4456        // resolver, so they cannot disagree.
4457        let maintenance_state = crate::maintenance::MaintenanceState::new();
4458        let flag_path = crate::maintenance::resolve_flag_file_path();
4459        if let Ok(Some(cfg)) = crate::maintenance::MaintenanceState::load_from_file(&flag_path) {
4460            maintenance_state.enable(cfg);
4461        }
4462        state.insert_extension(maintenance_state.clone());
4463
4464        let poller_state = maintenance_state.clone();
4465        tokio::spawn(async move {
4466            let path = crate::maintenance::resolve_flag_file_path();
4467            let interval = std::time::Duration::from_millis(500);
4468            loop {
4469                let poll_path = path.clone();
4470                let load_res = crate::time::spawn_blocking(move || {
4471                    crate::maintenance::MaintenanceState::load_from_file(&poll_path)
4472                })
4473                .await;
4474
4475                match load_res {
4476                    Ok(Ok(Some(cfg))) => {
4477                        if poller_state.get() != Some(cfg.clone()) {
4478                            poller_state.enable(cfg);
4479                        }
4480                    }
4481                    Ok(Ok(None)) => {
4482                        if poller_state.is_active() {
4483                            poller_state.disable();
4484                        }
4485                    }
4486                    Ok(Err(e)) => {
4487                        tracing::error!(error = %e, "failed to load maintenance flag file");
4488                    }
4489                    Err(e) => {
4490                        tracing::error!(error = %e, "maintenance poller task panicked");
4491                    }
4492                }
4493                tokio::time::sleep(interval).await;
4494            }
4495        });
4496
4497        // Resolve the canary deploy-version label (AUTUMN_DEPLOY_VERSION /
4498        // AUTUMN_CANARY) once at startup and publish it so the actuator metrics
4499        // endpoint can tag every metric family with version="stable|canary".
4500        let canary_state = crate::canary::CanaryState::from_env();
4501        if canary_state.is_canary() {
4502            tracing::info!(
4503                version = canary_state.version(),
4504                "canary: replica labelled as canary cohort"
4505            );
4506        }
4507        state.insert_extension(canary_state);
4508
4509        // A rollback flag present at startup means a controller already retired
4510        // this replica. Flip /ready to draining immediately so a supervisor
4511        // restart cannot put a rolled-back replica back into the canary cohort;
4512        // `canary_rollback_signal` then drives the clean drain → exit.
4513        if crate::canary::CanaryState::rollback_flag_present(std::path::Path::new(
4514            crate::canary::CANARY_ROLLBACK_FLAG_FILE,
4515        )) {
4516            tracing::warn!(
4517                "canary: rollback flag present at startup; /ready will report draining until \
4518                 the flag is cleared (`autumn canary promote`)"
4519            );
4520            state.begin_shutdown();
4521        }
4522
4523        // Clone class: this interceptor-wiring block (mail/job/db/channels,
4524        // including the channels -> presence -> collab backend swap, and
4525        // http) is byte-for-byte identical in `run()`, `run_build_mode()`
4526        // and `run_one_off_task_mode()`. It was born duplicated by #862,
4527        // whose own follow-up commit ("fix: wire interceptors into
4528        // one-off task mode") is a missed-fix this duplication already
4529        // produced once: the one-off task path first shipped without this
4530        // block, so its interceptors were silently inert. #1806 later had
4531        // to hand-repeat the channels/presence/collab swap in all three
4532        // copies again. Change one, change the other two --
4533        // `optional_interceptor_wiring_matches_across_all_three_run_modes`
4534        // below fails if they drift.
4535        #[cfg(feature = "mail")]
4536        if let Some(interceptor) = mail_interceptor {
4537            state.insert_extension(interceptor);
4538        }
4539        if let Some(interceptor) = job_interceptor {
4540            state.insert_extension(interceptor);
4541        }
4542        #[cfg(feature = "db")]
4543        if let Some(interceptor) = db_interceptor {
4544            state.insert_extension(interceptor);
4545        }
4546        #[cfg(feature = "ws")]
4547        if let Some(interceptor) = channels_interceptor {
4548            state.insert_extension(interceptor.clone());
4549            state.channels = crate::channels::Channels::with_shared_backend(std::sync::Arc::new(
4550                crate::channels::InterceptedChannelsBackend::new(
4551                    state.channels.backend().clone(),
4552                    vec![interceptor],
4553                ),
4554            ));
4555            #[cfg(feature = "presence")]
4556            {
4557                state.presence = crate::presence::Presence::new(state.channels.clone());
4558                // The collaboration hub holds the channel registry and the
4559                // presence tracker it was built with, so replacing either
4560                // leaves it publishing into the old backend (#1806).
4561                #[cfg(feature = "collab")]
4562                {
4563                    state.collab = crate::collab::CollabHub::new(
4564                        state.channels.clone(),
4565                        state.presence.clone(),
4566                    );
4567                }
4568            }
4569        }
4570        #[cfg(feature = "oauth2")]
4571        if let Some(interceptor) = http_interceptor {
4572            state.insert_extension(interceptor);
4573        }
4574
4575        // Populate the metrics source registry from builder registrations.
4576        // Duplicate names were already rejected in `metrics_source()`, so
4577        // all entries here are unique.
4578        for (name, source) in metrics_sources {
4579            if let Err(e) = state.metrics_source_registry.register(name, source) {
4580                tracing::warn!("{e}");
4581            }
4582        }
4583
4584        // Populate the health indicator registry from builder registrations.
4585        for (name, group, indicator) in health_indicators {
4586            if let Err(e) = state
4587                .health_indicator_registry
4588                .register(name, group, indicator)
4589            {
4590                tracing::warn!("{e}");
4591            }
4592        }
4593
4594        // Continuous SQLite replication (#1628). Resolved here, next to the other
4595        // indicator registrations, so lag and verification are baked into
4596        // `/actuator/health` — and so an indicator that stays non-healthy past
4597        // the grace period is escalated by the existing #1610 alerter with no
4598        // bespoke alert condition of its own. The destination is built on a
4599        // BLOCKING thread: its HTTP client must never be constructed inside the
4600        // async runtime. The loop itself is spawned at bind time below.
4601        #[cfg(feature = "db")]
4602        let mut replication_worker: Option<crate::replication::Replicator> = None;
4603        #[cfg(feature = "db")]
4604        if let Some(replication_config) = config.replication.clone() {
4605            let database_url = config
4606                .database
4607                .primary_url
4608                .clone()
4609                .or_else(|| config.database.url.clone())
4610                .unwrap_or_default();
4611            let profile = config.profile.clone().unwrap_or_else(|| "dev".to_owned());
4612            // Only a genuinely S3-backed app has a blob-storage bucket to clash
4613            // with; a leftover `[storage.s3]` bucket on the local backend is
4614            // inert and must not trip the distinct-destination guard (the same
4615            // rule #1619's offsite upload applies).
4616            #[cfg(feature = "storage")]
4617            let storage_destination = (config.storage.backend
4618                == crate::storage::StorageBackend::S3)
4619                .then(|| {
4620                    config
4621                        .storage
4622                        .s3
4623                        .bucket
4624                        .clone()
4625                        .map(|bucket| (bucket, config.storage.s3.endpoint.clone()))
4626                })
4627                .flatten();
4628            #[cfg(not(feature = "storage"))]
4629            let storage_destination: Option<(String, Option<String>)> = None;
4630            // The injected clock, not the wall clock: every artifact the
4631            // replicator stamps, and the health indicator's startup grace, are
4632            // read from it, so a test that freezes time moves them all (#1797).
4633            let clock = state.clock_arc();
4634            #[cfg(feature = "sqlite")]
4635            let fleet_lag_alert_after = replication_config
4636                .rpo()
4637                .saturating_mul(crate::replication::LAG_ALERT_MULTIPLIER);
4638
4639            let built = crate::time::spawn_blocking(move || {
4640                crate::replication::build(
4641                    &replication_config,
4642                    &database_url,
4643                    &profile,
4644                    storage_destination.as_ref().map(|(bucket, endpoint)| {
4645                        crate::replication::StorageDestination {
4646                            bucket,
4647                            endpoint: endpoint.as_deref(),
4648                        }
4649                    }),
4650                    clock,
4651                )
4652            })
4653            .await;
4654
4655            match built {
4656                Ok(Ok(runtime)) => {
4657                    tracing::info!(
4658                        database = %runtime.settings.database_path.display(),
4659                        destination = %runtime.status.snapshot().destination,
4660                        prefix = %runtime.settings.root,
4661                        sync_interval_secs = runtime.settings.sync_interval.as_secs(),
4662                        "continuous SQLite replication is enabled"
4663                    );
4664                    if let Err(e) = state.health_indicator_registry.register(
4665                        crate::replication::INDICATOR_NAME,
4666                        crate::actuator::IndicatorGroup::HealthOnly,
4667                        std::sync::Arc::clone(&runtime.indicator)
4668                            as std::sync::Arc<dyn crate::actuator::HealthIndicator>,
4669                    ) {
4670                        tracing::warn!("{e}");
4671                    }
4672                    // A SQLite fleet (ADR 0019 §5) ships each of its databases
4673                    // next to the control database, through the same destination.
4674                    #[cfg(feature = "sqlite")]
4675                    if let Err(e) =
4676                        start_fleet_replication(&state, &config, &runtime, fleet_lag_alert_after)
4677                    {
4678                        tracing::error!("SQLite fleet replication could not start: {e}");
4679                        std::process::exit(1);
4680                    }
4681                    replication_worker = Some(runtime.replicator);
4682                }
4683                Ok(Err(crate::replication::SetupError::Disabled)) => {}
4684                // A misconfigured durability story must not boot silently
4685                // half-working: the operator asked for replication and is not
4686                // getting it, which is exactly the situation this feature exists
4687                // to prevent.
4688                Ok(Err(e)) => {
4689                    tracing::error!("Continuous SQLite replication could not start: {e}");
4690                    #[cfg(feature = "managed-pg")]
4691                    crate::managed_pg::emergency_stop_async().await;
4692                    std::process::exit(1);
4693                }
4694                Err(e) => {
4695                    tracing::error!("Continuous SQLite replication setup panicked: {e}");
4696                    #[cfg(feature = "managed-pg")]
4697                    crate::managed_pg::emergency_stop_async().await;
4698                    std::process::exit(1);
4699                }
4700            }
4701        }
4702
4703        // When ACME is configured, register a `HealthOnly` indicator backed by a
4704        // shared status the renewal task writes. Built here (before the router)
4705        // so it is baked into `/actuator/health`; the same `AcmeStatus` handle is
4706        // reused by the renewal task spawned at bind time below.
4707        #[cfg(feature = "acme")]
4708        let acme_status: Option<crate::acme::renewal::AcmeStatus> =
4709            if let Some(acme_cfg) = config.server.tls.as_ref().and_then(|t| t.acme.as_ref()) {
4710                let status = crate::acme::renewal::AcmeStatus::new();
4711                let indicator = std::sync::Arc::new(
4712                    crate::acme::renewal::AcmeHealthIndicator::new(
4713                        status.clone(),
4714                        acme_cfg.renew_before_days,
4715                    )
4716                    // Which challenge is in play (and, for DNS-01, which provider)
4717                    // is the first thing an operator needs when issuance is failing
4718                    // — and is safe to publish: it names no credential (#1620).
4719                    .with_dns_provider(acme_cfg.dns.as_ref().map(|dns| dns.provider.as_str())),
4720                );
4721                if let Err(e) = state.health_indicator_registry.register(
4722                    "acme",
4723                    crate::actuator::IndicatorGroup::HealthOnly,
4724                    indicator,
4725                ) {
4726                    tracing::warn!("{e}");
4727                }
4728                Some(status)
4729            } else {
4730                None
4731            };
4732
4733        #[cfg(feature = "db")]
4734        configure_replica_migration_check(&state, replica_migration_check);
4735        #[cfg(feature = "db")]
4736        apply_replica_migration_readiness(&state, replica_readiness);
4737        if let Some(cache) = cache_backend {
4738            // Wrapped once, so the global and the state hold the same `Arc`.
4739            let cache = crate::cache::with_capsule_seam(cache);
4740            crate::cache::set_global_cache(cache.clone());
4741            state.shared_cache = Some(cache);
4742        } else {
4743            crate::cache::clear_global_cache();
4744        }
4745        state.insert_extension(RegisteredApiVersions(api_versions));
4746
4747        // Capture a clone of the registered reporter chain for the ACME renewal
4748        // task (spawned below) so each renewal failure reaches the same
4749        // Sentry/etc. sinks a request-path 5xx would. Empty is fine — failures
4750        // still log via `tracing` inside the loop.
4751        #[cfg(all(feature = "acme", feature = "reporting"))]
4752        let acme_reporters = error_reporters.clone();
4753
4754        // Install registered error reporters so the reporting layer (wired in
4755        // `apply_middleware`) can deliver panic + 5xx events. Empty is fine —
4756        // the layer falls back to the built-in `LogReporter`.
4757        #[cfg(feature = "reporting")]
4758        if !error_reporters.is_empty() {
4759            state.insert_extension(crate::reporting::RegisteredReporters(error_reporters));
4760        }
4761        // Apply deferred policy / scope registrations onto the live
4762        // app state. Done before the router is built so any panic
4763        // from double-registration surfaces during startup, not
4764        // mid-request.
4765        for register in policy_registrations {
4766            register(state.policy_registry());
4767        }
4768        // Now that registrations have been applied, verify that
4769        // every `#[repository(policy = X)]`-annotated route has
4770        // an X actually registered on the live registry. Catches
4771        // the "wired the macro arg, forgot the `.policy(...)`
4772        // builder call" footgun before any 500 lands.
4773        validate_repository_policies_registered(
4774            &all_routes,
4775            &scoped_groups,
4776            state.policy_registry(),
4777            &config,
4778        );
4779        #[cfg(feature = "mail")]
4780        if let Some(handle) = suppression_store {
4781            state.insert_extension(handle);
4782        }
4783        #[cfg(feature = "mail")]
4784        if let Some(handle) = mail_suppression_store {
4785            state.insert_extension(handle);
4786        }
4787        // Before the mailer: with `outbox.enabled`, `deliver_later` goes
4788        // through the outbox unless the app set its own queue (issue #3062).
4789        #[cfg(feature = "db")]
4790        {
4791            crate::outbox::install(&state, &config.outbox, outbox_handlers);
4792            if state.extension::<crate::outbox::OutboxRelay>().is_some()
4793                && let Err(error) = crate::outbox::ensure_relay_schema(&state).await
4794            {
4795                tracing::error!(error = %error, "Failed to create the outbox tables");
4796                exit_stop_managed_pg();
4797                std::process::exit(1);
4798            }
4799        }
4800        #[cfg(feature = "mail")]
4801        crate::mail::install_mailer_with_factory(
4802            &state,
4803            &config.mail,
4804            mail_delivery_queue_factory,
4805            true,
4806        )
4807        .unwrap_or_else(|error| {
4808            tracing::error!(error = %error, "Failed to configure mailer");
4809            exit_stop_managed_pg();
4810            std::process::exit(1);
4811        });
4812        #[cfg(feature = "mail")]
4813        state.insert_extension(crate::mail::MailPreviewRegistry::new(mail_previews));
4814        #[cfg(feature = "maud")]
4815        install_story_registry(&state, story_gallery);
4816        // Operator alerts: build the built-in mail/webhook channels from
4817        // `[alerts]` config, combine with any builder-registered channels, and
4818        // start the background evaluation loop. No-op when nothing is
4819        // configured. Installed after the mailer so the mail channel can bind
4820        // to the live `Mailer` extension.
4821        crate::alerts::install_from_config(&state, &config.alerts, alert_channels);
4822        // An MCP endpoint with no audit sink still serves tools; what it does
4823        // not do is leave a record that an agent called one. That is a property
4824        // of the deployment rather than of any grant, so it is said once, here,
4825        // where both are known -- and it is also carried in the agent-authority
4826        // manifest's `audit.sink_configured` (#1691 R9).
4827        #[cfg(feature = "mcp")]
4828        if mcp.is_some()
4829            && !audit_logger
4830                .as_ref()
4831                .is_some_and(|logger| logger.is_enabled())
4832        {
4833            tracing::warn!(
4834                target: "autumn.agent",
4835                "MCP is mounted with no audit sink installed: agent tool calls will be traced \
4836                 but not recorded. Install one with `AppBuilder::with_audit_sink(..)`. \
4837                 See docs/guide/agent-authority.md"
4838            );
4839        }
4840        if let Some(logger) = audit_logger {
4841            state.insert_extension::<crate::audit::AuditLogger>((*logger).clone());
4842        }
4843        #[cfg(feature = "i18n")]
4844        let custom_layers =
4845            install_i18n_bundle_layer(custom_layers, &state, i18n_bundle, &config.i18n);
4846
4847        // Install the preflighted blob store on the freshly-built
4848        // AppState, and remember the serving router so it gets merged
4849        // into the user's router below.
4850        #[cfg(feature = "storage")]
4851        let storage_router = storage_bootstrap.and_then(|b| b.install(&state));
4852        install_webhook_registry(&state, &config);
4853        run_state_initializers(state_initializers, &state);
4854        crate::cost::install(&state, &config);
4855        // A live-state block that could not be installed is a refusal to start,
4856        // not a silent fallback: the previous build is still serving and still
4857        // holds the only copy of that state (#1674). Checked here, in async
4858        // context, so a managed-Postgres child is stopped rather than orphaned.
4859        if let Some(failure) = state.extension::<crate::upgrade::LiveStateInstallFailure>() {
4860            tracing::error!("refusing to start: {}", failure.0);
4861            #[cfg(feature = "managed-pg")]
4862            crate::managed_pg::emergency_stop_async().await;
4863            std::process::exit(1);
4864        }
4865        finalize_event_bus(listeners, &mut jobs, &state);
4866
4867        // One `redis:<subsystem>` PING indicator per Redis-backed subsystem
4868        // (#3059). Here, the job set is final: with no jobs, no job runtime
4869        // starts, so `jobs` gets no indicator. The user indicators are already
4870        // registered: if two names are the same, the user indicator stays.
4871        #[cfg(feature = "redis")]
4872        {
4873            if jobs.is_empty() {
4874                unused_redis_subsystems.push("jobs");
4875            }
4876            unused_redis_subsystems.extend(crate::redis_health::unused_for_role(role));
4877            crate::redis_health::register_redis_health_indicators(
4878                &config,
4879                &state.health_indicator_registry,
4880                &crate::redis_health::app_pingers(&state),
4881                &unused_redis_subsystems,
4882            );
4883        }
4884
4885        let env = crate::config::OsEnv;
4886        let dist_dir = project_dir("dist", &env);
4887        let dist_ref = if dist_dir.exists() {
4888            Some(dist_dir.as_path())
4889        } else {
4890            None
4891        };
4892        #[cfg_attr(
4893            not(any(feature = "storage", feature = "inbound-mail")),
4894            allow(unused_mut)
4895        )]
4896        let mut merge_routers = merge_routers;
4897        #[cfg(feature = "storage")]
4898        if let Some(router) = storage_router {
4899            merge_routers.push(router);
4900        }
4901
4902        // Static routes are pre-rendered by requesting their single,
4903        // unprefixed path — never locale-aware — so they must stay excluded
4904        // from locale-prefix routing even when it's enabled (issue #1251).
4905        // Must run before both the sitemap generation below and the router
4906        // build further down, so the two agree on which paths are excluded.
4907        exclude_static_routes_from_locale_prefix(&mut config, &static_metas);
4908        // `state` already holds an `AutumnConfig` snapshot cloned inside
4909        // `build_state` above — captured BEFORE this mutation. Refresh it, or
4910        // `tenancy_middleware` (which reads config via
4911        // `state.extension::<AutumnConfig>()`) would see a stale copy missing
4912        // these auto-excluded static routes and could misjudge whether a
4913        // `/{locale}`-look-alike path was ever actually locale-prefixed
4914        // (Codex review).
4915        state.insert_extension(config.clone());
4916        state.insert_extension(crate::aeo::RegisteredAgentSkills(agent_skills));
4917        state.insert_extension(commerce_docs);
4918
4919        // Register SEO routes (/robots.txt and /sitemap.xml) when any SEO
4920        // configuration is present or dynamic sources are registered. With
4921        // neither, AEO serves a default `/robots.txt` from the router
4922        // fallback, so an application route at that path always wins.
4923        if !seo_sources.is_empty() || crate::seo::has_seo_config(&config.seo) {
4924            let seo_cfg = &config.seo;
4925            let raw_profile = config.profile.as_deref().unwrap_or("dev");
4926            let profile = crate::seo::effective_seo_profile(raw_profile, seo_cfg.robots.allow_all);
4927            // A static route that declared `seo(robots = "noindex")` must not
4928            // be advertised in sitemap.xml — otherwise the app tells crawlers
4929            // "here is this URL" and "do not index it" at the same time (#1182).
4930            let static_paths: Vec<&str> = static_metas
4931                .iter()
4932                .filter(|m| !crate::seo::defaults_exclude_from_sitemap(m.seo))
4933                .map(|m| m.path)
4934                .collect();
4935            let (robots_body, sitemap_body) = crate::seo::assemble_seo_bodies_with_policy(
4936                profile,
4937                seo_cfg.base_url.as_deref(),
4938                seo_cfg.robots.sitemap_url.as_deref(),
4939                &seo_cfg.robots.additional_rules,
4940                &seo_sources,
4941                &static_paths,
4942                sitemap_locale_config(&config),
4943                &crate::aeo::robots_policy(&config),
4944            )
4945            .await;
4946            let seo_router = crate::seo::build_seo_router_from_bodies(robots_body, sitemap_body);
4947            let is_seo_path = |p: &str| p == "/robots.txt" || p == "/sitemap.xml";
4948            let seo_collision = all_routes.iter().any(|r| is_seo_path(r.path))
4949                || static_metas.iter().any(|m| is_seo_path(m.path))
4950                || scoped_groups.iter().any(|g| {
4951                    let prefix = g.prefix.trim_end_matches('/');
4952                    g.routes
4953                        .iter()
4954                        .any(|r| is_seo_path(&format!("{prefix}{}", r.path)))
4955                })
4956                // Declared plugin routes belong in this check for the same reason
4957                // as the others. A sandboxed manifest may take `/robots.txt` as
4958                // its prefix — a `.` is legal inside a prefix segment — and its
4959                // routes nest after this router merges, so the two would overlap
4960                // and axum would panic at startup. The declared-route preflight
4961                // cannot catch it either: it compares against a claim set built
4962                // from config alone, and SEO also mounts when a source is
4963                // registered in code. Yielding matches what this site already does
4964                // for a custom `#[static_get("/robots.txt")]`, and the operator saw
4965                // the prefix on the consent screen before installing it.
4966                || declared_routes.iter().any(|r| {
4967                    // Only a GET can clash with the generated GETs: a declared
4968                    // POST or HEAD merges cleanly into the same `MethodRouter`
4969                    // (verified against axum 0.8.9, the same finding
4970                    // `reject_declared_framework_collisions` is written on), and
4971                    // a `WS` upgrade mounts as a GET. Yielding to a disjoint
4972                    // verb would hand an untrusted plugin a way to suppress
4973                    // robots.txt and sitemap.xml without even serving them.
4974                    (r.method.eq_ignore_ascii_case("GET")
4975                        || r.method.eq_ignore_ascii_case("WS"))
4976                        && is_seo_path(&r.path)
4977                });
4978            if seo_collision {
4979                tracing::warn!(
4980                    "seo: /robots.txt or /sitemap.xml is already registered by the application; \
4981                     skipping automatic SEO routes to prevent a startup panic"
4982                );
4983            } else {
4984                merge_routers.push(seo_router);
4985            }
4986        }
4987
4988        #[cfg(feature = "inbound-mail")]
4989        if let Some(ref im_router) = inbound_mail_router {
4990            let mut registered_inbound: std::collections::HashSet<String> =
4991                std::collections::HashSet::new();
4992            for (path, axum_router) in crate::inbound_mail::build_routes(im_router) {
4993                // Preflight collision check: if an annotated POST route already
4994                // claims this path, merging an opaque router at the same path
4995                // would cause Axum to panic at startup.  Warn and skip instead
4996                // so the application can still start and the conflict is visible.
4997                if all_routes
4998                    .iter()
4999                    .any(|r| r.method == http::Method::POST && r.path == path)
5000                    || scoped_groups.iter().any(|g| {
5001                        g.routes.iter().any(|r| {
5002                            r.method == http::Method::POST
5003                                && crate::router::join_nested_path(&g.prefix, r.path)
5004                                    == path.as_str()
5005                        })
5006                    })
5007                    || nest_routers.iter().any(|(nest_path, _)| {
5008                        let p = nest_path.as_str();
5009                        path.as_str() == p
5010                            || path.starts_with(p)
5011                                && (p.ends_with('/') || path.as_bytes().get(p.len()) == Some(&b'/'))
5012                    })
5013                {
5014                    tracing::warn!(
5015                        path = %path,
5016                        "inbound_mail: skipping webhook route — a POST handler is \
5017                         already registered at this path by the application"
5018                    );
5019                    continue;
5020                }
5021                // Also guard against two inbound endpoints sharing the same path,
5022                // which would cause the same Axum merge panic.
5023                if !registered_inbound.insert(path.clone()) {
5024                    tracing::warn!(
5025                        path = %path,
5026                        "inbound_mail: skipping duplicate inbound webhook path"
5027                    );
5028                    continue;
5029                }
5030                // Exempt each inbound webhook path from both CSRF and CAPTCHA:
5031                // these routes receive provider-signed POST requests that never
5032                // carry a CSRF or CAPTCHA token.
5033                config.security.csrf.exempt_paths.push(path.clone());
5034                config.security.captcha_exempt_paths.push(path);
5035                merge_routers.push(axum_router);
5036            }
5037        }
5038
5039        // Worker role does not serve user routes: build a probe-only router that
5040        // exposes just the framework liveness/readiness probes and the actuator,
5041        // so orchestrators can supervise the process and `/actuator/jobs` works.
5042        // Web and combined roles build the full application router. All the
5043        // route/router-context inputs assembled above are simply dropped in the
5044        // worker branch.
5045        // Publish the architecture graph this app serves (#1747) before the
5046        // router is built, so `/actuator/graph` answers from the first request
5047        // rather than after some later warm-up. This is the *serving* path —
5048        // the static-build and capsule-replay paths publish their own below.
5049        // `graph_installed_before_every_router_build` pins all three, because a
5050        // graph installed on only some of them is an endpoint that answers 503
5051        // in production while every unit test passes.
5052        //
5053        // A worker serves the probe-only router: it mounts no application route
5054        // at all, and drops every raw router the builder collected. Publishing
5055        // the full route table there would have `/actuator/graph` — which a
5056        // worker can expose — describe endpoints this process does not serve
5057        // (Codex round 4). The declared elements are still nodes, because they
5058        // are still compiled in; they simply report `mounted: false`, and the
5059        // completeness section names them, which is the honest answer to "what
5060        // does this process serve".
5061        let (graph_mounted, graph_opaque) = if role.serves_http() {
5062            (
5063                graph_mounted_routes(
5064                    &all_routes,
5065                    &scoped_groups,
5066                    &declared_routes,
5067                    &config,
5068                    #[cfg(feature = "openapi")]
5069                    openapi.as_ref(),
5070                ),
5071                omitted_router_count(
5072                    merge_routers.len(),
5073                    nest_routers.iter().map(|(prefix, _)| prefix.as_str()),
5074                    &declared_routes,
5075                ),
5076            )
5077        } else {
5078            (graph_worker_mounted_routes(&config), 0)
5079        };
5080        crate::graph::install(
5081            &state,
5082            crate::graph::manifest::audit(&graph_mounted, graph_opaque),
5083        );
5084
5085        let router_build = if role.serves_http() {
5086            crate::router::try_build_router_with_static_inner(
5087                all_routes,
5088                &config,
5089                state.clone(),
5090                dist_ref,
5091                crate::router::RouterContext {
5092                    exception_filters,
5093                    scoped_groups,
5094                    merge_routers,
5095                    nest_routers,
5096                    // The sandboxed-plugin manifests (and any other declared plugin
5097                    // routes) this builder collected. Handing them to the router build is
5098                    // what lets the duplicate-route preflight see inside an otherwise
5099                    // opaque `nest` mount and refuse a collision instead of panicking.
5100                    declared_routes,
5101                    custom_layers,
5102                    static_gate_layers,
5103                    #[cfg(feature = "maud")]
5104                    error_page_renderer,
5105                    session_store,
5106                    // Respect the [openapi] profile gate: if disabled in config,
5107                    // suppress the endpoint even when .openapi(...) was called.
5108                    #[cfg(feature = "openapi")]
5109                    openapi: if config.openapi_runtime.enabled {
5110                        openapi
5111                    } else {
5112                        None
5113                    },
5114                    #[cfg(feature = "mcp")]
5115                    mcp,
5116                    #[cfg(feature = "webmcp")]
5117                    webmcp,
5118                },
5119            )
5120        } else {
5121            crate::router::try_build_probe_only_router(&config, state.clone())
5122        };
5123        let router = router_build.unwrap_or_else(|error| {
5124            tracing::error!(error = %error, "Failed to build router");
5125            exit_stop_managed_pg();
5126            std::process::exit(1);
5127        });
5128
5129        // 7. Bind and initialize pre-serve runtime dependencies. Start listening
5130        // before the startup hooks finish, so `/startup` can report honest
5131        // progress.
5132        // Bind the configured transport. A `server.unix_socket` path selects a
5133        // Unix domain socket (local daemon mode); otherwise bind TCP on
5134        // `host:port`. `bound_desc` is the log description, and
5135        // `unix_socket_cleanup` is the socket to unlink on clean exit — axum does
5136        // not remove it — carried as `(path, dev, inode)` so cleanup can confirm
5137        // the file is still the one this process bound.
5138        // Load the `[push]` VAPID key once, before binding. A key that is present
5139        // but unusable — a typo, an env var that failed to interpolate, a
5140        // mismatched public/private pair — is a hard boot failure rather than a
5141        // quiet fallback to "push disabled". The failure this guards is an app
5142        // that starts cleanly, records subscriptions, and silently delivers
5143        // nothing (#1392). An app with no `[push]` block is unaffected.
5144        if let Err(e) = config.validate_push() {
5145            tracing::error!("Invalid [push] configuration: {e}");
5146            #[cfg(feature = "managed-pg")]
5147            crate::managed_pg::emergency_stop_async().await;
5148            std::process::exit(1);
5149        }
5150
5151        // Validate `[server.tls]` wiring before we bind anything, so a
5152        // misconfiguration is a clear pre-bind failure. Two cases fail fast:
5153        // (1) the section is present but this binary was built without the
5154        // `tls` feature — otherwise it would be silently ignored and the app
5155        // would serve plain HTTP on a port operators expect to be HTTPS;
5156        // (2) TLS is combined with a Unix socket, which the direct-HTTPS path
5157        // does not serve over (TLS terminates on `host:port`).
5158        if let Some(tls_cfg) = config.server.tls.as_ref() {
5159            // Reject an incoherent `[server.tls]` (both static + ACME, neither,
5160            // a half-set cert pair, an empty/wildcard ACME domain set, …) before
5161            // binding, with a named message. Pure config validation, so it runs
5162            // regardless of build features.
5163            if let Err(msg) = tls_cfg.validate() {
5164                tracing::error!("Invalid [server.tls] configuration: {msg}");
5165                #[cfg(feature = "managed-pg")]
5166                crate::managed_pg::emergency_stop_async().await;
5167                std::process::exit(1);
5168            }
5169            #[cfg(not(feature = "tls"))]
5170            {
5171                tracing::error!(
5172                    "[server.tls] is configured but this binary was built without the `tls` \
5173                     feature; rebuild with `--features tls`, or remove [server.tls] to serve \
5174                     plain HTTP"
5175                );
5176                #[cfg(feature = "managed-pg")]
5177                crate::managed_pg::emergency_stop_async().await;
5178                std::process::exit(1);
5179            }
5180            // `[server.tls.acme]` needs the `acme` feature; otherwise it would be
5181            // silently ignored and the app would serve a self-signed placeholder
5182            // (or fail) on a port operators expect to serve a real ACME cert.
5183            #[cfg(all(feature = "tls", not(feature = "acme")))]
5184            if tls_cfg.acme.is_some() {
5185                tracing::error!(
5186                    "[server.tls.acme] is configured but this binary was built without the \
5187                     `acme` feature; rebuild with `--features acme`, or configure a static \
5188                     cert_path/key_path instead"
5189                );
5190                #[cfg(feature = "managed-pg")]
5191                crate::managed_pg::emergency_stop_async().await;
5192                std::process::exit(1);
5193            }
5194            #[cfg(feature = "tls")]
5195            if config.server.unix_socket.is_some() {
5196                tracing::error!(
5197                    "[server.tls] cannot be combined with server.unix_socket; direct TLS \
5198                     terminates on host:port. Unset one of them"
5199                );
5200                #[cfg(feature = "managed-pg")]
5201                crate::managed_pg::emergency_stop_async().await;
5202                std::process::exit(1);
5203            }
5204        }
5205
5206        // Root shutdown token for all background tasks. Created before the bind
5207        // block so the TLS listener's background acceptor task can take a child
5208        // token and stop cleanly on shutdown (issue #1603).
5209        let server_shutdown = tokio_util::sync::CancellationToken::new();
5210
5211        // Carries the cert/key reload wiring from the TLS bind path to the
5212        // background reload task spawned once `server_shutdown` exists.
5213        #[cfg(feature = "tls")]
5214        let mut tls_reload_state: Option<crate::tls::CertReloader> = None;
5215        // The mTLS trust-store reloader (#1640), when
5216        // `[server.tls.client_auth]` is active. Spawned beside the certificate
5217        // reloader below so a CA rotation lands without a restart.
5218        #[cfg(feature = "tls")]
5219        let mut client_trust_reload: Option<crate::tls::client_auth::ClientTrustReloader> = None;
5220
5221        // Carries the ACME challenge listener + renewal task wiring from the TLS
5222        // bind path to the sibling tasks spawned once `server_shutdown` exists.
5223        #[cfg(feature = "acme")]
5224        let mut acme_bind_state: Option<AcmeBindState> = None;
5225
5226        // Where the app actually bound, as the readiness protocol spells it
5227        // (`<transport> <address>`) — distinct from `bound_desc`, which is a
5228        // human-facing log line carrying a scheme and any TLS note. The
5229        // supervisor reads this to write its address-discovery file, so it must
5230        // be the resolved address, not the configured one.
5231        let bound_endpoint: String;
5232        let (bound_listener, bound_desc, unix_socket_cleanup): (
5233            BoundListener,
5234            String,
5235            Option<(std::path::PathBuf, u64, u64)>,
5236        ) = if let Some(socket_path) = config.server.unix_socket.as_deref() {
5237            let _ = socket_path;
5238            #[cfg(unix)]
5239            {
5240                use std::os::unix::fs::PermissionsExt;
5241
5242                let path = std::path::Path::new(socket_path);
5243                if let Err(e) = prepare_unix_socket_path(path) {
5244                    tracing::error!(socket = %socket_path, "Failed to prepare unix socket: {e}");
5245                    // `setup_database` already started the managed Postgres child;
5246                    // `process::exit` skips `on_shutdown`, so stop it first.
5247                    #[cfg(feature = "managed-pg")]
5248                    crate::managed_pg::emergency_stop_async().await;
5249                    std::process::exit(1);
5250                }
5251                // Bind under an owner-only umask so the socket is `0600` from the
5252                // start. A plain bind would briefly leave it group- or
5253                // other-connectable, and a later `chmod` cannot revoke a
5254                // connection already established in that window. This matters for
5255                // a user-configured `server.unix_socket` in a shared dir.
5256                // `umask` is process-wide, so serialize save/bind/restore: a
5257                // concurrent UDS bind in the same process could otherwise
5258                // interleave the pairs and either reopen that window or leave
5259                // `0177` set permanently. The guard is released before the
5260                // `.await` in the error arm below.
5261                let bind_result = {
5262                    static UMASK_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
5263                    let _umask_guard = UMASK_LOCK
5264                        .lock()
5265                        .unwrap_or_else(std::sync::PoisonError::into_inner);
5266                    let prev_umask =
5267                        nix::sys::stat::umask(nix::sys::stat::Mode::from_bits_truncate(0o177));
5268                    let result = tokio::net::UnixListener::bind(path);
5269                    nix::sys::stat::umask(prev_umask);
5270                    result
5271                };
5272                let listener = match bind_result {
5273                    Ok(listener) => listener,
5274                    Err(e) => {
5275                        tracing::error!(socket = %socket_path, "Failed to bind unix socket: {e}");
5276                        #[cfg(feature = "managed-pg")]
5277                        crate::managed_pg::emergency_stop_async().await;
5278                        std::process::exit(1);
5279                    }
5280                };
5281                // Owner-only access, belt-and-suspenders after the umask bind.
5282                // Fail *closed* — if we cannot enforce `0600` (chmod error, an ACL
5283                // /filesystem that rejects it), refuse to serve rather than expose
5284                // a reachable control socket. Remove the socket we just bound so
5285                // nothing keeps listening on it.
5286                if let Err(e) =
5287                    std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600))
5288                {
5289                    tracing::error!(socket = %socket_path, "Failed to enforce owner-only permissions on unix socket: {e}");
5290                    let _ = std::fs::remove_file(path);
5291                    #[cfg(feature = "managed-pg")]
5292                    crate::managed_pg::emergency_stop_async().await;
5293                    std::process::exit(1);
5294                }
5295                // Capture the bound socket's identity so a later successor that
5296                // rebinds the same path isn't unlinked by our shutdown.
5297                let (dev, ino) = {
5298                    use std::os::unix::fs::MetadataExt;
5299                    std::fs::metadata(path).map_or((0, 0), |m| (m.dev(), m.ino()))
5300                };
5301                bound_endpoint = format!("unix {socket_path}");
5302                (
5303                    BoundListener::Unix(listener),
5304                    format!("unix:{socket_path}"),
5305                    Some((path.to_path_buf(), dev, ino)),
5306                )
5307            }
5308            #[cfg(not(unix))]
5309            {
5310                tracing::error!(
5311                    "server.unix_socket is only supported on Unix platforms; \
5312                     unset it or use server.host/server.port"
5313                );
5314                std::process::exit(1);
5315            }
5316        } else {
5317            // A successor that terminates TLS cannot take over a plaintext
5318            // listener: the socket it inherited is mid-conversation with HTTP
5319            // clients, and wrapping it in rustls fails every one of them while
5320            // both builds accept from the shared queue. Refuse the same way an
5321            // existing TLS or Unix listener is refused — before binding
5322            // anything, so the predecessor's wait ends on this process exiting.
5323            #[cfg(feature = "tls")]
5324            if config.server.tls.is_some() && crate::upgrade::handoff_requested() {
5325                tracing::error!(
5326                    "refusing to start: this build terminates TLS ([server.tls]) but was \
5327                     handed the previous build's plaintext listening socket. An in-place \
5328                     upgrade cannot change the transport — restart the process to apply it. \
5329                     The previous build keeps serving"
5330                );
5331                #[cfg(feature = "managed-pg")]
5332                crate::managed_pg::emergency_stop_async().await;
5333                std::process::exit(1);
5334            }
5335            let configured_addr = format!("{}:{}", config.server.host, config.server.port);
5336            let listener = bind_or_adopt_tcp_listener(&configured_addr).await;
5337            // Report where this process is *actually* listening, not what was
5338            // asked for: a socket inherited from a predecessor (#1674) keeps
5339            // that process's port, and `server.port = 0` resolves to whichever
5340            // ephemeral port the kernel picked.
5341            let addr = listener
5342                .local_addr()
5343                .map_or(configured_addr, |bound| bound.to_string());
5344            bound_endpoint = format!("tcp {}", dialable_endpoint(&addr));
5345            // When `[server.tls]` is set (and the `tls` feature is built in),
5346            // wrap the just-bound TCP listener in a rustls acceptor so the same
5347            // host:port serves HTTPS. Fail fast on any cert/key problem — the
5348            // pre-bind guard already rejected a Unix-socket combination and a
5349            // feature-less build, so reaching here with `tls = Some` means the
5350            // feature is on.
5351            #[cfg(feature = "tls")]
5352            {
5353                if let Some(tls_cfg) = config.server.tls.as_ref() {
5354                    // ACME mode: build the resolver from a stored cert if present,
5355                    // else a self-signed placeholder so `:443` binds immediately;
5356                    // the renewal task swaps the real cert in once issued.
5357                    #[cfg(feature = "acme")]
5358                    if let Some(acme_cfg) = tls_cfg.acme.as_ref() {
5359                        let https_port = config.server.port;
5360                        match build_acme_tls_listener(
5361                            listener,
5362                            tls_cfg,
5363                            acme_cfg,
5364                            config.tenancy.base_domain.as_deref(),
5365                            &config.credentials,
5366                            https_port,
5367                            acme_status.clone(),
5368                            server_shutdown.child_token(),
5369                        )
5370                        .await
5371                        {
5372                            Ok((tls_listener, bind_state)) => {
5373                                acme_bind_state = Some(bind_state);
5374                                (
5375                                    BoundListener::Tls(tls_listener),
5376                                    format!("https://{addr} (ACME)"),
5377                                    None,
5378                                )
5379                            }
5380                            Err(e) => {
5381                                tracing::error!(error = %e, "Failed to configure [server.tls.acme]");
5382                                #[cfg(feature = "managed-pg")]
5383                                crate::managed_pg::emergency_stop_async().await;
5384                                std::process::exit(1);
5385                            }
5386                        }
5387                    } else {
5388                        match build_tls_listener(listener, tls_cfg, server_shutdown.child_token()) {
5389                            Ok((tls_listener, reload, client_reload)) => {
5390                                tls_reload_state = Some(reload);
5391                                client_trust_reload = client_reload;
5392                                (
5393                                    BoundListener::Tls(tls_listener),
5394                                    format!("https://{addr}"),
5395                                    None,
5396                                )
5397                            }
5398                            Err(e) => {
5399                                tracing::error!(error = %e, "Failed to configure [server.tls]");
5400                                #[cfg(feature = "managed-pg")]
5401                                crate::managed_pg::emergency_stop_async().await;
5402                                std::process::exit(1);
5403                            }
5404                        }
5405                    }
5406                    #[cfg(not(feature = "acme"))]
5407                    match build_tls_listener(listener, tls_cfg, server_shutdown.child_token()) {
5408                        Ok((tls_listener, reload, client_reload)) => {
5409                            tls_reload_state = Some(reload);
5410                            client_trust_reload = client_reload;
5411                            (
5412                                BoundListener::Tls(tls_listener),
5413                                format!("https://{addr}"),
5414                                None,
5415                            )
5416                        }
5417                        Err(e) => {
5418                            tracing::error!(error = %e, "Failed to configure [server.tls]");
5419                            #[cfg(feature = "managed-pg")]
5420                            crate::managed_pg::emergency_stop_async().await;
5421                            std::process::exit(1);
5422                        }
5423                    }
5424                } else {
5425                    (BoundListener::Tcp(listener), addr, None)
5426                }
5427            }
5428            #[cfg(not(feature = "tls"))]
5429            {
5430                (BoundListener::Tcp(listener), addr, None)
5431            }
5432        };
5433
5434        let shutdown_timeout = config.server.shutdown_timeout_secs;
5435        if let Some(warning) = config.server.drain_window_warning() {
5436            tracing::warn!(
5437                shutdown_timeout_secs = shutdown_timeout,
5438                request_timeout_ms = config.server.timeouts.request_timeout_ms,
5439                "{warning}"
5440            );
5441        }
5442        let prestop_grace = config.server.prestop_grace_secs;
5443
5444        if let Err(error) = initialize_job_runtime(
5445            jobs,
5446            &state,
5447            &server_shutdown,
5448            &config.jobs,
5449            role.runs_workers(),
5450        ) {
5451            tracing::error!(error = %error, "job runtime initialization failed");
5452            // Post-DB failure: `process::exit` skips `on_shutdown`, so stop any
5453            // managed Postgres before bailing.
5454            #[cfg(feature = "managed-pg")]
5455            crate::managed_pg::emergency_stop_async().await;
5456            std::process::exit(1);
5457        }
5458
5459        // Embedded cluster control plane (#1762). Mirrors the
5460        // `crate::alerts::install_from_config` precedent — a no-op when
5461        // `[cluster]` is disabled — but installed here rather than beside alerts
5462        // because it owns a listener and two loops. A cluster that cannot bind or
5463        // start is a hard boot failure: a node that silently never joins would
5464        // serve its own private view of a counter it claims is cluster-wide.
5465        //
5466        // Its token is deliberately not a child of `server_shutdown`. That token
5467        // fires at phase 5, when the listener stops accepting, while in-flight
5468        // requests still drain for up to `shutdown_timeout_secs`. A request served
5469        // during that drain can still increment a cluster counter, and with the
5470        // push loop already departed the increment would land in a document
5471        // nothing replicates and die with the process. The cluster is therefore
5472        // cancelled after the drain completes (see `cluster_shutdown.cancel()`
5473        // below), inside the same budget.
5474        let cluster_shutdown = tokio_util::sync::CancellationToken::new();
5475        if let Err(error) =
5476            crate::cluster::install_from_config(&state, &config.cluster, &cluster_shutdown)
5477        {
5478            tracing::error!(error = %error, "cluster installation failed");
5479            #[cfg(feature = "managed-pg")]
5480            crate::managed_pg::emergency_stop_async().await;
5481            std::process::exit(1);
5482        }
5483
5484        #[cfg(feature = "db")]
5485        {
5486            #[cfg(feature = "ws")]
5487            crate::repository_commit_hooks::set_global_channels(state.channels().clone());
5488        }
5489
5490        // Draining durable after-commit hook rows is background execution, so gate
5491        // it on the process role exactly like the `#[job]` runtime above: a `web`
5492        // replica must not claim or execute hook rows, while `worker` and
5493        // `combined` replicas keep running it. The worker drains rows through a
5494        // Postgres queue (in-process kick, polling and row-locked claiming). Under the
5495        // `sqlite` feature the runtime pool is a SQLite pool the Postgres worker
5496        // cannot drive, so the worker is not spawned.
5497        #[cfg(all(feature = "db", not(feature = "sqlite")))]
5498        if role.runs_workers()
5499            && let Some(pool) = state.pool().cloned()
5500        {
5501            #[cfg(feature = "ws")]
5502            {
5503                let channels = state.channels().clone();
5504                crate::repository_commit_hooks::start_repository_commit_hook_worker(
5505                    pool,
5506                    Some(channels),
5507                    server_shutdown.child_token(),
5508                );
5509            }
5510            #[cfg(not(feature = "ws"))]
5511            crate::repository_commit_hooks::start_repository_commit_hook_worker(
5512                pool,
5513                server_shutdown.child_token(),
5514            );
5515        }
5516        // The outbox relay is background work too: only roles that run
5517        // workers send messages (issue #3062).
5518        #[cfg(feature = "db")]
5519        if role.runs_workers() {
5520            drop(crate::outbox::start_relay_worker(
5521                state.clone(),
5522                server_shutdown.child_token(),
5523            ));
5524        }
5525        // Repositories built over a shard pool (`with_pool`) enqueue durable
5526        // commit hooks into that shard's queue table; drain each one too — again
5527        // only on a role that runs workers, so a web replica leaves shard hook
5528        // rows for the worker tier.
5529        #[cfg(all(feature = "db", not(feature = "sqlite")))]
5530        if role.runs_workers()
5531            && let Some(shards) = state.shards()
5532        {
5533            for shard in shards.iter() {
5534                #[cfg(feature = "ws")]
5535                crate::repository_commit_hooks::start_repository_commit_hook_worker(
5536                    shard.primary_pool().clone(),
5537                    Some(state.channels().clone()),
5538                    server_shutdown.child_token(),
5539                );
5540                #[cfg(not(feature = "ws"))]
5541                crate::repository_commit_hooks::start_repository_commit_hook_worker(
5542                    shard.primary_pool().clone(),
5543                    server_shutdown.child_token(),
5544                );
5545            }
5546        }
5547        // SQLite durable commit-hook worker (#1996 item 5): the runtime pool is a
5548        // single-node SQLite pool the Postgres queue worker cannot drive, so the
5549        // SQLite worker (BEGIN IMMEDIATE claim + in-process Notify kick + poll
5550        // fallback) is spawned instead — same `role.runs_workers()` gate and same
5551        // `server_shutdown.child_token()` graceful-drain wiring as the Postgres tier.
5552        #[cfg(all(feature = "db", feature = "sqlite"))]
5553        if role.runs_workers()
5554            && let Some(pool) = state.pool().cloned()
5555        {
5556            #[cfg(feature = "ws")]
5557            {
5558                let channels = state.channels().clone();
5559                crate::repository_commit_hooks::start_repository_commit_hook_worker(
5560                    pool,
5561                    Some(channels),
5562                    server_shutdown.child_token(),
5563                );
5564            }
5565            #[cfg(not(feature = "ws"))]
5566            crate::repository_commit_hooks::start_repository_commit_hook_worker(
5567                pool,
5568                server_shutdown.child_token(),
5569            );
5570        }
5571        // SQLite fleet (ADR 0019): a commit-hook worker per open database, the
5572        // idle sweeper and the pending-work sweeper (issue 3234). Like
5573        // replication below, the idle sweeper's token is cancelled only after
5574        // the request drain, so the final close of every database (and its
5575        // last replication ship) comes after the last commit. The pending-work
5576        // sweeper stops with the listener, and is awaited before that close.
5577        #[cfg(feature = "sqlite")]
5578        let fleet_shutdown = tokio_util::sync::CancellationToken::new();
5579        #[cfg(feature = "sqlite")]
5580        let mut fleet_maintenance = start_fleet_runtime(
5581            &state,
5582            fleet_shutdown.clone(),
5583            Some(server_shutdown.child_token()),
5584        )
5585        .await;
5586
5587        // The replication loop runs on a dedicated OS thread, not a
5588        // `spawn_blocking` task: it lives for the whole process and does blocking
5589        // file, SQLite, and HTTP work, which would pin a thread in tokio's
5590        // blocking pool — sized for short tasks — forever. It stops with the
5591        // server and ships one final time on the way out.
5592        //
5593        // Its token is deliberately not a child of `server_shutdown`, which fires
5594        // at phase 5 when the listener stops accepting. Requests keep draining
5595        // after that and still commit, so the loop's final tick must come later:
5596        // the token is cancelled explicitly below, once the drain finishes. Same
5597        // reasoning as the cluster's token, for the same class of dropped write.
5598        #[cfg(feature = "db")]
5599        let replication_shutdown = tokio_util::sync::CancellationToken::new();
5600        // Signals that `Replicator::run` has returned — i.e. the final flush is
5601        // done. A channel rather than a `JoinHandle`: waiting on a join means
5602        // blocking, and a blocking wait that times out cannot be cancelled, so a
5603        // stuck upload would hold the runtime open past the shutdown budget it
5604        // was supposed to be bounded by. Dropping this receiver abandons the
5605        // thread instead, which is what the budget expiring is supposed to mean.
5606        #[cfg(feature = "db")]
5607        let mut replication_done: Option<tokio::sync::oneshot::Receiver<()>> = None;
5608        #[cfg(feature = "db")]
5609        if let Some(replicator) = replication_worker.take() {
5610            let replication_shutdown = replication_shutdown.clone();
5611            let (finished, waiter) = tokio::sync::oneshot::channel();
5612            match std::thread::Builder::new()
5613                .name("autumn-sqlite-replication".to_owned())
5614                .spawn(move || {
5615                    replicator.run(&replication_shutdown);
5616                    // The receiver is gone when shutdown stopped waiting; the
5617                    // flush still happened, so there is nothing to report.
5618                    let _ = finished.send(());
5619                }) {
5620                Ok(_handle) => replication_done = Some(waiter),
5621                // Serving on without the replicator is the worst of both
5622                // worlds: the pool has already disabled auto-checkpointing for
5623                // it, so the -wal would grow until the disk filled, and the
5624                // operator would believe they were replicating. Fail loudly
5625                // instead.
5626                Err(e) => {
5627                    tracing::error!("Could not start the SQLite replication thread: {e}");
5628                    #[cfg(feature = "managed-pg")]
5629                    crate::managed_pg::emergency_stop_async().await;
5630                    std::process::exit(1);
5631                }
5632            }
5633        }
5634
5635        // Replica lag (issue #3065): measure in the background, so a replica
5636        // that falls behind leaves read rotation without waiting for `/ready`.
5637        #[cfg(feature = "db")]
5638        let _replica_lag_monitor = spawn_replica_lag_monitor(&state, server_shutdown.child_token());
5639
5640        #[cfg(feature = "presence")]
5641        {
5642            let presence = state.presence().clone();
5643            let sweep_shutdown = server_shutdown.child_token();
5644            tokio::spawn(async move {
5645                let interval = std::time::Duration::from_secs(15);
5646                loop {
5647                    tokio::select! {
5648                        () = tokio::time::sleep(interval) => {
5649                            presence.sweep_expired();
5650                        }
5651                        () = sweep_shutdown.cancelled() => break,
5652                    }
5653                }
5654            });
5655        }
5656
5657        // TLS certificate hot-reload: poll the cert/key file mtimes on an
5658        // interval and swap the served certificate in place on change (e.g.
5659        // after a `certbot`/ACME renewal), so a renewal is picked up WITHOUT a
5660        // restart. A child of `server_shutdown`, exactly like the presence
5661        // sweep above, so it stops cleanly on shutdown. A failed reload logs an
5662        // error and keeps serving the previously loaded certificate — a bad
5663        // renewal never breaks the listener.
5664        #[cfg(feature = "tls")]
5665        if let Some(reload) = tls_reload_state.take() {
5666            let reload_shutdown = server_shutdown.child_token();
5667            tokio::spawn(async move {
5668                reload.run(reload_shutdown).await;
5669            });
5670        }
5671
5672        // mTLS trust-store hot reload (#1640): poll the client-CA bundle and
5673        // CRL and swap the verifier when either changes, so a CA rotation — or
5674        // a newly published revocation — lands without a restart and without
5675        // dropping established connections.
5676        #[cfg(feature = "tls")]
5677        if let Some(reload) = client_trust_reload.take() {
5678            let reload_shutdown = server_shutdown.child_token();
5679            tokio::spawn(async move {
5680                reload.run(reload_shutdown).await;
5681            });
5682        }
5683
5684        // ACME (issue #1608): bind the `:80` HTTP-01 challenge + HTTP→HTTPS
5685        // redirect listener and spawn the renewal loop, each a child of
5686        // `server_shutdown` so they tear down with the main server. The renewal
5687        // loop runs on every replica (a pure `web` replica must renew its own
5688        // cert) and leader-elects through the scheduler coordinator so only one
5689        // replica orders per certificate.
5690        #[cfg(feature = "acme")]
5691        if let Some(bind_state) = acme_bind_state.take() {
5692            let AcmeBindState {
5693                mut renewal_task,
5694                tokens,
5695                http_challenge_port,
5696                https_port,
5697                dns01,
5698                custom_domains,
5699                client_trust_reload: acme_client_trust_reload,
5700            } = bind_state;
5701            // Read before `custom_domains` is moved into the spawn below.
5702            let custom_domains_enabled = custom_domains.is_some();
5703            // The mTLS trust store rotates on this arm too (#1640). Spawned
5704            // HERE, not hoisted into the slot the static-cert arm fills: that
5705            // slot is drained above this block, so an assignment to it would
5706            // never be read and the ACME arm's reloader would never run.
5707            #[cfg(feature = "tls")]
5708            if let Some(reload) = acme_client_trust_reload {
5709                let reload_shutdown = server_shutdown.child_token();
5710                tokio::spawn(async move {
5711                    reload.run(reload_shutdown).await;
5712                });
5713            }
5714
5715            // The `:80` challenge/redirect listener, bound dual-stack so the CA
5716            // can validate HTTP-01 over IPv4 and IPv6 — an AAAA-only host is
5717            // otherwise unreachable on `:80`. Preferred: one `[::]` socket with
5718            // IPV6_V6ONLY=false; on a platform that refuses it, a separate IPv4
5719            // and IPv6 pair, each served below. A bind error under HTTP-01 is
5720            // fail-fast: `:80` needs CAP_NET_BIND_SERVICE and validation cannot
5721            // succeed without it.
5722            //
5723            // Under DNS-01 it is only a warning. The CA never connects here —
5724            // domain control is proved by a TXT record — so the listener is just
5725            // the HTTP→HTTPS redirect. Exiting would kill the deployment #1620
5726            // exists to serve: a container without CAP_NET_BIND_SERVICE using
5727            // DNS-01 because `:80` is unavailable. `autumn doctor` grades this the
5728            // same way, and the runtime must not refuse a config it passes.
5729            let challenge_listeners =
5730                match crate::acme::challenge::bind_challenge_listeners(http_challenge_port).await {
5731                    Ok(listeners) => listeners,
5732                    // Only DNS-01 WITHOUT custom domains can live without this
5733                    // listener. Tenant certificates are always ordered over
5734                    // HTTP-01 — the record lives in the tenant's zone, where
5735                    // this deployment holds no DNS credential — so continuing
5736                    // here would verify every tenant domain and then burn its
5737                    // issuance budget into permanent backoff, never activating.
5738                    Err(e) if dns01 && !custom_domains_enabled => {
5739                        tracing::warn!(
5740                            port = http_challenge_port,
5741                            error = %e,
5742                            "Could not bind the ACME challenge/redirect listener. DNS-01 issuance \
5743                             does not need it, so startup continues — but visitors who type \
5744                             http:// will not be redirected to HTTPS. Grant \
5745                             CAP_NET_BIND_SERVICE, or set [server.tls.acme] http_challenge_port \
5746                             to a port this process may bind"
5747                        );
5748                        Vec::new()
5749                    }
5750                    Err(e) => {
5751                        if dns01 {
5752                            tracing::error!(
5753                                port = http_challenge_port,
5754                                "Failed to bind the ACME HTTP-01 challenge listener: {e}. This \
5755                                 deployment issues its own certificate over DNS-01, which does \
5756                                 not need the listener — but [server.tls.acme.custom_domains] is \
5757                                 enabled, and a tenant's domain can only be validated over \
5758                                 HTTP-01. Grant CAP_NET_BIND_SERVICE, set [server.tls.acme] \
5759                                 http_challenge_port to a port a front-end forwards :80 to, or \
5760                                 disable custom domains"
5761                            );
5762                            #[cfg(feature = "managed-pg")]
5763                            crate::managed_pg::emergency_stop_async().await;
5764                            std::process::exit(1);
5765                        }
5766                        tracing::error!(
5767                            port = http_challenge_port,
5768                            "Failed to bind the ACME HTTP-01 challenge listener: {e}. Port \
5769                             {http_challenge_port} typically needs privilege (grant \
5770                             CAP_NET_BIND_SERVICE), or set [server.tls.acme] http_challenge_port \
5771                             to a port a front-end forwards :80 to"
5772                        );
5773                        #[cfg(feature = "managed-pg")]
5774                        crate::managed_pg::emergency_stop_async().await;
5775                        std::process::exit(1);
5776                    }
5777                };
5778            let challenge_router =
5779                crate::acme::challenge::challenge_router(tokens.clone(), https_port);
5780            // Serve every bound listener (one for dual-stack, two for the split
5781            // fallback), each a child of `server_shutdown` so they tear down with
5782            // the main server. The router is cheap to clone (shared Arc state).
5783            for challenge_listener in challenge_listeners {
5784                let router = challenge_router.clone();
5785                let challenge_shutdown = server_shutdown.child_token();
5786                // A public listener: same `[server.http]` limits (issue #3065).
5787                let limits = crate::http_server::HttpLimits::from(&config.server.http);
5788                tokio::spawn(async move {
5789                    if let Err(e) = crate::http_server::serve(
5790                        challenge_listener,
5791                        router.into_make_service(),
5792                        limits,
5793                        async move {
5794                            challenge_shutdown.cancelled().await;
5795                        },
5796                    )
5797                    .await
5798                    {
5799                        tracing::error!(
5800                            error = %e,
5801                            "ACME challenge listener stopped with an error"
5802                        );
5803                    }
5804                });
5805            }
5806
5807            // Build the coordinator for leader election (whatever the role) and
5808            // the reporter callback, then spawn the renewal loop.
5809            //
5810            // `leadership_degraded` marks the dangerous case: a distributed
5811            // backend was configured — multi-replica intent — but
5812            // `coordinator_from_config` could not build one (no DB pool, or no
5813            // `db` feature here) and fell back to a per-process coordinator. It is
5814            // keyed off both the configured backend and the actual fallback, so it
5815            // never fires for a genuinely single-replica `in_process` deployment.
5816            // When set, the renewal loop refuses to order (see `AcmeRenewalTask`)
5817            // rather than let every replica take a local lease and race the CA.
5818            let mut leadership_degraded = false;
5819            let coordinator =
5820                match crate::scheduler::coordinator_from_config(&config.scheduler, &state) {
5821                    Ok(c) => c,
5822                    Err(e) => {
5823                        tracing::warn!(
5824                            error = %e,
5825                            "ACME renewal: falling back to an in-process coordinator"
5826                        );
5827                        leadership_degraded = config.scheduler.backend.is_fleet_distributed();
5828                        std::sync::Arc::new(crate::scheduler::InProcessSchedulerCoordinator::new(
5829                            config.scheduler.resolved_replica_id(),
5830                        ))
5831                    }
5832                };
5833            renewal_task.leadership_degraded = leadership_degraded;
5834
5835            // A distributed scheduler backend means several processes serve
5836            // this app — a fleet on `postgres`, or processes on one host on
5837            // `sqlite`. ACME is not safe across either without care: see
5838            // `acme_fleet_warning` for the two hazards, which of them each
5839            // backend actually carries, and why DNS-01 retires only one of them
5840            // on a fleet. Warn loudly rather than silently mis-serving (#1620).
5841            if let Some(message) = acme_fleet_warning(config.scheduler.backend, dns01) {
5842                tracing::warn!(scheduler_backend = coordinator.backend(), "{message}");
5843            }
5844
5845            #[cfg(feature = "reporting")]
5846            let reporter = make_acme_reporter(acme_reporters);
5847            #[cfg(not(feature = "reporting"))]
5848            let reporter = make_acme_reporter();
5849            // Certificate renewal is a framework-scheduled operation, so a
5850            // failed issuance/renewal raises #1610's `scheduled_task_failure`
5851            // alert — reaching the operator's configured destination (email,
5852            // Slack, PagerDuty) rather than only the error-reporting sink. The
5853            // renew-before window (default 30 days) means this fires with weeks
5854            // of validity left, not at expiry (#1620).
5855            let reporter = compose_acme_alert_reporter(reporter, &state);
5856            renewal_task.recovery = Some(make_acme_alert_recovery(&state));
5857            let renewal_shutdown = server_shutdown.child_token();
5858            let renewal_coordinator = std::sync::Arc::clone(&coordinator);
5859            tokio::spawn(async move {
5860                renewal_task
5861                    .run(renewal_coordinator, reporter, renewal_shutdown)
5862                    .await;
5863            });
5864
5865            // Tenant custom domains (#1635): publish the registry so tenancy
5866            // resolution can route a connected `Host`, register the health
5867            // indicator and the retention pruner, and spawn the orchestrator.
5868            if let Some(cd) = custom_domains {
5869                spawn_custom_domain_task(
5870                    cd,
5871                    tokens,
5872                    std::sync::Arc::clone(&coordinator),
5873                    leadership_degraded,
5874                    &state,
5875                    server_shutdown.child_token(),
5876                );
5877            }
5878        }
5879
5880        tracing::info!(bound = %bound_desc, "Listening");
5881
5882        let server_shutdown_wait = server_shutdown.clone();
5883        // Wrap the built router with the HTML form method-override layer at the
5884        // very edge — outside path and method routing — so a plain browser
5885        // `<form method="post">` carrying `_method=PUT|PATCH|DELETE` reaches the
5886        // declared handler. In axum 0.8 `Router::layer` applies middleware per
5887        // registered method handler, which is too late: the inner `MethodRouter`
5888        // returns `405` before a layered service runs. Wrapping the whole router
5889        // as a `tower::Service` is the documented way to run middleware before
5890        // route matching. `TrustedProxiesLayer` must be outermost, stamped before
5891        // `MethodOverrideLayer` reads `ResolvedClientIdentity` for its
5892        // same-origin form check.
5893        let after_method = tower::Layer::layer(
5894            &crate::middleware::MethodOverrideLayer::new()
5895                .with_max_scan_bytes(config.security.upload.max_request_size_bytes),
5896            router,
5897        );
5898        let service = tower::Layer::layer(
5899            &crate::security::TrustedProxiesLayer::from_config(&config.security.trusted_proxies),
5900            after_method,
5901        );
5902        // Spawn the serve task per transport. The arms differ only in the
5903        // connect-info type baked into the make-service (`SocketAddr` for TCP,
5904        // `UdsConnectInfo` for Unix sockets); the shutdown wiring and resulting
5905        // `JoinHandle<io::Result<()>>` are identical. Handlers extracting
5906        // `ConnectInfo<SocketAddr>` are unsupported under a Unix socket — daemon
5907        // mode is local and loopback-equivalent.
5908        // A duplicate of the listening socket is kept aside so a `SIGUSR2`
5909        // in-place upgrade (#1674) can hand it to a successor while this process
5910        // keeps serving through the original. Only a plain TCP listener can be
5911        // handed over in this release. This runs up front, before the match
5912        // consumes `bound_listener` to build the (not yet spawned) accept-loop
5913        // future below.
5914        #[cfg(unix)]
5915        let mut handoff_socket: Option<crate::upgrade::HandoffSocket> = None;
5916        #[cfg(unix)]
5917        if let BoundListener::Tcp(listener) = &bound_listener {
5918            match crate::upgrade::HandoffSocket::from_listener(listener) {
5919                Ok(socket) => handoff_socket = Some(socket),
5920                Err(e) => tracing::warn!(
5921                    error = %e,
5922                    "could not duplicate the listening socket; in-place upgrade \
5923                     (SIGUSR2) will be refused for this process"
5924                ),
5925            }
5926        }
5927
5928        // Build the accept-loop future per transport; when it goes live
5929        // depends on how this process started (see below). The arms differ
5930        // only in the connect-info type baked into the
5931        // make-service (`SocketAddr` for TCP, `UdsConnectInfo` for Unix
5932        // sockets); the shutdown wiring and the resulting `io::Result<()>` are
5933        // identical. Handlers extracting `ConnectInfo<SocketAddr>` are
5934        // unsupported under a Unix socket — daemon mode is local and
5935        // loopback-equivalent.
5936        //
5937        // The deferred half of #2368: adopting the inherited fd still happens
5938        // up front (so a failure to adopt aborts early, as today), but during
5939        // an in-place upgrade the successor must not compete for connections
5940        // before it can actually serve them: every connection it wins in that
5941        // window is answered by the startup barrier with a 503 while the
5942        // predecessor is right there, healthy. The predecessor keeps serving
5943        // for the whole window; the successor's accept loop goes live once
5944        // `run_startup_hooks` has returned `Ok`.
5945        //
5946        // The deferral applies ONLY to an upgrade successor
5947        // (`handoff_requested()`): a cold start spawns its accept loop
5948        // immediately, as before, so `/live` and `/startup` stay reachable
5949        // behind the startup barrier while the hooks run — a hook that
5950        // outlasts a probe threshold must not read as a dead pod.
5951        // `[server.http]` limits for every listener kind (issue #3065).
5952        let http_limits = crate::http_server::HttpLimits::from(&config.server.http);
5953        let server_future: std::pin::Pin<
5954            Box<dyn std::future::Future<Output = std::io::Result<()>> + Send + 'static>,
5955        > = match bound_listener {
5956            BoundListener::Tcp(listener) => {
5957                // The listener is wrapped (`StopAcceptingOnShutdown`), so the
5958                // connect info is `TcpPeer`; this re-stamps the
5959                // `ConnectInfo<SocketAddr>` everything below reads.
5960                let service = tower::Layer::layer(
5961                    &axum::middleware::from_fn(crate::accept_drain::stamp_tcp_peer),
5962                    service,
5963                );
5964                let make_service =
5965                    axum::ServiceExt::<axum::extract::Request>::into_make_service_with_connect_info::<
5966                        crate::accept_drain::TcpPeer,
5967                    >(service);
5968                Box::pin(async move {
5969                    crate::http_server::serve(
5970                        crate::accept_drain::StopAcceptingOnShutdown::new(
5971                            listener,
5972                            server_shutdown_wait.clone(),
5973                        ),
5974                        make_service,
5975                        http_limits,
5976                        crate::accept_drain::drain_signal(server_shutdown_wait),
5977                    )
5978                    .await
5979                })
5980            }
5981            #[cfg(unix)]
5982            BoundListener::Unix(listener) => {
5983                // UDS requests carry no TCP peer, so stamp a loopback identity
5984                // before `TrustedProxiesLayer` runs — local daemon requests then
5985                // resolve a `ClientAddr` (and IP-based maintenance/rate-limit
5986                // behavior works) exactly like a localhost TCP connection.
5987                let service = tower::Layer::layer(
5988                    &axum::middleware::from_fn(stamp_loopback_connect_info),
5989                    service,
5990                );
5991                let make_service =
5992                    axum::ServiceExt::<axum::extract::Request>::into_make_service_with_connect_info::<
5993                        UdsConnectInfo,
5994                    >(service);
5995                Box::pin(async move {
5996                    crate::http_server::serve(
5997                        crate::accept_drain::StopAcceptingOnShutdown::new(
5998                            listener,
5999                            server_shutdown_wait.clone(),
6000                        ),
6001                        make_service,
6002                        http_limits,
6003                        crate::accept_drain::drain_signal(server_shutdown_wait),
6004                    )
6005                    .await
6006                })
6007            }
6008            // HTTPS arm: mirrors the TCP arm. The connect info is
6009            // `TlsConnectInfo` rather than a bare `SocketAddr` so the verified
6010            // mTLS client identity (#1640) rides along with the peer address;
6011            // `ClientIdentityLayer` immediately re-stamps
6012            // `ConnectInfo<SocketAddr>` from it, so `TrustedProxiesLayer` /
6013            // `ClientAddr` resolution, SSE and wss streaming, rate limiting and
6014            // shutdown wiring all behave exactly as on plain TCP. Only the
6015            // rustls handshake inside the listener's `accept` differs.
6016            #[cfg(feature = "tls")]
6017            BoundListener::Tls(mut listener) => {
6018                // The listener takes the `max_connections` slots before TCP
6019                // accept, so a connection counts during its TLS handshake too.
6020                let handoff = http_limits
6021                    .max_connections
6022                    .and_then(|max| listener.limit_connections(max));
6023                // Applied inside the connect-info layer (which
6024                // `into_make_service_with_connect_info` installs outermost), so
6025                // this sees `ConnectInfo<TlsConnectInfo>` and everything below
6026                // it sees `ConnectInfo<SocketAddr>` plus the identity.
6027                //
6028                // The route-level mTLS requirement (#1640) is deliberately NOT
6029                // applied here. It lives inside the router
6030                // (`build_client_cert_requirement_layer`), so the MCP dispatch
6031                // clone traverses it and a rejection flows through the rest of
6032                // the response stack. Only the identity plumbing belongs at
6033                // this boundary, because `ConnectInfo<TlsConnectInfo>` exists
6034                // nowhere else.
6035                let service =
6036                    tower::Layer::layer(&crate::tls::client_auth::ClientIdentityLayer, service);
6037                let make_service =
6038                    axum::ServiceExt::<axum::extract::Request>::into_make_service_with_connect_info::<
6039                        crate::tls::TlsConnectInfo,
6040                    >(service);
6041                Box::pin(async move {
6042                    crate::http_server::serve_with_handoff(
6043                        crate::accept_drain::StopAcceptingOnShutdown::new(
6044                            listener,
6045                            server_shutdown_wait.clone(),
6046                        ),
6047                        make_service,
6048                        http_limits,
6049                        handoff,
6050                        crate::accept_drain::drain_signal(server_shutdown_wait),
6051                    )
6052                    .await
6053                })
6054            }
6055        };
6056
6057        // Whether this process is the successor half of an in-place upgrade
6058        // (#1674). Only a successor shares its listening socket with a live
6059        // predecessor, so only a successor defers its accept loop past the
6060        // startup hooks (#2368). A cold start spawns immediately, exactly as
6061        // before this change.
6062        let defer_accept_loop = crate::upgrade::handoff_requested();
6063        // The accept-loop future while it is not yet live; `None` once it has
6064        // been taken to spawn `server_task`.
6065        let mut pending_accept: Option<
6066            std::pin::Pin<
6067                Box<dyn std::future::Future<Output = std::io::Result<()>> + Send + 'static>,
6068            >,
6069        > = None;
6070        let mut server_task: Option<tokio::task::JoinHandle<std::io::Result<()>>> = None;
6071        if defer_accept_loop {
6072            pending_accept = Some(server_future);
6073        } else {
6074            server_task = Some(tokio::spawn(server_future));
6075        }
6076
6077        // Cancelled by the in-place upgrade watcher once a successor has taken
6078        // over the listening socket; the drain below then runs without the
6079        // load-balancer choreography a real shutdown needs (#1674).
6080        let upgrade_cutover = tokio_util::sync::CancellationToken::new();
6081        let upgrade_cutover_wait = upgrade_cutover.clone();
6082
6083        // In-place upgrade watcher (#1674): on `SIGUSR2`, hand this process's
6084        // listening socket and designated live state to a freshly-execed build
6085        // and, once that build is serving, cancel `upgrade_cutover` so the
6086        // drain below runs.
6087        #[cfg(unix)]
6088        {
6089            let upgrade_config = config.server.upgrade.clone();
6090            let upgrade_state = state.clone();
6091            let cutover = upgrade_cutover.clone();
6092            let socket = handoff_socket.take();
6093            // A child of `server_shutdown`, so an ordinary drain ends the
6094            // watcher: it drops the duplicated listening socket (which would
6095            // otherwise keep the port bound and accepting into a queue nobody
6096            // serves for the whole drain), and drops any handoff in flight,
6097            // whose cleanup kills the half-started successor and unfreezes the
6098            // live state.
6099            let watcher_shutdown = server_shutdown.child_token();
6100            tokio::spawn(async move {
6101                tokio::select! {
6102                    () = watch_for_in_place_upgrade(
6103                        &upgrade_config,
6104                        upgrade_signal,
6105                        socket,
6106                        upgrade_state,
6107                        cutover,
6108                    ) => {}
6109                    () = watcher_shutdown.cancelled() => {
6110                        tracing::debug!(
6111                            "shutting down: in-place upgrade is no longer available in this \
6112                             process"
6113                        );
6114                    }
6115                }
6116            });
6117        }
6118
6119        let shutdown_state = state.clone();
6120        let shutdown_signal_token = server_shutdown.clone();
6121        let handler_shutdown = state.probes.shutdown_signal().clone();
6122        // Clone metrics so the drain-watchdog can record aborted requests.
6123        let shutdown_metrics = state.metrics.clone();
6124
6125        // Shared timestamp: set by shutdown_task when the listener is cancelled
6126        // (phase 5). Main reads it after server_task completes to measure only
6127        // actual drain time for hook budget — not the app's full uptime.
6128        let drain_started_at: std::sync::Arc<std::sync::OnceLock<crate::time::MonotonicInstant>> =
6129            std::sync::Arc::new(std::sync::OnceLock::new());
6130        let drain_started_clone = std::sync::Arc::clone(&drain_started_at);
6131        // The same moment on the real clock. The fleet's final flush bounds
6132        // blocking I/O, so it needs a reading a fixed or unadvanced injected
6133        // clock cannot freeze.
6134        let drain_started_real: std::sync::Arc<std::sync::OnceLock<std::time::Instant>> =
6135            std::sync::Arc::new(std::sync::OnceLock::new());
6136        let drain_started_real_clone = std::sync::Arc::clone(&drain_started_real);
6137        // The shutdown task does not capture `state`, so hand it its own handle
6138        // on the injected clock — the drain window is measured on the same
6139        // monotonic timeline `hook_budget` is computed from below.
6140        let drain_clock = state.clock_arc();
6141        let drain_clock_for_task = std::sync::Arc::clone(&drain_clock);
6142
6143        // Notified by main just before server_task.await (after startup hooks
6144        // complete). If SIGTERM arrives during startup hooks the watchdog waits
6145        // here so the drain deadline is always measured from when drain starts.
6146        let drain_phase_notify = std::sync::Arc::new(tokio::sync::Notify::new());
6147        let drain_phase_notify_for_watchdog = std::sync::Arc::clone(&drain_phase_notify);
6148        // Boolean companion so the watchdog can skip the wait when SIGTERM arrives
6149        // after startup has already finished (the common case).
6150        let server_entered_drain = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
6151        let server_entered_drain_for_watchdog = std::sync::Arc::clone(&server_entered_drain);
6152
6153        // Shutdown task: the rolling-deploy lifecycle phases.
6154        //
6155        //   1. SIGTERM / Ctrl-C received
6156        //   2. /ready → 503  (probe flips before listener closes)
6157        //   3. prestop_grace elapses  (load-balancer deregistration window)
6158        //   4. WebSocket sessions receive close frame
6159        //   5. TCP listener stops accepting new connections; jobs/scheduler
6160        //      stop dequeuing (they share server_shutdown CancellationToken)
6161        //   6. In-flight requests drain within shutdown_timeout_secs; past the
6162        //      deadline the watchdog exits with code 1 and records
6163        //      autumn_shutdown_aborted_requests_total.
6164        //
6165        // Phases 7-9 (on_shutdown hooks, telemetry flush, DB pool close) run in
6166        // main after server_task completes, within the remaining part of the same
6167        // shutdown_timeout_secs budget — not an additional window.
6168        let shutdown_task = tokio::spawn(async move {
6169            // Phase 1: Wait for an OS signal — or for an in-place upgrade
6170            // (#1674) whose successor is already serving on this same socket.
6171            let cause = shutdown_signal(upgrade_cutover_wait).await;
6172            let upgrade_cutover = matches!(cause, DrainCause::UpgradeCutover);
6173
6174            if upgrade_cutover {
6175                // Phases 2 and 3 exist to let a load balancer take this replica
6176                // out of rotation before its socket closes. An in-place upgrade
6177                // has no such gap to cover: the successor is already accepting
6178                // on the *same* listening socket, so flipping `/ready` to 503
6179                // would only make a live address look unhealthy, and the
6180                // prestop grace would delay the handover for nothing.
6181                tracing::info!(
6182                    phase = "upgrade_cutover",
6183                    shutdown_timeout_secs = shutdown_timeout,
6184                    "shutdown: successor is serving; draining without a readiness flip"
6185                );
6186            } else {
6187                tracing::info!(
6188                    phase = "signal_received",
6189                    prestop_grace_secs = prestop_grace,
6190                    shutdown_timeout_secs = shutdown_timeout,
6191                    "shutdown: graceful shutdown initiated"
6192                );
6193
6194                // Phase 2: flip /ready → 503 strictly before the listener closes.
6195                shutdown_state.begin_shutdown();
6196                tracing::info!(phase = "ready_draining", "shutdown: /ready now 503");
6197
6198                // Phase 3: prestop grace — wait for load balancers to deregister.
6199                if prestop_grace > 0 {
6200                    tokio::time::sleep(std::time::Duration::from_secs(prestop_grace)).await;
6201                }
6202            }
6203            tracing::info!(phase = "listener_stopping", "shutdown: stopping listener");
6204
6205            // Phase 4: send WebSocket close frames and cancel the
6206            // `ShutdownToken` of running handlers.
6207            handler_shutdown.cancel();
6208
6209            // Phase 5: stop listener and signal jobs/scheduler to stop dequeuing.
6210            // Record drain-start before cancelling so main gets the right hook
6211            // budget even in the startup-overlap case.
6212            let _ = drain_started_clone.set(drain_clock_for_task.monotonic());
6213            #[allow(
6214                clippy::disallowed_methods,
6215                reason = "real time: the fleet flush bounds blocking I/O"
6216            )]
6217            let _ = drain_started_real_clone.set(std::time::Instant::now());
6218            shutdown_signal_token.cancel();
6219
6220            // Phase 6: drain watchdog. If the in-flight drain exceeds the budget,
6221            // record the aborted count and force a non-zero exit before hooks run.
6222            //
6223            // Always measure the deadline from when the drain actually starts, so
6224            // in-flight requests get the full shutdown_timeout_secs window. On a
6225            // normal SIGTERM after startup, server_entered_drain is already true:
6226            // skip the wait and sleep the full budget. On a SIGTERM during hooks,
6227            // wait for the notify first. Without that wait, hooks completing just
6228            // before the watchdog fires would let it exit(1) at once, with no
6229            // fresh drain window for requests that arrived after the hooks.
6230            if !server_entered_drain_for_watchdog.load(std::sync::atomic::Ordering::Acquire) {
6231                tracing::warn!(
6232                    phase = "signal_during_startup",
6233                    "shutdown: SIGTERM during startup hooks; waiting for drain phase \
6234                     to begin before enforcing the drain deadline"
6235                );
6236                // Suspend until main fires notify_one() at drain start.
6237                // Orchestrator hard-kill backstop: if hooks never complete, the
6238                // orchestrator's kill_timeout / terminationGracePeriodSeconds kills us.
6239                drain_phase_notify_for_watchdog.notified().await;
6240            }
6241            tokio::time::sleep(std::time::Duration::from_secs(shutdown_timeout)).await;
6242            // Guard against the boundary race where server_task completes at
6243            // exactly the deadline before main has called shutdown_task.abort().
6244            // Zero active requests means drain completed cleanly; return and let
6245            // main complete the cleanup path.
6246            if shutdown_metrics.snapshot().http.requests_active == 0 {
6247                return;
6248            }
6249            let aborted = shutdown_metrics.snapshot().http.requests_active;
6250            shutdown_metrics.record_shutdown_aborted(aborted);
6251            tracing::error!(
6252                phase = "in_flight_drain",
6253                timeout_secs = shutdown_timeout,
6254                autumn_shutdown_aborted_requests_total = aborted,
6255                exit_code = 1,
6256                "shutdown: in_flight_drain phase exceeded deadline; terminating"
6257            );
6258            // The watchdog's `process::exit` skips the remaining `on_shutdown`
6259            // hooks — including a managed-Postgres `stop()` — so a drain that
6260            // overruns its budget would orphan the postmaster. Stop it here too.
6261            #[cfg(feature = "managed-pg")]
6262            crate::managed_pg::emergency_stop_async().await;
6263            std::process::exit(1);
6264        });
6265
6266        if let Err(error) = run_startup_hooks(&startup_hooks, state.clone()).await {
6267            tracing::error!(error = %error, "startup hook failed");
6268            // A cold start already spawned the accept loop above, so stop it.
6269            // An upgrade successor (#2368) never spawned one, so there is
6270            // nothing to abort: the predecessor simply keeps serving through
6271            // the adopted socket while this process exits, and no user ever
6272            // sees a 503.
6273            if let Some(task) = server_task.take() {
6274                task.abort();
6275            }
6276            server_shutdown.cancel();
6277            // `process::exit` skips `on_shutdown`; stop any managed Postgres.
6278            #[cfg(feature = "managed-pg")]
6279            crate::managed_pg::emergency_stop_async().await;
6280            std::process::exit(1);
6281        }
6282
6283        // Go live on the accept loop. An upgrade successor (#2368) deferred it
6284        // past the hooks; they have now succeeded, so it can serve the
6285        // connections it wins on the shared socket — the predecessor has been
6286        // serving them until now. A cold start spawned the loop up front, so
6287        // `/live` and `/startup` stayed reachable behind the startup barrier
6288        // while the hooks ran.
6289        let server_task: tokio::task::JoinHandle<std::io::Result<()>> =
6290            match (server_task.take(), pending_accept.take()) {
6291                (Some(task), _) => task,
6292                (None, Some(future)) => tokio::spawn(future),
6293                // Unreachable: `pending_accept` is `None` only once it has
6294                // been taken to spawn `server_task` on the cold-start path.
6295                (None, None) => {
6296                    unreachable!("accept-loop future consumed without spawning its task")
6297                }
6298            };
6299
6300        if !state.probes().is_shutting_down() {
6301            // Web role runs no cron scheduler (workers/combined only). Skipping
6302            // the scheduler must not regress readiness: mark_startup_complete and
6303            // signal_serve_ready below still run.
6304            if role.runs_workers() && !tasks.is_empty() {
6305                if let Some(warning) =
6306                    in_process_fleet_warning(&config, &tasks, &crate::config::OsEnv)
6307                {
6308                    tracing::warn!("{warning}");
6309                }
6310                let res = start_task_scheduler_with_config(
6311                    tasks,
6312                    &state,
6313                    &server_shutdown,
6314                    &config.scheduler,
6315                );
6316                if let Err(err) = res {
6317                    tracing::error!(error = %err, "scheduled task runtime initialization failed");
6318                    server_shutdown.cancel();
6319                    server_task.abort();
6320                    // `process::exit` skips `on_shutdown`; stop any managed Postgres.
6321                    #[cfg(feature = "managed-pg")]
6322                    crate::managed_pg::emergency_stop_async().await;
6323                    std::process::exit(1);
6324                }
6325            }
6326            state.probes().mark_startup_complete();
6327            // Release a predecessor that is waiting on this build (#1674). Only
6328            // now, with startup hooks done and the router serving, is it safe
6329            // for the old process to stop accepting — and only if this build
6330            // actually took over everything it was handed.
6331            #[cfg(unix)]
6332            {
6333                if let Err(reason) = crate::upgrade::verify_handover_complete() {
6334                    tracing::error!("{reason}");
6335                    // `process::exit` skips `on_shutdown`; stop any managed Postgres.
6336                    #[cfg(feature = "managed-pg")]
6337                    crate::managed_pg::emergency_stop_async().await;
6338                    std::process::exit(1);
6339                }
6340                // Publishing readiness is the last thing that can fail, so the
6341                // adopted state stays frozen until it succeeds. A readiness signal
6342                // that never reaches the predecessor — a full or read-only handoff
6343                // filesystem — means the predecessor times out and kills this
6344                // process, taking anything acknowledged meanwhile with it.
6345                // Refusing here ends that wait when this process exits, in ~20 ms
6346                // rather than the readiness timeout, and it resumes writable.
6347                // The unfreeze runs under the handoff lock.
6348                match crate::upgrade::publish_upgrade_readiness(|| {
6349                    unfreeze_adopted_live_state(&state);
6350                }) {
6351                    Ok(_) => {}
6352                    Err(error) => {
6353                        tracing::error!(
6354                            error = %error,
6355                            "refusing to start: this build took over but could not tell the \
6356                             previous build it is serving, so the handover cannot complete. \
6357                             The previous build keeps serving"
6358                        );
6359                        // `process::exit` skips `on_shutdown`; stop any managed Postgres.
6360                        #[cfg(feature = "managed-pg")]
6361                        crate::managed_pg::emergency_stop_async().await;
6362                        std::process::exit(1);
6363                    }
6364                }
6365            }
6366            signal_serve_ready(
6367                config
6368                    .server
6369                    .prestop_grace_secs
6370                    .saturating_add(config.server.shutdown_timeout_secs),
6371                &bound_endpoint,
6372            );
6373        }
6374
6375        // Signal the drain phase. The watchdog checks the flag for the common
6376        // case (SIGTERM arrives after startup) and waits on the notify for the
6377        // rare case (SIGTERM arrived during startup hooks). Both must be set so
6378        // the watchdog never re-enforces the deadline before drain actually starts.
6379        server_entered_drain.store(true, std::sync::atomic::Ordering::Release);
6380        drain_phase_notify.notify_one();
6381
6382        // Wait for the server to drain all in-flight requests.  The drain
6383        // watchdog in shutdown_task will force-exit if drain takes too long.
6384        let server_result = server_task.await.unwrap_or_else(|e| {
6385            tracing::error!("Server task join error: {e}");
6386            // `process::exit` skips the `on_shutdown` hooks, so stop a managed
6387            // Postgres child here to avoid orphaning it on an accept-loop/join
6388            // failure (direct/foreground runs have no CLI reaper).
6389            exit_stop_managed_pg();
6390            std::process::exit(1);
6391        });
6392        // Drain completed within the deadline; abort the watchdog.
6393        shutdown_task.abort();
6394        server_result.unwrap_or_else(|e| {
6395            tracing::error!("Server error: {e}");
6396            exit_stop_managed_pg();
6397            std::process::exit(1);
6398        });
6399
6400        // How much of `shutdown_timeout_secs` the drain has spent so far,
6401        // measured on the injected clock from the instant phase 5 recorded.
6402        // Read twice below — once for the departure wait, once for the hook
6403        // budget — because everything after the drain shares that one budget.
6404        let elapsed_since_drain_start = || {
6405            drain_started_at
6406                .get()
6407                .map_or(std::time::Duration::ZERO, |started| {
6408                    drain_clock.monotonic().saturating_duration_since(*started)
6409                })
6410        };
6411        let shutdown_budget = std::time::Duration::from_secs(shutdown_timeout);
6412
6413        // Phase 6a: the replication loop's final flush. Cancelling the token at
6414        // phase 5 only wakes that loop; the tick that ships the last committed
6415        // frames runs after it, and nothing waits for that tick unless this does.
6416        // Requests have drained, so no further transaction can commit and this is
6417        // the last flush there will be — the difference between a clean stop that
6418        // loses nothing and one that leaves the tail of the WAL only on a machine
6419        // that may be about to go away.
6420        //
6421        // Bounded by what the drain left of `shutdown_timeout_secs`, on the same
6422        // budget as the departure and the hooks below: a destination that has gone
6423        // away must not hold the process open past what a supervisor allows.
6424        // Overrunning it is loud rather than silent — the operator's RPO is at stake.
6425        // Phase 6a (fleet): close every open fleet database. Each close waits
6426        // for its in-flight connections, then runs the lifecycle hooks — a
6427        // fleet replicator ships the database's last frames there. Same budget
6428        // and same reasoning as the control replication flush below.
6429        #[cfg(feature = "sqlite")]
6430        if let Some(FleetRuntime {
6431            maintenance,
6432            sweeper,
6433            ownership,
6434        }) = fleet_maintenance.take()
6435        {
6436            let flush_budget = || {
6437                let real_elapsed = drain_started_real
6438                    .get()
6439                    .map_or(std::time::Duration::ZERO, std::time::Instant::elapsed);
6440                fleet_flush_budget(shutdown_budget, elapsed_since_drain_start(), real_elapsed)
6441            };
6442            // The sweeper's last pass ends before its databases close. It
6443            // spends from the same budget.
6444            if let Some(sweeper) = sweeper {
6445                let abort = sweeper.abort_handle();
6446                if tokio::time::timeout(flush_budget(), sweeper).await.is_err() {
6447                    abort.abort();
6448                    tracing::warn!("shutdown: the fleet sweeper did not stop within the budget");
6449                }
6450            }
6451            let wait = flush_budget();
6452            // Fix the cutoff now. The maintenance task may wake late, and the
6453            // wait above counts from here.
6454            if let Some(replication) = state
6455                .shards()
6456                .and_then(|s| s.fleet())
6457                .and_then(|f| f.replication())
6458            {
6459                replication.set_shutdown_deadline_in(wait);
6460            }
6461            fleet_shutdown.cancel();
6462            if tokio::time::timeout(wait, maintenance).await.is_ok() {
6463                tracing::info!("shutdown: SQLite fleet databases closed");
6464                // Only after the close: the last frames ship under the epoch.
6465                if let Some(ownership) = ownership {
6466                    ownership.finish().await;
6467                }
6468            } else {
6469                tracing::warn!(
6470                    timeout_secs = wait.as_secs(),
6471                    "shutdown: SQLite fleet databases did not all close within the shutdown \
6472                     budget; a replicated fleet's last frames may not be offsite"
6473                );
6474                // Renewals stop, so the ranges lapse. No release: a database
6475                // may still be open.
6476                if let Some(ownership) = ownership {
6477                    drop(ownership.abandon());
6478                }
6479            }
6480        }
6481
6482        #[cfg(feature = "db")]
6483        if let Some(waiter) = replication_done.take() {
6484            // Only now: every request that will ever commit has committed, so
6485            // the tick this releases is genuinely the last one.
6486            replication_shutdown.cancel();
6487            let wait = shutdown_budget.saturating_sub(elapsed_since_drain_start());
6488            match tokio::time::timeout(wait, waiter).await {
6489                Ok(Ok(())) => {
6490                    tracing::info!("shutdown: SQLite replication flushed and stopped");
6491                }
6492                Ok(Err(_)) => tracing::error!(
6493                    "shutdown: the SQLite replication thread ended without finishing; the \
6494                     frames committed since the last successful tick may not be offsite"
6495                ),
6496                // The thread is left running and the process exits without it.
6497                // Waiting further is what the budget exists to prevent, and a
6498                // blocking join could not be abandoned at all.
6499                Err(_) => tracing::warn!(
6500                    timeout_secs = wait.as_secs(),
6501                    "shutdown: SQLite replication did not finish its final flush within the \
6502                     shutdown budget; the last frames may not be offsite"
6503                ),
6504            }
6505        }
6506
6507        // Phase 6b: the cluster departs only now, once no request can still be
6508        // running — an increment accepted during the drain must have a push loop
6509        // left to replicate it. Ordering, all inside the one
6510        // `shutdown_timeout_secs` budget: drain → departure → hooks. The departure
6511        // is bounded by `LEAVE_BUDGET` inside the node, and this waits for it, but
6512        // only for what the drain left (`cluster_departure_wait`): a supervisor
6513        // times the process out on `shutdown_timeout_secs`, and an unconditional
6514        // wait after a slow drain would push past it. The hook budget below
6515        // subtracts this wait from the same clock reading, so the three phases add
6516        // up to the budget rather than budget + `LEAVE_BUDGET`. A departure that is
6517        // budgeted away leaves the peer to converge on the suspicion timeout, which
6518        // is the actual contract.
6519        if config.cluster.enabled {
6520            cluster_shutdown.cancel();
6521            let departure_wait =
6522                cluster_departure_wait(shutdown_budget, elapsed_since_drain_start());
6523            if !departure_wait.is_zero() {
6524                tokio::time::sleep(departure_wait).await;
6525            }
6526        }
6527
6528        // Phase 7: run on_shutdown hooks within the *remaining* portion of
6529        // shutdown_timeout_secs (drain + departure + hooks share one budget,
6530        // not three).
6531        // Plugin ordering: plugins register during build() before app hooks,
6532        // so app hooks run before plugin hooks (LIFO = last-registered first).
6533        let hook_budget = shutdown_budget.saturating_sub(elapsed_since_drain_start());
6534        run_shutdown_hooks_with_timeout(&shutdown_hooks, hook_budget, hook_budget).await;
6535        // If request drain consumed the whole `shutdown_timeout_secs`, the
6536        // managed-Postgres `on_shutdown` hook may have been budgeted away above.
6537        // Stop the cluster directly here (idempotent — a no-op once the hook
6538        // already stopped it) so a direct/foreground run, which has no CLI
6539        // reaper, never leaves the postmaster holding the data dir/port.
6540        #[cfg(feature = "managed-pg")]
6541        crate::managed_pg::emergency_stop_async().await;
6542
6543        // Remove the Unix socket file on clean exit; axum does not unlink it.
6544        // (An abnormal force-exit may leave it behind, but the next bind's
6545        // `prepare_unix_socket_path` reclaims a stale socket.) Only unlink if the
6546        // socket is still the one we bound — a successor that rebound the same
6547        // path after we closed has a different inode, and removing it would make
6548        // the new server unreachable.
6549        #[cfg(unix)]
6550        if let Some((path, dev, ino)) = &unix_socket_cleanup {
6551            use std::os::unix::fs::MetadataExt;
6552            let still_ours =
6553                std::fs::metadata(path).is_ok_and(|m| m.dev() == *dev && m.ino() == *ino);
6554            if still_ours {
6555                let _ = std::fs::remove_file(path);
6556            }
6557        }
6558        #[cfg(not(unix))]
6559        let _ = &unix_socket_cleanup;
6560
6561        tracing::info!(exit_code = 0, "shutdown: all phases completed cleanly");
6562    }
6563
6564    /// Render all registered static routes to `dist/` and exit.
6565    ///
6566    /// Triggered when `AUTUMN_BUILD_STATIC=1` is set (by `autumn build`).
6567    /// Builds the Axum router, renders each static route through it, and
6568    /// writes HTML + manifest to the `dist/` directory.
6569    #[allow(clippy::too_many_lines)]
6570    async fn run_build_mode(self) {
6571        let Self {
6572            routes,
6573            api_versions,
6574            route_sources: _,
6575            current_plugin: _,
6576            tasks: _,
6577            one_off_tasks: _,
6578            jobs: _,
6579            listeners,
6580            #[cfg(feature = "db")]
6581                outbox_handlers: _,
6582            static_metas,
6583            exception_filters: _,
6584            scoped_groups,
6585            merge_routers: _,
6586            nest_routers: _,
6587            custom_layers,
6588            static_gate_layers: _,
6589            startup_hooks: _,
6590            state_initializers,
6591            shutdown_hooks: _,
6592            extensions: _,
6593            registered_plugins: _,
6594            plugin_contracts: _,
6595            plugin_config_roots,
6596            #[cfg(feature = "maud")]
6597                error_page_renderer: _,
6598            #[cfg(feature = "db")]
6599                migrations: _,
6600            config_loader_factory,
6601            #[cfg(feature = "db")]
6602            pool_provider_factory,
6603            #[cfg(feature = "db")]
6604            shard_provider_factory,
6605            #[cfg(feature = "db")]
6606            shard_router,
6607            #[cfg(feature = "db")]
6608            directory_shard_router,
6609            telemetry_provider,
6610            session_store,
6611            #[cfg(feature = "ws")]
6612            channels_backend,
6613            #[cfg(feature = "storage")]
6614            blob_store,
6615            cache_backend,
6616            #[cfg(feature = "reporting")]
6617            error_reporters,
6618            alert_channels: _,
6619            #[cfg(feature = "openapi")]
6620            openapi,
6621            #[cfg(feature = "mcp")]
6622                mcp: _,
6623            #[cfg(feature = "webmcp")]
6624                webmcp: _,
6625            audit_logger: _,
6626            #[cfg(feature = "i18n")]
6627            i18n_bundle,
6628            #[cfg(feature = "i18n")]
6629            i18n_auto_load,
6630            #[cfg(feature = "embed-assets")]
6631            embedded_static,
6632            #[cfg(all(feature = "embed-assets", feature = "i18n"))]
6633            embedded_locales,
6634            policy_registrations,
6635            #[cfg(feature = "mail")]
6636            mail_delivery_queue_factory,
6637            #[cfg(feature = "mail")]
6638            suppression_store,
6639            #[cfg(feature = "mail")]
6640            mail_suppression_store,
6641            #[cfg(feature = "mail")]
6642            mount_unsubscribe_endpoint,
6643            #[cfg(feature = "mail")]
6644            mail_previews,
6645            #[cfg(feature = "maud")]
6646            story_gallery,
6647            declared_routes: _,
6648            plugin_asset_bundles: _,
6649            idempotency_enabled,
6650            #[cfg(feature = "mail")]
6651            mail_interceptor,
6652            job_interceptor,
6653            #[cfg(feature = "db")]
6654            db_interceptor,
6655            #[cfg(feature = "ws")]
6656            channels_interceptor,
6657            #[cfg(feature = "oauth2")]
6658            http_interceptor,
6659            seo_sources,
6660            agent_skills,
6661            commerce_docs,
6662            metrics_sources,
6663            health_indicators,
6664            #[cfg(feature = "inbound-mail")]
6665                inbound_mail_router: _,
6666        } = self;
6667
6668        let _ = &api_versions;
6669        let _ = &metrics_sources;
6670        let _ = &health_indicators;
6671        let all_routes = routes;
6672
6673        // Load config (same as normal startup)
6674        let (mut config, telemetry_guard) = load_config_and_telemetry(
6675            config_loader_factory,
6676            telemetry_provider,
6677            plugin_config_roots,
6678        )
6679        .await;
6680
6681        #[cfg(feature = "mail")]
6682        apply_mail_builder_overrides(&mut config, mount_unsubscribe_endpoint);
6683        if idempotency_enabled {
6684            let env_disabled = std::env::var("AUTUMN_IDEMPOTENCY__ENABLED")
6685                .is_ok_and(|v| matches!(v.to_lowercase().as_str(), "false" | "0" | "no" | "off"));
6686            // Only apply the builder default when neither the env var nor the
6687            // loaded config file explicitly sets enabled = false.
6688            if !env_disabled && config.idempotency.enabled != Some(false) {
6689                config.idempotency.enabled = Some(true);
6690            }
6691        }
6692
6693        // Register the embedded `static/` tree (if any) before the router is
6694        // built so `/static/*` serves from the binary and `asset_url()` resolves
6695        // against the embedded manifest, then prefer embedded locales over disk
6696        // auto-loading when no explicit bundle was provided.
6697        #[cfg(feature = "embed-assets")]
6698        register_embedded_static_dir(embedded_static);
6699
6700        #[cfg(all(feature = "embed-assets", feature = "i18n"))]
6701        let i18n_bundle = embedded_i18n_bundle(i18n_bundle, embedded_locales, &config);
6702
6703        #[cfg(feature = "i18n")]
6704        let i18n_bundle =
6705            resolve_i18n_bundle(i18n_bundle, i18n_auto_load, &config, &crate::config::OsEnv);
6706
6707        // Snapshot ApiDocs before all_routes is moved into the router builder.
6708        // Includes top-level routes and scoped groups (with prefixed paths) so
6709        // the emitted dist/openapi.json matches what the runtime spec serves.
6710        #[cfg(feature = "openapi")]
6711        let api_docs_snapshot: Vec<crate::openapi::ApiDoc> = {
6712            let mut docs: Vec<crate::openapi::ApiDoc> = all_routes
6713                .iter()
6714                .map(|r| {
6715                    let mut doc = r.api_doc.clone();
6716                    doc.api_version = r.api_version;
6717                    doc.sunset_opt_out = r.sunset_opt_out;
6718                    doc
6719                })
6720                .collect();
6721            for group in &scoped_groups {
6722                // Mirror the same normalization as the runtime OpenAPI builder:
6723                // use join_nested_path for correct trailing-slash handling, and
6724                // merge prefix path params so they appear in the operation.
6725                let prefix_params = crate::router::extract_path_params(&group.prefix);
6726                for route in &group.routes {
6727                    let mut doc = route.api_doc.clone();
6728                    doc.api_version = route.api_version;
6729                    doc.sunset_opt_out = route.sunset_opt_out;
6730                    let full = crate::router::join_nested_path(&group.prefix, route.api_doc.path);
6731                    doc.path = Box::leak(full.into_boxed_str());
6732                    if !prefix_params.is_empty() {
6733                        let mut merged: Vec<&'static str> = prefix_params
6734                            .iter()
6735                            .map(|p| &*Box::leak(p.clone().into_boxed_str()))
6736                            .collect();
6737                        merged.extend_from_slice(doc.path_params);
6738                        doc.path_params = Box::leak(merged.into_boxed_slice());
6739                    }
6740                    docs.push(doc);
6741                }
6742            }
6743            docs
6744        };
6745
6746        if static_metas.is_empty() {
6747            let allow_empty = std::env::var(BUILD_STATIC_ALLOW_EMPTY_ENV).as_deref() == Ok("1");
6748            if allow_empty {
6749                let dist_dir = project_dir("dist", &crate::config::OsEnv);
6750                if let Err(error) = clear_stale_static_output(&dist_dir) {
6751                    eprintln!("Failed to remove stale {}: {error}", dist_dir.display());
6752                    std::process::exit(1);
6753                }
6754                eprintln!("No static routes registered. Nothing to render.");
6755            } else {
6756                eprintln!("No static routes registered. Nothing to build.");
6757                eprintln!("Hint: use .static_routes(static_routes![...]) on your AppBuilder.");
6758            }
6759            std::process::exit(no_static_routes_exit_code(allow_empty));
6760        }
6761
6762        // Fail-fast on invalid session config — only when no custom store
6763        // was installed. Symmetrical to the same check in run() so static
6764        // builds don't run migrations against a doomed boot either.
6765        fail_fast_on_invalid_session_config(&config, session_store.is_some());
6766        fail_fast_on_invalid_signing_secret(&config);
6767        fail_fast_on_missing_encryption_keys(&config);
6768        fail_fast_on_invalid_trusted_hosts(&config);
6769
6770        // Preflight the configured BlobStore the same way `run()` does.
6771        // Static routes can read presigned URLs out of `BlobStoreState`
6772        // during pre-rendering (e.g. `<img src=blob.url()>`); without
6773        // the bootstrap they'd 500 during `autumn build` even though
6774        // the server path works. A custom store from `.with_blob_store()`
6775        // bypasses config-driven instantiation.
6776        #[cfg(feature = "storage")]
6777        let storage_bootstrap = blob_store.map_or_else(
6778            || preflight_storage(&config),
6779            |store| {
6780                Some(StorageBootstrap {
6781                    store,
6782                    serving: None,
6783                })
6784            },
6785        );
6786
6787        // Build state (with DB if configured)
6788        #[cfg(feature = "db")]
6789        let database = setup_database(
6790            &config,
6791            vec![],
6792            pool_provider_factory,
6793            shard_provider_factory,
6794            shard_router,
6795            directory_shard_router,
6796            RepositoryCommitHookQueueMigrationMode::StaticBuild,
6797        )
6798        .await
6799        .unwrap_or_else(|e| {
6800            eprintln!("{e}");
6801            std::process::exit(1);
6802        });
6803        #[cfg(feature = "db")]
6804        let pool = database.topology;
6805        #[cfg(feature = "db")]
6806        let shards = database.shards;
6807        #[cfg(feature = "db")]
6808        let replica_readiness = database.replica_readiness;
6809        #[cfg(feature = "db")]
6810        let replica_migration_check = database.replica_migration_check;
6811
6812        let mut state = build_state(
6813            &config,
6814            #[cfg(feature = "db")]
6815            pool.as_ref(),
6816            #[cfg(feature = "db")]
6817            shards,
6818            #[cfg(feature = "ws")]
6819            channels_backend,
6820        );
6821        if let Some(buf) = telemetry_guard.log_buffer.clone() {
6822            state.insert_extension(buf);
6823        }
6824        // Wire the live-subscriber reload handle into the loggers actuator so
6825        // `PUT /actuator/loggers/{name}` affects the running subscriber, not
6826        // just an in-memory map (issue #1044).
6827        if let Some(handle) = telemetry_guard.filter_reload.clone() {
6828            state.log_levels().attach_reload_handle(handle);
6829        }
6830        state.insert_extension(RegisteredApiVersions(api_versions.clone()));
6831        // Clone class: this interceptor-wiring block (mail/job/db/channels,
6832        // including the channels -> presence -> collab backend swap, and
6833        // http) is byte-for-byte identical in `run()`, `run_build_mode()`
6834        // and `run_one_off_task_mode()`. It was born duplicated by #862,
6835        // whose own follow-up commit ("fix: wire interceptors into
6836        // one-off task mode") is a missed-fix this duplication already
6837        // produced once: the one-off task path first shipped without this
6838        // block, so its interceptors were silently inert. #1806 later had
6839        // to hand-repeat the channels/presence/collab swap in all three
6840        // copies again. Change one, change the other two --
6841        // `optional_interceptor_wiring_matches_across_all_three_run_modes`
6842        // below fails if they drift.
6843        #[cfg(feature = "mail")]
6844        if let Some(interceptor) = mail_interceptor {
6845            state.insert_extension(interceptor);
6846        }
6847        if let Some(interceptor) = job_interceptor {
6848            state.insert_extension(interceptor);
6849        }
6850        #[cfg(feature = "db")]
6851        if let Some(interceptor) = db_interceptor {
6852            state.insert_extension(interceptor);
6853        }
6854        #[cfg(feature = "ws")]
6855        if let Some(interceptor) = channels_interceptor {
6856            state.insert_extension(interceptor.clone());
6857            state.channels = crate::channels::Channels::with_shared_backend(std::sync::Arc::new(
6858                crate::channels::InterceptedChannelsBackend::new(
6859                    state.channels.backend().clone(),
6860                    vec![interceptor],
6861                ),
6862            ));
6863            #[cfg(feature = "presence")]
6864            {
6865                state.presence = crate::presence::Presence::new(state.channels.clone());
6866                // The collaboration hub holds the channel registry and the
6867                // presence tracker it was built with, so replacing either
6868                // leaves it publishing into the old backend (#1806).
6869                #[cfg(feature = "collab")]
6870                {
6871                    state.collab = crate::collab::CollabHub::new(
6872                        state.channels.clone(),
6873                        state.presence.clone(),
6874                    );
6875                }
6876            }
6877        }
6878        #[cfg(feature = "oauth2")]
6879        if let Some(interceptor) = http_interceptor {
6880            state.insert_extension(interceptor);
6881        }
6882        #[cfg(feature = "db")]
6883        configure_replica_migration_check(&state, replica_migration_check);
6884        #[cfg(feature = "db")]
6885        apply_replica_migration_readiness(&state, replica_readiness);
6886        if let Some(cache) = cache_backend {
6887            // Wrapped once, so the global and the state hold the same `Arc`.
6888            let cache = crate::cache::with_capsule_seam(cache);
6889            crate::cache::set_global_cache(cache.clone());
6890            state.shared_cache = Some(cache);
6891        } else {
6892            crate::cache::clear_global_cache();
6893        }
6894        #[cfg(feature = "reporting")]
6895        if !error_reporters.is_empty() {
6896            state.insert_extension(crate::reporting::RegisteredReporters(error_reporters));
6897        }
6898        // Static-site builds are short-lived and don't run the request loop,
6899        // so deliver_later is never invoked. install_mailer_with_factory skips
6900        // the queue factory when enforce_durable_guard is false (the factory
6901        // may open Redis/Harvest connections unavailable here), and the guard
6902        // itself is bypassed too — the Mailer is still installed so static
6903        // routes that extract `Mailer` for immediate `send` calls resolve.
6904        #[cfg(feature = "mail")]
6905        if let Some(handle) = suppression_store {
6906            state.insert_extension(handle);
6907        }
6908        #[cfg(feature = "mail")]
6909        if let Some(handle) = mail_suppression_store {
6910            state.insert_extension(handle);
6911        }
6912        #[cfg(feature = "mail")]
6913        crate::mail::install_mailer_with_factory(
6914            &state,
6915            &config.mail,
6916            mail_delivery_queue_factory,
6917            false,
6918        )
6919        .unwrap_or_else(|error| {
6920            eprintln!("Failed to configure mailer: {error}");
6921            exit_stop_managed_pg();
6922            std::process::exit(1);
6923        });
6924        #[cfg(feature = "mail")]
6925        state.insert_extension(crate::mail::MailPreviewRegistry::new(mail_previews));
6926        #[cfg(feature = "maud")]
6927        install_story_registry(&state, story_gallery);
6928        // run_build_mode used ProbeState::default(), which does not start as pending
6929        state.probes = crate::probe::ProbeState::default();
6930        state.apply_health_config(&config.health);
6931
6932        // Apply deferred policy and scope registrations onto the live app state,
6933        // as `run()` does. Static routes can carry `#[authorize]` checks or sit
6934        // behind `#[repository(policy = ..., scope = ...)]` index endpoints;
6935        // without registering here, every such pre-render call would 500 at build
6936        // time with "no policy/scope registered", and `render_static_routes` would
6937        // treat that as a build failure even though `.policy(...)`/`.scope(...)`
6938        // was configured on the builder.
6939        for register in policy_registrations {
6940            register(state.policy_registry());
6941        }
6942
6943        #[cfg(feature = "i18n")]
6944        let custom_layers =
6945            install_i18n_bundle_layer(custom_layers, &state, i18n_bundle, &config.i18n);
6946
6947        // #2405: render through the pre-layer router — the same layer
6948        // composition the ISR regeneration path uses
6949        // (`partition_custom_layers_for_static_render`, shared with the SSG
6950        // serve path) — so the recorded Content-Type and the body on disk are
6951        // the handler's own, not the app layer stack's post-layer output. The
6952        // serve path applies the drained layers to the cached response at
6953        // request time, outside the static-first middleware, so recording the
6954        // post-layer output both double-applies the layers (once at
6955        // generation, once per request) and — because ISR's type guard sees
6956        // the pre-layer response — refuses every regeneration for an app with
6957        // a Content-Type-rewriting layer, freezing the route until the next
6958        // build. The drained set is dropped: this process exits after the
6959        // render; serving is a separate invocation.
6960        let (custom_layers, _drained) =
6961            crate::router::partition_custom_layers_for_static_render(custom_layers);
6962
6963        // Install the preflighted storage and remember the serving
6964        // router so static generation hits the same `/_blobs/...`
6965        // routes the server path serves.
6966        #[cfg(feature = "storage")]
6967        let storage_router = storage_bootstrap.and_then(|b| b.install(&state));
6968        install_webhook_registry(&state, &config);
6969        run_state_initializers(state_initializers, &state);
6970        crate::cost::install(&state, &config);
6971        // Static generation has no job runtime, so register only sync listeners.
6972        // Durable listeners are dropped entirely (not just their jobs) so a
6973        // static route publishing such an event is a clean no-op for the durable
6974        // side effect rather than a "job runtime not initialized" error.
6975        let sync_listeners: Vec<_> = listeners
6976            .into_iter()
6977            .filter(|listener| listener.mode == crate::events::DispatchMode::Sync)
6978            .collect();
6979        finalize_event_bus(sync_listeners, &mut Vec::new(), &state);
6980
6981        // Build the router for static rendering. Use the inner builder
6982        // so the custom session store installed via with_session_store(...)
6983        // is honored during static generation — apps that swap in a custom
6984        // store specifically to avoid Redis/external backends at build time
6985        // would otherwise silently fall back to the config-driven backend.
6986        // Custom Tower layers registered via .layer(...) are deliberately
6987        // NOT applied here (#2405): they were drained above, so the render
6988        // sees the handler's own response — the same pre-layer composition
6989        // ISR regeneration uses. The serve path applies those layers to the
6990        // cached response at request time, which is what makes the recorded
6991        // Content-Type and the body on disk mean "what the handler declared"
6992        // rather than "what the layer stack happened to produce".
6993        #[cfg_attr(not(feature = "storage"), allow(unused_mut))]
6994        let mut merge_routers: Vec<axum::Router<AppState>> = Vec::new();
6995        #[cfg(feature = "storage")]
6996        if let Some(router) = storage_router {
6997            merge_routers.push(router);
6998        }
6999        // Static routes are pre-rendered by requesting their single,
7000        // unprefixed path — never locale-aware — so they must stay excluded
7001        // from locale-prefix routing even when it's enabled (issue #1251).
7002        exclude_static_routes_from_locale_prefix(&mut config, &static_metas);
7003        // Refresh the AppState-stored config snapshot — see the matching
7004        // comment in `run()` (Codex review).
7005        state.insert_extension(config.clone());
7006        state.insert_extension(crate::aeo::RegisteredAgentSkills(agent_skills));
7007        state.insert_extension(commerce_docs);
7008        // Publish the architecture graph this app serves (#1747) before the
7009        // router is built, so `/actuator/graph` can answer from the first
7010        // request rather than after some later warm-up.
7011        // The agent documents list the API that `dist/openapi.json` (written
7012        // below) describes. The build writes that file at the site root
7013        // whatever path the runtime serves it on, and writes no Swagger UI. The
7014        // router below always mounts it, so the census sees the same config
7015        // with the runtime gate open.
7016        #[cfg(feature = "openapi")]
7017        let static_openapi = openapi
7018            .clone()
7019            .map(|api| api.openapi_json_path("/openapi.json").swagger_ui_path(None));
7020        #[cfg(feature = "openapi")]
7021        let census_config = {
7022            let mut census = config.clone();
7023            census.openapi_runtime.enabled = true;
7024            census
7025        };
7026        #[cfg(not(feature = "openapi"))]
7027        let census_config = config.clone();
7028        crate::graph::install(
7029            &state,
7030            crate::graph::manifest::audit(
7031                &graph_mounted_routes(
7032                    &all_routes,
7033                    &scoped_groups,
7034                    &[],
7035                    &census_config,
7036                    #[cfg(feature = "openapi")]
7037                    static_openapi.as_ref(),
7038                ),
7039                // The static-build path builds its router with no nest mounts and
7040                // no declared plugin routes (see the `RouterContext` below), so the
7041                // merge count is the whole opaque surface here.
7042                omitted_router_count(merge_routers.len(), std::iter::empty::<&str>(), &[]),
7043            ),
7044        );
7045        // The router build installs the AEO site in the state's shared
7046        // extensions; the agent documents below read its home `Link`s.
7047        let aeo_state = state.clone();
7048        let router = crate::router::try_build_router_inner(
7049            all_routes,
7050            &config,
7051            state,
7052            crate::router::RouterContext {
7053                exception_filters: Vec::new(),
7054                scoped_groups,
7055                merge_routers,
7056                nest_routers: Vec::new(),
7057                declared_routes: Vec::new(),
7058                custom_layers,
7059                static_gate_layers: Vec::new(),
7060                #[cfg(feature = "maud")]
7061                error_page_renderer: None,
7062                session_store,
7063                #[cfg(feature = "openapi")]
7064                openapi: static_openapi,
7065                #[cfg(feature = "mcp")]
7066                mcp: None,
7067                #[cfg(feature = "webmcp")]
7068                webmcp: None,
7069            },
7070        )
7071        .unwrap_or_else(|error| {
7072            eprintln!("Failed to build router: {error}");
7073            exit_stop_managed_pg();
7074            std::process::exit(1);
7075        });
7076
7077        let env = crate::config::OsEnv;
7078        let dist_dir = project_dir("dist", &env);
7079
7080        eprintln!("Building {} static route(s)...", static_metas.len());
7081
7082        match crate::static_gen::render_static_routes(router.clone(), &static_metas, &dist_dir)
7083            .await
7084        {
7085            Ok(()) => {
7086                eprintln!(
7087                    "\n  \u{2713} Static build complete \u{2192} {}",
7088                    dist_dir.display()
7089                );
7090            }
7091            Err(e) => {
7092                eprintln!("\n  \u{2717} Static build failed: {e}");
7093                exit_stop_managed_pg();
7094                std::process::exit(1);
7095            }
7096        }
7097
7098        // Agent documents (llms.txt, skills, ARD, ...). Their URLs are
7099        // absolute, so they need `[seo] base_url`.
7100        if config.aeo.enabled {
7101            let home_links = aeo_state
7102                .extension::<crate::aeo::AeoSite>()
7103                .map(|site| site.negotiate.home_links.clone())
7104                .unwrap_or_default();
7105            match crate::aeo::write_static_documents(
7106                router,
7107                config.seo.base_url.as_deref(),
7108                &home_links,
7109                &dist_dir,
7110            )
7111            .await
7112            {
7113                Ok(paths) => {
7114                    for path in paths {
7115                        eprintln!("  \u{2713} AEO: {path} written");
7116                    }
7117                    if config.seo.base_url.is_none() {
7118                        eprintln!(
7119                            "  \u{2139} AEO: set [seo] base_url to write the agent documents"
7120                        );
7121                    }
7122                }
7123                Err(e) => eprintln!("  \u{26A0} Failed to write the agent documents: {e}"),
7124            }
7125        }
7126
7127        // When OpenAPI is configured, write the spec to dist/ so consumers
7128        // can retrieve a machine-readable API contract alongside the HTML.
7129        #[cfg(feature = "openapi")]
7130        if let Some(mut openapi_config) = openapi {
7131            openapi_config.api_versions = api_versions;
7132            let openapi_config =
7133                openapi_config.session_cookie_name(config.session.cookie_name.clone());
7134            let docs: Vec<&crate::openapi::ApiDoc> = api_docs_snapshot.iter().collect();
7135            let spec = crate::openapi::generate_spec(&openapi_config, &docs);
7136            let written = crate::openapi::write_openapi_spec_to_dist(&spec, &dist_dir)
7137                .and_then(|()| crate::aeo::commerce::write_mpp_spec(&spec, &config, &dist_dir));
7138            match written {
7139                Ok(()) => {
7140                    eprintln!(
7141                        "  \u{2713} OpenAPI spec written \u{2192} {}/openapi.json",
7142                        dist_dir.display()
7143                    );
7144                }
7145                Err(e) => {
7146                    eprintln!("  \u{26A0} Failed to write OpenAPI spec: {e}");
7147                }
7148            }
7149        }
7150
7151        // Write robots.txt and sitemap.xml to dist/ — when SEO is configured,
7152        // dynamic sources are registered, or AEO is on; never overwrite files
7153        // already produced by a custom #[static_get("/robots.txt")] route.
7154        if !seo_sources.is_empty() || crate::seo::has_seo_config(&config.seo) || config.aeo.enabled
7155        {
7156            let seo_cfg = &config.seo;
7157            let raw_profile = config.profile.as_deref().unwrap_or("dev");
7158            // Here for AEO alone, the file is AEO's: it classes profiles as
7159            // the server's AEO `robots.txt` does, so a custom production
7160            // profile does not ship `Disallow: /`.
7161            let aeo_only = seo_sources.is_empty() && !crate::seo::has_seo_config(seo_cfg);
7162            let profile = if aeo_only {
7163                crate::aeo::robots_profile(&config)
7164            } else {
7165                crate::seo::effective_seo_profile(raw_profile, seo_cfg.robots.allow_all)
7166            };
7167            // A static route that declared `seo(robots = "noindex")` must not
7168            // be advertised in sitemap.xml — otherwise the app tells crawlers
7169            // "here is this URL" and "do not index it" at the same time (#1182).
7170            let static_paths: Vec<&str> = static_metas
7171                .iter()
7172                .filter(|m| !crate::seo::defaults_exclude_from_sitemap(m.seo))
7173                .map(|m| m.path)
7174                .collect();
7175            let (robots_body, sitemap_body) = crate::seo::assemble_seo_bodies_with_policy(
7176                profile,
7177                seo_cfg.base_url.as_deref(),
7178                seo_cfg.robots.sitemap_url.as_deref(),
7179                &seo_cfg.robots.additional_rules,
7180                &seo_sources,
7181                &static_paths,
7182                sitemap_locale_config(&config),
7183                &crate::aeo::robots_policy(&config),
7184            )
7185            .await;
7186            // Write each file only if it wasn't already produced by a
7187            // custom #[static_get] route.
7188            let robots_path = dist_dir.join("robots.txt");
7189            let sitemap_path = dist_dir.join("sitemap.xml");
7190            if robots_path.exists() {
7191                eprintln!(
7192                    "  \u{2713} SEO: robots.txt already present (custom static route), skipping"
7193                );
7194            } else {
7195                match tokio::fs::write(&robots_path, robots_body).await {
7196                    Ok(()) => eprintln!(
7197                        "  \u{2713} SEO: robots.txt written \u{2192} {}",
7198                        robots_path.display()
7199                    ),
7200                    Err(e) => eprintln!("  \u{26A0} Failed to write robots.txt: {e}"),
7201                }
7202            }
7203            if sitemap_path.exists() {
7204                eprintln!(
7205                    "  \u{2713} SEO: sitemap.xml already present (custom static route), skipping"
7206                );
7207            } else {
7208                match tokio::fs::write(&sitemap_path, sitemap_body).await {
7209                    Ok(()) => eprintln!(
7210                        "  \u{2713} SEO: sitemap.xml written \u{2192} {}",
7211                        sitemap_path.display()
7212                    ),
7213                    Err(e) => eprintln!("  \u{26A0} Failed to write sitemap.xml: {e}"),
7214                }
7215            }
7216        }
7217
7218        // Build finished: stop the managed Postgres child `setup_database` may
7219        // have started. Build mode discards the app's `on_shutdown` hooks, so
7220        // without this even a *successful* `autumn build` would leak the cluster.
7221        #[cfg(feature = "managed-pg")]
7222        crate::managed_pg::emergency_stop_async().await;
7223    }
7224
7225    /// Dump the application's architecture graph as JSON and exit.
7226    ///
7227    /// Triggered when `AUTUMN_DUMP_GRAPH=1` is set (by `autumn graph`).
7228    /// Does not connect to a database or bind a TCP port.
7229    async fn run_dump_graph_mode(self) {
7230        let (mounted, opaque) = self.dump_graph_surface().await;
7231        crate::graph::manifest::print_manifest_dump(&crate::graph::manifest::audit(
7232            &mounted, opaque,
7233        ));
7234    }
7235
7236    /// The served surface the graph dump joins against.
7237    ///
7238    /// The framework's mounts depend on configuration, so this reads the app's
7239    /// own config through the builder's loader and plugin roots, as normal
7240    /// startup does. It starts no telemetry and opens no database. A config
7241    /// that fails to load falls back to defaults, so the graph still dumps.
7242    /// A worker role gets the worker census, as at runtime.
7243    async fn dump_graph_surface(self) -> (Vec<crate::graph::MountedRoute>, usize) {
7244        let opaque = self.graph_opaque_router_count();
7245        let config =
7246            match try_load_config(self.config_loader_factory, self.plugin_config_roots).await {
7247                Ok(config) => config,
7248                Err(error) => {
7249                    // Stdout carries the graph, so the warning goes to stderr.
7250                    eprintln!("warning: config failed to load, graph uses defaults: {error}");
7251                    crate::config::AutumnConfig::default()
7252                }
7253            };
7254        if !config.role.serves_http() {
7255            return (graph_worker_mounted_routes(&config), 0);
7256        }
7257        let mounted = graph_mounted_routes(
7258            &self.routes,
7259            &self.scoped_groups,
7260            &self.declared_routes,
7261            &config,
7262            #[cfg(feature = "openapi")]
7263            self.openapi.as_ref(),
7264        );
7265        (mounted, opaque)
7266    }
7267
7268    /// Raw `merge`/`nest` routers whose endpoints the graph cannot enumerate.
7269    ///
7270    /// The same count `autumn routes audit` hard-fails its coverage gate on
7271    /// (`omitted_router_count`), so the graph and the route audit cannot
7272    /// disagree about how much of the served surface is opaque.
7273    fn graph_opaque_router_count(&self) -> usize {
7274        omitted_router_count(
7275            self.merge_routers.len(),
7276            self.nest_routers.iter().map(|(prefix, _)| prefix.as_str()),
7277            &self.declared_routes,
7278        )
7279    }
7280
7281    /// Dump the agent-authority manifest as one marker-prefixed JSON line and
7282    /// exit.
7283    ///
7284    /// Triggered when `AUTUMN_DUMP_AGENT_AUTHORITY=1` is set (by `autumn agents
7285    /// manifest`). Takes `&self` rather than consuming the builder: it reads
7286    /// the route table and the audit-sink status and touches nothing else, so
7287    /// there is no database to open and no port to bind.
7288    fn run_dump_agent_authority_mode(&self) {
7289        // Whether agent invocations have anywhere to be recorded is a property
7290        // of the deployment, not of any grant, and it belongs in the document
7291        // rather than in a startup line nobody reads (#1691 R9).
7292        let audit_sink_configured = self
7293            .audit_logger
7294            .as_ref()
7295            .is_some_and(|logger| logger.is_enabled());
7296        // The whole-API MCP hatch, when the `mcp` feature is compiled in and
7297        // the app opted into it. Without it every route `expose_all_as_mcp()`
7298        // exposes would be missing from the document entirely.
7299        #[cfg(feature = "mcp")]
7300        let expose_all = self.mcp.as_ref().is_some_and(|rt| rt.expose_all);
7301        #[cfg(not(feature = "mcp"))]
7302        let expose_all = false;
7303        // The WebMCP hatch, likewise. A WebMCP tool runs in the user's own
7304        // session, so the document lists it apart from the `/mcp` tools.
7305        #[cfg(feature = "webmcp")]
7306        let webmcp_expose_all = self.webmcp.is_some_and(|rt| rt.expose_all);
7307        #[cfg(not(feature = "webmcp"))]
7308        let webmcp_expose_all = false;
7309        let exposure = AgentExposure {
7310            mcp_expose_all: expose_all,
7311            webmcp_expose_all,
7312        };
7313        // Top-level routes carry their own path; a scoped group's children do
7314        // not -- the group's prefix is applied at mount time, so the path on
7315        // the `Route` is the child path alone. Passing that through would
7316        // record `/items` for a tool an agent calls at `/api/v1/items`, and a
7317        // scope rename would then produce no drift at all.
7318        let mut routes: Vec<crate::agent_authority::manifest::RouteSummary> = self
7319            .routes
7320            .iter()
7321            .map(|route| agent_authority_route_summary(route, None, exposure))
7322            .chain(self.scoped_groups.iter().flat_map(|group| {
7323                group.routes.iter().map(move |route| {
7324                    agent_authority_route_summary(route, Some(&group.prefix), exposure)
7325                })
7326            }))
7327            .collect();
7328        // The projector keeps the first route per tool name; so does this.
7329        crate::agent_authority::manifest::dedupe_mcp_tool_names(&mut routes);
7330        crate::agent_authority::manifest::print_manifest_dump(
7331            &crate::agent_authority::manifest::build(&routes, audit_sink_configured),
7332        );
7333    }
7334
7335    /// Dump the application's route listing as JSON and exit.
7336    ///
7337    /// Triggered when `AUTUMN_DUMP_ROUTES=1` is set (by `autumn routes`).
7338    /// Exits with code 0 on success, code 1 on JSON serialization failure.
7339    /// Does not connect to a database or bind a TCP port.
7340    #[allow(clippy::too_many_lines)]
7341    async fn run_dump_routes_mode(self) {
7342        let Self {
7343            routes,
7344            api_versions,
7345            route_sources,
7346            scoped_groups,
7347            merge_routers,
7348            nest_routers,
7349            declared_routes,
7350            config_loader_factory,
7351            telemetry_provider,
7352            #[cfg(feature = "openapi")]
7353            openapi,
7354            plugin_config_roots,
7355            plugin_contracts,
7356            ..
7357        } = self;
7358
7359        // Validate that all versioned routes use a registered API version
7360        let registered_versions: std::collections::HashSet<&str> =
7361            api_versions.iter().map(|av| av.version.as_str()).collect();
7362
7363        for route in &routes {
7364            if let Some(ver) = route
7365                .api_version
7366                .filter(|ver| !registered_versions.contains(*ver))
7367            {
7368                eprintln!(
7369                    "Failed to build router: route '{}' uses unregistered API version '{}'",
7370                    route.name, ver
7371                );
7372                std::process::exit(1);
7373            }
7374        }
7375
7376        for group in &scoped_groups {
7377            for route in &group.routes {
7378                if let Some(ver) = route
7379                    .api_version
7380                    .filter(|ver| !registered_versions.contains(*ver))
7381                {
7382                    eprintln!(
7383                        "Failed to build router: route '{}' uses unregistered API version '{}'",
7384                        route.name, ver
7385                    );
7386                    std::process::exit(1);
7387                }
7388            }
7389        }
7390
7391        // Raw Axum routers registered via `.merge()`/`.nest()` are opaque: no
7392        // public API enumerates their routes, so they are omitted from the listing
7393        // and hard-fail `autumn routes audit`, since their auth posture cannot be
7394        // proven. The exception is a `.nest(prefix, router)` whose endpoints were
7395        // declared through `declare_plugin_routes`: those are enumerable, folded
7396        // into `declared_routes`, and must not count as omitted. Every `.merge()`
7397        // is rootless and always counts; a bare `.nest()` with no covering
7398        // declaration stays opaque and counts.
7399        let hidden = omitted_router_count(
7400            merge_routers.len(),
7401            nest_routers.iter().map(|(prefix, _)| prefix.as_str()),
7402            &declared_routes,
7403        );
7404        if hidden > 0 {
7405            eprintln!(
7406                "[autumn routes] warning: {hidden} raw router(s) added via \
7407                 .merge()/.nest() are not enumerable and are omitted from this listing"
7408            );
7409            // Machine-readable marker consumed by `autumn routes audit` to
7410            // hard-fail the coverage gate: omitted routes can't be proven.
7411            eprintln!(
7412                "{marker}{hidden}",
7413                marker = crate::route_listing::OMITTED_ROUTES_MARKER
7414            );
7415        }
7416
7417        let (config, _telemetry_guard) = load_config_and_telemetry(
7418            config_loader_factory,
7419            telemetry_provider,
7420            plugin_config_roots,
7421        )
7422        .await;
7423
7424        // Emit the resolved security configuration for the manifest's `declared`
7425        // dimensions (CSRF, security headers). Gated on `AUTUMN_DUMP_SECURITY`
7426        // so only `autumn routes audit` sees it — kept off stdout so the
7427        // routes-only JSON parse path stays byte-compatible.
7428        if is_dump_security_mode() {
7429            let security = crate::route_listing::SecurityDump::from_config(&config);
7430            match serde_json::to_string(&security) {
7431                Ok(json) => eprintln!(
7432                    "{marker}{json}",
7433                    marker = crate::route_listing::SECURITY_CONFIG_MARKER
7434                ),
7435                Err(e) => eprintln!("Failed to serialize security config: {e}"),
7436            }
7437        }
7438
7439        // Emit the plugin compatibility contracts declared by this app's
7440        // plugins (issue #1601). Gated on `AUTUMN_DUMP_PLUGIN_CONTRACT` so only
7441        // `autumn plugin-check` sees it, and emitted even when the array is
7442        // empty: the CLI distinguishes "this binary declares no contracts" from
7443        // "this binary predates the marker" by the line's presence.
7444        if is_dump_plugin_contract_mode() {
7445            match serde_json::to_string(&plugin_contracts) {
7446                Ok(json) => eprintln!(
7447                    "{marker}{json}",
7448                    marker = crate::plugin_contract::PLUGIN_CONTRACT_MARKER
7449                ),
7450                Err(e) => eprintln!("Failed to serialize plugin contracts: {e}"),
7451            }
7452        }
7453
7454        let mut infos = match crate::route_listing::collect_route_infos(
7455            &routes,
7456            &route_sources,
7457            &scoped_groups,
7458            &api_versions,
7459        ) {
7460            Ok(infos) => infos,
7461            Err(e) => {
7462                eprintln!("Failed to build router: {e}");
7463                std::process::exit(1);
7464            }
7465        };
7466        infos.extend(declared_routes);
7467        crate::route_listing::append_framework_routes(&mut infos, &config);
7468        #[cfg(feature = "openapi")]
7469        if let Some(ref oa) = openapi {
7470            crate::route_listing::append_openapi_routes(&mut infos, oa);
7471        }
7472        crate::route_listing::sort_route_infos(&mut infos);
7473
7474        let json = serde_json::to_string_pretty(&infos).unwrap_or_else(|e| {
7475            eprintln!("Failed to serialize route listing: {e}");
7476            std::process::exit(1);
7477        });
7478        println!("{json}");
7479        std::process::exit(0);
7480    }
7481
7482    /// Dump the generated `OpenAPI` document as JSON and exit.
7483    ///
7484    /// Triggered when `AUTUMN_DUMP_OPENAPI=1` is set (by
7485    /// `autumn openapi export`). Does not connect to a database or bind a TCP
7486    /// port.
7487    ///
7488    /// The document is built through the exact same pair the `/openapi.json`
7489    /// route uses — [`crate::router::collect_openapi_docs`] then the spec
7490    /// generator — so an exported spec and a served one cannot drift. Config is
7491    /// loaded the same way a normal boot loads it, because the session cookie
7492    /// name feeds the `SessionAuth` security scheme.
7493    ///
7494    /// Like the served route this evaluates deprecation/sunset state against
7495    /// the current instant ([`crate::openapi::generate_spec`] passes
7496    /// `Utc::now()`), so an export is reproducible except across a declared
7497    /// deprecation or sunset date — which is a real contract change a `--check`
7498    /// diff should surface, not noise to suppress.
7499    ///
7500    /// Exits 0 on success, 1 on serialization failure, and 2 when the app has
7501    /// no spec to emit (reported on the
7502    /// [`OPENAPI_UNAVAILABLE_MARKER`](crate::openapi::OPENAPI_UNAVAILABLE_MARKER)
7503    /// protocol).
7504    #[cfg(feature = "openapi")]
7505    async fn run_dump_openapi_mode(self) {
7506        let Self {
7507            routes,
7508            scoped_groups,
7509            api_versions,
7510            openapi,
7511            config_loader_factory,
7512            plugin_config_roots,
7513            merge_routers,
7514            nest_routers,
7515            declared_routes,
7516            #[cfg(feature = "mcp")]
7517            mcp,
7518            #[cfg(feature = "mail")]
7519            mount_unsubscribe_endpoint,
7520            policy_registrations,
7521            tasks,
7522            ..
7523        } = self;
7524
7525        let Some(openapi_config) = openapi else {
7526            eprintln!(
7527                "{marker}{reason}",
7528                marker = crate::openapi::OPENAPI_UNAVAILABLE_MARKER,
7529                reason = crate::openapi::OPENAPI_UNAVAILABLE_UNCONFIGURED,
7530            );
7531            std::process::exit(2);
7532        };
7533
7534        // Config only: `TelemetryProvider::init` can reach a collector or read
7535        // production credentials, and telemetry cannot affect the document, so
7536        // an export advertised as touching nothing must not run it.
7537        #[cfg_attr(
7538            not(feature = "mail"),
7539            expect(unused_mut, reason = "only `mail` mutates it")
7540        )]
7541        let mut config = load_config_only(config_loader_factory, plugin_config_roots).await;
7542
7543        // The builder flag must land BEFORE the collision checks below, exactly
7544        // as it does on the serving path: it is what decides whether
7545        // `/_autumn/unsubscribe` is claimed, and a check run against the
7546        // unmodified config would approve a mount that startup rejects.
7547        #[cfg(feature = "mail")]
7548        apply_mail_builder_overrides(&mut config, mount_unsubscribe_endpoint);
7549
7550        // Run the SERVING PATH'S OWN preflight before emitting anything. An
7551        // export that skips a check the router enforces lets `--check` pass for
7552        // an application that cannot start — and worse, does so QUIETLY:
7553        // `generate_spec` keys operations by (path, method), so a duplicate
7554        // silently DROPS the earlier one and the document describes a subset of
7555        // the API as though it were the whole of it.
7556        //
7557        // Every one of these calls the router's own function rather than
7558        // re-deriving its rule. A second copy would drift, and a preflight that
7559        // disagreed with the router about what it rejects would be worse than
7560        // none. The two that used to be inline in `build_router_pre_state` and
7561        // `build_openapi_router` were extracted for exactly this, so there is
7562        // still one definition per rule.
7563        //
7564        // Ordered as the serving path orders them, so an app with more than one
7565        // problem reports the same first error either way.
7566        // The config-only preconditions first, in `run()`'s order: a split role
7567        // on a non-durable jobs backend, or a duplicate scheduled task name,
7568        // stops startup before anything route-shaped is even looked at.
7569        let tasks = merge_framework_scheduled_tasks(tasks, &config);
7570        if let Err(message) = validate_config_preconditions(&config, &tasks) {
7571            eprintln!("\u{2717} Cannot export a spec for an app that cannot start: {message}");
7572            std::process::exit(1);
7573        }
7574
7575        // `.policy::<R, _>(...)` / `.scope::<R, _>(...)` are DEFERRED closures the
7576        // serving path replays onto live state before checking that every
7577        // `#[repository(policy = X)]` route actually has an X registered. The
7578        // export dropped them and checked only that the macro argument existed,
7579        // so an app that declares a policy but forgets the builder call — which
7580        // refuses to start under a production profile — still exported a
7581        // contract `--check` would approve.
7582        //
7583        // A throwaway `PolicyRegistry` is enough: the check only ever reads the
7584        // registry, and building real `AppState` would open the database this
7585        // command promises not to touch.
7586        let export_registry = crate::authorization::PolicyRegistry::default();
7587        for register in policy_registrations {
7588            register(&export_registry);
7589        }
7590        validate_repository_policies_registered(&routes, &scoped_groups, &export_registry, &config);
7591
7592        let mcp_mount_path: Option<&str> = {
7593            #[cfg(feature = "mcp")]
7594            {
7595                mcp.as_ref().map(|rt| rt.mount_path.as_str())
7596            }
7597            #[cfg(not(feature = "mcp"))]
7598            {
7599                None
7600            }
7601        };
7602        if let Err(error) = export_preflight(&ExportPreflight {
7603            routes: &routes,
7604            scoped_groups: &scoped_groups,
7605            api_versions: &api_versions,
7606            openapi_config: &openapi_config,
7607            merge_routers: &merge_routers,
7608            nest_routers: &nest_routers,
7609            declared_routes: &declared_routes,
7610            config: &config,
7611            mcp_mount_path,
7612        }) {
7613            eprintln!("\u{2717} Cannot export a spec for a router that cannot be built: {error}");
7614            std::process::exit(1);
7615        }
7616
7617        let mut openapi_config = openapi_config;
7618        openapi_config.api_versions = api_versions;
7619        let openapi_config = openapi_config.session_cookie_name(config.session.cookie_name.clone());
7620
7621        let docs = crate::router::collect_openapi_docs(&routes, &scoped_groups);
7622        let refs: Vec<&crate::openapi::ApiDoc> = docs.iter().collect();
7623        let spec = crate::openapi::generate_spec(&openapi_config, &refs);
7624
7625        let json = crate::aeo::commerce::spec_json_with_mpp(&spec, &config).unwrap_or_else(|e| {
7626            eprintln!("Failed to serialize OpenAPI spec: {e}");
7627            std::process::exit(1);
7628        });
7629        println!("{json}");
7630        std::process::exit(0);
7631    }
7632
7633    /// Dump the effective drained-queue manifest as TOML and exit.
7634    ///
7635    /// Triggered when `AUTUMN_DUMP_JOBS=1` is set (by `autumn jobs manifest`).
7636    /// Emits a single top-level `queues = [...]` array — the configured
7637    /// `[jobs.queues]` set unioned with every `#[job(queue = "…")]`-declared
7638    /// queue, ordered highest priority first exactly as the runtime drains — so a
7639    /// topology-aware `autumn doctor` consumes the ground-truth set the app runs
7640    /// with. Does not connect to a database or bind a TCP port. Always exits 0.
7641    async fn run_dump_jobs_mode(self) {
7642        let Self {
7643            jobs,
7644            listeners,
7645            config_loader_factory,
7646            telemetry_provider,
7647            plugin_config_roots,
7648            ..
7649        } = self;
7650
7651        let (config, _telemetry_guard) = load_config_and_telemetry(
7652            config_loader_factory,
7653            telemetry_provider,
7654            plugin_config_roots,
7655        )
7656        .await;
7657
7658        // Fold in the synthesized durable-listener jobs exactly as the boot path
7659        // does, so the manifest reflects the same effective drained-queue set the
7660        // runtime drains (including the `default` queue those jobs land on).
7661        let manifest = dump_jobs_manifest(&config.jobs.queues, jobs, listeners);
7662        print!("{manifest}");
7663        std::process::exit(0);
7664    }
7665
7666    /// Dump registered one-off tasks as JSON and exit.
7667    ///
7668    /// Triggered by `AUTUMN_LIST_TASKS=1` from `autumn task --list`.
7669    fn run_list_one_off_tasks_mode(self) {
7670        let Self { one_off_tasks, .. } = self;
7671
7672        if let Err(error) = crate::task::validate_unique_one_off_task_names(&one_off_tasks) {
7673            eprintln!("Invalid task registration: {error}");
7674            std::process::exit(1);
7675        }
7676
7677        let listing = crate::task::list_one_off_tasks(&one_off_tasks);
7678        let json = serde_json::to_string_pretty(&listing).unwrap_or_else(|error| {
7679            eprintln!("Failed to serialize task listing: {error}");
7680            std::process::exit(1);
7681        });
7682        println!("{json}");
7683        std::process::exit(0);
7684    }
7685
7686    /// Apply pending embedded migrations and exit (the `AUTUMN_MIGRATE=1`
7687    /// one-shot), WITHOUT starting the HTTP server or binding a port.
7688    ///
7689    /// Reuses the exact applier the startup auto-migration path uses
7690    /// ([`run_pending_locked`](crate::migrate::run_pending_locked), the public
7691    /// wrapper over the same locked engine `auto_migrate` drives) and the same
7692    /// framework-migration fold ([`migrations_with_repository_framework_migrations`]),
7693    /// so the applied set matches a normal boot. Unlike that path it applies
7694    /// regardless of profile — the deploy invokes it explicitly — and it targets
7695    /// the writable primary(ies) only (control primary + each shard primary),
7696    /// exactly like `autumn migrate` / the deploy DB preflight; replicas are never
7697    /// migration targets. The framework-internal directory / shard-map guard
7698    /// tables are deliberately NOT applied here: the app applies them
7699    /// unconditionally at startup, so the candidate's own boot creates them.
7700    ///
7701    /// Exits 0 after applying (printing a redacted count — never a URL or secret)
7702    /// and 1 on the first failure, so a failed migration aborts the deploy before
7703    /// cutover with the old release still serving (AC-3).
7704    #[cfg(feature = "db")]
7705    #[allow(clippy::too_many_lines)]
7706    async fn run_migrate_only_mode(self) {
7707        let Self {
7708            migrations,
7709            config_loader_factory,
7710            telemetry_provider,
7711            plugin_config_roots,
7712            ..
7713        } = self;
7714
7715        // The telemetry guard is dropped at end of scope; a migrate-only run does
7716        // not need tracing wired, but loading config the same way keeps env/profile
7717        // resolution identical to a normal boot.
7718        let (config, _telemetry_guard) = load_config_and_telemetry(
7719            config_loader_factory,
7720            telemetry_provider,
7721            plugin_config_roots,
7722        )
7723        .await;
7724
7725        // Fold in the framework migration sets a normal boot would apply, using the
7726        // SAME helper as `setup_database`, so the applied set is identical.
7727        let migrations = Arc::new(migrations_with_repository_framework_migrations(
7728            migrations,
7729            crate::repository_commit_hooks::has_repository_commit_hook_descriptors(),
7730            crate::version_history::has_versioned_repository_descriptors(),
7731            crate::derivation::has_derivation_descriptors(),
7732            RepositoryCommitHookQueueMigrationMode::Runtime,
7733        ));
7734        #[cfg(feature = "sqlite")]
7735        let fleet_migrations = Arc::clone(&migrations);
7736
7737        // Writable targets only: the control primary, then each shard primary.
7738        let control_url = config.database.effective_primary_url().map(str::to_owned);
7739        let lock_policy = crate::migrate::MigrationLockPolicy::from_config(&config.database);
7740        let shard_targets: Vec<(String, String)> = config
7741            .database
7742            .shards
7743            .iter()
7744            .map(|shard| (format!("shard:{}", shard.name), shard.primary_url.clone()))
7745            .collect();
7746
7747        if migrations.is_empty() || (control_url.is_none() && shard_targets.is_empty()) {
7748            eprintln!(
7749                "autumn migrate: no database configured or no migrations registered — nothing to apply"
7750            );
7751            std::process::exit(0);
7752        }
7753
7754        // SQLite migrate-only guard (#1614, PR3). Sharding is Postgres-only, so a
7755        // `sqlite:` control target with shards configured, or any `sqlite:` shard
7756        // target, fails fast here with the actionable sharding error. It is the
7757        // same `sqlite_sharding_unsupported_guard` normal boot applies, so the two
7758        // paths cannot drift. A plain `sqlite:` control target with no shards is
7759        // not gated: the SQLite apply path in the loop below handles its
7760        // migrations. An all-Postgres or empty-shard configuration is never gated.
7761        #[cfg(feature = "sqlite")]
7762        {
7763            let sqlite_guard_shard_urls: Vec<&str> =
7764                shard_targets.iter().map(|(_, url)| url.as_str()).collect();
7765            if let Err(e) = sqlite_sharding_unsupported_guard(
7766                control_url.as_deref(),
7767                !shard_targets.is_empty(),
7768                &sqlite_guard_shard_urls,
7769            ) {
7770                eprintln!("autumn migrate: {e}");
7771                // `process::exit` skips `on_shutdown`/`Drop`; stop any managed
7772                // Postgres child first, mirroring `apply_pending_or_exit`.
7773                #[cfg(feature = "managed-pg")]
7774                crate::managed_pg::emergency_stop();
7775                std::process::exit(1);
7776            }
7777        }
7778
7779        // Computed once, on the FINAL registered set (after the fold above),
7780        // so a version collision resolves automatically instead of one
7781        // migration silently never applying — see
7782        // `compute_migration_disambiguation`. `migration_sets_for_disambiguation`
7783        // folds in the two standalone shard control sets too, matching
7784        // `run_startup_migrations`, so both paths reach the same decision
7785        // regardless of which runs first.
7786        let disambiguation_sets =
7787            migration_sets_for_disambiguation(&migrations, config.database.has_shards());
7788        let disambiguated = crate::migrate::compute_migration_disambiguation(&disambiguation_sets);
7789        #[cfg(feature = "sqlite")]
7790        let sqlite_history_sets = crate::migrate::sqlite_collision_pairs(&disambiguation_sets);
7791
7792        // The diesel harness and the advisory-lock poll block, so apply off the
7793        // Tokio worker threads. Each target's failure exits non-zero from inside.
7794        let applied_total = crate::time::spawn_blocking(move || {
7795            let mut total = 0_usize;
7796            if let Some(url) = &control_url {
7797                // SQLite single-writer control target (issue #1614, PR3): apply with
7798                // NO advisory lock via the SQLite harness. Sharding is rejected above,
7799                // so a SQLite control target here is always unsharded (shard_targets
7800                // is empty). Every non-SQLite target keeps the byte-identical locked
7801                // Postgres applier.
7802                #[cfg(feature = "sqlite")]
7803                let is_sqlite_control = crate::config::DatabaseBackend::detect(url)
7804                    == Some(crate::config::DatabaseBackend::Sqlite);
7805                #[cfg(not(feature = "sqlite"))]
7806                let is_sqlite_control = false;
7807                if is_sqlite_control {
7808                    // A migration this database already ran under a version the
7809                    // map now gives a substitute keeps its record, moved to that
7810                    // substitute, rather than running twice (same as the CLI's
7811                    // SQLite path).
7812                    #[cfg(feature = "sqlite")]
7813                    if let Err(error) = crate::migrate::adopt_sqlite_collision_history(
7814                        url,
7815                        &sqlite_history_sets,
7816                        &disambiguated,
7817                    ) {
7818                        eprintln!(
7819                            "autumn migrate: could not move an already-applied migration's \
7820                             version record (target control): {error}"
7821                        );
7822                        std::process::exit(1);
7823                    }
7824                    #[cfg(feature = "sqlite")]
7825                    for (_, mig) in migrations.iter() {
7826                        total += apply_pending_sqlite_or_exit(
7827                            url,
7828                            crate::migrate::DisambiguatedMigrations::new(mig, &disambiguated),
7829                            "control",
7830                        );
7831                    }
7832                } else {
7833                    for (_, mig) in migrations.iter() {
7834                        total += apply_pending_or_exit(
7835                            url,
7836                            crate::migrate::DisambiguatedMigrations::new(mig, &disambiguated),
7837                            "control",
7838                            lock_policy,
7839                        );
7840                    }
7841                }
7842            }
7843            // Shards hold tenant data, not the control-plane schema; skip the
7844            // control framework set for shard targets (mirrors `run_startup_migrations`).
7845            // A `sqlite:` shard is rejected by the guard above, so every shard here
7846            // is Postgres.
7847            for (label, url) in &shard_targets {
7848                for (_, mig) in migrations
7849                    .iter()
7850                    .filter(|(_, mig)| !migration_set_is_control_framework(mig))
7851                {
7852                    total += apply_pending_or_exit(
7853                        url,
7854                        crate::migrate::DisambiguatedMigrations::new(mig, &disambiguated),
7855                        label,
7856                        lock_policy,
7857                    );
7858                }
7859            }
7860            total
7861        })
7862        .await
7863        .unwrap_or_else(|error| {
7864            eprintln!("autumn migrate: migration task panicked: {error}");
7865            std::process::exit(1);
7866        });
7867
7868        // A SQLite fleet (ADR 0019): every tenant / slot database on disk gets
7869        // the same non-control sets, so a boot that does not auto-apply finds
7870        // them current instead of refusing them.
7871        #[cfg(feature = "sqlite")]
7872        let applied_total = applied_total + migrate_fleet_or_exit(&config, fleet_migrations).await;
7873
7874        eprintln!(
7875            "autumn migrate: applied {applied_total} pending migration(s); database is up to date"
7876        );
7877        std::process::exit(0);
7878    }
7879
7880    /// The `AUTUMN_MIGRATE=1` one-shot on a build compiled WITHOUT database
7881    /// support: there is nothing to migrate, so report and exit 0 (never starting
7882    /// the server) so a DB-free app's deploy still runs the step harmlessly.
7883    #[cfg(not(feature = "db"))]
7884    #[allow(clippy::unused_async)]
7885    async fn run_migrate_only_mode(self) {
7886        eprintln!("autumn migrate: this build has no database support — nothing to migrate");
7887        std::process::exit(0);
7888    }
7889
7890    /// Count (never delete) the rows every registered
7891    /// `#[repository(..., retention(...))]` policy would sweep right now,
7892    /// print the report as JSON, and exit.
7893    ///
7894    /// Triggered by `AUTUMN_RETENTION_DRY_RUN=1` from `autumn retention
7895    /// --dry-run` (issue #1342). Boots just enough context to query the
7896    /// database — no HTTP listener, no job/mail/i18n machinery — mirroring
7897    /// `run_migrate_only_mode`'s minimal footprint.
7898    #[cfg(feature = "db")]
7899    async fn run_retention_dry_run_mode(self) {
7900        let Self {
7901            tasks,
7902            migrations,
7903            config_loader_factory,
7904            telemetry_provider,
7905            plugin_config_roots,
7906            pool_provider_factory,
7907            shard_provider_factory,
7908            shard_router,
7909            directory_shard_router,
7910            #[cfg(feature = "ws")]
7911            channels_backend,
7912            ..
7913        } = self;
7914
7915        let model_filter = retention_dry_run_model_filter_from_env();
7916        // Resolve/validate the requested policy selection BEFORE connecting
7917        // to the database (#1342 review round 9): an app with no
7918        // retention(...) policies at all, or a --model that names nothing
7919        // registered, can answer without ever opening a connection — every
7920        // check here reads only the compile-time-registered descriptor set.
7921        // A real database only gets touched once we know a policy actually
7922        // needs to be counted.
7923        let descriptors =
7924            match crate::retention::resolve_retention_descriptors(model_filter.as_deref()) {
7925                Ok(descriptors) => descriptors,
7926                Err(error) => {
7927                    eprintln!("retention dry-run: {error}");
7928                    std::process::exit(1);
7929                }
7930            };
7931
7932        // `resolve_retention_descriptors` validates collisions only among
7933        // retention-generated task names. It cannot see hand-declared `tasks![...]`
7934        // entries, which real boot merges in and validates through
7935        // `validate_unique_scheduled_task_names`. Without this check a dry run could
7936        // report success for a policy whose generated name collides with a
7937        // hand-declared task, while real boot panics on that collision. `tasks` is
7938        // carried into this mode, rather than discarded via `..`, so this check can
7939        // run.
7940        //
7941        // Merged against every registered retention descriptor, not just the
7942        // `descriptors` a `--model` filter narrowed to: real boot has no filter
7943        // concept, so a hand-declared task colliding with an unselected policy's
7944        // generated name would still panic real boot.
7945        if let Err(error) =
7946            merge_and_validate_task_names(&crate::retention::all_retention_descriptors(), tasks)
7947        {
7948            eprintln!("retention dry-run: {error}");
7949            std::process::exit(1);
7950        }
7951
7952        if descriptors.is_empty() {
7953            println!("{RETENTION_DRY_RUN_JSON_PREFIX}[]");
7954            std::process::exit(0);
7955        }
7956
7957        let (config, _telemetry_guard) = load_config_and_telemetry(
7958            config_loader_factory,
7959            telemetry_provider,
7960            plugin_config_roots,
7961        )
7962        .await;
7963
7964        // A `match`, not `.unwrap_or_else` (#1342 review round 15): a
7965        // managed-Postgres provider may have already started its postmaster
7966        // by the time `setup_database` fails (e.g. `pool_size = 0` failing
7967        // the deadpool build in `create_pool`, after the postmaster is up).
7968        // `emergency_stop_async()` is async, and `.unwrap_or_else`'s closure
7969        // can't `.await` — matching the same restructuring every later exit
7970        // in this function already uses (#1342 review round 6).
7971        let database = match setup_database(
7972            &config,
7973            migrations,
7974            pool_provider_factory,
7975            shard_provider_factory,
7976            shard_router,
7977            directory_shard_router,
7978            RepositoryCommitHookQueueMigrationMode::Runtime,
7979        )
7980        .await
7981        {
7982            Ok(database) => database,
7983            Err(error) => {
7984                eprintln!("{error}");
7985                #[cfg(feature = "managed-pg")]
7986                crate::managed_pg::emergency_stop_async().await;
7987                std::process::exit(1);
7988            }
7989        };
7990
7991        let state = build_state(
7992            &config,
7993            database.topology.as_ref(),
7994            database.shards,
7995            #[cfg(feature = "ws")]
7996            channels_backend,
7997        );
7998
7999        // `process::exit` below skips `on_shutdown` — including a managed-Postgres
8000        // `stop()` — so every exit from this one-shot would otherwise leave the
8001        // postmaster `setup_database` may have started running, with its data
8002        // directory locked for later commands (#1342 review round 6). Stop it
8003        // explicitly before every exit rather than relying on `on_shutdown`.
8004        match crate::retention::run_retention_dry_run(&state, model_filter.as_deref()).await {
8005            Ok(reports) => match serde_json::to_string(&reports) {
8006                Ok(json) => {
8007                    println!("{RETENTION_DRY_RUN_JSON_PREFIX}{json}");
8008                    #[cfg(feature = "managed-pg")]
8009                    crate::managed_pg::emergency_stop_async().await;
8010                    std::process::exit(0);
8011                }
8012                Err(error) => {
8013                    eprintln!("retention dry-run: failed to serialize report: {error}");
8014                    #[cfg(feature = "managed-pg")]
8015                    crate::managed_pg::emergency_stop_async().await;
8016                    std::process::exit(1);
8017                }
8018            },
8019            Err(error) => {
8020                eprintln!("retention dry-run: {error}");
8021                #[cfg(feature = "managed-pg")]
8022                crate::managed_pg::emergency_stop_async().await;
8023                std::process::exit(1);
8024            }
8025        }
8026    }
8027
8028    /// Report or enforce the unified `[retention]` policy over every
8029    /// framework-owned dataset, print the report as JSON, and exit
8030    /// (issue #1605).
8031    ///
8032    /// Triggered by `AUTUMN_DB_RETENTION=report|purge` from `autumn db
8033    /// retention`. Boots just enough context to answer honestly — the
8034    /// resolved config, the database, and the app's own state initializers,
8035    /// which is what installs the [`crate::gdpr::GdprRegistry`] a legal hold
8036    /// lives in and the [`crate::audit::AuditLogger`] the sweep records
8037    /// through — but no HTTP listener and no job/scheduler machinery.
8038    ///
8039    /// Running inside the app rather than from the standalone CLI is what
8040    /// makes the report trustworthy: it calls the *same*
8041    /// [`crate::data_retention::run_retention`] the scheduled sweep calls, so
8042    /// what the CLI prints and what the app enforces cannot drift.
8043    #[allow(clippy::too_many_lines)]
8044    async fn run_framework_retention_mode(self, mode: FrameworkRetentionMode) {
8045        let Self {
8046            state_initializers,
8047            audit_logger,
8048            config_loader_factory,
8049            telemetry_provider,
8050            plugin_config_roots,
8051            #[cfg(feature = "db")]
8052            migrations,
8053            #[cfg(feature = "db")]
8054            pool_provider_factory,
8055            #[cfg(feature = "db")]
8056            shard_provider_factory,
8057            #[cfg(feature = "db")]
8058            shard_router,
8059            #[cfg(feature = "db")]
8060            directory_shard_router,
8061            #[cfg(feature = "ws")]
8062            channels_backend,
8063            ..
8064        } = self;
8065
8066        let dataset_filter = framework_retention_dataset_from_env();
8067        // Reject a mistyped `--dataset` before opening a connection: the
8068        // registry is compile-time, so this needs no database at all.
8069        if let Some(filter) = dataset_filter.as_deref()
8070            && crate::data_retention::RetentionDataset::from_key(filter).is_none()
8071        {
8072            let known: Vec<&str> = crate::data_retention::RETENTION_DATASETS
8073                .iter()
8074                .map(|dataset| dataset.key())
8075                .collect();
8076            eprintln!(
8077                "autumn db retention: unknown dataset {filter:?}; known datasets: {}",
8078                known.join(", ")
8079            );
8080            std::process::exit(1);
8081        }
8082
8083        let (config, _telemetry_guard) = load_config_and_telemetry(
8084            config_loader_factory,
8085            telemetry_provider,
8086            plugin_config_roots,
8087        )
8088        .await;
8089
8090        // The CLI guard only knows the profile name it passes down. Check the
8091        // profile this app resolved, next to the deletion.
8092        if let Some(refusal) = framework_purge_refusal(
8093            mode,
8094            config.profile.as_deref(),
8095            std::env::var(FRAMEWORK_RETENTION_FORCE_ENV).ok().as_deref(),
8096        ) {
8097            eprintln!("autumn db retention: {refusal}");
8098            std::process::exit(1);
8099        }
8100
8101        #[cfg(feature = "db")]
8102        let database = match setup_database(
8103            &config,
8104            migrations,
8105            pool_provider_factory,
8106            shard_provider_factory,
8107            shard_router,
8108            directory_shard_router,
8109            RepositoryCommitHookQueueMigrationMode::Runtime,
8110        )
8111        .await
8112        {
8113            Ok(database) => database,
8114            Err(error) => {
8115                eprintln!("{error}");
8116                #[cfg(feature = "managed-pg")]
8117                crate::managed_pg::emergency_stop_async().await;
8118                std::process::exit(1);
8119            }
8120        };
8121
8122        let state = build_state(
8123            &config,
8124            #[cfg(feature = "db")]
8125            database.topology.as_ref(),
8126            #[cfg(feature = "db")]
8127            database.shards,
8128            #[cfg(feature = "ws")]
8129            channels_backend,
8130        );
8131        // `AppBuilder::with_audit_sink(...)` installs its logger here, not via
8132        // a state initializer. Skipping it would leave an on-demand purge with
8133        // no audit record at all, and would make
8134        // `--dataset audit_archives` a silent no-op for exactly the apps that
8135        // use the first-class builder.
8136        if let Some(logger) = audit_logger {
8137            state.insert_extension::<crate::audit::AuditLogger>((*logger).clone());
8138        }
8139        // Custom domains (#2652): the `custom_domains` retention dataset looks
8140        // up the same pruner the server installs, but the TLS bind path that
8141        // installs it never runs in a one-shot `run_framework_retention_mode`
8142        // — without this the dataset always reported "custom domains are not
8143        // enabled". The pruner carries no ACME issuer: pruning only touches
8144        // the registry and the certificate store, so no order is placed and
8145        // no CA is contacted.
8146        #[cfg(feature = "acme")]
8147        install_custom_domain_retention_pruner(
8148            config.server.tls.as_ref().and_then(|tls| tls.acme.as_ref()),
8149            config.tenancy.base_domain.as_deref(),
8150            &state,
8151            mode != FrameworkRetentionMode::Report,
8152        )
8153        .await;
8154        // The app's own state initializers are what install the GDPR registry
8155        // a legal hold lives in. Skipping them would make `autumn db
8156        // retention` report a sweep the running app would actually refuse.
8157        run_state_initializers(state_initializers, &state);
8158
8159        let options = crate::data_retention::RetentionRunOptions {
8160            dry_run: mode == FrameworkRetentionMode::Report,
8161            dataset: dataset_filter.as_deref(),
8162        };
8163        // `process::exit` skips `on_shutdown`, so a managed-Postgres
8164        // postmaster `setup_database` may have started must be stopped
8165        // explicitly before every exit below.
8166        match crate::data_retention::run_retention(&state, &options).await {
8167            Ok(reports) => match serde_json::to_string(&reports) {
8168                Ok(json) => {
8169                    println!("{FRAMEWORK_RETENTION_JSON_PREFIX}{json}");
8170                    #[cfg(feature = "managed-pg")]
8171                    crate::managed_pg::emergency_stop_async().await;
8172                    // A dataset that failed is reported in its own row rather
8173                    // than aborting the run, but the command as a whole must
8174                    // still exit non-zero so a scripted purge doesn't look
8175                    // successful.
8176                    let exit_code = i32::from(reports.iter().any(|r| r.error.is_some()));
8177                    std::process::exit(exit_code);
8178                }
8179                Err(error) => {
8180                    eprintln!("autumn db retention: failed to serialize report: {error}");
8181                    #[cfg(feature = "managed-pg")]
8182                    crate::managed_pg::emergency_stop_async().await;
8183                    std::process::exit(1);
8184                }
8185            },
8186            Err(error) => {
8187                eprintln!("autumn db retention: {error}");
8188                #[cfg(feature = "managed-pg")]
8189                crate::managed_pg::emergency_stop_async().await;
8190                std::process::exit(1);
8191            }
8192        }
8193    }
8194
8195    /// Run `AUTUMN_DATA_CAPSULE=export|import|verify` and exit (issue #1811).
8196    ///
8197    /// All modes boot the app's state initializers, which install the
8198    /// [`crate::gdpr::GdprRegistry`] and any `CapsuleService`. Export and
8199    /// import also use the database and the blob store. Verify uses only the
8200    /// signer: it runs no migration, and its built-in pools never connect. It
8201    /// still builds the pools as the other modes do, with the app's own pool
8202    /// provider, so an initializer can install its `CapsuleService`.
8203    #[allow(clippy::too_many_lines)]
8204    async fn run_data_capsule_mode(self, mode: DataCapsuleMode) {
8205        let Self {
8206            state_initializers,
8207            config_loader_factory,
8208            telemetry_provider,
8209            plugin_config_roots,
8210            #[cfg(feature = "db")]
8211            migrations,
8212            #[cfg(feature = "db")]
8213            pool_provider_factory,
8214            #[cfg(feature = "db")]
8215            shard_provider_factory,
8216            #[cfg(feature = "db")]
8217            shard_router,
8218            #[cfg(feature = "db")]
8219            directory_shard_router,
8220            #[cfg(feature = "ws")]
8221            channels_backend,
8222            #[cfg(feature = "storage")]
8223            blob_store,
8224            ..
8225        } = self;
8226
8227        let Some(path) = std::env::var_os(DATA_CAPSULE_PATH_ENV)
8228            .filter(|p| !p.is_empty())
8229            .map(std::path::PathBuf::from)
8230        else {
8231            eprintln!("autumn data capsule: {DATA_CAPSULE_PATH_ENV} is not set");
8232            std::process::exit(1);
8233        };
8234        let subject = std::env::var(DATA_CAPSULE_SUBJECT_ENV).unwrap_or_default();
8235        if mode == DataCapsuleMode::Export && subject.is_empty() {
8236            eprintln!("autumn data capsule export: {DATA_CAPSULE_SUBJECT_ENV} is not set");
8237            std::process::exit(1);
8238        }
8239
8240        let (config, _telemetry_guard) = load_config_and_telemetry(
8241            config_loader_factory,
8242            telemetry_provider,
8243            plugin_config_roots,
8244        )
8245        .await;
8246        // A capsule holds personal data: use the same secret rules as a server.
8247        fail_fast_on_invalid_signing_secret(&config);
8248
8249        // Verify runs no migration. It keeps the pools and the blob store, so
8250        // an initializer that builds a service from them still works.
8251        #[cfg(feature = "db")]
8252        let verify = mode == DataCapsuleMode::Verify;
8253
8254        #[cfg(feature = "storage")]
8255        let storage_bootstrap = blob_store.map_or_else(
8256            || preflight_storage(&config),
8257            |store| {
8258                Some(StorageBootstrap {
8259                    store,
8260                    serving: None,
8261                })
8262            },
8263        );
8264
8265        #[cfg(feature = "db")]
8266        let (topology, shards) = if verify {
8267            verify_database_parts(
8268                &config,
8269                pool_provider_factory,
8270                shard_provider_factory,
8271                shard_router,
8272                directory_shard_router,
8273            )
8274            .await
8275        } else {
8276            match setup_database(
8277                &config,
8278                migrations,
8279                pool_provider_factory,
8280                shard_provider_factory,
8281                shard_router,
8282                directory_shard_router,
8283                RepositoryCommitHookQueueMigrationMode::Runtime,
8284            )
8285            .await
8286            {
8287                Ok(database) => (database.topology, database.shards),
8288                Err(error) => {
8289                    eprintln!("{error}");
8290                    #[cfg(feature = "managed-pg")]
8291                    crate::managed_pg::emergency_stop_async().await;
8292                    std::process::exit(1);
8293                }
8294            }
8295        };
8296
8297        let state = build_state(
8298            &config,
8299            #[cfg(feature = "db")]
8300            topology.as_ref(),
8301            #[cfg(feature = "db")]
8302            shards,
8303            #[cfg(feature = "ws")]
8304            channels_backend,
8305        );
8306        #[cfg(feature = "storage")]
8307        if let Some(bootstrap) = storage_bootstrap {
8308            let _ = bootstrap.install(&state);
8309        }
8310        run_state_initializers(state_initializers, &state);
8311
8312        let result = match mode {
8313            // Verify needs only the signer. An installed `CapsuleService` can
8314            // bring its own, so it runs after the state initializers too.
8315            DataCapsuleMode::Verify => {
8316                crate::gdpr::portability::CapsuleService::signer_for_state(&state)
8317                    .and_then(|signer| crate::gdpr::portability::verify_dir(&path, &signer))
8318                    .map(|report| serde_json::json!(report))
8319            }
8320            DataCapsuleMode::Export | DataCapsuleMode::Import => {
8321                match crate::gdpr::portability::CapsuleService::from_state(&state) {
8322                    Ok(service) if mode == DataCapsuleMode::Export => service
8323                        .export_to(&subject, &path)
8324                        .await
8325                        .map(|report| serde_json::json!(report)),
8326                    Ok(service) => service
8327                        .import_from(&path)
8328                        .await
8329                        .map(|summary| serde_json::json!(summary)),
8330                    Err(error) => Err(error),
8331                }
8332            }
8333        };
8334        // `process::exit` skips `on_shutdown`: stop a managed postmaster first.
8335        #[cfg(feature = "managed-pg")]
8336        crate::managed_pg::emergency_stop_async().await;
8337        emit_data_capsule_report(mode, &result);
8338    }
8339
8340    /// The `AUTUMN_RETENTION_DRY_RUN=1` one-shot on a build compiled WITHOUT
8341    /// database support: there is nothing to sweep, so report and exit 0
8342    /// (never starting the server).
8343    ///
8344    /// Still prints `[]` to stdout (framed by
8345    /// [`RETENTION_DRY_RUN_JSON_PREFIX`]) — `autumn retention --dry-run`
8346    /// always looks for that framed report line, so silently printing
8347    /// nothing here would surface as a parse failure instead of the
8348    /// intended "no policies" result.
8349    #[cfg(not(feature = "db"))]
8350    #[allow(clippy::unused_async)]
8351    async fn run_retention_dry_run_mode(self) {
8352        eprintln!(
8353            "autumn retention --dry-run: this build has no database support — nothing to report"
8354        );
8355        println!("{RETENTION_DRY_RUN_JSON_PREFIX}[]");
8356        std::process::exit(0);
8357    }
8358
8359    /// Run a registered one-off task with full application context and exit.
8360    ///
8361    /// Triggered by `AUTUMN_RUN_TASK=<name>` from `autumn task <name>`.
8362    #[allow(clippy::too_many_lines)]
8363    #[allow(clippy::cognitive_complexity)]
8364    async fn run_one_off_task_mode(self, requested_name: String) {
8365        let Self {
8366            one_off_tasks,
8367            mut jobs,
8368            listeners,
8369            #[cfg(feature = "db")]
8370            outbox_handlers,
8371            #[cfg(feature = "i18n")]
8372            custom_layers,
8373            #[cfg(not(feature = "i18n"))]
8374                custom_layers: _,
8375            startup_hooks,
8376            state_initializers,
8377            shutdown_hooks,
8378            config_loader_factory,
8379            #[cfg(feature = "db")]
8380            migrations,
8381            #[cfg(feature = "db")]
8382            pool_provider_factory,
8383            #[cfg(feature = "db")]
8384            shard_provider_factory,
8385            #[cfg(feature = "db")]
8386            shard_router,
8387            #[cfg(feature = "db")]
8388            directory_shard_router,
8389            telemetry_provider,
8390            session_store,
8391            #[cfg(feature = "ws")]
8392            channels_backend,
8393            #[cfg(feature = "storage")]
8394            blob_store,
8395            audit_logger,
8396            #[cfg(feature = "i18n")]
8397            i18n_bundle,
8398            #[cfg(feature = "i18n")]
8399            i18n_auto_load,
8400            #[cfg(feature = "embed-assets")]
8401            embedded_static,
8402            #[cfg(all(feature = "embed-assets", feature = "i18n"))]
8403            embedded_locales,
8404            policy_registrations,
8405            cache_backend,
8406            #[cfg(feature = "mail")]
8407            mail_delivery_queue_factory,
8408            #[cfg(feature = "mail")]
8409            suppression_store,
8410            #[cfg(feature = "mail")]
8411            mail_suppression_store,
8412            #[cfg(feature = "mail")]
8413                mount_unsubscribe_endpoint: _,
8414            #[cfg(feature = "mail")]
8415            mail_interceptor,
8416            job_interceptor,
8417            #[cfg(feature = "db")]
8418            db_interceptor,
8419            #[cfg(feature = "ws")]
8420            channels_interceptor,
8421            #[cfg(feature = "oauth2")]
8422            http_interceptor,
8423            plugin_config_roots,
8424            ..
8425        } = self;
8426
8427        if let Err(error) = crate::task::validate_unique_one_off_task_names(&one_off_tasks) {
8428            eprintln!("Invalid task registration: {error}");
8429            std::process::exit(1);
8430        }
8431
8432        let Some((task_name, task_handler)) = one_off_tasks
8433            .iter()
8434            .find(|task| task.name == requested_name)
8435            .map(|task| (task.name.clone(), task.handler))
8436        else {
8437            eprintln!("No one-off task named '{requested_name}' is registered.");
8438            print_available_one_off_tasks(&one_off_tasks);
8439            std::process::exit(1);
8440        };
8441
8442        let args = one_off_task_args_from_env().unwrap_or_else(|error| {
8443            eprintln!("Invalid task args: {error}");
8444            std::process::exit(1);
8445        });
8446
8447        let (config, telemetry_guard) = load_config_and_telemetry(
8448            config_loader_factory,
8449            telemetry_provider,
8450            plugin_config_roots,
8451        )
8452        .await;
8453
8454        // Register the embedded `static/` tree (if any) before the router is
8455        // built so `/static/*` serves from the binary and `asset_url()` resolves
8456        // against the embedded manifest, then prefer embedded locales over disk
8457        // auto-loading when no explicit bundle was provided.
8458        #[cfg(feature = "embed-assets")]
8459        register_embedded_static_dir(embedded_static);
8460
8461        #[cfg(all(feature = "embed-assets", feature = "i18n"))]
8462        let i18n_bundle = embedded_i18n_bundle(i18n_bundle, embedded_locales, &config);
8463
8464        #[cfg(feature = "i18n")]
8465        let i18n_bundle =
8466            resolve_i18n_bundle(i18n_bundle, i18n_auto_load, &config, &crate::config::OsEnv);
8467
8468        fail_fast_on_invalid_session_config(&config, session_store.is_some());
8469        fail_fast_on_invalid_signing_secret(&config);
8470        fail_fast_on_missing_encryption_keys(&config);
8471        fail_fast_on_invalid_trusted_hosts(&config);
8472
8473        #[cfg(feature = "storage")]
8474        let storage_bootstrap = blob_store.map_or_else(
8475            || preflight_storage(&config),
8476            |store| {
8477                Some(StorageBootstrap {
8478                    store,
8479                    serving: None,
8480                })
8481            },
8482        );
8483
8484        #[cfg(feature = "db")]
8485        let database = setup_database(
8486            &config,
8487            migrations,
8488            pool_provider_factory,
8489            shard_provider_factory,
8490            shard_router,
8491            directory_shard_router,
8492            RepositoryCommitHookQueueMigrationMode::Runtime,
8493        )
8494        .await
8495        .unwrap_or_else(|error| {
8496            eprintln!("{error}");
8497            std::process::exit(1);
8498        });
8499        #[cfg(feature = "db")]
8500        let pool = database.topology;
8501        #[cfg(feature = "db")]
8502        let shards = database.shards;
8503        #[cfg(feature = "db")]
8504        let replica_readiness = database.replica_readiness;
8505        #[cfg(feature = "db")]
8506        let replica_migration_check = database.replica_migration_check;
8507
8508        let mut state = build_state(
8509            &config,
8510            #[cfg(feature = "db")]
8511            pool.as_ref(),
8512            #[cfg(feature = "db")]
8513            shards,
8514            #[cfg(feature = "ws")]
8515            channels_backend,
8516        );
8517        if let Some(buf) = telemetry_guard.log_buffer.clone() {
8518            state.insert_extension(buf);
8519        }
8520        // Wire the live-subscriber reload handle into the loggers actuator so
8521        // `PUT /actuator/loggers/{name}` affects the running subscriber, not
8522        // just an in-memory map (issue #1044).
8523        if let Some(handle) = telemetry_guard.filter_reload.clone() {
8524            state.log_levels().attach_reload_handle(handle);
8525        }
8526        // Clone class: this interceptor-wiring block (mail/job/db/channels,
8527        // including the channels -> presence -> collab backend swap, and
8528        // http) is byte-for-byte identical in `run()`, `run_build_mode()`
8529        // and `run_one_off_task_mode()`. It was born duplicated by #862,
8530        // whose own follow-up commit ("fix: wire interceptors into
8531        // one-off task mode") is a missed-fix this duplication already
8532        // produced once: the one-off task path first shipped without this
8533        // block, so its interceptors were silently inert. #1806 later had
8534        // to hand-repeat the channels/presence/collab swap in all three
8535        // copies again. Change one, change the other two --
8536        // `optional_interceptor_wiring_matches_across_all_three_run_modes`
8537        // below fails if they drift.
8538        #[cfg(feature = "mail")]
8539        if let Some(interceptor) = mail_interceptor {
8540            state.insert_extension(interceptor);
8541        }
8542        if let Some(interceptor) = job_interceptor {
8543            state.insert_extension(interceptor);
8544        }
8545        #[cfg(feature = "db")]
8546        if let Some(interceptor) = db_interceptor {
8547            state.insert_extension(interceptor);
8548        }
8549        #[cfg(feature = "ws")]
8550        if let Some(interceptor) = channels_interceptor {
8551            state.insert_extension(interceptor.clone());
8552            state.channels = crate::channels::Channels::with_shared_backend(std::sync::Arc::new(
8553                crate::channels::InterceptedChannelsBackend::new(
8554                    state.channels.backend().clone(),
8555                    vec![interceptor],
8556                ),
8557            ));
8558            #[cfg(feature = "presence")]
8559            {
8560                state.presence = crate::presence::Presence::new(state.channels.clone());
8561                // The collaboration hub holds the channel registry and the
8562                // presence tracker it was built with, so replacing either
8563                // leaves it publishing into the old backend (#1806).
8564                #[cfg(feature = "collab")]
8565                {
8566                    state.collab = crate::collab::CollabHub::new(
8567                        state.channels.clone(),
8568                        state.presence.clone(),
8569                    );
8570                }
8571            }
8572        }
8573        #[cfg(feature = "oauth2")]
8574        if let Some(interceptor) = http_interceptor {
8575            state.insert_extension(interceptor);
8576        }
8577        #[cfg(feature = "db")]
8578        configure_replica_migration_check(&state, replica_migration_check);
8579        #[cfg(feature = "db")]
8580        apply_replica_migration_readiness(&state, replica_readiness);
8581        if let Some(cache) = cache_backend {
8582            // Wrapped once, so the global and the state hold the same `Arc`.
8583            let cache = crate::cache::with_capsule_seam(cache);
8584            crate::cache::set_global_cache(cache.clone());
8585            state.shared_cache = Some(cache);
8586        } else {
8587            crate::cache::clear_global_cache();
8588        }
8589
8590        for register in policy_registrations {
8591            register(state.policy_registry());
8592        }
8593
8594        #[cfg(feature = "mail")]
8595        if let Some(handle) = suppression_store {
8596            state.insert_extension(handle);
8597        }
8598        #[cfg(feature = "mail")]
8599        if let Some(handle) = mail_suppression_store {
8600            state.insert_extension(handle);
8601        }
8602        // A one-off task writes to the outbox; the server relay sends it.
8603        #[cfg(feature = "db")]
8604        {
8605            crate::outbox::install(&state, &config.outbox, outbox_handlers);
8606            if state.extension::<crate::outbox::OutboxRelay>().is_some()
8607                && let Err(error) = crate::outbox::ensure_relay_schema(&state).await
8608            {
8609                eprintln!("Failed to create the outbox tables: {error}");
8610                exit_stop_managed_pg();
8611                std::process::exit(1);
8612            }
8613        }
8614        #[cfg(feature = "mail")]
8615        crate::mail::install_mailer_with_factory(
8616            &state,
8617            &config.mail,
8618            mail_delivery_queue_factory,
8619            true,
8620        )
8621        .unwrap_or_else(|error| {
8622            eprintln!("Failed to configure mailer: {error}");
8623            exit_stop_managed_pg();
8624            std::process::exit(1);
8625        });
8626
8627        if let Some(logger) = audit_logger {
8628            state.insert_extension::<crate::audit::AuditLogger>((*logger).clone());
8629        }
8630
8631        #[cfg(feature = "i18n")]
8632        let _custom_layers =
8633            install_i18n_bundle_layer(custom_layers, &state, i18n_bundle, &config.i18n);
8634
8635        #[cfg(feature = "storage")]
8636        let _storage_router = storage_bootstrap.and_then(|bootstrap| bootstrap.install(&state));
8637        run_state_initializers(state_initializers, &state);
8638        crate::cost::install(&state, &config);
8639        finalize_event_bus(listeners, &mut jobs, &state);
8640
8641        let task_shutdown = tokio_util::sync::CancellationToken::new();
8642        if let Err(error) = initialize_job_runtime(jobs, &state, &task_shutdown, &config.jobs, true)
8643        {
8644            eprintln!("job runtime initialization failed: {error}");
8645            #[cfg(feature = "managed-pg")]
8646            crate::managed_pg::emergency_stop_async().await;
8647            std::process::exit(1);
8648        }
8649
8650        #[cfg(feature = "db")]
8651        {
8652            #[cfg(feature = "ws")]
8653            crate::repository_commit_hooks::set_global_channels(state.channels().clone());
8654        }
8655
8656        // Postgres-only durable commit-hook worker; not spawned under sqlite
8657        // (the runtime pool is a SQLite pool the Postgres worker cannot drive).
8658        #[cfg(all(feature = "db", not(feature = "sqlite")))]
8659        if let Some(pool) = state.pool().cloned() {
8660            #[cfg(feature = "ws")]
8661            {
8662                let channels = state.channels().clone();
8663                crate::repository_commit_hooks::start_repository_commit_hook_worker(
8664                    pool,
8665                    Some(channels),
8666                    task_shutdown.child_token(),
8667                );
8668            }
8669            #[cfg(not(feature = "ws"))]
8670            crate::repository_commit_hooks::start_repository_commit_hook_worker(
8671                pool,
8672                task_shutdown.child_token(),
8673            );
8674        }
8675        // Repositories built over a shard pool (`with_pool`) enqueue durable
8676        // commit hooks into that shard's queue table; drain each one too.
8677        #[cfg(all(feature = "db", not(feature = "sqlite")))]
8678        if let Some(shards) = state.shards() {
8679            for shard in shards.iter() {
8680                #[cfg(feature = "ws")]
8681                crate::repository_commit_hooks::start_repository_commit_hook_worker(
8682                    shard.primary_pool().clone(),
8683                    Some(state.channels().clone()),
8684                    task_shutdown.child_token(),
8685                );
8686                #[cfg(not(feature = "ws"))]
8687                crate::repository_commit_hooks::start_repository_commit_hook_worker(
8688                    shard.primary_pool().clone(),
8689                    task_shutdown.child_token(),
8690                );
8691            }
8692        }
8693        // SQLite durable commit-hook worker on the one-off task-runner path
8694        // (#1996 item 5); see the server-path spawn above for the rationale.
8695        #[cfg(all(feature = "db", feature = "sqlite"))]
8696        if let Some(pool) = state.pool().cloned() {
8697            #[cfg(feature = "ws")]
8698            {
8699                let channels = state.channels().clone();
8700                crate::repository_commit_hooks::start_repository_commit_hook_worker(
8701                    pool,
8702                    Some(channels),
8703                    task_shutdown.child_token(),
8704                );
8705            }
8706            #[cfg(not(feature = "ws"))]
8707            crate::repository_commit_hooks::start_repository_commit_hook_worker(
8708                pool,
8709                task_shutdown.child_token(),
8710            );
8711        }
8712        #[cfg(feature = "sqlite")]
8713        let fleet_maintenance =
8714            start_fleet_runtime(&state, task_shutdown.child_token(), None).await;
8715
8716        if let Err(error) = run_startup_hooks(&startup_hooks, state.clone()).await {
8717            eprintln!("startup hook failed: {error}");
8718            task_shutdown.cancel();
8719            #[cfg(feature = "managed-pg")]
8720            crate::managed_pg::emergency_stop_async().await;
8721            std::process::exit(1);
8722        }
8723        state.probes().mark_startup_complete();
8724
8725        tracing::info!(task = %task_name, "Running one-off task");
8726        let span = tracing::info_span!("one_off_task", task = %task_name);
8727        #[cfg(feature = "oauth2")]
8728        let result = {
8729            use crate::interceptor::{ACTIVE_HTTP_INTERCEPTORS, HttpInterceptor};
8730            let interceptors: Vec<std::sync::Arc<dyn HttpInterceptor>> = state
8731                .extension::<std::sync::Arc<dyn HttpInterceptor>>()
8732                .map(|interceptor_arc| vec![(*interceptor_arc).clone()])
8733                .unwrap_or_default();
8734            ACTIVE_HTTP_INTERCEPTORS
8735                .scope(
8736                    interceptors,
8737                    (task_handler)(state.clone(), args).instrument(span),
8738                )
8739                .await
8740        };
8741        #[cfg(not(feature = "oauth2"))]
8742        let result = (task_handler)(state.clone(), args).instrument(span).await;
8743
8744        task_shutdown.cancel();
8745        // Close the fleet databases the task opened (and ship their last
8746        // frames when the fleet replicates) before the process exits.
8747        #[cfg(feature = "sqlite")]
8748        if let Some(runtime) = fleet_maintenance {
8749            let _ = runtime.maintenance.await;
8750            if let Some(ownership) = runtime.ownership {
8751                ownership.finish().await;
8752            }
8753        }
8754        run_shutdown_hooks(&shutdown_hooks).await;
8755        // If the generated `pg.stop()` hook errored/timed out it keeps the
8756        // handle for a retry, but a one-off task then exits — so retry the stop
8757        // here (idempotent; a no-op once the hook stopped it cleanly) to avoid
8758        // orphaning the postmaster on the data dir/port.
8759        #[cfg(feature = "managed-pg")]
8760        crate::managed_pg::emergency_stop_async().await;
8761
8762        match result {
8763            Ok(()) => {
8764                tracing::info!(task = %task_name, "One-off task completed");
8765            }
8766            Err(error) => {
8767                tracing::error!(task = %task_name, error = %error, "One-off task failed");
8768                eprintln!("Task '{task_name}' failed: {error}");
8769                for cause in error.source_chain() {
8770                    eprintln!("Caused by: {cause}");
8771                }
8772                std::process::exit(1);
8773            }
8774        }
8775    }
8776
8777    /// Replay a recorded failure capsule against this application and exit with
8778    /// the verdict (issue #1598).
8779    ///
8780    /// Triggered by `AUTUMN_REPLAY_CAPSULE=<path>`, which `autumn replay` sets.
8781    /// The capsule supplies the request, every clock reading and every database
8782    /// answer, so this path must not reach anything outside it. Exits `0` when
8783    /// the recorded failure reproduced, `1` when it did not (a different
8784    /// outcome, code that left the recorded database tape, or code that
8785    /// reached the recorded outcome without asking for all of it), `2` when the
8786    /// capsule was refused and nothing ran at all — which includes any capsule
8787    /// marked truncated, such as one whose request used an unrecorded
8788    /// `[[database.shards]]` connection.
8789    ///
8790    /// # Deltas from [`run`](Self::run)
8791    ///
8792    /// **Kept**, because a replay against a stripped-down app is not a replay:
8793    /// the app's own configuration, the route table and scoped groups, merged
8794    /// and nested routers, custom Tower layers, the exception-filter chain, the
8795    /// error-page renderer, the i18n bundle, the custom session store, policy
8796    /// and scope registrations, state initializers, the DB interceptor, and the
8797    /// real router builder ([`try_build_router_inner`](crate::router::try_build_router_inner)).
8798    /// Kept app code is *not* fail-closed: a state initializer that dials an
8799    /// external service itself (a feature-flag SDK with its own HTTP stack)
8800    /// still dials it here — the offline guarantees below cover the
8801    /// framework's seams, and the guide's "Limitations" section says so.
8802    ///
8803    /// **Dropped** (F15 — a replay is offline by construction):
8804    ///
8805    /// * `setup_database` is never called: no pool is dialled and no migration
8806    ///   runs. The pool is
8807    ///   [`pool_from_capsule`](crate::capsule::pool_from_capsule), which answers
8808    ///   from the capsule's recorded wire traffic over an in-process pipe.
8809    /// * Capture is never armed — `[failure_capture]` is forced off, so no
8810    ///   [`CaptureLayer`](crate::capsule::CaptureLayer) is installed, no
8811    ///   request carries a capture scope, and the replay neither points the
8812    ///   checkout marker at the stub pool nor writes a capsule of itself.
8813    /// * The session store is forced to memory and the process cache is
8814    ///   cleared, so no Redis or external cache is dialled. A store installed
8815    ///   with [`with_session_store`](Self::with_session_store) is **dropped**
8816    ///   rather than forwarded: it outranks the config in `apply_session_layer`,
8817    ///   so passing it through would let a replay reach — and write to — the
8818    ///   application's real session backend.
8819    /// * Channels are forced in-process and a backend installed with
8820    ///   [`with_channels_backend`](Self::with_channels_backend) is **dropped**,
8821    ///   for the same reason as the session store: the Redis backend spawns a
8822    ///   publisher and a listener against the application's live fan-out as
8823    ///   soon as the state is built.
8824    /// * A config loader installed with
8825    ///   [`with_config_loader`](Self::with_config_loader) is **dropped**:
8826    ///   the documented implementations call AWS Secrets Manager, Vault,
8827    ///   Consul, or an HTTP endpoint, and an offline replay must neither
8828    ///   contact production infrastructure nor abort because it is
8829    ///   unreachable. Configuration loads from local files and the
8830    ///   environment; the secrets a loader fetches feed subsystems replay
8831    ///   forces off or serves from the capsule.
8832    /// * The global request timeout is cleared. A replay's inputs all come from
8833    ///   the capsule, but the timeout layer runs on real tokio timers, so a
8834    ///   breakpoint held in a debugger would otherwise cancel the handler and
8835    ///   report a mismatch that never happened. (A per-route
8836    ///   `#[timeout(...)]` override lives in the route table and still applies.)
8837    /// * Outbound HTTP is blocked wholesale
8838    ///   (`http_client::block_outbound_for_replay`). The state carries the
8839    ///   application's real `reqwest` client, and a capsule records no HTTP
8840    ///   responses — so a handler that calls a third party gets a clear error
8841    ///   naming the block, which surfaces as a mismatch rather than as a
8842    ///   request to a live service.
8843    /// * No job runtime, no scheduler, no startup/shutdown hooks, and only
8844    ///   *sync* event listeners (a durable listener needs the job runtime).
8845    /// * A replay mailer, whose sends are served from the capsule and never
8846    ///   delivered (`mail::install_replay_mailer`).
8847    /// * No storage preflight, no fail-fast configuration gates: a machine
8848    ///   replaying a production capsule generally has none of that configured,
8849    ///   and none of it is on the recorded path. A handler that extracts one of
8850    ///   those subsystems is reported as a mismatch rather than killing the
8851    ///   replay.
8852    /// * No port is bound.
8853    #[cfg(feature = "reporting")]
8854    #[allow(clippy::too_many_lines)]
8855    async fn run_replay_mode(self, capsule_path: String) {
8856        let Self {
8857            routes,
8858            api_versions,
8859            listeners,
8860            exception_filters,
8861            scoped_groups,
8862            merge_routers,
8863            nest_routers,
8864            // Bound rather than dropped with the rest: a nested router is
8865            // opaque, so these declarations are the ONLY thing that lets the
8866            // collision preflight see inside a sandboxed plugin's mount.
8867            // Replaying with the nests but without them would skip the checks
8868            // and reach the axum mount panic the preflight exists to replace
8869            // with a refusal.
8870            declared_routes,
8871            custom_layers,
8872            state_initializers,
8873            config_loader_factory,
8874            telemetry_provider,
8875            // Kept (rather than dropped with the rest of the runtime) so a
8876            // *job-entry* capsule can dispatch the recorded job's handler
8877            // (#1634). No job runtime, scheduler or backend is started: the
8878            // handler is called directly, exactly once, with the recorded
8879            // payload — through the application's `JobInterceptor` when it
8880            // registered one, since that is part of how the recorded run
8881            // executed and dropping it would replay a different path.
8882            jobs,
8883            job_interceptor,
8884            // F15: deliberately *not* destructured. A store installed with
8885            // `with_session_store(...)` outranks `config.session.backend` in
8886            // `apply_session_layer`, so forwarding it would let a replay dial —
8887            // and mutate — the application's live Redis or database session
8888            // backend, or fail 503 when that backend is unreachable. Replay
8889            // builds its router with no custom store, so the memory backend
8890            // `force_offline_replay_config` sets is what actually applies.
8891            session_store: _replay_ignores_custom_session_store,
8892            policy_registrations,
8893            #[cfg(feature = "db")]
8894            db_interceptor,
8895            // F15, same reasoning as the session store: a backend installed
8896            // with `with_channels_backend(...)` outranks `config.channels`, so
8897            // forwarding it would let a replay publish into — and subscribe to
8898            // — the application's live Redis fan-out. Replay builds its state
8899            // with no custom backend, so the in-process backend
8900            // `force_offline_replay_config` selects is what actually applies.
8901            #[cfg(feature = "ws")]
8902                channels_backend: _replay_ignores_custom_channels_backend,
8903            #[cfg(feature = "i18n")]
8904            i18n_bundle,
8905            #[cfg(feature = "i18n")]
8906            i18n_auto_load,
8907            #[cfg(feature = "maud")]
8908            error_page_renderer,
8909            #[cfg(feature = "embed-assets")]
8910            embedded_static,
8911            #[cfg(all(feature = "embed-assets", feature = "i18n"))]
8912            embedded_locales,
8913            plugin_config_roots,
8914            agent_skills,
8915            commerce_docs,
8916            // Read only for whether it is set: calling it could reach a live
8917            // queue. The replay mailer gets a refusing queue in its place.
8918            #[cfg(feature = "mail")]
8919            mail_delivery_queue_factory,
8920            // Read only for whether it is set: replay builds no cache backend.
8921            cache_backend,
8922            ..
8923        } = self;
8924        #[cfg(feature = "mail")]
8925        let builder_mail_queue = mail_delivery_queue_factory.is_some();
8926
8927        // Nothing outside the capsule may be reached from here on: the router
8928        // this rebuilds is the real one, with the application's real outbound
8929        // HTTP client in its state (AC4).
8930        #[cfg(feature = "http-client")]
8931        crate::http_client::block_outbound_for_replay();
8932        // Egress outside the HTTP client (`capsule::guard_egress`), in every
8933        // build.
8934        crate::capsule::boundary::block_egress_for_replay();
8935        // A cache call with no tape (a state initializer) must not reach a
8936        // live backend either.
8937        crate::cache::block_backends_for_replay();
8938
8939        let path = std::path::PathBuf::from(&capsule_path);
8940        let capsule = match crate::capsule::load_capsule(&path) {
8941            Ok(capsule) => capsule,
8942            Err(error) => {
8943                std::process::exit(crate::capsule::print_refusal(&error.to_string(), &path))
8944            }
8945        };
8946        if let Some(reason) = crate::capsule::refusal_reason(&capsule) {
8947            std::process::exit(crate::capsule::print_refusal(&reason, &path));
8948        }
8949
8950        // F15: the configured telemetry provider — the default OTLP batch
8951        // exporter, or a custom Datadog/Sentry initializer — is replaced with
8952        // a logging-only one. An offline replay must not flush spans to a live
8953        // collector, and must not abort before its verdict because that
8954        // collector is unreachable from the machine doing the replaying.
8955        let _replay_ignores_custom_telemetry_provider = telemetry_provider;
8956        // A custom config loader is a live service call — the documented
8957        // implementations reach AWS Secrets Manager, Vault, Consul, or an HTTP
8958        // endpoint — and an offline replay must neither contact production
8959        // infrastructure nor abort because it is unreachable. Configuration comes
8960        // from the local files and environment instead. The values replay consumes
8961        // — routes, middleware, the filter list, the profile — live there, and the
8962        // secrets a loader fetches feed subsystems replay forces off or serves from
8963        // the capsule.
8964        let _replay_ignores_custom_config_loader = config_loader_factory;
8965        let (mut config, telemetry_guard) = load_config_and_telemetry(
8966            None,
8967            Some(Box::new(crate::telemetry::ReplayTelemetryProvider)),
8968            plugin_config_roots,
8969        )
8970        .await;
8971        force_offline_replay_config(&mut config);
8972
8973        // A capsule from a *different application* can replay "successfully"
8974        // against this one when both expose the same route shape — a verdict
8975        // that looks authoritative and is meaningless. Same-name is not
8976        // provable (service names drift), so this warns loudly instead of
8977        // refusing; the human summary makes the mismatch impossible to miss.
8978        if let Some(recorded_app) = capsule.app.name.as_deref() {
8979            let this_app = config.telemetry.service_name.as_str();
8980            if recorded_app != this_app {
8981                tracing::warn!(
8982                    recorded_app,
8983                    this_app,
8984                    "the capsule was recorded by a different application; a verdict against \
8985                     this one may be meaningless"
8986                );
8987                eprintln!(
8988                    "warning: capsule was recorded by application {recorded_app:?}, but this \
8989                     build is {this_app:?} — if this is not a renamed service, the verdict \
8990                     below compares apples to oranges"
8991                );
8992            }
8993        }
8994
8995        #[cfg(feature = "embed-assets")]
8996        register_embedded_static_dir(embedded_static);
8997        #[cfg(all(feature = "embed-assets", feature = "i18n"))]
8998        let i18n_bundle = embedded_i18n_bundle(i18n_bundle, embedded_locales, &config);
8999        #[cfg(feature = "i18n")]
9000        let i18n_bundle =
9001            resolve_i18n_bundle(i18n_bundle, i18n_auto_load, &config, &crate::config::OsEnv);
9002
9003        // One log, shared: the stub connections write into it while the router
9004        // runs, and the verdict reads it once the router is done.
9005        let divergences = std::sync::Arc::new(crate::capsule::DivergenceLog::new());
9006        #[cfg(feature = "db")]
9007        let topology = replay_database_topology(&capsule, &divergences, &path);
9008
9009        let mut state = build_state(
9010            &config,
9011            #[cfg(feature = "db")]
9012            topology.as_ref(),
9013            #[cfg(feature = "db")]
9014            None,
9015            #[cfg(feature = "ws")]
9016            None,
9017        );
9018
9019        // Time, randomness and the effect seams are all inputs like any other:
9020        // serve what the capture took, in order (#1598 for the clock, #1634
9021        // for entropy and the effect tape).
9022        let fixtures = crate::capsule::ReplayFixtures::from_capsule(&capsule);
9023        state = state.with_clock(fixtures.clock());
9024        state = state.with_entropy(fixtures.entropy());
9025        if let Some(buf) = telemetry_guard.log_buffer.clone() {
9026            state.insert_extension(buf);
9027        }
9028        // No startup barrier is applied on this path (`try_build_router_inner`
9029        // does not add one), and a replayed request should meet the app as a
9030        // warm process, not one still starting.
9031        state.probes = crate::probe::ProbeState::default();
9032        state.apply_health_config(&config.health);
9033        state.insert_extension(RegisteredApiVersions(api_versions));
9034        // The AEO documents must replay as captured.
9035        state.insert_extension(crate::aeo::RegisteredAgentSkills(agent_skills));
9036        state.insert_extension(commerce_docs);
9037        #[cfg(feature = "db")]
9038        if let Some(interceptor) = db_interceptor {
9039            state.insert_extension(interceptor);
9040        }
9041        // In place of the builder's backend, which replay does not build: in
9042        // the global cache and the state, as `with_cache_backend` does.
9043        state.shared_cache = crate::cache::install_replay_cache(cache_backend.is_some());
9044
9045        for register in policy_registrations {
9046            register(state.policy_registry());
9047        }
9048
9049        #[cfg(feature = "i18n")]
9050        let custom_layers =
9051            install_i18n_bundle_layer(custom_layers, &state, i18n_bundle, &config.i18n);
9052
9053        install_webhook_registry(&state, &config);
9054        // Production's `outbox::install` gives `deliver_later` a queue when
9055        // `outbox.enabled` and the app has a database. Replay runs no relay,
9056        // so the replay mailer stands in for that queue too.
9057        #[cfg(all(feature = "mail", feature = "db"))]
9058        let builder_mail_queue =
9059            builder_mail_queue || crate::outbox::installs_mail_queue(&state, &config.outbox);
9060        // Before the initializers, as production installs its mailer, so an
9061        // initializer that reads the `Mailer` sees the same configuration.
9062        #[cfg(feature = "mail")]
9063        crate::mail::install_replay_mailer(&state, &config.mail, builder_mail_queue);
9064        run_state_initializers(state_initializers, &state);
9065        // Again after the initializers, so it replaces a live `Mailer` one of
9066        // them installed; before the router state is cloned, so a job replay
9067        // gets it too (#2351 item 7).
9068        #[cfg(feature = "mail")]
9069        crate::mail::install_replay_mailer(&state, &config.mail, builder_mail_queue);
9070        // After the initializers: a cache that only a startup hook installed,
9071        // which replay does not run. Production's initializers did not see it.
9072        crate::cache::install_late_replay_cache(&state, &capsule.effects);
9073        crate::cost::install(&state, &config);
9074        // Durable listeners need the job runtime this path never starts, so —
9075        // as in static builds — only sync listeners are registered, and a
9076        // durable side effect is a clean no-op.
9077        let sync_listeners: Vec<_> = listeners
9078            .into_iter()
9079            .filter(|listener| listener.mode == crate::events::DispatchMode::Sync)
9080            .collect();
9081        finalize_event_bus(sync_listeners, &mut Vec::new(), &state);
9082        // Refresh the AppState-stored config snapshot — see the matching
9083        // comment in `run()`.
9084        state.insert_extension(config.clone());
9085
9086        // Cloned before the builder consumes it, so a job capsule can dispatch
9087        // its handler against the same rebuilt state the router serves from.
9088        let router_state = state.clone();
9089        // See the matching call in `run()`: the graph is published before the
9090        // router is built so `/actuator/graph` answers from the first request.
9091        crate::graph::install(
9092            &state,
9093            crate::graph::manifest::audit(
9094                &graph_mounted_routes(
9095                    &routes,
9096                    &scoped_groups,
9097                    &declared_routes,
9098                    &config,
9099                    // The replay router mounts no `OpenAPI` routes.
9100                    #[cfg(feature = "openapi")]
9101                    None,
9102                ),
9103                omitted_router_count(
9104                    merge_routers.len(),
9105                    nest_routers.iter().map(|(prefix, _)| prefix.as_str()),
9106                    &declared_routes,
9107                ),
9108            ),
9109        );
9110        let router = crate::router::try_build_router_inner(
9111            routes,
9112            &config,
9113            state,
9114            crate::router::RouterContext {
9115                exception_filters,
9116                scoped_groups,
9117                merge_routers,
9118                nest_routers,
9119                declared_routes,
9120                custom_layers,
9121                static_gate_layers: Vec::new(),
9122                #[cfg(feature = "maud")]
9123                error_page_renderer,
9124                session_store: None,
9125                #[cfg(feature = "openapi")]
9126                openapi: None,
9127                #[cfg(feature = "mcp")]
9128                mcp: None,
9129                #[cfg(feature = "webmcp")]
9130                webmcp: None,
9131            },
9132        )
9133        .unwrap_or_else(|error| {
9134            std::process::exit(crate::capsule::print_refusal(
9135                &format!("the application's router could not be rebuilt: {error}"),
9136                &path,
9137            ))
9138        });
9139
9140        // A job capsule has no request to drive: dispatch the recorded job's
9141        // handler instead, with the same clock, entropy and effect tape.
9142        let outcome = if let Some(job) = capsule.job.as_ref() {
9143            let Some(info) = jobs.iter().find(|info| info.name == job.name) else {
9144                std::process::exit(crate::capsule::print_refusal(
9145                    &format!(
9146                        "the capsule records a failure in job {:?}, which this build does not \
9147                         register; replay it against the build that ran it, or add the job to \
9148                         `AppBuilder::jobs()`",
9149                        job.name
9150                    ),
9151                    &path,
9152                ));
9153            };
9154            let handler = info.handler;
9155            let job_state = router_state.clone();
9156            if let Some(interceptor) = job_interceptor {
9157                job_state.insert_extension(interceptor);
9158            }
9159            let job_name = job.name.clone();
9160            let dispatch: crate::capsule::JobDispatch = Box::new(move |payload| {
9161                Box::pin(async move {
9162                    crate::job::run_handler_with_interceptor(&job_name, handler, job_state, payload)
9163                        .await
9164                })
9165            });
9166            crate::capsule::execute_job(dispatch, &capsule, divergences, &fixtures).await
9167        } else {
9168            crate::capsule::execute(router, &capsule, divergences, &fixtures).await
9169        };
9170        std::process::exit(crate::capsule::print_verdict(&outcome, &path));
9171    }
9172}
9173
9174pub(crate) fn is_static_build_mode() -> bool {
9175    std::env::var("AUTUMN_BUILD_STATIC").as_deref() == Ok("1")
9176}
9177
9178/// Set to `1` by `autumn build` when the build has another output (an edge
9179/// capsule), so a static build with no static routes is not an error.
9180pub(crate) const BUILD_STATIC_ALLOW_EMPTY_ENV: &str = "AUTUMN_BUILD_STATIC_ALLOW_EMPTY";
9181
9182/// Remove `dist/` from an earlier build. An edge-only build renders no pages,
9183/// so old ones must not ship with it. A missing `dist/` is not an error.
9184fn clear_stale_static_output(dist_dir: &std::path::Path) -> std::io::Result<()> {
9185    match std::fs::remove_dir_all(dist_dir) {
9186        Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
9187        other => other,
9188    }
9189}
9190
9191/// The exit code of a static build that finds no static routes.
9192const fn no_static_routes_exit_code(allow_empty: bool) -> i32 {
9193    if allow_empty { 0 } else { 1 }
9194}
9195
9196/// Stop a managed Postgres child from a synchronous `process::exit` path in a
9197/// non-server entrypoint (static build, one-off task). Those modes don't run
9198/// `on_shutdown` before their failure exits, and `process::exit` skips `Drop`,
9199/// so a managed cluster started by `setup_database` would otherwise be orphaned
9200/// on the data dir/port.
9201///
9202/// These call sites run on a Tokio worker thread; the (blocking, own-runtime)
9203/// `emergency_stop` would panic if entered there, so run it on a fresh thread
9204/// with no ambient runtime. No-op unless the `managed-pg` feature is active.
9205// The body is empty without `managed-pg` (so it can't be `const` with it).
9206#[allow(clippy::missing_const_for_fn)]
9207fn exit_stop_managed_pg() {
9208    #[cfg(feature = "managed-pg")]
9209    {
9210        let _ = std::thread::spawn(crate::managed_pg::emergency_stop).join();
9211    }
9212}
9213
9214pub(crate) fn is_dump_routes_mode() -> bool {
9215    std::env::var("AUTUMN_DUMP_ROUTES").as_deref() == Ok("1")
9216}
9217
9218/// Whether the process should dump the generated `OpenAPI` document and exit.
9219///
9220/// Set by `autumn openapi export`. Unlike the routes dump this is checked even
9221/// when the `openapi` feature is off, so the CLI gets an explicit "no spec here"
9222/// answer instead of a booted server.
9223pub(crate) fn is_dump_openapi_mode() -> bool {
9224    std::env::var("AUTUMN_DUMP_OPENAPI").as_deref() == Ok("1")
9225}
9226
9227/// Whether the dump should also emit the declared plugin contracts
9228/// ([`PLUGIN_CONTRACT_MARKER`](crate::plugin_contract::PLUGIN_CONTRACT_MARKER)).
9229///
9230/// Set by `autumn plugin-check`, which needs the contracts to report
9231/// experimental-surface use. The plain `autumn routes` listing does not set it,
9232/// so its stderr is unchanged.
9233pub(crate) fn is_dump_plugin_contract_mode() -> bool {
9234    std::env::var("AUTUMN_DUMP_PLUGIN_CONTRACT").as_deref() == Ok("1")
9235}
9236
9237/// Whether the dump should also emit the resolved security configuration
9238/// ([`SECURITY_CONFIG_MARKER`](crate::route_listing::SECURITY_CONFIG_MARKER)).
9239///
9240/// Set by `autumn routes audit` (which needs the CSRF / headers config to build
9241/// the `declared` manifest dimensions) but not by the plain `autumn routes`
9242/// listing, so that command's stderr stays free of the marker line.
9243pub(crate) fn is_dump_security_mode() -> bool {
9244    std::env::var("AUTUMN_DUMP_SECURITY").as_deref() == Ok("1")
9245}
9246
9247pub(crate) fn is_dump_jobs_mode() -> bool {
9248    std::env::var("AUTUMN_DUMP_JOBS").as_deref() == Ok("1")
9249}
9250
9251/// The mounted-route table the architecture graph joins against (issue #1747).
9252///
9253/// Top-level routes carry their own path; a scoped group's children do not --
9254/// the group's prefix is applied at mount time -- so each child is summarised
9255/// with its group prefix.
9256fn graph_mounted_routes(
9257    routes: &[Route],
9258    scoped_groups: &[ScopedGroup],
9259    declared_routes: &[crate::route_listing::RouteInfo],
9260    config: &crate::config::AutumnConfig,
9261    #[cfg(feature = "openapi")] openapi: Option<&crate::openapi::OpenApiConfig>,
9262) -> Vec<crate::graph::MountedRoute> {
9263    // The framework's own mounts — probes, the actuator, htmx assets, the docs
9264    // UI — are served by `router.rs` and belong in the census: the manifest and
9265    // the guide both promise a framework endpoint is *named* in
9266    // `unmodelled_mounted_routes`, and without them the completeness report
9267    // systematically understated the served surface (Codex round 5). Built with
9268    // the same helper `autumn routes` uses, so the two cannot disagree about
9269    // what the framework mounts.
9270    let mut framework: Vec<crate::route_listing::RouteInfo> = Vec::new();
9271    crate::route_listing::append_framework_routes(&mut framework, config);
9272    // The router merges the `OpenAPI` routes only when the runtime gate is open.
9273    #[cfg(feature = "openapi")]
9274    if let Some(openapi) = openapi.filter(|_| config.openapi_runtime.enabled) {
9275        crate::route_listing::append_openapi_routes(&mut framework, openapi);
9276    }
9277
9278    routes
9279        .iter()
9280        .map(|route| graph_route_summary(route, None))
9281        .chain(scoped_groups.iter().flat_map(|group| {
9282            group
9283                .routes
9284                .iter()
9285                .map(move |route| graph_route_summary(route, Some(&group.prefix)))
9286        }))
9287        .chain(declared_routes.iter().map(graph_declared_route_summary))
9288        // Framework endpoints are named only; their auth is not a declaration.
9289        .chain(framework.iter().map(|route| crate::graph::MountedRoute {
9290            auth_declared: false,
9291            ..graph_declared_route_summary(route)
9292        }))
9293        .collect()
9294}
9295
9296/// The census of a process that does not serve HTTP routes (a worker).
9297///
9298/// A worker builds `try_build_probe_only_router`: probes and actuator only.
9299fn graph_worker_mounted_routes(
9300    config: &crate::config::AutumnConfig,
9301) -> Vec<crate::graph::MountedRoute> {
9302    let mut framework: Vec<crate::route_listing::RouteInfo> = Vec::new();
9303    crate::route_listing::append_probe_and_actuator_routes(&mut framework, config);
9304    // Framework endpoints are named only; their auth is not a declaration.
9305    framework
9306        .iter()
9307        .map(|route| crate::graph::MountedRoute {
9308            auth_declared: false,
9309            ..graph_declared_route_summary(route)
9310        })
9311        .collect()
9312}
9313
9314/// The architecture-graph view of a route declared through
9315/// `declare_plugin_routes` (issue #1747).
9316///
9317/// These are real served endpoints behind an otherwise opaque `nest` mount, and
9318/// declaring them is what stops `omitted_router_count` counting that nest as
9319/// unenumerable. Without them here the graph would report *neither* an opaque
9320/// router nor a mounted route for that surface — a hole that reads as complete
9321/// coverage. They carry no `#[route]` descriptor in this binary, so they land
9322/// in `unmodelled_mounted_routes`: named, which is the honest answer, rather
9323/// than silently absent.
9324fn graph_declared_route_summary(
9325    route: &crate::route_listing::RouteInfo,
9326) -> crate::graph::MountedRoute {
9327    let mut roles = route.roles.clone();
9328    roles.sort();
9329    roles.dedup();
9330    let mut scopes = route.scopes.clone();
9331    scopes.sort();
9332    scopes.dedup();
9333    crate::graph::MountedRoute {
9334        method: route.method.clone(),
9335        path: route.path.clone(),
9336        handler: route.handler.clone(),
9337        module_path: route.module.clone().unwrap_or_default(),
9338        auth: crate::graph::RouteAuth {
9339            secured: route.classification == crate::route_listing::RouteClassification::Gated,
9340            roles,
9341            scopes,
9342            policy: route.policy,
9343            // A `RouteInfo`'s `classification` already comes from
9344            // `route_listing::classify`, which folds a repository's own policy
9345            // and scope guard into `Gated` — so unlike the `Route` path above
9346            // there is nothing further to recover here, and no separate
9347            // repository scope to name.
9348            repository_scope: false,
9349            public: route.classification == crate::route_listing::RouteClassification::Public,
9350        },
9351        // A `RouteInfo` carries no repository metadata: these are plugin and
9352        // framework mounts, which no `#[repository]` generated.
9353        repository_api: None,
9354        // Only a proven posture is a declaration. An unclassified route has
9355        // none, so it stays name-only. The caller clears this for framework
9356        // endpoints.
9357        auth_declared: matches!(
9358            route.classification,
9359            crate::route_listing::RouteClassification::Gated
9360                | crate::route_listing::RouteClassification::Public
9361        ),
9362    }
9363}
9364
9365/// The architecture-graph view of one mounted route (issue #1747).
9366///
9367/// The *mounted* path, not the declared one: a scoped group's children carry
9368/// only their child path on the `Route`, and recording `/items` for a route an
9369/// operator calls at `/api/v1/items` would make a scope rename invisible.
9370/// `join_nested_path` is the same helper the `OpenAPI` collector and the
9371/// agent-authority manifest use, so the three cannot disagree about where a
9372/// route lives.
9373///
9374/// The auth posture is read straight off the route's `ApiDoc` rather than
9375/// derived a second time here: `#[secured]`/`#[authorize]`/`#[public]` already
9376/// populate it, and a second derivation is a second thing to drift.
9377fn graph_route_summary(route: &Route, scope_prefix: Option<&str>) -> crate::graph::MountedRoute {
9378    let mut roles: Vec<String> = route
9379        .api_doc
9380        .required_roles
9381        .iter()
9382        .map(|r| (*r).to_owned())
9383        .collect();
9384    roles.sort();
9385    roles.dedup();
9386    let mut scopes: Vec<String> = route
9387        .api_doc
9388        .required_scopes
9389        .iter()
9390        .map(|s| (*s).to_owned())
9391        .collect();
9392    scopes.sort();
9393    scopes.dedup();
9394    crate::graph::MountedRoute {
9395        method: route.method.to_string(),
9396        path: scope_prefix.map_or_else(
9397            || route.path.to_string(),
9398            |prefix| crate::router::join_nested_path(prefix, route.path),
9399        ),
9400        handler: route.name.to_owned(),
9401        module_path: route.api_doc.module_path.to_owned(),
9402        // A repository auto-API route registers its guard on the *repository*,
9403        // not on the generated handler, so `ApiDoc` alone reports it as
9404        // unauthenticated. `route_listing::classify` — the derivation `autumn
9405        // routes audit` proves the posture with — ORs in the repository's own
9406        // `has_policy` and treats a registered `scope_check` as gated. The graph
9407        // claims to state that same posture, so it has to read the same two
9408        // sources; reading `ApiDoc` alone made a `#[repository(api = "...",
9409        // policy = ...)]` endpoint serialize as `auth: none`.
9410        auth: crate::graph::RouteAuth {
9411            secured: route.api_doc.secured,
9412            roles,
9413            scopes,
9414            policy: route.api_doc.has_policy
9415                || route
9416                    .repository
9417                    .as_ref()
9418                    .is_some_and(|meta| meta.has_policy),
9419            repository_scope: route
9420                .repository
9421                .as_ref()
9422                .is_some_and(|meta| meta.scope_check.is_some()),
9423            public: route.api_doc.public,
9424        },
9425        // Declared ownership, straight off the route the `#[repository(api =
9426        // "...")]` macro generated. Never re-derived from the served path.
9427        repository_api: route
9428            .repository
9429            .as_ref()
9430            .map(|meta| crate::graph::RepositoryApiOwner {
9431                api: meta.api_path.to_owned(),
9432                model: meta.resource_type_name.to_owned(),
9433            }),
9434        auth_declared: false,
9435    }
9436}
9437
9438/// The whole-API hatches the agent-authority manifest reads.
9439#[derive(Debug, Clone, Copy, Default)]
9440struct AgentExposure {
9441    /// `expose_all_as_mcp()` was called.
9442    mcp_expose_all: bool,
9443    /// `expose_all_as_webmcp()` was called.
9444    webmcp_expose_all: bool,
9445}
9446
9447/// The slice of a [`Route`] the agent-authority manifest needs (#1691).
9448///
9449/// Built here rather than in `agent_authority::manifest` so that module needs
9450/// no dependency on the router, and unconditionally rather than behind the
9451/// `openapi` feature: which handlers an agent can reach is not an
9452/// documentation concern.
9453///
9454/// `expose_all` is the app's whole-API MCP hatch. It has to be threaded in:
9455/// deriving tool-ness from `#[api_doc(mcp)]` alone made every route
9456/// `expose_all_as_mcp()` swept up invisible to this document — in neither
9457/// `actions` nor `ungoverned_tools` — while the document's own `excluded`
9458/// section claimed they surfaced there (#1691 P2-6). The same call also
9459/// applies the JSON-out eligibility gate, so an HTML route someone tagged
9460/// `#[api_doc(mcp)]` is no longer reported as a tool it will never become.
9461fn agent_authority_route_summary(
9462    route: &Route,
9463    scope_prefix: Option<&str>,
9464    exposure: AgentExposure,
9465) -> crate::agent_authority::manifest::RouteSummary {
9466    // One string, used both for the row and for the predicate, so the two can
9467    // never disagree about a route's verb.
9468    let method = route.method.to_string();
9469    // The path an agent actually calls. `join_nested_path` is the same helper
9470    // the OpenAPI collector uses for scoped groups, so the manifest and the
9471    // spec cannot disagree about where a route lives.
9472    let path = scope_prefix.map_or_else(
9473        || route.path.to_string(),
9474        |prefix| crate::router::join_nested_path(prefix, route.path),
9475    );
9476    let input = crate::agent_authority::manifest::McpExposureInput {
9477        method: &method,
9478        hidden: route.api_doc.hidden,
9479        mcp_tool: route.api_doc.mcp_tool,
9480        mcp_exclude: route.api_doc.mcp_exclude,
9481        mcp_stream: route.api_doc.mcp_stream,
9482        has_response_schema: route.api_doc.response.is_some(),
9483        success_status: route.api_doc.success_status,
9484        expose_all: exposure.mcp_expose_all,
9485    };
9486    let exposed_by = crate::agent_authority::manifest::mcp_exposure(&input);
9487    let webmcp_exposed_by = crate::agent_authority::manifest::webmcp_exposure(
9488        &input,
9489        route.api_doc.webmcp_tool,
9490        route.api_doc.webmcp_exclude,
9491        exposure.webmcp_expose_all,
9492    );
9493    crate::agent_authority::manifest::RouteSummary {
9494        method,
9495        path,
9496        handler: route.name,
9497        // The name an MCP client actually calls: `#[api_doc(operation_id =
9498        // "...")]` renames the tool without renaming the handler.
9499        operation_id: route.api_doc.operation_id,
9500        module_path: route.api_doc.module_path,
9501        mcp_tool: exposed_by.is_some(),
9502        exposed_by,
9503        duplicate_operation_id: false,
9504        webmcp_tool: webmcp_exposed_by.is_some(),
9505        webmcp_exposed_by,
9506        // Filled by the route macro from the handler's `#[agent_operable]`
9507        // marker, in either attribute order.
9508        agent_authority: route.api_doc.agent_authority,
9509    }
9510}
9511
9512pub(crate) fn is_list_one_off_tasks_mode() -> bool {
9513    std::env::var("AUTUMN_LIST_TASKS").as_deref() == Ok("1")
9514}
9515
9516/// Whether `AUTUMN_MIGRATE=1` requests the migrate-only one-shot: apply pending
9517/// embedded migrations and exit without starting the HTTP server. Set by
9518/// `autumn deploy`'s redeploy cutover (issue #1607) so migrations land before
9519/// traffic is flipped to the new release.
9520pub(crate) fn is_migrate_only_mode() -> bool {
9521    std::env::var("AUTUMN_MIGRATE").as_deref() == Ok("1")
9522}
9523
9524/// The `autumn db retention` one-shot mode requested by
9525/// `AUTUMN_DB_RETENTION`, if any (issue #1605).
9526///
9527/// `report` counts what is eligible and deletes nothing; `purge` enforces the
9528/// policy immediately. Any other value is rejected loudly rather than
9529/// defaulting to either — guessing wrong in one direction deletes data the
9530/// operator did not ask to delete.
9531pub(crate) fn framework_retention_mode_from_env() -> Option<FrameworkRetentionMode> {
9532    let raw = std::env::var(FRAMEWORK_RETENTION_ENV).ok()?;
9533    match raw.trim() {
9534        "" => None,
9535        "report" => Some(FrameworkRetentionMode::Report),
9536        "purge" => Some(FrameworkRetentionMode::Purge),
9537        other => {
9538            // Warn and fall through to a normal boot rather than exiting.
9539            // This var is read on *every* start, so exiting here would let a
9540            // stray value in a wrapping script or a shared environment take a
9541            // production app down at boot. Mirrors `AUTUMN_ROLE`'s handling
9542            // of an unrecognized value.
9543            eprintln!(
9544                "Warning: {FRAMEWORK_RETENTION_ENV}={other:?} is not valid (expected \"report\" \
9545                 or \"purge\"), ignoring"
9546            );
9547            None
9548        }
9549    }
9550}
9551
9552/// The `autumn data capsule` one-shot mode (issue #1811).
9553#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9554pub(crate) enum DataCapsuleMode {
9555    /// Write a capsule for one subject.
9556    Export,
9557    /// Verify, then import a capsule.
9558    Import,
9559    /// Verify a capsule only.
9560    Verify,
9561}
9562
9563impl DataCapsuleMode {
9564    const fn as_str(self) -> &'static str {
9565        match self {
9566            Self::Export => "export",
9567            Self::Import => "import",
9568            Self::Verify => "verify",
9569        }
9570    }
9571}
9572
9573/// The env var `autumn data capsule` sets to select the one-shot mode.
9574pub(crate) const DATA_CAPSULE_ENV: &str = "AUTUMN_DATA_CAPSULE";
9575/// The subject id of `autumn data capsule export`.
9576pub(crate) const DATA_CAPSULE_SUBJECT_ENV: &str = "AUTUMN_DATA_CAPSULE_SUBJECT";
9577/// The capsule directory of `autumn data capsule`.
9578pub(crate) const DATA_CAPSULE_PATH_ENV: &str = "AUTUMN_DATA_CAPSULE_PATH";
9579/// Line prefix of the JSON report, matched by `autumn-cli/src/data_capsule.rs`.
9580pub(crate) const DATA_CAPSULE_JSON_PREFIX: &str = "AUTUMN_DATA_CAPSULE_REPORT=";
9581
9582/// The `autumn data capsule` mode requested by `AUTUMN_DATA_CAPSULE`, if any.
9583///
9584/// An unknown value gives a warning and a normal boot, the same as
9585/// `AUTUMN_DB_RETENTION`: a stray value must not stop a server.
9586pub(crate) fn data_capsule_mode_from_env() -> Option<DataCapsuleMode> {
9587    parse_data_capsule_mode(&std::env::var(DATA_CAPSULE_ENV).ok()?)
9588}
9589
9590fn parse_data_capsule_mode(raw: &str) -> Option<DataCapsuleMode> {
9591    match raw.trim() {
9592        "" => None,
9593        "export" => Some(DataCapsuleMode::Export),
9594        "import" => Some(DataCapsuleMode::Import),
9595        "verify" => Some(DataCapsuleMode::Verify),
9596        other => {
9597            eprintln!(
9598                "Warning: {DATA_CAPSULE_ENV}={other:?} is not valid (expected \"export\", \
9599                 \"import\" or \"verify\"), ignoring"
9600            );
9601            None
9602        }
9603    }
9604}
9605
9606/// The JSON report line of `autumn data capsule`.
9607fn data_capsule_report_line(
9608    mode: DataCapsuleMode,
9609    result: &Result<serde_json::Value, crate::gdpr::portability::DataCapsuleError>,
9610) -> String {
9611    let body = match result {
9612        Ok(report) => serde_json::json!({"ok": true, "mode": mode.as_str(), "report": report}),
9613        Err(error) => {
9614            serde_json::json!({"ok": false, "mode": mode.as_str(), "error": error.to_string()})
9615        }
9616    };
9617    format!("{DATA_CAPSULE_JSON_PREFIX}{body}")
9618}
9619
9620/// Print the report line and exit: `0` on success, else `1`.
9621fn emit_data_capsule_report(
9622    mode: DataCapsuleMode,
9623    result: &Result<serde_json::Value, crate::gdpr::portability::DataCapsuleError>,
9624) -> ! {
9625    println!("{}", data_capsule_report_line(mode, result));
9626    if let Err(error) = result {
9627        eprintln!("autumn data capsule {}: {error}", mode.as_str());
9628    }
9629    std::process::exit(i32::from(result.is_err()));
9630}
9631
9632/// The database state that verify gives to the state initializers.
9633///
9634/// Verify runs no migration. It builds the pools as export and import do,
9635/// with the app's `with_pool_provider` and `with_shard_provider` when it has
9636/// them, so an initializer sees the same `pool()` and `shards()`, for example
9637/// to build its own `CapsuleService`. The built-in pools are lazy: they
9638/// connect, and a `SQLite` pool creates its file, only at the first checkout,
9639/// which verify never makes. A provider that fails gives no state: verify then
9640/// falls back to the signer of the configuration.
9641#[cfg(feature = "db")]
9642async fn verify_database_parts(
9643    config: &AutumnConfig,
9644    pool_provider: Option<PoolProviderFactory>,
9645    shard_provider: Option<ShardProviderFactory>,
9646    shard_router: Option<Arc<dyn crate::sharding::ShardRouter>>,
9647    directory_shard_router: bool,
9648) -> (
9649    Option<crate::db::DatabaseTopology>,
9650    Option<crate::sharding::ShardSet>,
9651) {
9652    let topology = match pool_provider {
9653        Some(factory) => factory(config.database.clone()).await,
9654        None => crate::db::create_topology(&config.database),
9655    }
9656    .ok()
9657    .flatten();
9658    let use_directory_router = shard_router.is_none()
9659        && (directory_shard_router || config.database.directory_shard_router);
9660    // No invalidation listener: verify opens no connection of its own.
9661    let shards = resolve_shard_set(
9662        config,
9663        shard_router,
9664        shard_provider,
9665        use_directory_router,
9666        false,
9667        topology.as_ref(),
9668    )
9669    .await
9670    .ok()
9671    .flatten();
9672    (topology, shards)
9673}
9674
9675#[cfg(test)]
9676mod data_capsule_mode_tests {
9677    use super::*;
9678
9679    #[cfg(all(feature = "db", not(feature = "sqlite")))]
9680    #[tokio::test]
9681    async fn verify_gives_initializers_lazy_pools_and_shards() {
9682        // A state initializer can build a routed store from `shards()`, so
9683        // verify keeps the shape of the state. The pools must not connect:
9684        // nothing listens on port 9.
9685        let mut config = AutumnConfig::default();
9686        config.database.url = Some("postgres://u:p@127.0.0.1:9/none".to_owned());
9687        config.database.shards = vec![crate::config::ShardConfig {
9688            name: "s1".to_owned(),
9689            primary_url: crate::test_urls::primary("s1"),
9690            slots: None,
9691            replica_url: None,
9692            primary_pool_size: None,
9693            replica_pool_size: None,
9694            replica_fallback: None,
9695        }];
9696        let (topology, shards) = verify_database_parts(&config, None, None, None, false).await;
9697        assert!(topology.is_some(), "a lazy control pool");
9698        assert_eq!(shards.map(|s| s.len()), Some(1), "the configured shard");
9699    }
9700
9701    #[cfg(feature = "sqlite")]
9702    #[tokio::test]
9703    async fn verify_gives_sqlite_initializers_a_lazy_pool_without_a_file() {
9704        // An initializer can build its `CapsuleService` from `pool()`. The
9705        // pool must not open the database: SQLite would create the file.
9706        let dir = tempfile::tempdir().unwrap();
9707        let path = dir.path().join("app.db");
9708        let mut config = AutumnConfig::default();
9709        config.database.url = Some(format!("sqlite://{}", path.display()));
9710        let (topology, _) = verify_database_parts(&config, None, None, None, false).await;
9711        assert!(topology.is_some(), "a lazy pool");
9712        assert!(!path.exists(), "verify creates no database file");
9713    }
9714
9715    #[cfg(feature = "db")]
9716    #[tokio::test]
9717    async fn verify_uses_the_pool_provider_of_the_app() {
9718        // The app has no database URL: only its `with_pool_provider` knows
9719        // the pool, as on export and import. Verify must ask it too.
9720        let dir = tempfile::tempdir().unwrap();
9721        let url = if cfg!(feature = "sqlite") {
9722            format!("sqlite://{}", dir.path().join("app.db").display())
9723        } else {
9724            "postgres://u:p@127.0.0.1:9/none".to_owned()
9725        };
9726        let provider: PoolProviderFactory = Box::new(move |mut database| {
9727            Box::pin(async move {
9728                database.url = Some(url);
9729                crate::db::create_topology(&database)
9730            })
9731        });
9732        let config = AutumnConfig::default();
9733        let (topology, _) = verify_database_parts(&config, Some(provider), None, None, false).await;
9734        assert!(topology.is_some(), "the topology of the provider");
9735    }
9736
9737    #[test]
9738    fn mode_parses_the_three_values_and_ignores_others() {
9739        assert_eq!(
9740            parse_data_capsule_mode("export"),
9741            Some(DataCapsuleMode::Export)
9742        );
9743        assert_eq!(
9744            parse_data_capsule_mode(" import "),
9745            Some(DataCapsuleMode::Import)
9746        );
9747        assert_eq!(
9748            parse_data_capsule_mode("verify"),
9749            Some(DataCapsuleMode::Verify)
9750        );
9751        assert_eq!(parse_data_capsule_mode(""), None);
9752        assert_eq!(parse_data_capsule_mode("purge"), None);
9753    }
9754
9755    #[test]
9756    fn report_line_carries_the_prefix_and_the_outcome() {
9757        let ok = data_capsule_report_line(
9758            DataCapsuleMode::Export,
9759            &Ok(serde_json::json!({"records": 3})),
9760        );
9761        let json: serde_json::Value =
9762            serde_json::from_str(ok.strip_prefix(DATA_CAPSULE_JSON_PREFIX).unwrap()).unwrap();
9763        assert_eq!(json["ok"], true);
9764        assert_eq!(json["mode"], "export");
9765        assert_eq!(json["report"]["records"], 3);
9766
9767        let err = data_capsule_report_line(
9768            DataCapsuleMode::Verify,
9769            &Err(crate::gdpr::portability::DataCapsuleError::MissingSigningSecret),
9770        );
9771        let json: serde_json::Value =
9772            serde_json::from_str(err.strip_prefix(DATA_CAPSULE_JSON_PREFIX).unwrap()).unwrap();
9773        assert_eq!(json["ok"], false);
9774        assert!(json["error"].as_str().unwrap().contains("signing secret"));
9775    }
9776}
9777
9778/// The env var `autumn db retention` sets to select the one-shot mode.
9779pub(crate) const FRAMEWORK_RETENTION_ENV: &str = "AUTUMN_DB_RETENTION";
9780
9781/// Why an on-demand purge must stop, if it must. `force_env` is the value of
9782/// [`FRAMEWORK_RETENTION_FORCE_ENV`]. Report mode is never refused.
9783fn framework_purge_refusal(
9784    mode: FrameworkRetentionMode,
9785    profile: Option<&str>,
9786    force_env: Option<&str>,
9787) -> Option<String> {
9788    if mode != FrameworkRetentionMode::Purge {
9789        return None;
9790    }
9791    crate::data_retention::purge_refusal(profile, force_env == Some("1"))
9792}
9793
9794/// Set to `1` by `autumn db retention --force`.
9795pub(crate) const FRAMEWORK_RETENTION_FORCE_ENV: &str = "AUTUMN_DB_RETENTION_FORCE";
9796
9797/// The env var `autumn db retention --dataset <key>` sets to narrow the run.
9798pub(crate) const FRAMEWORK_RETENTION_DATASET_ENV: &str = "AUTUMN_DB_RETENTION_DATASET";
9799
9800/// Line prefix framing the framework retention report on stdout, matched
9801/// verbatim by `autumn-cli/src/db/retention.rs`.
9802pub(crate) const FRAMEWORK_RETENTION_JSON_PREFIX: &str = "AUTUMN_DB_RETENTION_REPORT=";
9803
9804/// What `autumn db retention` asked the app to do.
9805#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9806pub(crate) enum FrameworkRetentionMode {
9807    /// Count what is eligible; delete nothing.
9808    Report,
9809    /// Enforce the policy now.
9810    Purge,
9811}
9812
9813/// The `--dataset` filter, if one was passed. Blank is treated as absent so a
9814/// wrapping script exporting an empty value cannot turn into a not-found
9815/// error.
9816pub(crate) fn framework_retention_dataset_from_env() -> Option<String> {
9817    std::env::var(FRAMEWORK_RETENTION_DATASET_ENV)
9818        .ok()
9819        .map(|value| value.trim().to_owned())
9820        .filter(|value| !value.is_empty())
9821}
9822
9823/// Whether `AUTUMN_RETENTION_DRY_RUN=1` requests the retention dry-run
9824/// one-shot: count (never delete) what every declared `retention(...)`
9825/// policy would sweep and exit. Set by `autumn retention --dry-run`
9826/// (issue #1342).
9827pub(crate) fn is_retention_dry_run_mode() -> bool {
9828    std::env::var("AUTUMN_RETENTION_DRY_RUN").as_deref() == Ok("1")
9829}
9830
9831/// Line prefix framing the retention dry-run's machine-readable JSON report
9832/// on stdout, matched verbatim by `autumn-cli/src/retention.rs`.
9833///
9834/// Regression (#1342 review round 14): the dry-run one-shot's stdout is not
9835/// otherwise guaranteed to contain nothing but the JSON report — the default
9836/// `dev` profile initializes a stdout-backed tracing formatter, and Diesel's
9837/// `MigrationHarness` writes pending-migration progress directly to stdout
9838/// (`HarnessWithOutput::write_to_stdout`), both of which run before the
9839/// report line prints whenever anything is pending. Parsing the *entire*
9840/// captured stdout as one JSON blob (the original approach) then fails on
9841/// any of that incidental output. Framing the report as the one line
9842/// starting with this prefix lets the CLI find it regardless of what else
9843/// landed on stdout, without having to redirect every one-shot's logging or
9844/// Diesel's hardcoded migration-progress writer — both shared with other
9845/// boot paths (e.g. `autumn migrate`) that want stdout output.
9846const RETENTION_DRY_RUN_JSON_PREFIX: &str = "AUTUMN_RETENTION_DRY_RUN_REPORT=";
9847
9848/// `AUTUMN_RETENTION_MODEL=<name>` narrows the dry-run report to one model's
9849/// policy. Set by `autumn retention --dry-run --model <name>`.
9850#[cfg(feature = "db")]
9851fn retention_dry_run_model_filter_from_env() -> Option<String> {
9852    std::env::var("AUTUMN_RETENTION_MODEL")
9853        .ok()
9854        .map(|value| value.trim().to_owned())
9855        .filter(|value| !value.is_empty())
9856}
9857
9858/// Combines `descriptors`' generated task names with `tasks` (hand-declared
9859/// via `tasks![...]`) and checks the merged set for a collision — the same
9860/// merge-then-validate step real boot performs (`tasks.extend(...)` followed
9861/// by `validate_unique_scheduled_task_names`, in `AppBuilder::build`) before
9862/// ever spawning a scheduler loop.
9863///
9864/// A free function (rather than inlined into `run_retention_dry_run_mode`)
9865/// so the collision case is unit-testable against synthetic descriptors and
9866/// tasks without booting an `AppBuilder` or touching the process-global
9867/// `inventory` registry (#1342 review round 18).
9868#[cfg(feature = "db")]
9869fn merge_and_validate_task_names(
9870    descriptors: &[&crate::retention::RetentionSweepDescriptor],
9871    tasks: Vec<crate::task::TaskInfo>,
9872) -> Result<(), String> {
9873    let mut merged: Vec<crate::task::TaskInfo> = descriptors
9874        .iter()
9875        .map(|descriptor| (descriptor.task_info)())
9876        .collect();
9877    merged.extend(tasks);
9878    crate::task::validate_unique_scheduled_task_names(&merged)
9879}
9880
9881/// Whether `AUTUMN_REPLAY_CAPSULE=<path>` requests the capsule-replay one-shot:
9882/// rebuild the app offline, replay the recorded request, print the verdict and
9883/// exit without starting the HTTP server. Set by `autumn replay` (issue #1598).
9884pub(crate) fn is_replay_mode() -> bool {
9885    replay_capsule_from_env().is_some()
9886}
9887
9888/// The capsule path `AUTUMN_REPLAY_CAPSULE` names, if it names one.
9889fn replay_capsule_from_env() -> Option<String> {
9890    std::env::var("AUTUMN_REPLAY_CAPSULE")
9891        .ok()
9892        .as_deref()
9893        .and_then(normalize_replay_capsule)
9894}
9895
9896/// Trim a raw `AUTUMN_REPLAY_CAPSULE` value; a blank one selects no mode.
9897fn normalize_replay_capsule(value: &str) -> Option<String> {
9898    let trimmed = value.trim();
9899    (!trimmed.is_empty()).then(|| trimmed.to_owned())
9900}
9901
9902/// Force the configuration knobs a replay must not honour (F15).
9903///
9904/// A capsule is replayed on a laptop or in CI, where the recording app's Redis,
9905/// object storage and mail transport are not reachable — and where reaching
9906/// them would be a side effect of *diagnosing* a failure, not part of it. Every
9907/// other setting stays the application's own, because status codes, CSRF,
9908/// locale routing and error rendering all depend on them.
9909#[cfg(feature = "reporting")]
9910fn force_offline_replay_config(config: &mut AutumnConfig) {
9911    // Capturing the replay would write a capsule of a capsule, and arming
9912    // capture would point the connection-checkout marker at the stub pool.
9913    config.failure_capture.enabled = false;
9914    // Sessions in process memory: no Redis is dialled, and the
9915    // memory-in-production warning is noise for a one-shot replay.
9916    config.session.backend = crate::session::SessionBackend::Memory;
9917    config.session.allow_memory_in_production = true;
9918    // Channels in process memory for the same reason: the Redis backend spawns
9919    // a publisher and a listener task against the application's live fan-out
9920    // the moment the state is built.
9921    config.channels.backend = crate::config::ChannelBackend::InProcess;
9922    // Every other config-driven store the request path can reach. A replay does
9923    // not merely dial these, it writes to them: a rate-limit bucket is
9924    // decremented, an idempotency key and its lock are taken, a submit token is
9925    // consumed, a webhook replay key is inserted and deleted. Against the recording
9926    // deployment's Redis, diagnosing a failure would change production state, and
9927    // an unreachable backend would manufacture a 429 or 503 the recorded run never
9928    // produced. A replay is a read of the past; it writes nothing anywhere.
9929    config.security.rate_limit.backend = crate::security::config::RateLimitBackend::Memory;
9930    config.idempotency.backend = crate::config::IdempotencyBackend::Memory;
9931    config.idempotency.allow_memory_in_production = true;
9932    // Submit tokens inherit the idempotency backend when unset, so the
9933    // in-memory store is pinned explicitly rather than left to that inheritance.
9934    config.security.submit_token.backend = Some(crate::config::IdempotencyBackend::Memory);
9935    config.security.webhooks.replay.backend = crate::webhook::WebhookReplayBackend::Memory;
9936    config.security.webhooks.replay.allow_memory_in_production = true;
9937    // A `#[cached]` handler would otherwise read — and populate — the
9938    // application's shared cache.
9939    config.cache.backend = crate::config::CacheBackend::Memory;
9940    // A handler that enqueues a job would otherwise put it on the real queue,
9941    // where a live worker would run it. The replay never starts a job runtime,
9942    // so the enqueue lands in a process-local queue nothing drains.
9943    "local".clone_into(&mut config.jobs.backend);
9944    // No wall-clock deadline. Everything a replay consumes comes from the capsule,
9945    // including the clock the handler reads, but the request-timeout layer runs on
9946    // real tokio timers — so how long the replay takes is the one thing still
9947    // measured in real seconds. That matters as soon as someone attaches a
9948    // debugger: a breakpoint held longer than the app's `request_timeout_ms`
9949    // cancels the handler mid-replay and prints a mismatch that is an artefact of
9950    // the debugging session. A per-route `#[timeout(...)]` override still applies —
9951    // it is part of the route table, not the configuration.
9952    config.server.timeouts.request_timeout_ms = None;
9953    // A shadow mirror has its own client and is not a handler effect.
9954    config.shadow.enabled = false;
9955}
9956
9957/// The database topology a replay runs against: an in-process pool answering
9958/// from the capsule's recorded wire traffic, or none when the capsule recorded
9959/// no database work at all.
9960#[cfg(all(feature = "reporting", feature = "db", not(feature = "sqlite")))]
9961fn replay_database_topology(
9962    capsule: &crate::capsule::Capsule,
9963    divergences: &std::sync::Arc<crate::capsule::DivergenceLog>,
9964    capsule_path: &std::path::Path,
9965) -> Option<crate::db::DatabaseTopology> {
9966    // "This request issued no queries" and "this application has no database" are
9967    // different facts, and only the capsule tells them apart. A handler or state
9968    // initializer that checks `state.pool()`, or replica availability, before
9969    // querying would otherwise meet `None` while replaying an application that had
9970    // a pool in production, take a branch it never took, and report a mismatch no
9971    // code caused. A capsule recorded before `db_roles` existed carries none and
9972    // falls back to the old tape-only behaviour.
9973    if capsule.db.is_none() && capsule.db_roles.is_empty() {
9974        return None;
9975    }
9976    let pool = crate::capsule::pool_from_capsule(capsule, std::sync::Arc::clone(divergences))
9977        .unwrap_or_else(|error| {
9978            std::process::exit(crate::capsule::print_refusal(
9979                &format!("the capsule's database tape could not be served: {error}"),
9980                capsule_path,
9981            ))
9982        });
9983    // Rebuild the topology with the shape the recording had: replica-recorded
9984    // tapes get their own stub pool, so a write-then-read request claims each
9985    // tape from the role it was recorded on instead of funnelling reads into
9986    // the primary stub and diverging on both sides.
9987    let replica =
9988        crate::capsule::replica_pool_from_capsule(capsule, std::sync::Arc::clone(divergences))
9989            .unwrap_or_else(|error| {
9990                std::process::exit(crate::capsule::print_refusal(
9991                    &format!("the capsule's replica tape could not be served: {error}"),
9992                    capsule_path,
9993                ))
9994            });
9995    Some(crate::db::DatabaseTopology::from_pools(pool, replica))
9996}
9997
9998/// `SQLite` builds have no wire capture and no wire replay (F18), so a capsule
9999/// carrying a database tape — which is `PostgreSQL` protocol traffic — cannot be
10000/// replayed by this binary.
10001#[cfg(all(feature = "reporting", feature = "db", feature = "sqlite"))]
10002fn replay_database_topology(
10003    capsule: &crate::capsule::Capsule,
10004    _divergences: &std::sync::Arc<crate::capsule::DivergenceLog>,
10005    capsule_path: &std::path::Path,
10006) -> Option<crate::db::DatabaseTopology> {
10007    if capsule.db.is_some() {
10008        std::process::exit(crate::capsule::print_refusal(
10009            "the capsule carries a PostgreSQL database tape, but this binary was built with the \
10010             `sqlite` backend, which has neither wire capture nor wire replay. Replay it with a \
10011             PostgreSQL build of the application.",
10012            capsule_path,
10013        ));
10014    }
10015    None
10016}
10017
10018fn one_off_task_name_from_env() -> Option<String> {
10019    std::env::var("AUTUMN_RUN_TASK")
10020        .ok()
10021        .map(|value| value.trim().to_owned())
10022        .filter(|value| !value.is_empty())
10023}
10024
10025fn one_off_task_args_from_env() -> Result<Vec<String>, String> {
10026    match std::env::var("AUTUMN_TASK_ARGS_JSON") {
10027        Ok(raw) if !raw.trim().is_empty() => serde_json::from_str(&raw)
10028            .map_err(|error| format!("AUTUMN_TASK_ARGS_JSON must be a JSON string array: {error}")),
10029        _ => Ok(Vec::new()),
10030    }
10031}
10032
10033fn print_available_one_off_tasks(tasks: &[crate::task::OneOffTaskInfo]) {
10034    let listing = crate::task::list_one_off_tasks(tasks);
10035    if listing.is_empty() {
10036        eprintln!("No one-off tasks are registered. Add .one_off_tasks(one_off_tasks![...]).");
10037        return;
10038    }
10039
10040    eprintln!("Available tasks:");
10041    for task in listing {
10042        if task.description.is_empty() {
10043            eprintln!("  {}", task.name);
10044        } else {
10045            eprintln!("  {:<24} {}", task.name, task.description);
10046        }
10047    }
10048}
10049
10050/// Start scheduled tasks in background Tokio tasks.
10051///
10052/// Each task runs in its own spawned task with error logging.
10053/// Uses `tokio::time` for fixed-delay scheduling and `croner` for cron-based
10054/// scheduling. The `shutdown` token is used to stop cron loops gracefully when
10055/// the server receives a termination signal.
10056#[allow(clippy::cast_possible_truncation)]
10057#[allow(clippy::cognitive_complexity)]
10058#[allow(dead_code)]
10059fn start_task_scheduler(
10060    tasks: Vec<crate::task::TaskInfo>,
10061    state: &AppState,
10062    shutdown: &tokio_util::sync::CancellationToken,
10063) {
10064    if let Err(error) = start_task_scheduler_with_config(
10065        tasks,
10066        state,
10067        shutdown,
10068        &crate::config::SchedulerConfig::default(),
10069    ) {
10070        tracing::error!(error = %error, "scheduled task runtime initialization failed");
10071    }
10072}
10073
10074#[allow(clippy::cast_possible_truncation)]
10075#[allow(clippy::cognitive_complexity)]
10076pub(crate) fn start_task_scheduler_with_config(
10077    tasks: Vec<crate::task::TaskInfo>,
10078    state: &AppState,
10079    shutdown: &tokio_util::sync::CancellationToken,
10080    scheduler_config: &crate::config::SchedulerConfig,
10081) -> crate::AutumnResult<()> {
10082    tracing::info!(count = tasks.len(), "Starting scheduled tasks");
10083    let coordinator = crate::scheduler::coordinator_from_config(scheduler_config, state)?;
10084    let lease_ttl = std::time::Duration::from_secs(scheduler_config.lease_ttl_secs);
10085    for task_info in &tasks {
10086        let schedule_desc = task_info.schedule.to_string();
10087        tracing::info!(
10088            name = %task_info.name,
10089            schedule = %schedule_desc,
10090            coordination = %task_info.coordination,
10091            scheduler_backend = coordinator.backend(),
10092            replica_id = coordinator.replica_id(),
10093            lease_ttl_secs = scheduler_config.lease_ttl_secs,
10094            "Registered task"
10095        );
10096    }
10097
10098    let mut cron_tasks: Vec<CronTaskSpec> = Vec::new();
10099
10100    for task_info in tasks {
10101        let state = state.clone();
10102        let name = task_info.name.clone();
10103        let handler = task_info.handler;
10104        let coordination = task_info.coordination;
10105        let schedule_desc = task_info.schedule.to_string();
10106        state.task_registry.register_scheduled(
10107            &name,
10108            &schedule_desc,
10109            coordination,
10110            coordinator.backend(),
10111            coordinator.replica_id(),
10112        );
10113
10114        match task_info.schedule {
10115            crate::task::Schedule::FixedDelay(delay) => {
10116                let coordinator = Arc::clone(&coordinator);
10117                let shutdown = shutdown.child_token();
10118                crate::sim::spawn_app_task(&state.clone(), async move {
10119                    loop {
10120                        state.task_registry.record_next_run_at(
10121                            &name,
10122                            &format_next_task_run_after(state.clock().now(), delay),
10123                        );
10124                        tokio::select! {
10125                            // Fixed branch order, so a sim replays it (#3067).
10126                            biased;
10127                            () = shutdown.cancelled() => break,
10128                            () = tokio::time::sleep(delay) => {
10129                                execute_fixed_delay_task(
10130                                    name.clone(),
10131                                    state.clone(),
10132                                    handler,
10133                                    delay,
10134                                    coordination,
10135                                    Arc::clone(&coordinator),
10136                                    lease_ttl,
10137                                    shutdown.clone(),
10138                                )
10139                                .await;
10140                            }
10141                        }
10142                    }
10143                });
10144            }
10145            crate::task::Schedule::Cron {
10146                expression,
10147                timezone,
10148            } => {
10149                cron_tasks.push(CronTaskSpec {
10150                    name,
10151                    expression,
10152                    timezone,
10153                    coordination,
10154                    handler,
10155                });
10156            }
10157        }
10158    }
10159
10160    run_cron_scheduler(cron_tasks, state, shutdown, &coordinator, lease_ttl);
10161
10162    Ok(())
10163}
10164
10165/// The boot warning for an `in_process` scheduler that runs fleet tasks while
10166/// a hint shows more than one replica. Each replica would run each tick
10167/// (issue #3052). `None` when there is nothing to warn about.
10168fn in_process_fleet_warning(
10169    config: &AutumnConfig,
10170    tasks: &[crate::task::TaskInfo],
10171    env: &dyn crate::config::Env,
10172) -> Option<String> {
10173    let has_fleet_task = tasks
10174        .iter()
10175        .any(|task| task.coordination == crate::task::TaskCoordination::Fleet);
10176    if !has_fleet_task {
10177        return None;
10178    }
10179    let replicas = env.var("AUTUMN_REPLICAS").ok();
10180    let kubernetes = env.var("KUBERNETES_SERVICE_HOST").is_ok();
10181    let hint = crate::scheduler::in_process_fleet_hint(
10182        config.scheduler.backend,
10183        &config.jobs.backend,
10184        replicas.as_deref(),
10185        kubernetes,
10186    )?;
10187    let fix = if cfg!(feature = "sqlite") {
10188        "sqlite"
10189    } else {
10190        "postgres"
10191    };
10192    Some(format!(
10193        "scheduler.backend = \"in_process\" runs each fleet tick on every replica \
10194         ({hint}). If more than one replica runs, set scheduler.backend = \"{fix}\". \
10195         See docs/guide/scheduled-multi-replica.md"
10196    ))
10197}
10198
10199#[allow(unused_variables, clippy::needless_pass_by_value)]
10200fn send_ws_sys_task_msg(
10201    state: &AppState,
10202    event: &str,
10203    name: &str,
10204    extra: Vec<(&str, serde_json::Value)>,
10205) {
10206    #[cfg(feature = "ws")]
10207    {
10208        // ⚡ Bolt Optimization:
10209        // Use serde_json::json! to avoid multiple String allocations (`.to_string()`)
10210        // and repetitive `Map::insert` calls for `sys:tasks` websocket messages.
10211        let mut msg = serde_json::json!({
10212            "event": event,
10213            "task": name,
10214            "timestamp": state.clock().now().to_rfc3339(),
10215        });
10216        if let Some(map) = msg.as_object_mut() {
10217            for (k, v) in extra {
10218                map.insert(k.to_string(), v);
10219            }
10220        }
10221        let _ = state.channels().sender("sys:tasks").send(msg.to_string());
10222    }
10223}
10224
10225/// Milliseconds elapsed since `start` on the app's injected monotonic clock.
10226///
10227/// Every scheduled-task duration goes through here so a `#[sim_test]` reports
10228/// virtual run times (and two same-seed runs agree) instead of real ones.
10229fn task_duration_ms(state: &AppState, start: crate::time::MonotonicInstant) -> u64 {
10230    let elapsed = state.monotonic().saturating_duration_since(start);
10231    u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX)
10232}
10233
10234async fn execute_task_result(
10235    state: &AppState,
10236    handler: crate::task::TaskHandler,
10237    start: crate::time::MonotonicInstant,
10238    name: &str,
10239    schedule: &'static str,
10240    waited: bool,
10241) -> Result<u64, (u64, String)> {
10242    // A tick is work a sim drain must see (issue #2967).
10243    crate::sim::note_drain_progress();
10244    // A fresh span per run so OTLP-enabled deployments see each invocation
10245    // as its own trace rather than inheriting whatever was current on the
10246    // scheduler thread.
10247    let task_span = tracing::info_span!(
10248        parent: None,
10249        "scheduled_task",
10250        otel.kind = "internal",
10251        task = %name,
10252        schedule = schedule,
10253    );
10254    // The handler is called in the first poll, so `catch_unwind` below also
10255    // catches a handler that panics before it returns its future, and the
10256    // meter records that tick too (issue #1720).
10257    let handler_state = state.clone();
10258    let future = async move { (handler)(handler_state).instrument(task_span).await };
10259    let run = crate::cost::WorkRun {
10260        tenant: None,
10261        waited,
10262    };
10263    let future = crate::cost::meter_work(state, crate::cost::WorkKind::Task, name, run, future);
10264    let result = std::panic::AssertUnwindSafe(future).catch_unwind().await;
10265    let duration_ms = task_duration_ms(state, start);
10266
10267    match result {
10268        Ok(Ok(())) => Ok(duration_ms),
10269        // `message`, not `Display`: this string is stored as the task's
10270        // `last_error`, sent in alerts, and broadcast on `sys:tasks`.
10271        Ok(Err(e)) => Err((duration_ms, e.message())),
10272        Err(panic) => Err((duration_ms, format_scheduled_task_panic(panic.as_ref()))),
10273    }
10274}
10275
10276fn format_scheduled_task_panic(panic: &(dyn Any + Send)) -> String {
10277    let detail = panic
10278        .downcast_ref::<String>()
10279        .map(String::as_str)
10280        .or_else(|| panic.downcast_ref::<&'static str>().copied())
10281        .unwrap_or("non-string panic payload");
10282    format!("scheduled task handler panicked: {detail}")
10283}
10284
10285/// Run one tick with `tick` as its [`crate::scheduler::current_tick`], and
10286/// stop it after `lease_ttl` when one is set.
10287#[allow(
10288    clippy::too_many_arguments,
10289    reason = "one tick: its task, schedule, lease and cost wait"
10290)]
10291async fn execute_task_result_with_optional_lease_ttl(
10292    state: &AppState,
10293    handler: crate::task::TaskHandler,
10294    start: crate::time::MonotonicInstant,
10295    name: &str,
10296    schedule: &'static str,
10297    lease_ttl: Option<std::time::Duration>,
10298    tick: crate::scheduler::ScheduledTick,
10299    waited: bool,
10300) -> Result<u64, (u64, String)> {
10301    let run = execute_task_result(state, handler, start, name, schedule, waited);
10302    // A scheduled task runs outside any request, so its framework
10303    // transactions get the configured timeouts from here (#3057).
10304    #[cfg(feature = "db")]
10305    let run = crate::db::scope_background_tx_timeouts(state, run);
10306    let run = crate::scheduler::with_tick(tick, run);
10307    let Some(lease_ttl) = lease_ttl else {
10308        return run.await;
10309    };
10310
10311    tokio::time::timeout(lease_ttl, run)
10312        .await
10313        .unwrap_or_else(|_| {
10314            let duration_ms = task_duration_ms(state, start);
10315            Err((
10316                duration_ms,
10317                format!(
10318                    "scheduled task exceeded lease TTL of {}s",
10319                    lease_ttl.as_secs()
10320                ),
10321            ))
10322        })
10323}
10324
10325/// Handle the execution of a single fixed-delay task.
10326#[allow(clippy::cognitive_complexity)]
10327#[allow(
10328    clippy::too_many_arguments,
10329    reason = "one tick: its task, schedule, lease and cost gate"
10330)]
10331async fn execute_fixed_delay_task(
10332    name: String,
10333    state: AppState,
10334    handler: crate::task::TaskHandler,
10335    delay: std::time::Duration,
10336    coordination: crate::task::TaskCoordination,
10337    coordinator: Arc<dyn crate::scheduler::SchedulerCoordinator>,
10338    lease_ttl: std::time::Duration,
10339    shutdown: tokio_util::sync::CancellationToken,
10340) {
10341    // Cost gate (#1720). The loop awaits this run, so later ticks wait too.
10342    let gate = CostGate {
10343        shutdown,
10344        waiting: None,
10345        waited: std::sync::atomic::AtomicBool::new(false),
10346    };
10347    let wait_first = CostGate::waits_before_lease(&*coordinator);
10348    let (lease, tick_key) = loop {
10349        if wait_first && !gate.wait(&state, &name).await {
10350            return;
10351        }
10352        let tick_key = crate::scheduler::fixed_delay_tick_key(
10353            &name,
10354            delay,
10355            crate::time::clock_unix_duration(state.clock()),
10356        );
10357        let lease = match coordinator
10358            // Pass the delay: replica timers can reach one bucket a full delay apart.
10359            .try_acquire_for_period(&name, &tick_key, coordination, delay)
10360            .await
10361        {
10362            Ok(Some(lease)) => lease,
10363            Ok(None) => {
10364                tracing::debug!(task = %name, tick = %tick_key, "Scheduled task tick already claimed");
10365                return;
10366            }
10367            Err(error) => {
10368                tracing::warn!(task = %name, tick = %tick_key, error = %error, "Failed to acquire scheduled task lease");
10369                return;
10370            }
10371        };
10372        // The signal can rise while an expiring lease attempt waits: free the
10373        // claim (the tick has not run), wait again, and retry the same key.
10374        if wait_first && CostGate::must_wait(&state, &name) {
10375            CostGate::free_unrun_claim(lease, &name, &tick_key).await;
10376            continue;
10377        }
10378        break (lease, tick_key);
10379    };
10380    state
10381        .task_registry
10382        .record_leader(&name, lease.leader_id(), &tick_key);
10383    if !wait_first && !gate.wait(&state, &name).await {
10384        release_task_lease(lease, &name, &tick_key).await;
10385        return;
10386    }
10387    tracing::debug!(task = %name, "Running scheduled task");
10388    state.task_registry.record_start(&name);
10389
10390    send_ws_sys_task_msg(&state, "started", &name, vec![]);
10391
10392    let start = state.monotonic();
10393    let lease_ttl = lease_ttl_for_run(&lease, coordination, lease_ttl);
10394    let tick = crate::scheduler::ScheduledTick::new(&tick_key, &lease);
10395    match execute_task_result_with_optional_lease_ttl(
10396        &state,
10397        handler,
10398        start,
10399        &name,
10400        "fixed_delay",
10401        lease_ttl,
10402        tick,
10403        gate.waited(),
10404    )
10405    .await
10406    {
10407        Ok(duration_ms) => {
10408            state.task_registry.record_success(&name, duration_ms);
10409            crate::alerts::notify_scheduled_task_recovered(&state, &name);
10410            tracing::debug!(task = %name, "Task completed");
10411            send_ws_sys_task_msg(
10412                &state,
10413                "success",
10414                &name,
10415                vec![("duration_ms", serde_json::json!(duration_ms))],
10416            );
10417        }
10418        Err((duration_ms, error_str)) => {
10419            state
10420                .task_registry
10421                .record_failure(&name, duration_ms, &error_str);
10422            crate::alerts::notify_scheduled_task_failure(&state, &name, &error_str);
10423            tracing::warn!(task = %name, error = %error_str, "Task failed");
10424            send_ws_sys_task_msg(
10425                &state,
10426                "failure",
10427                &name,
10428                vec![
10429                    ("duration_ms", serde_json::json!(duration_ms)),
10430                    ("error", serde_json::json!(error_str)),
10431                ],
10432            );
10433        }
10434    }
10435
10436    if let Err(error) = lease.release().await {
10437        tracing::warn!(task = %name, tick = %tick_key, error = %error, "Failed to release scheduled task lease");
10438    }
10439}
10440
10441/// The cost-signal gate for one scheduled tick (issue #1720).
10442struct CostGate {
10443    shutdown: tokio_util::sync::CancellationToken,
10444    /// `true` while the tick waits. The cron loop folds later ticks into it.
10445    waiting: Option<Arc<std::sync::atomic::AtomicBool>>,
10446    /// `true` after the tick waited in a real window. The cost meter reads it.
10447    waited: std::sync::atomic::AtomicBool,
10448}
10449
10450impl CostGate {
10451    /// Wait while the task must defer. Return `false` on shutdown.
10452    async fn wait(&self, state: &AppState, name: &str) -> bool {
10453        crate::cost::wait_while_deferred(
10454            state,
10455            crate::cost::WorkKind::Task,
10456            name,
10457            &self.shutdown,
10458            self.waiting.as_deref(),
10459            &self.waited,
10460        )
10461        .await
10462    }
10463
10464    fn waited(&self) -> bool {
10465        self.waited.load(std::sync::atomic::Ordering::Acquire)
10466    }
10467
10468    /// Free a claim whose tick has not run, so a retry of the same key can
10469    /// take it again. Every replica then resumes against that one key, so the
10470    /// tick runs one time. A tick that ran keeps its claim
10471    /// ([`SchedulerLease::release`](crate::scheduler::SchedulerLease::release)).
10472    /// A failed delete is tried again a few times. If it still fails, the
10473    /// claim stays until it expires: the tick may then not run, but it never
10474    /// runs twice.
10475    async fn free_unrun_claim(lease: crate::scheduler::SchedulerLease, name: &str, tick_key: &str) {
10476        const ATTEMPTS: u32 = 5;
10477        for attempt in 1..=ATTEMPTS {
10478            match lease.free().await {
10479                Ok(()) => break,
10480                Err(error) if attempt < ATTEMPTS => {
10481                    tracing::debug!(task = %name, tick = %tick_key, attempt, error = %error, "Retrying the free of a deferred tick's claim");
10482                    tokio::time::sleep(std::time::Duration::from_millis(200 * u64::from(attempt)))
10483                        .await;
10484                }
10485                Err(error) => {
10486                    tracing::warn!(task = %name, tick = %tick_key, error = %error, "Failed to free the claim of a deferred tick; the tick may not run");
10487                }
10488            }
10489        }
10490        release_task_lease(lease, name, tick_key).await;
10491    }
10492
10493    /// `true` when the tick waits before it takes its lease.
10494    ///
10495    /// A `SQLite` lease and a Postgres tick row (#3052) expire, and a wait can
10496    /// be longer. So with those backends the tick waits first, then takes the
10497    /// lease and checks the signal again. An in-process lease does not expire,
10498    /// so the tick waits while it holds it.
10499    fn waits_before_lease(coordinator: &dyn crate::scheduler::SchedulerCoordinator) -> bool {
10500        matches!(coordinator.backend(), "sqlite" | "postgres")
10501    }
10502
10503    /// `true` when the task must wait now: it is deferrable and the signal is
10504    /// high.
10505    fn must_wait(state: &AppState, name: &str) -> bool {
10506        crate::cost::deferral_signal(state, crate::cost::WorkKind::Task, name).is_some()
10507    }
10508
10509    /// The tick no longer needs the reservation: later ticks run again. Call it
10510    /// on every way out of the lease and wait steps.
10511    fn release(&self) {
10512        if let Some(flag) = &self.waiting {
10513            flag.store(false, std::sync::atomic::Ordering::Release);
10514        }
10515    }
10516}
10517
10518async fn release_task_lease(lease: crate::scheduler::SchedulerLease, name: &str, tick_key: &str) {
10519    if let Err(error) = lease.release().await {
10520        tracing::warn!(task = %name, tick = %tick_key, error = %error, "Failed to release scheduled task lease");
10521    }
10522}
10523
10524/// One cron occurrence to run.
10525#[derive(Debug, Clone, Copy)]
10526struct CronTick {
10527    /// The occurrence's scheduled time, in Unix seconds.
10528    unix_secs: u64,
10529    /// Time from this occurrence to the next. The loop still runs a late
10530    /// occurrence inside this window, so the claim must hold for all of it.
10531    window: std::time::Duration,
10532}
10533
10534impl CronTick {
10535    /// `true` when `now_unix_secs` is at or after the next occurrence. A zero
10536    /// window (no next occurrence) is never past.
10537    const fn is_past_window(self, now_unix_secs: u64) -> bool {
10538        !self.window.is_zero()
10539            && now_unix_secs >= self.unix_secs.saturating_add(self.window.as_secs())
10540    }
10541
10542    /// `true`, with a debug log, when the state clock is past the window.
10543    fn is_late(self, state: &AppState, name: &str, tick_key: &str) -> bool {
10544        let late = self.is_past_window(crate::time::clock_unix_secs(state.clock()));
10545        if late {
10546            tracing::debug!(task = %name, tick = %tick_key, "Cron task tick is past its window");
10547        }
10548        late
10549    }
10550}
10551
10552/// Handle the execution of a single cron task.
10553#[allow(clippy::cognitive_complexity)]
10554#[allow(
10555    clippy::too_many_arguments,
10556    reason = "one tick: its task, schedule, lease and cost gate"
10557)]
10558async fn execute_cron_task(
10559    name: String,
10560    state: AppState,
10561    handler: crate::task::TaskHandler,
10562    coordination: crate::task::TaskCoordination,
10563    coordinator: Arc<dyn crate::scheduler::SchedulerCoordinator>,
10564    lease_ttl: std::time::Duration,
10565    occurrence: CronTick,
10566    gate: CostGate,
10567) {
10568    let scheduled_key = crate::scheduler::cron_tick_key(&name, occurrence.unix_secs);
10569    // Cost gate (#1720), as in `execute_fixed_delay_task`. The fold
10570    // reservation holds until this tick has its lease and is done waiting.
10571    let wait_first = CostGate::waits_before_lease(&*coordinator);
10572    let tick_key = scheduled_key;
10573    let lease = loop {
10574        if wait_first && !gate.wait(&state, &name).await {
10575            gate.release();
10576            return;
10577        }
10578        // A cost wait can outlast the tick row (#3071). Another replica can
10579        // have run this occurrence, and the prune can have deleted its row.
10580        // Do not claim the occurrence after its window.
10581        if gate.waited() && occurrence.is_late(&state, &name, &tick_key) {
10582            gate.release();
10583            return;
10584        }
10585        let lease = match coordinator
10586            .try_acquire_for_period(&name, &tick_key, coordination, occurrence.window)
10587            .await
10588        {
10589            Ok(Some(lease)) => lease,
10590            Ok(None) => {
10591                tracing::debug!(task = %name, tick = %tick_key, "Cron task tick already claimed");
10592                gate.release();
10593                return;
10594            }
10595            Err(error) => {
10596                tracing::warn!(task = %name, tick = %tick_key, error = %error, "Failed to acquire cron task lease");
10597                gate.release();
10598                return;
10599            }
10600        };
10601        // As in `execute_fixed_delay_task`: the signal can rise during a
10602        // lease attempt that waited first.
10603        if wait_first && CostGate::must_wait(&state, &name) {
10604            CostGate::free_unrun_claim(lease, &name, &tick_key).await;
10605            continue;
10606        }
10607        break lease;
10608    };
10609    // The claim can wait too (a pool checkout, a slow query). Check the window
10610    // again: a claim after the window can have found a pruned row of a tick
10611    // that ran (#3071). Keep the row, so no replica runs the tick again.
10612    if occurrence.is_late(&state, &name, &tick_key) {
10613        gate.release();
10614        release_task_lease(lease, &name, &tick_key).await;
10615        return;
10616    }
10617    state
10618        .task_registry
10619        .record_leader(&name, lease.leader_id(), &tick_key);
10620    if !wait_first && !gate.wait(&state, &name).await {
10621        gate.release();
10622        release_task_lease(lease, &name, &tick_key).await;
10623        return;
10624    }
10625    gate.release();
10626    tracing::debug!(task = %name, "Running cron task");
10627    state.task_registry.record_start(&name);
10628
10629    send_ws_sys_task_msg(&state, "started", &name, vec![]);
10630
10631    let start = state.monotonic();
10632    let lease_ttl = lease_ttl_for_run(&lease, coordination, lease_ttl);
10633    let tick = crate::scheduler::ScheduledTick::new(&tick_key, &lease);
10634    match execute_task_result_with_optional_lease_ttl(
10635        &state,
10636        handler,
10637        start,
10638        &name,
10639        "cron",
10640        lease_ttl,
10641        tick,
10642        gate.waited(),
10643    )
10644    .await
10645    {
10646        Ok(duration_ms) => {
10647            state.task_registry.record_success(&name, duration_ms);
10648            crate::alerts::notify_scheduled_task_recovered(&state, &name);
10649            tracing::debug!(task = %name, "Cron task completed");
10650            send_ws_sys_task_msg(
10651                &state,
10652                "success",
10653                &name,
10654                vec![("duration_ms", serde_json::json!(duration_ms))],
10655            );
10656        }
10657        Err((duration_ms, error_str)) => {
10658            state
10659                .task_registry
10660                .record_failure(&name, duration_ms, &error_str);
10661            crate::alerts::notify_scheduled_task_failure(&state, &name, &error_str);
10662            tracing::warn!(task = %name, error = %error_str, "Cron task failed");
10663            send_ws_sys_task_msg(
10664                &state,
10665                "failure",
10666                &name,
10667                vec![
10668                    ("duration_ms", serde_json::json!(duration_ms)),
10669                    ("error", serde_json::json!(error_str)),
10670                ],
10671            );
10672        }
10673    }
10674
10675    if let Err(error) = lease.release().await {
10676        tracing::warn!(task = %name, tick = %tick_key, error = %error, "Failed to release cron task lease");
10677    }
10678}
10679
10680struct CronTaskSpec {
10681    name: String,
10682    expression: String,
10683    timezone: Option<String>,
10684    coordination: crate::task::TaskCoordination,
10685    handler: crate::task::TaskHandler,
10686}
10687
10688fn lease_ttl_for_run(
10689    lease: &crate::scheduler::SchedulerLease,
10690    coordination: crate::task::TaskCoordination,
10691    lease_ttl: std::time::Duration,
10692) -> Option<std::time::Duration> {
10693    (coordination == crate::task::TaskCoordination::Fleet && lease.backend() == "postgres")
10694        .then_some(lease_ttl)
10695}
10696
10697fn run_cron_scheduler(
10698    tasks: Vec<CronTaskSpec>,
10699    state: &AppState,
10700    shutdown: &tokio_util::sync::CancellationToken,
10701    coordinator: &Arc<dyn crate::scheduler::SchedulerCoordinator>,
10702    lease_ttl: std::time::Duration,
10703) {
10704    if tasks.is_empty() {
10705        return;
10706    }
10707
10708    tracing::info!(count = tasks.len(), "Cron scheduler started");
10709    for task in tasks {
10710        let state = state.clone();
10711        let coordinator = Arc::clone(coordinator);
10712        let shutdown = shutdown.child_token();
10713        crate::sim::spawn_app_task(&state.clone(), async move {
10714            run_cron_task_loop(task, state, shutdown, coordinator, lease_ttl).await;
10715        });
10716    }
10717}
10718
10719#[allow(clippy::cognitive_complexity)]
10720async fn run_cron_task_loop(
10721    task: CronTaskSpec,
10722    state: AppState,
10723    shutdown: tokio_util::sync::CancellationToken,
10724    coordinator: Arc<dyn crate::scheduler::SchedulerCoordinator>,
10725    lease_ttl: std::time::Duration,
10726) {
10727    let CronTaskSpec {
10728        name,
10729        expression,
10730        timezone,
10731        coordination,
10732        handler,
10733    } = task;
10734
10735    let cron = match parse_cron_expression(&expression) {
10736        Ok(cron) => cron,
10737        Err(error) => {
10738            tracing::error!(task = %name, expression = %expression, error = %error, "Failed to create cron job");
10739            return;
10740        }
10741    };
10742    let timezone = timezone
10743        .as_deref()
10744        .and_then(|timezone| {
10745            timezone.parse::<chrono_tz::Tz>().map_or_else(
10746                |_| {
10747                    tracing::warn!(task = %name, timezone = %timezone, "Unrecognized timezone; falling back to UTC");
10748                    None
10749                },
10750                Some,
10751            )
10752        })
10753        .unwrap_or(chrono_tz::UTC);
10754    let mut cursor = state.clock().now().with_timezone(&timezone);
10755    let deferring = Arc::new(std::sync::atomic::AtomicBool::new(false));
10756
10757    loop {
10758        let now = state.clock().now().with_timezone(&timezone);
10759        let scheduled_at = match next_cron_occurrence_after(&cron, &cursor, &now) {
10760            Ok(scheduled_at) => scheduled_at,
10761            Err(error) => {
10762                tracing::error!(task = %name, expression = %expression, error = %error, "Failed to compute next cron tick");
10763                return;
10764            }
10765        };
10766        state.task_registry.record_next_run_at(
10767            &name,
10768            &scheduled_at.with_timezone(&chrono::Utc).to_rfc3339(),
10769        );
10770        let sleep_for = cron_sleep_duration_until(state.clock().now(), &scheduled_at);
10771        tokio::select! {
10772            // Fixed branch order, so a sim replays it (#3067).
10773            biased;
10774            () = shutdown.cancelled() => break,
10775            () = tokio::time::sleep(sleep_for) => {
10776                let woke_at = state.clock().now().with_timezone(&timezone);
10777                match cron_occurrence_is_overdue(&cron, &scheduled_at, &woke_at) {
10778                    Ok(true) => {
10779                        tracing::warn!(
10780                            task = %name,
10781                            scheduled_at = %scheduled_at,
10782                            woke_at = %woke_at,
10783                            "Skipping overdue cron task tick"
10784                        );
10785                        cursor = woke_at;
10786                        continue;
10787                    }
10788                    Ok(false) => {}
10789                    Err(error) => {
10790                        tracing::error!(task = %name, expression = %expression, error = %error, "Failed to evaluate cron tick lateness");
10791                        return;
10792                    }
10793                }
10794                // Cost gate (#1720): while a tick of this task waits for the
10795                // cost signal, later ticks fold into it.
10796                if deferring.load(std::sync::atomic::Ordering::Acquire) {
10797                    tracing::debug!(task = %name, "cron tick folds into the tick that waits for the cost signal");
10798                    cursor = scheduled_at;
10799                    continue;
10800                }
10801                // Reserve the tick before the lease attempt, whatever the signal
10802                // is now: a slow lease must not let the next ticks spawn and
10803                // pile up behind a signal that rises meanwhile. The tick clears
10804                // the mark when it stops to wait or to take its lease.
10805                if crate::cost::is_deferrable(crate::cost::WorkKind::Task, &name) {
10806                    deferring.store(true, std::sync::atomic::Ordering::Release);
10807                }
10808                let occurrence = CronTick {
10809                    unix_secs: u64::try_from(scheduled_at.timestamp()).unwrap_or_default(),
10810                    window: cron_occurrence_window(&cron, &scheduled_at),
10811                };
10812                crate::sim::spawn_app_task(&state, execute_cron_task(
10813                    name.clone(),
10814                    state.clone(),
10815                    handler,
10816                    coordination,
10817                    Arc::clone(&coordinator),
10818                    lease_ttl,
10819                    occurrence,
10820                    CostGate {
10821                        shutdown: shutdown.clone(),
10822                        waiting: Some(Arc::clone(&deferring)),
10823                        waited: std::sync::atomic::AtomicBool::new(false),
10824                    },
10825                ));
10826                cursor = scheduled_at;
10827            }
10828        }
10829    }
10830}
10831
10832/// Render the next fixed-delay run time, `delay` after `now`.
10833///
10834/// `now` is passed in (rather than read here) so the caller supplies it from the
10835/// app's injected clock and the rendered `next_run_at` follows virtual time
10836/// under a `#[sim_test]`.
10837///
10838/// Goes through [`crate::job::due_at_from`] for the addition rather than `now +
10839/// delay`. Once `now` comes from an injected clock it is no longer bounded by
10840/// real time, and chrono's `Add` **panics** on overflow — a clock pinned near
10841/// `DateTime::MAX_UTC` would kill this scheduler task before its first run, for
10842/// a value that only ends up in a log line. `due_at_from` already owns that
10843/// clamp for the job queue's deadlines; sharing it keeps the two from drifting
10844/// apart, and makes an unrepresentable delay render as the far future rather
10845/// than as `now` (which would read as "runs immediately").
10846fn format_next_task_run_after(
10847    now: chrono::DateTime<chrono::Utc>,
10848    delay: std::time::Duration,
10849) -> String {
10850    crate::job::due_at_from(now, delay).to_rfc3339()
10851}
10852
10853/// Parses a `#[scheduled(cron = "...")]` expression.
10854///
10855/// croner 4 rejects the shorthand step syntax 3.x accepted (`5/5 * * * *`,
10856/// `/10 * * * *`) unless `sloppy_ranges` is on. Keep it on so an expression
10857/// that scheduled under the earlier release still schedules under this one.
10858fn parse_cron_expression(expression: &str) -> Result<croner::Cron, croner::errors::CronError> {
10859    croner::parser::CronParser::builder()
10860        .sloppy_ranges(true)
10861        .build()
10862        .parse(expression)
10863}
10864
10865fn next_cron_occurrence_after<Tz: chrono::TimeZone>(
10866    cron: &croner::Cron,
10867    cursor: &chrono::DateTime<Tz>,
10868    now: &chrono::DateTime<Tz>,
10869) -> Result<chrono::DateTime<Tz>, croner::errors::CronError> {
10870    let anchor = if cursor < now { now } else { cursor };
10871    cron.find_next_occurrence(anchor, false)
10872}
10873
10874fn cron_occurrence_is_overdue<Tz: chrono::TimeZone>(
10875    cron: &croner::Cron,
10876    scheduled_at: &chrono::DateTime<Tz>,
10877    now: &chrono::DateTime<Tz>,
10878) -> Result<bool, croner::errors::CronError> {
10879    let next_after_scheduled = cron.find_next_occurrence(scheduled_at, false)?;
10880    Ok(&next_after_scheduled <= now)
10881}
10882
10883/// Time from `scheduled_at` to the next occurrence: the window in which the
10884/// loop still runs a late `scheduled_at` (see [`cron_occurrence_is_overdue`]).
10885/// Zero when no next occurrence exists.
10886fn cron_occurrence_window<Tz: chrono::TimeZone>(
10887    cron: &croner::Cron,
10888    scheduled_at: &chrono::DateTime<Tz>,
10889) -> std::time::Duration {
10890    cron.find_next_occurrence(scheduled_at, false)
10891        .ok()
10892        .and_then(|next| next.signed_duration_since(scheduled_at).to_std().ok())
10893        .unwrap_or_default()
10894}
10895
10896/// How long to sleep from `now` until `scheduled_at`, saturating at zero for a
10897/// target already in the past.
10898///
10899/// `now` is passed in so the caller supplies it from the app's injected clock.
10900/// Note the caller must keep this in step with `tokio::time::sleep`: under a
10901/// `#[sim_test]` both the injected clock and tokio's timer wheel are advanced
10902/// together by `Sim::advance`, which is what keeps the loop from spinning.
10903fn cron_sleep_duration_until<Tz: chrono::TimeZone>(
10904    now: chrono::DateTime<chrono::Utc>,
10905    scheduled_at: &chrono::DateTime<Tz>,
10906) -> std::time::Duration {
10907    scheduled_at
10908        .with_timezone(&chrono::Utc)
10909        .signed_duration_since(now)
10910        .to_std()
10911        .unwrap_or_default()
10912}
10913
10914async fn run_startup_hooks(hooks: &[StartupHook], state: AppState) -> crate::AutumnResult<()> {
10915    for hook in hooks {
10916        hook(state.clone()).await?;
10917    }
10918    Ok(())
10919}
10920
10921/// Install a designated live-state block into `state`, adopting the snapshot a
10922/// predecessor handed over when this process was started by an in-place
10923/// upgrade (issue #1674).
10924///
10925/// A snapshot this build cannot account for is a hard startup failure, not a
10926/// silent fallback to `initial`: the predecessor is still serving and still
10927/// holds the only copy of that state, so exiting here abandons the upgrade and
10928/// keeps the data. On a cold start there is no snapshot and `initial` is used.
10929fn install_live_state<T>(
10930    state: &AppState,
10931    initial: T,
10932    decode: fn(&crate::upgrade::StateEnvelope) -> Result<T, crate::upgrade::AdoptError>,
10933) where
10934    T: crate::upgrade::LiveState,
10935{
10936    if let Some(existing) = state.extension::<crate::upgrade::LiveStateRegistry>() {
10937        state.insert_extension(crate::upgrade::LiveStateInstallFailure(format!(
10938            "an app may designate only one block of live state for in-place upgrades, but \
10939             both {} and {} were designated; carrying just one of two designated blocks \
10940             would be the silent state loss this feature exists to prevent",
10941            existing.type_name(),
10942            std::any::type_name::<T>(),
10943        )));
10944        return;
10945    }
10946
10947    let value = match crate::upgrade::carried_snapshot_path() {
10948        None => initial,
10949        Some(path) => {
10950            match crate::upgrade::read_snapshot(&path).and_then(|envelope| {
10951                let decoded = decode(&envelope);
10952                if decoded.is_ok() {
10953                    tracing::info!(
10954                        state = std::any::type_name::<T>(),
10955                        from_version = envelope.version,
10956                        to_version = T::VERSION,
10957                        generation = envelope.generation,
10958                        "adopted the live state handed over by the previous build"
10959                    );
10960                }
10961                decoded
10962            }) {
10963                Ok(value) => value,
10964                Err(error) => {
10965                    state.insert_extension(crate::upgrade::LiveStateInstallFailure(format!(
10966                        "the previous build's live state cannot be carried into this one \
10967                         ({}): {error}. The previous build keeps serving; fix the \
10968                         migration (autumn_web::state_migration!) and try the upgrade again",
10969                        std::any::type_name::<T>(),
10970                    )));
10971                    return;
10972                }
10973            }
10974        }
10975    };
10976
10977    // A successor installs its state **frozen**. It starts accepting the moment
10978    // it adopts the socket — before its startup hooks have run — and until it
10979    // signals readiness the upgrade can still be abandoned, at which point the
10980    // predecessor resumes from the snapshot it took. A write acknowledged in
10981    // that window would die with this process: refuse it instead, so the
10982    // client's retry lands on whichever process is actually keeping state.
10983    // `unfreeze_adopted_live_state` lifts it at the readiness point.
10984    let handle = if crate::upgrade::handoff_requested() {
10985        crate::upgrade::LiveStateHandle::new_frozen(value)
10986    } else {
10987        crate::upgrade::LiveStateHandle::new(value)
10988    };
10989    state.insert_extension(crate::upgrade::LiveStateRegistry::new(&handle));
10990    state.insert_extension(handle);
10991}
10992
10993/// Make an adopted live-state block writable, once this build has finished
10994/// starting up and is about to release its predecessor (#1674).
10995///
10996/// A no-op on a cold start (the block was never frozen) and for an app that
10997/// designated none.
10998fn unfreeze_adopted_live_state(state: &AppState) {
10999    if !crate::upgrade::handoff_requested() {
11000        return;
11001    }
11002    if let Some(registry) = state.extension::<crate::upgrade::LiveStateRegistry>() {
11003        // Journal before unfreezing: no write can precede the journal, so the
11004        // predecessor can take back every write this build acknowledges.
11005        #[cfg(unix)]
11006        if let Some(path) = crate::upgrade::journal_path() {
11007            registry.start_journal(path);
11008        }
11009        registry.unfreeze();
11010    }
11011}
11012
11013fn run_state_initializers(initializers: Vec<StateInitializer>, state: &AppState) {
11014    for initializer in initializers {
11015        initializer(state);
11016    }
11017}
11018
11019/// Wire the typed event bus into the app at build time.
11020///
11021/// Builds the [`EventRegistry`](crate::events::EventRegistry) from registered
11022/// listeners, installs it onto `state` for the [`Events`](crate::events::Events)
11023/// extractor, appends a job per durable listener so they ride the job runtime
11024/// (retry + DLQ + restart-safety), and initializes the process-global bus used
11025/// by the module-level `events::publish`.
11026/// Build the [`EventRegistry`](crate::events::EventRegistry) from `listeners` and
11027/// append the synthesized `default`-queue [`JobInfo`](crate::job::JobInfo) for
11028/// each durable listener to `jobs`, returning the registry.
11029///
11030/// This is the pure, DB-free half of [`finalize_event_bus`]: it needs no live
11031/// `AppState` or database, only the listener set. Both the boot path (through
11032/// `finalize_event_bus`, which additionally wires the global bus onto live state)
11033/// and the deliberately DB-free `AUTUMN_DUMP_JOBS=1` dump path
11034/// ([`dump_jobs_manifest`]) funnel through here, so the emitted manifest can
11035/// never omit the durable-listener jobs the runtime actually drains.
11036fn synthesize_durable_listener_jobs(
11037    listeners: Vec<crate::events::ListenerInfo>,
11038    jobs: &mut Vec<crate::job::JobInfo>,
11039) -> crate::events::EventRegistry {
11040    let registry = crate::events::EventRegistry::from_listeners(listeners);
11041    jobs.extend(registry.durable_job_infos());
11042    registry
11043}
11044
11045fn finalize_event_bus(
11046    listeners: Vec<crate::events::ListenerInfo>,
11047    jobs: &mut Vec<crate::job::JobInfo>,
11048    state: &AppState,
11049) {
11050    let registry = synthesize_durable_listener_jobs(listeners, jobs);
11051    state.insert_extension(registry.clone());
11052    crate::events::init_global_event_bus(&registry, state, None);
11053}
11054
11055/// Compute the `AUTUMN_DUMP_JOBS=1` jobs manifest for the dump path.
11056///
11057/// Mirrors the boot path's job set: the builder's registered `jobs` PLUS the
11058/// synthesized `default`-queue jobs that [`finalize_event_bus`] appends for
11059/// durable listeners before the runtime starts. Without folding those in, an app
11060/// that registers a durable listener and configures `[jobs.queues]` without
11061/// `default` would emit a manifest omitting `default`, letting a topology-aware
11062/// `autumn doctor` accept a fleet where no tier drains the durable-listener jobs.
11063///
11064/// Only the pure job-synthesis half of `finalize_event_bus` runs here
11065/// (via [`synthesize_durable_listener_jobs`]); the dump path is deliberately
11066/// DB-free, so the live-state/global-bus wiring is skipped. Factored out so the
11067/// boot and dump paths share one job-preparation seam and so it is unit testable
11068/// without the process-exiting dump entrypoint.
11069fn dump_jobs_manifest(
11070    cfg: &crate::config::JobQueuesConfig,
11071    mut jobs: Vec<crate::job::JobInfo>,
11072    listeners: Vec<crate::events::ListenerInfo>,
11073) -> String {
11074    synthesize_durable_listener_jobs(listeners, &mut jobs);
11075    crate::job::render_jobs_manifest(cfg, &jobs)
11076}
11077
11078fn initialize_job_runtime(
11079    jobs: Vec<crate::job::JobInfo>,
11080    state: &AppState,
11081    shutdown: &tokio_util::sync::CancellationToken,
11082    config: &crate::config::JobConfig,
11083    run_workers: bool,
11084) -> crate::AutumnResult<()> {
11085    crate::job::clear_global_job_client();
11086    if jobs.is_empty() {
11087        Ok(())
11088    } else {
11089        crate::job::start_runtime(jobs, state, shutdown, config, run_workers)
11090    }
11091}
11092
11093/// Bind the configured TCP address — or adopt the listening socket a
11094/// predecessor handed over during an in-place upgrade (issue #1674).
11095///
11096/// Adopting is what makes the cutover connectionless-loss-free: the socket is
11097/// never closed and re-opened, so a connection queued on it while both builds
11098/// are alive is served by whichever one accepts it. Exits (rather than falling
11099/// back to `bind`) if an inherited socket turns out to be unusable: the
11100/// predecessor still holds the real one and is still serving, so a successor
11101/// that cannot adopt must abandon the upgrade, not race it for the port.
11102async fn bind_or_adopt_tcp_listener(addr: &str) -> tokio::net::TcpListener {
11103    #[cfg(unix)]
11104    {
11105        if let Some(inherited) = crate::upgrade::adopt_inherited_listener() {
11106            match tokio::net::TcpListener::from_std(inherited) {
11107                Ok(listener) => {
11108                    // The socket is the predecessor's, so a `server.host` /
11109                    // `server.port` change in the new build does NOT take
11110                    // effect: say so rather than let an operator believe a
11111                    // narrowed bind address is live.
11112                    if let Ok(bound) = listener.local_addr()
11113                        && bound.to_string() != addr
11114                    {
11115                        tracing::warn!(
11116                            inherited = %bound,
11117                            configured = %addr,
11118                            "the inherited listening socket does not match this build's \
11119                             configured address; an in-place upgrade cannot change where the \
11120                             app listens — restart the process to apply it"
11121                        );
11122                    }
11123                    return listener;
11124                }
11125                Err(e) => {
11126                    tracing::error!("Failed to adopt the inherited listening socket: {e}");
11127                    #[cfg(feature = "managed-pg")]
11128                    crate::managed_pg::emergency_stop_async().await;
11129                    std::process::exit(1);
11130                }
11131            }
11132        }
11133        // A predecessor handed us a socket and we could not take it: binding
11134        // instead would either collide with the process still serving that
11135        // address, or — worse, with an ephemeral port — succeed somewhere else
11136        // entirely and release the predecessor to drain away from the address
11137        // clients are using.
11138        if crate::upgrade::handoff_requested() {
11139            tracing::error!(
11140                "refusing to start: this build was handed the previous build's listening \
11141                 socket but could not adopt it (see the error above). The previous build \
11142                 keeps serving"
11143            );
11144            #[cfg(feature = "managed-pg")]
11145            crate::managed_pg::emergency_stop_async().await;
11146            std::process::exit(1);
11147        }
11148    }
11149    match tokio::net::TcpListener::bind(addr).await {
11150        Ok(listener) => listener,
11151        Err(e) => {
11152            tracing::error!(addr = %addr, "Failed to bind: {e}");
11153            // Stop the managed Postgres child started by `setup_database`
11154            // before bailing; `process::exit` skips `on_shutdown`.
11155            #[cfg(feature = "managed-pg")]
11156            crate::managed_pg::emergency_stop_async().await;
11157            std::process::exit(1);
11158        }
11159    }
11160}
11161
11162/// A bound network listener for the server, abstracting over the transport.
11163///
11164/// `run()` binds one of these based on `config.server.unix_socket`: a TCP
11165/// listener on `host:port` (the default) or a Unix domain socket (local
11166/// daemon mode). The two carry different connect-info types, so the serve
11167/// task is spawned per-variant.
11168enum BoundListener {
11169    /// TCP listener on `host:port`.
11170    Tcp(tokio::net::TcpListener),
11171    /// Unix domain socket listener (local daemon transport).
11172    #[cfg(unix)]
11173    Unix(tokio::net::UnixListener),
11174    /// TLS-terminating listener on `host:port` (direct HTTPS, issue #1603).
11175    #[cfg(feature = "tls")]
11176    Tls(crate::tls::TlsListener),
11177}
11178
11179/// Bind a TLS-terminating listener over `tcp`, loading and validating the
11180/// configured certificate and key (fail-fast on any problem).
11181#[cfg(feature = "tls")]
11182fn build_tls_listener(
11183    tcp: tokio::net::TcpListener,
11184    cfg: &crate::config::TlsConfig,
11185    shutdown: tokio_util::sync::CancellationToken,
11186) -> Result<
11187    (
11188        crate::tls::TlsListener,
11189        crate::tls::CertReloader,
11190        Option<crate::tls::client_auth::ClientTrustReloader>,
11191    ),
11192    crate::tls::TlsError,
11193> {
11194    let provider = crate::tls::crypto_provider();
11195    // The pre-bind `TlsConfig::validate()` guarantees both paths are set in
11196    // static-cert mode (the only mode that reaches this function; ACME mode is
11197    // handled separately), so unwrapping here is validated, not hopeful.
11198    let cert_path = cfg
11199        .cert_path
11200        .as_deref()
11201        .expect("validated: static [server.tls] sets cert_path");
11202    let key_path = cfg
11203        .key_path
11204        .as_deref()
11205        .expect("validated: static [server.tls] sets key_path");
11206    // One call, so the reload baseline is stat'd before the certificate is
11207    // loaded: a renewal landing in that gap must read as a change on the next
11208    // poll, not as the baseline (which would serve the superseded certificate
11209    // until the following renewal).
11210    let (resolver, reload) = crate::tls::CertReloader::load(
11211        cert_path.to_path_buf(),
11212        key_path.to_path_buf(),
11213        std::sync::Arc::clone(&provider),
11214        crate::tls::now_unix(),
11215        // A zero interval would busy-loop; clamp to at least one second.
11216        std::time::Duration::from_secs(cfg.reload_interval_secs.max(1)),
11217    )?;
11218    let (client_verifier, client_reload) = build_client_auth(cfg, &provider)?;
11219    let server_config = crate::tls::build_server_config_with_client_auth(
11220        std::sync::Arc::clone(&provider),
11221        std::sync::Arc::clone(&resolver) as std::sync::Arc<dyn rustls::server::ResolvesServerCert>,
11222        client_verifier,
11223    )?;
11224    // A zero handshake timeout would drop every connection instantly; clamp to
11225    // at least one second, mirroring the reload-interval clamp above.
11226    let handshake_timeout = std::time::Duration::from_secs(cfg.handshake_timeout_secs.max(1));
11227    let listener = crate::tls::TlsListener::new(tcp, server_config, handshake_timeout, shutdown);
11228    Ok((listener, reload, client_reload))
11229}
11230
11231/// The mTLS wiring `build_client_auth` hands back: the verifier the listener
11232/// enforces, and the reloader that rotates its trust store. Both `None` when
11233/// client auth is off.
11234#[cfg(feature = "tls")]
11235type ClientAuthWiring = (
11236    Option<std::sync::Arc<dyn rustls::server::danger::ClientCertVerifier>>,
11237    Option<crate::tls::client_auth::ClientTrustReloader>,
11238);
11239
11240/// Build the mTLS client-certificate verifier and its trust-store reloader from
11241/// `[server.tls.client_auth]` (issue #1640).
11242///
11243/// `(None, None)` — the identical #1603 server-only path — whenever the section
11244/// is absent or `mode = "off"`. Any problem with the bundle or CRL is returned
11245/// so the caller fails fast at boot with the path in the message.
11246#[cfg(feature = "tls")]
11247fn build_client_auth(
11248    cfg: &crate::config::TlsConfig,
11249    provider: &std::sync::Arc<rustls::crypto::CryptoProvider>,
11250) -> Result<ClientAuthWiring, crate::tls::TlsError> {
11251    if !cfg.client_auth_active() {
11252        return Ok((None, None));
11253    }
11254    // `client_auth_active()` is true only for a present section with a mode
11255    // other than `off`, and `ClientAuthConfig::validate()` (run pre-bind)
11256    // guarantees such a section names a bundle.
11257    let client_auth = cfg
11258        .client_auth
11259        .as_ref()
11260        .expect("validated: an active client_auth section is present");
11261    let bundle = client_auth
11262        .ca_bundle_path
11263        .clone()
11264        .expect("validated: an active client_auth section sets ca_bundle_path");
11265    let (verifier, reloader) = crate::tls::client_auth::ClientTrustReloader::load(
11266        bundle,
11267        client_auth.crl_path.clone(),
11268        client_auth.mode,
11269        std::sync::Arc::clone(provider),
11270        // A zero interval would busy-loop; clamp to at least one second.
11271        std::time::Duration::from_secs(client_auth.reload_interval_secs.max(1)),
11272    )?;
11273    Ok((Some(verifier), Some(reloader)))
11274}
11275
11276/// Carries the ACME challenge-listener + renewal-task wiring from the bind path
11277/// to the sibling tasks spawned once `server_shutdown` exists.
11278#[cfg(feature = "acme")]
11279struct AcmeBindState {
11280    renewal_task: crate::acme::renewal::AcmeRenewalTask,
11281    tokens: crate::acme::challenge::Http01Tokens,
11282    http_challenge_port: u16,
11283    https_port: u16,
11284    /// Whether DNS-01 is configured. Decides whether a failure to bind the
11285    /// challenge/redirect port is fatal (HTTP-01) or a warning (DNS-01, where
11286    /// the CA never connects to this host).
11287    dns01: bool,
11288    /// The tenant custom-domain wiring (#1635), present exactly when
11289    /// `[server.tls.acme.custom_domains] enabled = true`.
11290    custom_domains: Option<CustomDomainBindState>,
11291    /// The mTLS trust-store reloader (#1640), present exactly when
11292    /// `[server.tls.client_auth]` is active. Spawned beside the ACME renewal
11293    /// task, so a CA rotation lands without a restart on this arm too.
11294    client_trust_reload: Option<crate::tls::client_auth::ClientTrustReloader>,
11295}
11296
11297/// Everything the custom-domain orchestrator needs, built at bind time so the
11298/// SNI resolver and the loop share one registry and one certificate cache.
11299#[cfg(feature = "acme")]
11300struct CustomDomainBindState {
11301    registry: std::sync::Arc<crate::custom_domain::CustomDomainRegistry>,
11302    cache: std::sync::Arc<crate::custom_domain::CustomDomainCertCache>,
11303    store: std::sync::Arc<crate::acme::store::FsAcmeStore>,
11304    provider: std::sync::Arc<rustls::crypto::CryptoProvider>,
11305    config: crate::config::CustomDomainsConfig,
11306    /// The enclosing `[server.tls.acme]`, so the per-domain issuer orders on
11307    /// the SAME account and directory as the deployment's own certificate.
11308    acme: crate::config::AcmeConfig,
11309}
11310
11311/// Build a TLS listener for ACME mode: serve a stored certificate if one is
11312/// present, else a self-signed placeholder so `:443` binds immediately. The
11313/// returned [`AcmeBindState`] carries everything the renewal task and challenge
11314/// listener need.
11315#[cfg(feature = "acme")]
11316#[allow(
11317    clippy::too_many_arguments,
11318    clippy::too_many_lines,
11319    reason = "one bind-time assembly of the ACME listener, its store, its \
11320              placeholder and the custom-domain registry; splitting it would \
11321              only scatter the ordering these steps depend on"
11322)]
11323async fn build_acme_tls_listener(
11324    tcp: tokio::net::TcpListener,
11325    tls_cfg: &crate::config::TlsConfig,
11326    acme_cfg: &crate::config::AcmeConfig,
11327    tenancy_base_domain: Option<&str>,
11328    credentials: &crate::credentials::CredentialsStore,
11329    https_port: u16,
11330    status: Option<crate::acme::renewal::AcmeStatus>,
11331    shutdown: tokio_util::sync::CancellationToken,
11332) -> Result<(crate::tls::TlsListener, AcmeBindState), String> {
11333    use crate::acme::store::{AcmeStore, CertId, FsAcmeStore};
11334
11335    let provider = crate::tls::crypto_provider();
11336    let cert_id = CertId::from_domains(&acme_cfg.domains);
11337    let directory_label = crate::acme::directory_label(&acme_cfg.directory);
11338    let store: std::sync::Arc<dyn AcmeStore> = std::sync::Arc::new(FsAcmeStore::new(
11339        acme_cfg.cache_dir.clone(),
11340        directory_label,
11341    ));
11342    let status = status.unwrap_or_default();
11343
11344    // Prefer a valid stored certificate; fall back to a self-signed placeholder
11345    // so the port comes up while the first issuance runs in the background.
11346    let (initial, serving_stored_cert) = match store.load_cert(&cert_id).await {
11347        Ok(Some(stored)) => match crate::tls::certified_key_from_pem(
11348            stored.chain_pem.as_bytes(),
11349            stored.key_pem.as_bytes(),
11350            &provider,
11351        ) {
11352            Ok(ck) => {
11353                if let Ok(not_after) =
11354                    crate::tls::leaf_not_after_from_pem(stored.chain_pem.as_bytes())
11355                {
11356                    status.set_cert_not_after(not_after);
11357                }
11358                (ck, true)
11359            }
11360            Err(e) => {
11361                tracing::warn!(
11362                    "stored ACME certificate is unusable ({e}); serving a self-signed \
11363                     placeholder until the renewal task issues a real one"
11364                );
11365                (acme_placeholder_key(&acme_cfg.domains, &provider)?, false)
11366            }
11367        },
11368        Ok(None) => (acme_placeholder_key(&acme_cfg.domains, &provider)?, false),
11369        Err(e) => {
11370            tracing::warn!(
11371                "failed to read the stored ACME certificate ({e}); serving a self-signed \
11372                 placeholder"
11373            );
11374            (acme_placeholder_key(&acme_cfg.domains, &provider)?, false)
11375        }
11376    };
11377
11378    let resolver = std::sync::Arc::new(crate::tls::ReloadableCertResolver::new(initial));
11379
11380    // Tenant custom domains (#1635): the registry is hydrated from disk before
11381    // the listener binds, so a restart routes and serves every connected domain
11382    // on the first request rather than after the first orchestrator tick. The
11383    // SNI resolver wraps — rather than replaces — the operator's own resolver,
11384    // so the deployment's certificate keeps serving its own names unchanged.
11385    let custom_domains = match acme_cfg.custom_domains.as_ref() {
11386        Some(cd_cfg) if cd_cfg.enabled => {
11387            let reserved = custom_domain_reserved_names(acme_cfg, tenancy_base_domain);
11388            let registry = std::sync::Arc::new(
11389                crate::custom_domain::CustomDomainRegistry::new(
11390                    std::sync::Arc::new(crate::custom_domain::FsCustomDomainStore::new(
11391                        cd_cfg.store_dir.clone(),
11392                    )),
11393                    cd_cfg.max_domains,
11394                )
11395                .with_reserved(reserved),
11396            );
11397            match registry.load().await {
11398                Ok(count) => tracing::info!(count, "loaded tenant custom domains"),
11399                // A registry that cannot be read is not fatal to the
11400                // deployment: its own certificate still serves. It IS fatal to
11401                // custom domains, though — an index that hydrated nothing
11402                // cannot tell whether a hostname is already owned, so
11403                // `register` refuses until a load succeeds rather than
11404                // overwriting the durable record of whoever holds it.
11405                Err(e) => tracing::error!(
11406                    "failed to load the tenant custom-domain registry: {e}; connected domains will \
11407                     not route and no new domain can be connected until this is fixed and the \
11408                     process restarted"
11409                ),
11410            }
11411            Some(CustomDomainBindState {
11412                registry,
11413                cache: std::sync::Arc::new(crate::custom_domain::CustomDomainCertCache::new(
11414                    cd_cfg.cert_cache_size,
11415                )),
11416                store: std::sync::Arc::new(FsAcmeStore::new(
11417                    acme_cfg.cache_dir.clone(),
11418                    crate::acme::directory_label(&acme_cfg.directory),
11419                )),
11420                provider: std::sync::Arc::clone(&provider),
11421                config: cd_cfg.clone(),
11422                acme: acme_cfg.clone(),
11423            })
11424        }
11425        _ => None,
11426    };
11427
11428    let cert_resolver: std::sync::Arc<dyn rustls::server::ResolvesServerCert> =
11429        custom_domains.as_ref().map_or_else(
11430            || {
11431                std::sync::Arc::clone(&resolver)
11432                    as std::sync::Arc<dyn rustls::server::ResolvesServerCert>
11433            },
11434            |cd| {
11435                std::sync::Arc::new(
11436                    crate::custom_domain::SniCertResolver::new(
11437                        std::sync::Arc::clone(&resolver),
11438                        acme_cfg.domains.clone(),
11439                        std::sync::Arc::clone(&cd.registry),
11440                        std::sync::Arc::clone(&cd.cache),
11441                    )
11442                    // A domain evicted from the bounded cache (or never warmed,
11443                    // in a deployment with more domains than the cache holds)
11444                    // loads its certificate here rather than stopping being
11445                    // served.
11446                    .with_source(std::sync::Arc::new(
11447                        crate::acme::tenant_domains::FsSniCertSource::new(
11448                            std::sync::Arc::clone(&cd.store),
11449                            std::sync::Arc::clone(&provider),
11450                        ),
11451                    )),
11452                )
11453            },
11454        );
11455    // Client auth is orthogonal to how the SERVER's certificate is provisioned
11456    // (#1640), so the ACME arm wires the same verifier the static-cert arm does.
11457    // Without this a `[server.tls.client_auth] mode = "required"` deployment on
11458    // ACME would boot, report healthy, and never request a certificate — the
11459    // one misconfiguration that fails OPEN.
11460    let (client_verifier, client_reload) =
11461        build_client_auth(tls_cfg, &provider).map_err(|e| e.to_string())?;
11462    let server_config = crate::tls::build_server_config_with_client_auth(
11463        std::sync::Arc::clone(&provider),
11464        cert_resolver,
11465        client_verifier,
11466    )
11467    .map_err(|e| e.to_string())?;
11468    let handshake_timeout = std::time::Duration::from_secs(tls_cfg.handshake_timeout_secs.max(1));
11469    let listener = crate::tls::TlsListener::new(tcp, server_config, handshake_timeout, shutdown);
11470
11471    let tokens = crate::acme::challenge::Http01Tokens::new();
11472    // `[server.tls.acme.dns]` selects DNS-01, the only challenge a CA will
11473    // validate for a wildcard identifier (#1620). Built here, at bind time, so a
11474    // missing or malformed provider credential fails startup with an actionable
11475    // message instead of surfacing as a failed order 30 days later.
11476    let dns = build_dns_challenge(acme_cfg, credentials)?;
11477    let renewal_task = crate::acme::renewal::AcmeRenewalTask {
11478        resolver,
11479        provider,
11480        store,
11481        cert_id,
11482        tokens: tokens.clone(),
11483        status,
11484        config: acme_cfg.clone(),
11485        serving_stored_cert,
11486        // Filled in at the renewal spawn site once the scheduler coordinator has
11487        // been built and any distributed → in-process fallback is known.
11488        leadership_degraded: false,
11489        renew_window_misconfigured: std::sync::atomic::AtomicBool::new(false),
11490        dns,
11491        // Filled in at the renewal spawn site, where `AppState` (and so the
11492        // operator alerter) is in scope.
11493        recovery: None,
11494    };
11495    Ok((
11496        listener,
11497        AcmeBindState {
11498            renewal_task,
11499            tokens,
11500            http_challenge_port: acme_cfg.http_challenge_port,
11501            https_port,
11502            dns01: acme_cfg.dns.is_some(),
11503            custom_domains,
11504            client_trust_reload: client_reload,
11505        },
11506    ))
11507}
11508
11509/// The multi-replica ACME warning for this deployment, or `None` when the
11510/// scheduler backend says the deployment is single-replica.
11511///
11512/// One condition guards two distinct hazards, and DNS-01 only removes the
11513/// first:
11514///
11515/// 1. The HTTP-01 token map is per-process, so behind a load balancer the CA's
11516///    `:80` request can land on a replica that never minted the token (404).
11517/// 2. The certificate store is local disk, so replicas that did not win the
11518///    renewal lease never see the issued certificate and keep serving the
11519///    self-signed placeholder.
11520///
11521/// DNS-01 proves control through a TXT record, which retires (1) — but it does
11522/// not distribute certificates, so (2) still mis-serves TLS on every replica
11523/// but the leader. Warn either way; only the text differs (issue #1620).
11524///
11525/// The `sqlite` backend coordinates processes on **one** host (issue #1907), so
11526/// hazard (2) does not apply: every process reads the same `cache_dir` on the
11527/// same disk. Hazard (1) still does, because the token map is per-process — so
11528/// it gets its own, narrower message, and DNS-01 clears it entirely.
11529///
11530/// Keyed off the *configured* backend (operator intent) rather than the built
11531/// coordinator, so the warning still fires when `coordinator_from_config` fell
11532/// back to in-process after a Postgres error — exactly the case where the fleet
11533/// is multi-replica but this process degraded.
11534#[cfg(feature = "acme")]
11535const fn acme_fleet_warning(
11536    backend: crate::config::SchedulerBackend,
11537    dns01: bool,
11538) -> Option<&'static str> {
11539    if !backend.is_fleet_distributed() {
11540        return None;
11541    }
11542    if matches!(backend, crate::config::SchedulerBackend::Sqlite) {
11543        if dns01 {
11544            // DNS-01 needs no :80 challenge, and the store is already shared.
11545            return None;
11546        }
11547        return Some(
11548            "ACME HTTP-01 validation is not safe across the processes scheduler.backend = \
11549             \"sqlite\" coordinates: the token store is per-process, so a proxy may route the \
11550             CA's :80 challenge to a process without the token (404). The certificate store is \
11551             not at risk — every process on the host reads the same [server.tls.acme] \
11552             cache_dir. Serve ACME from one process, terminate TLS at the proxy, or use DNS-01 \
11553             (#1620)",
11554        );
11555    }
11556    Some(if dns01 {
11557        "ACME DNS-01 issuance is fleet-safe, but the on-disk certificate store is not: only \
11558         the replica holding the renewal lease writes the issued certificate, and the others \
11559         cannot adopt it from a non-shared store, so they keep serving the self-signed \
11560         placeholder. Run ACME on a single host, or point [server.tls.acme] cache_dir at \
11561         storage every replica shares (#1620)"
11562    } else {
11563        "ACME HTTP-01 validation is not fleet-safe with the local on-disk token store: behind \
11564         a load balancer the CA's :80 challenge may reach a replica without the token (404), \
11565         and non-leader replicas cannot adopt issued certificates from a non-shared store. Run \
11566         ACME on a single host, or terminate TLS at a shared proxy. DNS-01 removes the :80 \
11567         hazard but not the store one, so it needs a shared [server.tls.acme] cache_dir too \
11568         (#1620)"
11569    })
11570}
11571
11572/// Build the DNS-01 challenge wiring for `[server.tls.acme.dns]`, if configured.
11573///
11574/// The provider credential is read from the encrypted credentials store (or the
11575/// documented `AUTUMN_ACME_DNS_*` environment variables) — never from
11576/// `autumn.toml`, which has no field that could hold one. A missing or blank
11577/// credential is an error **here**, at bind time, rather than a failed order
11578/// discovered when the certificate is already near expiry (issue #1620).
11579#[cfg(feature = "acme")]
11580fn build_dns_challenge(
11581    acme_cfg: &crate::config::AcmeConfig,
11582    credentials: &crate::credentials::CredentialsStore,
11583) -> Result<Option<crate::acme::renewal::DnsChallenge>, String> {
11584    use crate::acme::dns;
11585
11586    let Some(dns_cfg) = acme_cfg.dns.as_ref() else {
11587        return Ok(None);
11588    };
11589    let resolvers = dns_cfg.resolver_addrs()?;
11590    let credential = dns::DnsCredential::resolve(dns_cfg, credentials, &dns::process_env);
11591    // One bounded HTTP timeout for the provider API and one for each DNS probe,
11592    // so neither can park the renewal loop: the whole propagation wait is
11593    // already bounded, and a black-holed provider API must not outlive it.
11594    let transport: std::sync::Arc<dyn dns::http::HttpTransport> = std::sync::Arc::new(
11595        dns::http::ReqwestTransport::new(std::time::Duration::from_secs(30))?,
11596    );
11597    let provider = dns::build_provider(dns_cfg, &credential, transport)?;
11598    Ok(Some(crate::acme::renewal::DnsChallenge {
11599        provider,
11600        lookup: std::sync::Arc::new(dns::resolver::UdpDnsLookup::new(
11601            std::time::Duration::from_secs(5),
11602        )),
11603        resolvers,
11604        propagation_timeout: std::time::Duration::from_secs(dns_cfg.propagation_timeout_secs),
11605        poll_interval: std::time::Duration::from_secs(dns_cfg.poll_interval_secs),
11606    }))
11607}
11608
11609/// Build a `CertifiedKey` from a fresh self-signed placeholder for `domains`.
11610#[cfg(feature = "acme")]
11611fn acme_placeholder_key(
11612    domains: &[String],
11613    provider: &rustls::crypto::CryptoProvider,
11614) -> Result<std::sync::Arc<rustls::sign::CertifiedKey>, String> {
11615    let placeholder = crate::acme::renewal::self_signed_placeholder(domains)?;
11616    crate::tls::certified_key_from_pem(
11617        placeholder.chain_pem.as_bytes(),
11618        placeholder.key_pem.as_bytes(),
11619        provider,
11620    )
11621}
11622
11623/// Build the ACME renewal failure reporter: dispatch each failure as an
11624/// [`ErrorEvent`](crate::reporting::ErrorEvent) to the registered reporter chain
11625/// on a detached task, so failures reach Sentry/etc. when configured (failures
11626/// also always log via `tracing` inside the loop).
11627#[cfg(all(feature = "acme", feature = "reporting"))]
11628fn make_acme_reporter(
11629    reporters: Vec<std::sync::Arc<dyn crate::reporting::ErrorReporter>>,
11630) -> crate::acme::renewal::ReporterFn {
11631    std::sync::Arc::new(move |message: String| {
11632        if reporters.is_empty() {
11633            return;
11634        }
11635        let reporters = reporters.clone();
11636        tokio::spawn(async move {
11637            let event = crate::reporting::ErrorEvent {
11638                status: axum::http::StatusCode::INTERNAL_SERVER_ERROR,
11639                message,
11640                problem_type: None,
11641                request_id: None,
11642                route: Some("acme-renewal".to_owned()),
11643                method: None,
11644                panic: None,
11645                capsule: None,
11646            };
11647            for reporter in &reporters {
11648                reporter.report(&event).await;
11649            }
11650        });
11651    })
11652}
11653
11654/// The scheduled-operation name ACME renewal alerts are keyed on. Stable: the
11655/// trigger and its recovery must share it, or the recovery cannot clear the
11656/// outstanding alert.
11657#[cfg(feature = "acme")]
11658const ACME_RENEWAL_TASK_NAME: &str = "acme-renewal";
11659
11660/// Wrap `inner` so every ACME failure ALSO raises #1610's
11661/// `scheduled_task_failure` operator alert.
11662///
11663/// Composition rather than replacement: the error-reporting chain (Sentry et al)
11664/// still sees the failure, and the alerter is an independent destination an
11665/// operator actually watches.
11666#[cfg(feature = "acme")]
11667fn compose_acme_alert_reporter(
11668    inner: crate::acme::renewal::ReporterFn,
11669    state: &AppState,
11670) -> crate::acme::renewal::ReporterFn {
11671    let state = state.clone();
11672    std::sync::Arc::new(move |message: String| {
11673        crate::alerts::notify_scheduled_task_failure(&state, ACME_RENEWAL_TASK_NAME, &message);
11674        inner(message);
11675    })
11676}
11677
11678/// The hostnames custom-domain registration must never claim: the
11679/// deployment's own ACME domains, the tenancy base domain, and the ingress
11680/// hostname.
11681///
11682/// Shared by the TLS bind path and the one-shot `autumn db retention` path so
11683/// the two registries agree on what is reserved (#2652).
11684#[cfg(feature = "acme")]
11685fn custom_domain_reserved_names(
11686    acme_cfg: &crate::config::AcmeConfig,
11687    tenancy_base_domain: Option<&str>,
11688) -> Vec<String> {
11689    // Reserve everything this deployment already serves. Without it a
11690    // tenant registers another tenant's subdomain — which the
11691    // operator's own wildcard already points here, so it verifies and
11692    // issues — and every request for that host then resolves to
11693    // whoever registered it.
11694    let mut reserved = acme_cfg.domains.clone();
11695    reserved.extend(tenancy_base_domain.map(ToOwned::to_owned));
11696    // The ingress hostname is the sharpest of the three. It ALREADY
11697    // resolves to the ingress addresses, so a tenant who registers it
11698    // needs no DNS change at all: verification passes on the first
11699    // tick, HTTP-01 validates, and from then on every request to the
11700    // deployment's own infrastructure hostname routes to that tenant.
11701    // It is not necessarily covered by `domains` — an operator may run
11702    // ingress under a separate infrastructure zone — so it is reserved
11703    // explicitly rather than by assuming overlap.
11704    if let Some(cd_cfg) = acme_cfg.custom_domains.as_ref() {
11705        reserved.extend(cd_cfg.ingress_hostname.clone());
11706    }
11707    reserved
11708}
11709
11710/// Install the custom-domain retention pruner for a one-shot
11711/// `autumn db retention` run (#2652).
11712///
11713/// The `dyn CustomDomainPruner` extension is normally installed by
11714/// [`spawn_custom_domain_task`] on the TLS bind path, which a one-shot run
11715/// never takes — without this the `custom_domains` dataset always reported
11716/// "custom domains are not enabled". The pruner carries no ACME issuer:
11717/// pruning only touches the registry and the certificate store, so no order
11718/// is placed and no CA is contacted.
11719#[cfg(feature = "acme")]
11720async fn install_custom_domain_retention_pruner(
11721    acme: Option<&crate::config::AcmeConfig>,
11722    tenancy_base_domain: Option<&str>,
11723    state: &AppState,
11724    migrate: bool,
11725) {
11726    let Some(acme_cfg) = acme else {
11727        return;
11728    };
11729    let Some(cd_cfg) = acme_cfg.custom_domains.as_ref() else {
11730        return;
11731    };
11732    if !cd_cfg.enabled {
11733        return;
11734    }
11735    let registry = std::sync::Arc::new(
11736        crate::custom_domain::CustomDomainRegistry::new(
11737            std::sync::Arc::new(crate::custom_domain::FsCustomDomainStore::new(
11738                cd_cfg.store_dir.clone(),
11739            )),
11740            cd_cfg.max_domains,
11741        )
11742        .with_reserved(custom_domain_reserved_names(acme_cfg, tenancy_base_domain)),
11743    );
11744    // A report (`--dry-run` or no flag) must write nothing: `load` gives a
11745    // pre-token record its ownership token, which also resets its status and
11746    // registration time — the very fields retention eligibility is judged on.
11747    // Only a purge, which is about to write anyway, runs the boot migration.
11748    let loaded = if migrate {
11749        registry.load().await
11750    } else {
11751        registry.load_without_migration().await
11752    };
11753    match loaded {
11754        Ok(count) => tracing::info!(count, "loaded tenant custom domains"),
11755        // A registry that cannot be read is not fatal to the deployment: its
11756        // own certificate still serves. It IS fatal to pruning, though — an
11757        // index that hydrated nothing cannot tell whether a hostname is
11758        // already owned, so `prune` refuses rather than deleting every stored
11759        // key.
11760        Err(e) => tracing::error!(
11761            "failed to load the tenant custom-domain registry: {e}; the custom_domains \
11762             retention dataset will refuse to prune until this is fixed"
11763        ),
11764    }
11765    let store = std::sync::Arc::new(crate::acme::store::FsAcmeStore::new(
11766        acme_cfg.cache_dir.clone(),
11767        crate::acme::directory_label(&acme_cfg.directory),
11768    ));
11769    let pruner = crate::acme::tenant_domains::PruneOnlyCustomDomainPruner {
11770        registry,
11771        cache: std::sync::Arc::new(crate::custom_domain::CustomDomainCertCache::new(
11772            cd_cfg.cert_cache_size,
11773        )),
11774        certs: std::sync::Arc::clone(&store) as std::sync::Arc<dyn crate::acme::store::AcmeStore>,
11775        limiter: std::sync::Arc::new(crate::custom_domain::IssuanceLimiter::new(
11776            cd_cfg.issuance_per_domain_per_day,
11777            cd_cfg.issuance_global_per_hour,
11778            cd_cfg.failure_backoff_secs,
11779            cd_cfg.max_failure_backoff_secs,
11780        )),
11781        cert_store_paths: Some(store),
11782        // The deployment's own certificate shares this store and has no
11783        // registry record; naming it keeps the retention prune from deleting
11784        // the certificate the listener is serving.
11785        retained_cert_ids: std::iter::once(
11786            crate::acme::store::CertId::from_domains(&acme_cfg.domains)
11787                .as_str()
11788                .to_owned(),
11789        )
11790        .collect(),
11791        reporter: make_custom_domain_reporter(state),
11792        recovery: Some(make_custom_domain_recovery(state)),
11793    };
11794    state
11795        .insert_extension(std::sync::Arc::new(pruner)
11796            as std::sync::Arc<dyn crate::custom_domain::CustomDomainPruner>);
11797}
11798
11799/// Publish the custom-domain registry and spawn its orchestrator (#1635).
11800///
11801/// The registry goes into `AppState` so tenancy resolution can route a
11802/// connected `Host`; the health indicator and the retention pruner are
11803/// registered from the same handles the loop mutates, so the three can never
11804/// disagree about a domain's state.
11805#[cfg(feature = "acme")]
11806fn spawn_custom_domain_task(
11807    bind_state: CustomDomainBindState,
11808    tokens: crate::acme::challenge::Http01Tokens,
11809    coordinator: std::sync::Arc<dyn crate::scheduler::SchedulerCoordinator>,
11810    leadership_degraded: bool,
11811    state: &AppState,
11812    shutdown: tokio_util::sync::CancellationToken,
11813) {
11814    let CustomDomainBindState {
11815        registry,
11816        cache,
11817        store,
11818        provider,
11819        config,
11820        acme,
11821    } = bind_state;
11822
11823    state.insert_extension(std::sync::Arc::clone(&registry));
11824    if let Err(e) = state.health_indicator_registry.register(
11825        "custom_domains",
11826        crate::actuator::IndicatorGroup::HealthOnly,
11827        std::sync::Arc::new(crate::custom_domain::CustomDomainHealthIndicator::new(
11828            std::sync::Arc::clone(&registry),
11829        )),
11830    ) {
11831        tracing::warn!("{e}");
11832    }
11833
11834    let issuer = std::sync::Arc::new(crate::acme::tenant_domains::AcmeDomainIssuer::new(
11835        acme.clone(),
11836        std::sync::Arc::clone(&store) as std::sync::Arc<dyn crate::acme::store::AcmeStore>,
11837        tokens,
11838    ));
11839    let task = std::sync::Arc::new(crate::acme::tenant_domains::CustomDomainTask {
11840        registry,
11841        cache,
11842        certs: std::sync::Arc::clone(&store) as std::sync::Arc<dyn crate::acme::store::AcmeStore>,
11843        provider,
11844        // `validate` already refused an unparseable entry at boot.
11845        verifier: std::sync::Arc::new(crate::custom_domain::SystemDomainVerifier::new(
11846            config.resolver_addrs().unwrap_or_default(),
11847        )),
11848        issuer,
11849        limiter: std::sync::Arc::new(crate::custom_domain::IssuanceLimiter::new(
11850            config.issuance_per_domain_per_day,
11851            config.issuance_global_per_hour,
11852            config.failure_backoff_secs,
11853            config.max_failure_backoff_secs,
11854        )),
11855        ingress: config.ingress(),
11856        renew_before_days: acme.renew_before_days,
11857        // A per-domain failure is a framework-scheduled operation failing, so
11858        // it raises #1610's alert naming the domain and its tenant.
11859        reporter: make_custom_domain_reporter(state),
11860        recovery: Some(make_custom_domain_recovery(state)),
11861        coordinator,
11862        leadership_degraded,
11863        cert_store_paths: Some(store),
11864        // The deployment's own certificate shares this store and has no
11865        // registry record; naming it keeps the retention prune from deleting
11866        // the certificate the listener is serving.
11867        retained_cert_ids: std::iter::once(
11868            crate::acme::store::CertId::from_domains(&acme.domains)
11869                .as_str()
11870                .to_owned(),
11871        )
11872        .collect(),
11873    });
11874
11875    // The pruner is the retention half of the task, not the task itself: the
11876    // same small type the one-shot `autumn db retention` path installs (#2652).
11877    state.insert_extension(std::sync::Arc::new(task.pruner())
11878        as std::sync::Arc<dyn crate::custom_domain::CustomDomainPruner>);
11879
11880    let interval = std::time::Duration::from_secs(config.poll_interval_secs.max(1));
11881    tokio::spawn(async move {
11882        task.run(interval, shutdown).await;
11883    });
11884}
11885
11886/// The reporter a custom-domain failure is dispatched through.
11887#[cfg(feature = "acme")]
11888fn make_custom_domain_reporter(state: &AppState) -> crate::acme::tenant_domains::ReporterFn {
11889    let state = state.clone();
11890    std::sync::Arc::new(move |message: String| {
11891        crate::alerts::notify_scheduled_task_failure(&state, CUSTOM_DOMAIN_TASK_NAME, &message);
11892    })
11893}
11894
11895/// The callback that clears an outstanding custom-domain alert.
11896#[cfg(feature = "acme")]
11897fn make_custom_domain_recovery(state: &AppState) -> crate::acme::tenant_domains::RecoveryFn {
11898    let state = state.clone();
11899    std::sync::Arc::new(move || {
11900        crate::alerts::notify_scheduled_task_recovered(&state, CUSTOM_DOMAIN_TASK_NAME);
11901    })
11902}
11903
11904/// The scheduled-task name a custom-domain failure alert is keyed on.
11905#[cfg(feature = "acme")]
11906const CUSTOM_DOMAIN_TASK_NAME: &str = "custom_domain_certificates";
11907
11908/// The callback that clears an outstanding ACME renewal alert once issuance
11909/// succeeds again.
11910#[cfg(feature = "acme")]
11911fn make_acme_alert_recovery(state: &AppState) -> crate::acme::renewal::RecoveryFn {
11912    let state = state.clone();
11913    std::sync::Arc::new(move || {
11914        crate::alerts::notify_scheduled_task_recovered(&state, ACME_RENEWAL_TASK_NAME);
11915    })
11916}
11917
11918/// The no-op ACME reporter used when the `reporting` feature is off (failures
11919/// still log via `tracing`).
11920#[cfg(all(feature = "acme", not(feature = "reporting")))]
11921fn make_acme_reporter() -> crate::acme::renewal::ReporterFn {
11922    std::sync::Arc::new(|_message: String| {})
11923}
11924
11925/// Connection info for a Unix-domain-socket request.
11926///
11927/// axum's `into_make_service_with_connect_info::<C>` requires `C:
11928/// Connected<IncomingStream>`. Unlike TCP there is no peer `SocketAddr` for a
11929/// Unix socket, so this carries no data — it exists purely to satisfy the
11930/// connect-info bound on the UDS serve path.
11931#[cfg(unix)]
11932#[derive(Clone, Debug)]
11933struct UdsConnectInfo;
11934
11935#[cfg(unix)]
11936impl
11937    axum::extract::connect_info::Connected<
11938        axum::serve::IncomingStream<'_, tokio::net::UnixListener>,
11939    > for UdsConnectInfo
11940{
11941    fn connect_info(_stream: axum::serve::IncomingStream<'_, tokio::net::UnixListener>) -> Self {
11942        Self
11943    }
11944}
11945
11946/// The UDS serve path wraps its listener in `StopAcceptingOnShutdown` (see
11947/// `accept_drain`), so the connect info must be available for the wrapper too.
11948#[cfg(unix)]
11949impl
11950    axum::extract::connect_info::Connected<
11951        axum::serve::IncomingStream<
11952            '_,
11953            crate::accept_drain::StopAcceptingOnShutdown<tokio::net::UnixListener>,
11954        >,
11955    > for UdsConnectInfo
11956{
11957    fn connect_info(
11958        _stream: axum::serve::IncomingStream<
11959            '_,
11960            crate::accept_drain::StopAcceptingOnShutdown<tokio::net::UnixListener>,
11961        >,
11962    ) -> Self {
11963        Self
11964    }
11965}
11966
11967#[cfg(unix)]
11968impl
11969    axum::extract::connect_info::Connected<
11970        crate::http_server::IncomingStream<
11971            '_,
11972            crate::accept_drain::StopAcceptingOnShutdown<tokio::net::UnixListener>,
11973        >,
11974    > for UdsConnectInfo
11975{
11976    fn connect_info(
11977        _stream: crate::http_server::IncomingStream<
11978            '_,
11979            crate::accept_drain::StopAcceptingOnShutdown<tokio::net::UnixListener>,
11980        >,
11981    ) -> Self {
11982        Self
11983    }
11984}
11985
11986/// Interval between replica lag checks: half the limit, from 250ms to 5s.
11987#[cfg(feature = "db")]
11988fn replica_lag_check_interval(max_lag: std::time::Duration) -> std::time::Duration {
11989    (max_lag / 2).clamp(
11990        std::time::Duration::from_millis(250),
11991        std::time::Duration::from_secs(5),
11992    )
11993}
11994
11995/// Measure the replica lag once and record it. One step of the monitor.
11996#[cfg(feature = "db")]
11997async fn sample_replica_lag(
11998    replica: &diesel_async::pooled_connection::deadpool::Pool<crate::db::RuntimeConnection>,
11999    probes: &crate::probe::ProbeState,
12000    max_lag: std::time::Duration,
12001    budget: std::time::Duration,
12002) {
12003    match replica.get().await {
12004        Ok(conn) => {
12005            crate::probe::refresh_replica_lag_bounded(probes, conn, budget, |conn| {
12006                Box::pin(crate::db::measure_replica_lag(conn, max_lag))
12007            })
12008            .await;
12009        }
12010        // A full pool is not a stale replica. Keep the last sample: it ages
12011        // out on its own (see `ProbeState::replica_status`).
12012        Err(error) => tracing::debug!(
12013            target: "autumn::db",
12014            error = %crate::db_url::redact_targets_in_message(&error.to_string()),
12015            "replica lag check skipped: no replica connection"
12016        ),
12017    }
12018}
12019
12020#[cfg(all(feature = "db", feature = "test-support"))]
12021impl AppState {
12022    /// Take one replica lag sample, as the background monitor does. For
12023    /// tests that do not run the monitor. Does nothing without a replica
12024    /// pool or `database.replica_max_lag_ms`.
12025    #[doc(hidden)]
12026    pub async fn sample_replica_lag_for_test(&self) {
12027        let (Some(replica), Some(max_lag)) = (self.replica_pool(), self.probes().replica_max_lag())
12028        else {
12029            return;
12030        };
12031        let budget = crate::probe::replica_lag_query_budget(max_lag);
12032        sample_replica_lag(replica, self.probes(), max_lag, budget).await;
12033    }
12034}
12035
12036/// Measure the replica lag until `shutdown`. `None` when there is no replica
12037/// or no `database.replica_max_lag_ms`.
12038#[cfg(feature = "db")]
12039fn spawn_replica_lag_monitor(
12040    state: &AppState,
12041    shutdown: tokio_util::sync::CancellationToken,
12042) -> Option<tokio::task::JoinHandle<()>> {
12043    let replica = state.replica_pool()?.clone();
12044    let max_lag = state.probes().replica_max_lag()?;
12045    let probes = state.probes().clone();
12046    let interval = replica_lag_check_interval(max_lag);
12047    // A query that hangs (for example on a half-open TCP connection) must not
12048    // keep the monitor from its next sample.
12049    let budget = crate::probe::replica_lag_query_budget(max_lag);
12050    Some(tokio::spawn(async move {
12051        loop {
12052            sample_replica_lag(&replica, &probes, max_lag, budget).await;
12053            tokio::select! {
12054                () = tokio::time::sleep(interval) => {}
12055                () = shutdown.cancelled() => break,
12056            }
12057        }
12058    }))
12059}
12060
12061/// Stamp a loopback peer (`127.0.0.1`) on Unix-domain-socket requests.
12062///
12063/// A UDS connection has no TCP peer `SocketAddr`, so without this the
12064/// trusted-proxy resolver and the [`ClientAddr`](crate::extract::ClientAddr)
12065/// extractor resolve no client address — breaking any route or middleware that
12066/// requires `ClientAddr` and any IP-based maintenance/rate-limit behavior. Local
12067/// daemon requests are loopback-equivalent, so present them as a `127.0.0.1`
12068/// connection (matching how an equivalent localhost TCP request is treated).
12069/// Installed before `TrustedProxiesLayer` on the UDS serve path only.
12070#[cfg(unix)]
12071async fn stamp_loopback_connect_info(
12072    mut req: axum::extract::Request,
12073    next: axum::middleware::Next,
12074) -> axum::response::Response {
12075    if req
12076        .extensions()
12077        .get::<axum::extract::ConnectInfo<std::net::SocketAddr>>()
12078        .is_none()
12079    {
12080        let loopback =
12081            std::net::SocketAddr::new(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST), 0);
12082        req.extensions_mut()
12083            .insert(axum::extract::ConnectInfo(loopback));
12084    }
12085    next.run(req).await
12086}
12087
12088/// Signal `autumn serve --daemon`'s supervisor that startup is complete.
12089///
12090/// The CLI passes a path via `AUTUMN_SERVE_READY_FILE` and polls for it; we
12091/// create it here, immediately after [`mark_startup_complete`], so the
12092/// supervisor's notion of "ready" means the socket is bound and serving *and*
12093/// startup hooks/migrations have finished — with no dependence on the app's HTTP
12094/// middleware (the startup barrier, maintenance mode, rate limiting, or custom
12095/// health paths, which an HTTP readiness probe would all have to thread).
12096///
12097/// Line one is the app's *resolved* graceful-drain budget in seconds
12098/// (`prestop_grace_secs + shutdown_timeout_secs`). The supervisor records this so
12099/// `autumn serve stop` waits for the budget the app will actually drain for —
12100/// even when a custom `with_config_loader` set it — instead of reconstructing it
12101/// from TOML/env and risking a premature `SIGKILL`.
12102///
12103/// Line two is where the app *actually* bound, as `<transport> <address>`. The
12104/// supervisor cannot derive this: `server.port = 0` resolves in the kernel, a
12105/// socket adopted from a predecessor keeps that process's port, and a custom
12106/// `with_config_loader` can put the app anywhere. Reporting it is what lets
12107/// `autumn serve --daemon` write an address-discovery file on Windows, where
12108/// there is no Unix socket path to hand the app in advance.
12109///
12110/// Best-effort: a write failure only delays readiness detection until the
12111/// supervisor's timeout, and a non-daemon run leaves the variable unset (no-op).
12112///
12113/// [`mark_startup_complete`]: crate::probe::ProbeState::mark_startup_complete
12114fn signal_serve_ready(drain_budget_secs: u64, bound_endpoint: &str) {
12115    let Some(path) = std::env::var_os("AUTUMN_SERVE_READY_FILE") else {
12116        return;
12117    };
12118    if path.is_empty() {
12119        return;
12120    }
12121    let path = std::path::PathBuf::from(path);
12122    // Write to a temp sibling and rename into place so the supervisor — which
12123    // polls for the file's existence and then reads the budget from it — never
12124    // observes a half-written file: it appears atomically with its full
12125    // contents. A plain `write` would make the path exist before the bytes land.
12126    let mut tmp = path.clone();
12127    tmp.as_mut_os_string().push(".tmp");
12128    if let Err(e) = std::fs::write(&tmp, serve_ready_payload(drain_budget_secs, bound_endpoint))
12129        .and_then(|()| std::fs::rename(&tmp, &path))
12130    {
12131        let _ = std::fs::remove_file(&tmp);
12132        tracing::warn!(error = %e, path = %path.display(),
12133            "could not write serve readiness file");
12134    }
12135}
12136
12137/// Turn a bound address into one a client can actually dial.
12138///
12139/// A wildcard bind is a *bind*, never a dial address — the same rule
12140/// `[cluster] advertise_addr` already enforces, for the same reason: handing a
12141/// peer `0.0.0.0` gives it something nothing can reach, and the failure then
12142/// looks like a network fault rather than the address it is. The production
12143/// smart default binds `0.0.0.0`, so without this a `--release` daemon publishes
12144/// an undialable `serve.addr` while reporting a successful start.
12145///
12146/// Only the *host* is rewritten, to loopback of the matching family; the
12147/// resolved port is what the supervisor could not have known and is preserved
12148/// exactly. An address that is already specific passes through untouched, and so
12149/// does one that will not parse — better to publish what we bound than to invent
12150/// something.
12151fn dialable_endpoint(addr: &str) -> String {
12152    let Ok(parsed) = addr.parse::<std::net::SocketAddr>() else {
12153        return addr.to_owned();
12154    };
12155    if !parsed.ip().is_unspecified() {
12156        return addr.to_owned();
12157    }
12158    match parsed {
12159        std::net::SocketAddr::V4(_) => format!("127.0.0.1:{}", parsed.port()),
12160        std::net::SocketAddr::V6(_) => format!("[::1]:{}", parsed.port()),
12161    }
12162}
12163
12164/// The readiness file's contents: the drain budget, then the bound endpoint.
12165///
12166/// Two lines rather than one structured value so the first line stays exactly
12167/// what it has always been — a bare integer — and the endpoint is additive. The
12168/// endpoint is `<transport> <address>` split on the FIRST space only, so a Unix
12169/// socket path containing spaces survives the round trip. An empty endpoint
12170/// emits no second line at all, rather than a blank one a reader could mistake
12171/// for an address. The last line, `capability shutdown-signal`, shows that the
12172/// app watches the shutdown-signal file; see
12173/// [`serve_ready_advertises_shutdown_signal`].
12174///
12175/// Public because it *is* the wire format between the app and its supervisor:
12176/// `autumn-cli`'s `parse_ready_payload` round-trips against this exact function,
12177/// so the two cannot drift into disagreement the way two hand-written parsers
12178/// would.
12179#[must_use]
12180pub fn serve_ready_payload(drain_budget_secs: u64, bound_endpoint: &str) -> String {
12181    let endpoint = bound_endpoint.trim();
12182    if endpoint.is_empty() {
12183        return format!("{drain_budget_secs}\n{SHUTDOWN_SIGNAL_CAPABILITY_LINE}");
12184    }
12185    format!("{drain_budget_secs}\n{endpoint}\n{SHUTDOWN_SIGNAL_CAPABILITY_LINE}")
12186}
12187
12188/// The ready-file line that says this app watches `AUTUMN_SHUTDOWN_SIGNAL_FILE`.
12189///
12190/// Written only after the watcher is armed. A supervisor that finds a ready file
12191/// without it and without an endpoint line knows the app predates the signal
12192/// file (#2458). An older supervisor reads `capability` as an unknown transport
12193/// and ignores the line. Only `autumn dev` on non-Unix reads it; elsewhere the
12194/// line is harmless.
12195const SHUTDOWN_SIGNAL_CAPABILITY_LINE: &str = "capability shutdown-signal";
12196
12197/// Whether a ready file says the app drains when asked through the signal file.
12198#[must_use]
12199pub fn serve_ready_advertises_shutdown_signal(payload: &str) -> bool {
12200    payload
12201        .lines()
12202        .any(|line| line.trim() == SHUTDOWN_SIGNAL_CAPABILITY_LINE)
12203}
12204
12205/// Prepare a Unix-socket path for binding: remove a *stale* socket left by a
12206/// previous run, but refuse to touch a non-socket file (guards against
12207/// clobbering a regular file) or a socket with a **live** listener (probed via
12208/// `connect`; clobbering it would silently make that service unreachable —
12209/// instead we fail like a TCP `EADDRINUSE`). A missing path is fine.
12210///
12211/// # Errors
12212///
12213/// Returns an error if the path exists and is not a socket, names a live
12214/// listener, or the stale socket cannot be removed.
12215#[cfg(unix)]
12216fn prepare_unix_socket_path(path: &std::path::Path) -> std::io::Result<()> {
12217    use std::os::unix::fs::FileTypeExt;
12218    match std::fs::symlink_metadata(path) {
12219        Ok(meta) if meta.file_type().is_socket() => {
12220            match std::os::unix::net::UnixStream::connect(path) {
12221                // A successful connect means another process is listening here.
12222                Ok(_) => Err(std::io::Error::new(
12223                    std::io::ErrorKind::AddrInUse,
12224                    format!(
12225                        "refusing to bind unix socket: {} is already in use by a \
12226                         live listener",
12227                        path.display()
12228                    ),
12229                )),
12230                // `ECONNREFUSED` (no listener) — or the path vanishing — means the
12231                // socket is stale; reclaim it.
12232                Err(e)
12233                    if matches!(
12234                        e.kind(),
12235                        std::io::ErrorKind::ConnectionRefused | std::io::ErrorKind::NotFound
12236                    ) =>
12237                {
12238                    std::fs::remove_file(path)
12239                }
12240                // `EACCES`/`EPERM` (or any other error): the socket may be a live,
12241                // operator-managed listener whose mode/ACL denies us. Connecting
12242                // failed, but liveness is unproven — refuse rather than clobber a
12243                // possibly-live service.
12244                Err(e) => Err(std::io::Error::new(
12245                    std::io::ErrorKind::AddrInUse,
12246                    format!(
12247                        "refusing to bind unix socket: cannot determine whether {} \
12248                         is live ({e}); not removing it",
12249                        path.display()
12250                    ),
12251                )),
12252            }
12253        }
12254        Ok(_) => Err(std::io::Error::new(
12255            std::io::ErrorKind::AlreadyExists,
12256            format!(
12257                "refusing to bind unix socket: {} exists and is not a socket",
12258                path.display()
12259            ),
12260        )),
12261        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
12262        Err(e) => Err(e),
12263    }
12264}
12265
12266/// What the fleet's final flush may spend of the shutdown budget. The larger
12267/// of the injected-clock and real elapsed times counts: the flush bounds
12268/// blocking I/O, so a fixed or unadvanced injected clock (which reads zero
12269/// while a real drain runs) must not hand it a fresh full budget, and a
12270/// simulated clock that ran ahead of real time still spends its virtual time.
12271#[cfg(feature = "sqlite")]
12272fn fleet_flush_budget(
12273    shutdown_budget: std::time::Duration,
12274    injected_elapsed: std::time::Duration,
12275    real_elapsed: std::time::Duration,
12276) -> std::time::Duration {
12277    shutdown_budget.saturating_sub(injected_elapsed.max(real_elapsed))
12278}
12279
12280/// How long to wait for the cluster's departure notice, given how much of the
12281/// shutdown budget the request drain already spent.
12282///
12283/// The departure itself is bounded by `LEAVE_BUDGET` inside the node; this
12284/// clamps the *wait* to what is left of `shutdown_timeout_secs`, so the leave
12285/// notice is a slice of the shutdown budget rather than an extension of it.
12286/// Without the clamp a drain that ran to its deadline would still be followed
12287/// by a full `LEAVE_BUDGET` sleep, and a supervisor timing the process out at
12288/// `shutdown_timeout_secs` could `SIGKILL` it mid-hook.
12289///
12290/// Returns zero when the drain has already consumed the budget: the departure
12291/// is skipped and the peer converges on the suspicion timeout, which is the
12292/// documented contract for a shutdown that overran.
12293fn cluster_departure_wait(
12294    shutdown_budget: std::time::Duration,
12295    drain_elapsed: std::time::Duration,
12296) -> std::time::Duration {
12297    crate::cluster::LEAVE_BUDGET.min(shutdown_budget.saturating_sub(drain_elapsed))
12298}
12299
12300async fn run_shutdown_hooks(hooks: &[ShutdownHook]) {
12301    for hook in hooks.iter().rev() {
12302        hook().await;
12303    }
12304}
12305
12306/// Run shutdown hooks in reverse-registration order (LIFO), enforcing a
12307/// per-hook timeout and a hard total-budget ceiling.
12308///
12309/// Plugin ordering rule: plugins register hooks during `build()`, which is
12310/// called before any app `on_shutdown` calls, so app hooks run **before**
12311/// plugin hooks (LIFO means last-registered runs first).
12312///
12313/// Overruns are logged at WARN but do not block the remaining budget.
12314async fn run_shutdown_hooks_with_timeout(
12315    hooks: &[ShutdownHook],
12316    per_hook_budget: std::time::Duration,
12317    total_budget: std::time::Duration,
12318) {
12319    let deadline = tokio::time::Instant::now() + total_budget;
12320    for hook in hooks.iter().rev() {
12321        let remaining = deadline.saturating_duration_since(tokio::time::Instant::now());
12322        if remaining.is_zero() {
12323            tracing::warn!("shutdown: total hook budget exhausted; skipping remaining hooks");
12324            break;
12325        }
12326        let timeout = remaining.min(per_hook_budget);
12327        // Hook overruns are intentionally non-fatal (exit 0 per ADR addendum).
12328        // Only drain deadline exhaustion (phase 6) triggers exit(1).
12329        if tokio::time::timeout(timeout, hook()).await.is_err() {
12330            tracing::warn!(
12331                per_hook_budget_ms = timeout.as_millis(),
12332                "shutdown: hook overran per-hook timeout; continuing with remaining budget"
12333            );
12334        }
12335    }
12336}
12337
12338/// Log a structured startup transparency report.
12339///
12340/// Activated by setting `AUTUMN_SHOW_CONFIG=1` (or `autumn dev --show-config`).
12341/// Prints all registered routes, scheduled tasks, active middleware, and
12342/// resolved configuration to the `INFO` log so developers can see exactly
12343/// what the macros and conventions configured.
12344#[allow(clippy::cognitive_complexity)]
12345fn log_startup_transparency(
12346    routes: &[Route],
12347    tasks: &[crate::task::TaskInfo],
12348    scoped_groups: &[ScopedGroup],
12349    config: &AutumnConfig,
12350) {
12351    tracing::info!(
12352        "Registered routes:{}",
12353        format_route_lines(routes, scoped_groups, config)
12354    );
12355
12356    if let Some(task_lines) = format_task_lines(tasks) {
12357        tracing::info!("Scheduled tasks:{task_lines}");
12358    }
12359
12360    tracing::info!("Active middleware: {}", format_middleware_list(config));
12361
12362    tracing::info!("Configuration:{}", format_config_summary(config));
12363}
12364
12365/// Fail the boot fast (before any DB side effects) when the default
12366/// session backend is misconfigured.
12367///
12368/// `AutumnConfig::validate()` is intentionally session-agnostic so that a
12369/// custom [`SessionStore`](crate::session::SessionStore) installed via
12370/// [`AppBuilder::with_session_store`] can override an otherwise-invalid
12371/// `session.backend = "redis"`-without-`redis.url` config. But when no
12372/// custom store is installed, the config-driven path will fail later in
12373/// `build_session_layer` — and by then, `setup_database` has already run
12374/// migrations, leaving DB side effects from a doomed boot. This helper
12375/// runs the same `backend_plan` check `build_session_layer` does, but
12376/// before any side effects, and only when the override path is inactive.
12377fn fail_fast_on_invalid_session_config(config: &AutumnConfig, has_custom_session_store: bool) {
12378    if has_custom_session_store {
12379        return;
12380    }
12381    if let Err(error) = config.session.backend_plan(config.profile.as_deref()) {
12382        eprintln!("Invalid session backend config: {error}");
12383        std::process::exit(1);
12384    }
12385}
12386
12387/// Resolve at-rest column-encryption keys at boot (#805).
12388///
12389/// On success this installs the process-global key ring. When encrypted columns
12390/// are registered but the key material under `active_record_encryption` is
12391/// missing or malformed, the behaviour mirrors the signing-secret check (#597):
12392/// a **hard failure in production** (the server must not bind with unusable
12393/// encryption), but only a **warning in dev/test** so zero-config local
12394/// development and the example apps continue to run. Apps that do not opt into
12395/// encrypted columns are unaffected (no registered columns -> no-op).
12396fn fail_fast_on_missing_encryption_keys(config: &AutumnConfig) {
12397    if let Err(diagnostic) = crate::encryption::init_attribute_encryption(config.credentials()) {
12398        let is_production = matches!(config.profile.as_deref(), Some("prod" | "production"));
12399        if is_production {
12400            eprintln!("Attribute encryption misconfiguration: {diagnostic}");
12401            std::process::exit(1);
12402        }
12403        eprintln!(
12404            "warning: attribute encryption is not fully configured (dev): {diagnostic}\n  \
12405             note: encrypted-column reads/writes will fail until keys are set; \
12406             this is a hard error in production."
12407        );
12408    }
12409}
12410
12411/// Fail immediately if the signing secret is misconfigured for the active profile.
12412///
12413/// In production, a missing, too-short, or demo-valued signing secret is a
12414/// hard failure — the server must not bind. In dev/test the check is skipped
12415/// so zero-config local development continues to work.
12416fn fail_fast_on_invalid_signing_secret(config: &AutumnConfig) {
12417    use crate::security::config::validate_signing_secret;
12418
12419    let is_production = matches!(config.profile.as_deref(), Some("prod" | "production"));
12420    let secret = config.security.signing_secret.secret.as_deref();
12421
12422    if let Err(error) = validate_signing_secret(secret, is_production) {
12423        eprintln!("Invalid signing secret configuration: {error}");
12424        eprintln!(
12425            "  hint: generate a secret with `openssl rand -hex 32` and set \
12426             AUTUMN_SECURITY__SIGNING_SECRET"
12427        );
12428        std::process::exit(1);
12429    }
12430
12431    // Previous secrets accepted during rotation must meet the same bar as the
12432    // current secret — a weak previous key can still be used to forge tokens.
12433    if is_production {
12434        for (i, prev) in config
12435            .security
12436            .signing_secret
12437            .previous_secrets
12438            .iter()
12439            .enumerate()
12440        {
12441            if let Err(error) = validate_signing_secret(Some(prev.as_str()), true) {
12442                eprintln!("Invalid signing secret configuration: previous_secrets[{i}]: {error}");
12443                eprintln!(
12444                    "  hint: every previous secret must meet the same entropy requirement \
12445                     as the current secret"
12446                );
12447                std::process::exit(1);
12448            }
12449        }
12450    }
12451}
12452
12453fn fail_fast_on_invalid_webhook_config(config: &AutumnConfig) {
12454    let is_production = matches!(config.profile.as_deref(), Some("prod" | "production"));
12455    if let Err(error) = config.security.webhooks.validate(is_production) {
12456        eprintln!("Invalid signed webhook configuration: {error}");
12457        std::process::exit(1);
12458    }
12459}
12460
12461fn fail_fast_on_invalid_trusted_hosts(config: &AutumnConfig) {
12462    let is_production = matches!(config.profile.as_deref(), Some("prod" | "production"));
12463    if !is_production {
12464        return;
12465    }
12466    let hosts: Vec<String> = config
12467        .security
12468        .trusted_hosts
12469        .hosts
12470        .iter()
12471        .map(|h| h.trim().to_owned())
12472        .filter(|h| !h.is_empty())
12473        .collect();
12474    if hosts.is_empty() {
12475        eprintln!(
12476            "[security.trusted_hosts] is required in production; set hosts = [\"example.com\"] or explicit entries"
12477        );
12478        std::process::exit(1);
12479    }
12480    if hosts.iter().any(|h| h == "*") {
12481        tracing::warn!("trusted host validation disabled via wildcard '*' in production");
12482    }
12483}
12484
12485fn fail_fast_on_invalid_idempotency_config(config: &AutumnConfig) {
12486    if !config.idempotency.enabled.unwrap_or(false) {
12487        return;
12488    }
12489    let is_production = matches!(config.profile.as_deref(), Some("prod" | "production"));
12490    if is_production
12491        && config.idempotency.backend == crate::config::IdempotencyBackend::Memory
12492        && !config.idempotency.allow_memory_in_production
12493    {
12494        eprintln!(
12495            "The in-memory idempotency backend is not safe for multi-replica production use.\n\
12496             Set `[idempotency] backend = \"redis\"` in autumn.toml, or set \
12497             `allow_memory_in_production = true` to suppress this check."
12498        );
12499        std::process::exit(1);
12500    }
12501    #[cfg(feature = "redis")]
12502    if config.idempotency.backend == crate::config::IdempotencyBackend::Redis {
12503        let url_missing = config
12504            .idempotency
12505            .redis
12506            .url
12507            .as_deref()
12508            .is_none_or(|u| u.trim().is_empty());
12509        if url_missing {
12510            eprintln!(
12511                "Redis idempotency backend requires a connection URL.\n\
12512                 Set AUTUMN_IDEMPOTENCY__REDIS__URL or `[idempotency.redis] url` in autumn.toml."
12513            );
12514            std::process::exit(1);
12515        }
12516    }
12517}
12518
12519pub(crate) fn install_webhook_registry(state: &AppState, config: &AutumnConfig) {
12520    if let Err(error) =
12521        crate::webhook::install_registry_from_config(state, &config.security.webhooks)
12522    {
12523        eprintln!("Invalid signed webhook configuration: {error}");
12524        std::process::exit(1);
12525    }
12526}
12527
12528/// Constructed [`BlobStore`](crate::storage::BlobStore) plus the
12529/// optional axum router that serves signed URLs for the Local backend.
12530/// Returned by [`preflight_storage`] before any DB side effects so a
12531/// doomed boot can't run migrations first; installed onto
12532/// [`AppState`] later via [`StorageBootstrap::install`].
12533#[cfg(feature = "storage")]
12534struct StorageBootstrap {
12535    store: crate::storage::SharedBlobStore,
12536    serving: Option<axum::Router<AppState>>,
12537}
12538
12539#[cfg(feature = "storage")]
12540impl StorageBootstrap {
12541    /// Install the preflighted store on `AppState` and return the
12542    /// optional serving router so the caller can merge it into the
12543    /// app router.
12544    fn install(self, state: &AppState) -> Option<axum::Router<AppState>> {
12545        state.insert_extension::<crate::storage::BlobStoreState>(
12546            crate::storage::BlobStoreState::new(self.store),
12547        );
12548        self.serving
12549    }
12550}
12551
12552/// Provision the configured [`BlobStore`](crate::storage::BlobStore)
12553/// before any database side effects. Construction is the side-effecting
12554/// step (creates + canonicalizes the storage root, may
12555/// `process::exit(1)` on a misconfiguration); we deliberately run it
12556/// before `setup_database` so a doomed boot doesn't apply migrations
12557/// first. Installation onto `AppState` happens later via
12558/// [`StorageBootstrap::install`].
12559#[cfg(feature = "storage")]
12560#[allow(clippy::too_many_lines)] // Single switch over backend variants reads as one unit.
12561fn preflight_storage(config: &AutumnConfig) -> Option<StorageBootstrap> {
12562    use crate::storage::StorageBackendPlan;
12563
12564    let plan = config
12565        .storage
12566        .backend_plan(config.profile.as_deref())
12567        .unwrap_or_else(|error| {
12568            // Cover the cases `backend_plan` rejects up front:
12569            // `LocalInProduction` (prod + local without ack),
12570            // `MissingS3Bucket`/`MissingS3Region`/`S3FeatureDisabled`.
12571            // Each is a configuration mistake — fail the boot loudly
12572            // rather than running migrations and then dying.
12573            tracing::error!(%error, "invalid storage backend config; aborting startup");
12574            std::process::exit(1);
12575        });
12576
12577    match plan {
12578        StorageBackendPlan::Disabled => None,
12579        StorageBackendPlan::Local {
12580            provider_id,
12581            root,
12582            mount_path,
12583            default_url_expiry_secs,
12584            warn_in_production,
12585        } => Some(bootstrap_local_storage(
12586            config,
12587            &provider_id,
12588            &root,
12589            &mount_path,
12590            default_url_expiry_secs,
12591            warn_in_production,
12592        )),
12593        StorageBackendPlan::S3 { .. } => {
12594            // `storage.backend = "s3"` requires the `autumn-storage-s3` plugin.
12595            // Construct an `S3BlobStore` and register it with `.with_blob_store()`
12596            // before calling `.run()` — when you do, the custom store bypasses
12597            // this path entirely and `preflight_storage` is never called.
12598            tracing::error!(
12599                "storage.backend=s3 requires the `autumn-storage-s3` plugin. \
12600                 Add it to your Cargo.toml, build an S3BlobStore from your config, \
12601                 and call `.with_blob_store(store)` on your AppBuilder. \
12602                 Aborting startup."
12603            );
12604            std::process::exit(1);
12605        }
12606    }
12607}
12608
12609#[cfg(feature = "storage")]
12610fn bootstrap_local_storage(
12611    config: &AutumnConfig,
12612    provider_id: &str,
12613    root: &std::path::Path,
12614    mount_path: &str,
12615    default_url_expiry_secs: u64,
12616    warn_in_production: bool,
12617) -> StorageBootstrap {
12618    use crate::storage::{LocalBlobStore, SharedBlobStore, local::SigningKey};
12619
12620    if warn_in_production {
12621        tracing::warn!(
12622            "prod profile is using the local-disk blob store; \
12623             bytes won't survive replica turnover. Set \
12624             storage.backend=s3 or storage.allow_local_in_production=true \
12625             to acknowledge"
12626        );
12627    }
12628
12629    // Signing key precedence:
12630    // 1. security.signing_secret (canonical, shared with session/CSRF)
12631    // 2. storage.local.signing_key (legacy override — still respected)
12632    // 3. Random ephemeral key (dev only — warns in prod)
12633    let (signing_key, previous_signing_keys) = config
12634        .security
12635        .signing_secret
12636        .secret
12637        .as_deref()
12638        .filter(|s| !s.is_empty())
12639        .map_or_else(
12640            || {
12641                config
12642                    .storage
12643                    .local
12644                    .signing_key
12645                    .as_deref()
12646                    .filter(|s| !s.is_empty())
12647                    .map_or_else(
12648                        || {
12649                            if matches!(config.profile.as_deref(), Some("prod" | "production")) {
12650                                tracing::warn!(
12651                                    "no signing secret configured in prod; blob URL signatures \
12652                                     won't survive a process restart. Set \
12653                                     AUTUMN_SECURITY__SIGNING_SECRET."
12654                                );
12655                            }
12656                            (SigningKey::random(), vec![])
12657                        },
12658                        |legacy| (SigningKey::new(legacy.as_bytes().to_vec()), vec![]),
12659                    )
12660            },
12661            |secret| {
12662                let current = SigningKey::new(secret.as_bytes().to_vec());
12663                let previous = config
12664                    .security
12665                    .signing_secret
12666                    .previous_secrets
12667                    .iter()
12668                    .map(|s| SigningKey::new(s.as_bytes().to_vec()))
12669                    .collect::<Vec<_>>();
12670                (current, previous)
12671            },
12672        );
12673
12674    let store = match LocalBlobStore::new(
12675        provider_id.to_string(),
12676        root.to_path_buf(),
12677        mount_path.to_string(),
12678        std::time::Duration::from_secs(default_url_expiry_secs),
12679        signing_key,
12680        previous_signing_keys,
12681    ) {
12682        Ok(store) => store,
12683        Err(err) => {
12684            // The operator explicitly chose `storage.backend = "local"`
12685            // — a non-writable root means uploads can't possibly
12686            // work, so abort the boot rather than letting upload
12687            // handlers serve 500s after deploy.
12688            tracing::error!(
12689                error = %err,
12690                root = %root.display(),
12691                "failed to initialize local blob store; aborting startup"
12692            );
12693            std::process::exit(1);
12694        }
12695    };
12696
12697    let serving = crate::storage::local::serve_router(&store);
12698    let arc: SharedBlobStore = std::sync::Arc::new(store);
12699
12700    tracing::info!(
12701        provider = %provider_id,
12702        root = %root.display(),
12703        mount = %mount_path,
12704        "Local blob store mounted"
12705    );
12706
12707    StorageBootstrap {
12708        store: arc,
12709        serving: Some(serving),
12710    }
12711}
12712async fn load_config_and_telemetry(
12713    config_loader: Option<ConfigLoaderFactory>,
12714    telemetry_provider: Option<Box<dyn crate::telemetry::TelemetryProvider>>,
12715    plugin_config_roots: BTreeSet<String>,
12716) -> (AutumnConfig, crate::telemetry::TelemetryGuard) {
12717    let config = load_config_only(config_loader, plugin_config_roots).await;
12718
12719    // 2. Initialize logging/telemetry via the installed provider, falling
12720    //    back to the default `tracing-subscriber + OTLP` initializer.
12721    let provider: Box<dyn crate::telemetry::TelemetryProvider> = telemetry_provider
12722        .unwrap_or_else(|| Box::new(crate::telemetry::TracingOtlpTelemetryProvider::new()));
12723    let telemetry_guard = provider
12724        .init(&config.log, &config.telemetry, config.profile.as_deref())
12725        .unwrap_or_else(|error| {
12726            eprintln!("Failed to initialize telemetry: {error}");
12727            std::process::exit(1);
12728        });
12729
12730    (config, telemetry_guard)
12731}
12732
12733/// Resolve the effective configuration WITHOUT initializing telemetry.
12734///
12735/// Split out of [`load_config_and_telemetry`] for the one-shot dump modes that
12736/// need config but must not touch the outside world. A custom
12737/// `TelemetryProvider::init` may open a collector connection, read credentials
12738/// or otherwise reach production resources, and telemetry cannot influence what
12739/// those modes emit — so `autumn openapi export`, advertised as binding no port
12740/// and opening no database, must not trigger it either (issue #802).
12741///
12742/// Everything up to and including [`AutumnConfig::apply_retention_caps`] is
12743/// shared with the telemetry-initializing path, so the config the two resolve is
12744/// identical.
12745async fn load_config_only(
12746    config_loader: Option<ConfigLoaderFactory>,
12747    plugin_config_roots: BTreeSet<String>,
12748) -> AutumnConfig {
12749    try_load_config(config_loader, plugin_config_roots)
12750        .await
12751        .unwrap_or_else(|e| {
12752            eprintln!("Failed to load configuration: {e}");
12753            std::process::exit(1);
12754        })
12755}
12756
12757/// [`load_config_only`] that returns a load error instead of exiting.
12758async fn try_load_config(
12759    config_loader: Option<ConfigLoaderFactory>,
12760    plugin_config_roots: BTreeSet<String>,
12761) -> Result<AutumnConfig, crate::config::ConfigError> {
12762    // 1. Load configuration via the installed loader, falling back to the
12763    //    five-layer TOML + env default.
12764    //
12765    // A custom `config_loader` factory owns its whole load and strict-config
12766    // handling, bypassing the default TOML path, so it does not receive the
12767    // declared plugin config roots; it accepts its own plugin-owned sections. The
12768    // default `TomlEnvConfigLoader` does get the roots, so `server.strict_config`
12769    // treats a plugin-declared `[root]` such as `[media]` as known-and-opaque
12770    // rather than an unknown-key hard error.
12771    let mut config = match config_loader {
12772        Some(factory) => factory().await,
12773        None => {
12774            crate::config::TomlEnvConfigLoader::new()
12775                .with_plugin_config_roots(plugin_config_roots)
12776                .load()
12777                .await
12778        }
12779    }?;
12780
12781    // `autumn serve --daemon` binds the app on a private Unix socket and then
12782    // discovers/health-probes it by path. A custom `with_config_loader` can
12783    // construct its `ServerConfig` from scratch and silently drop the
12784    // `AUTUMN_SERVER__UNIX_SOCKET` env override, leaving the daemon on TCP where
12785    // the supervisor can't reach it. The CLI therefore also passes the socket
12786    // out-of-band via `AUTUMN_SERVE_FORCE_UNIX_SOCKET`, applied here *after* the
12787    // loader runs so no loader can drop it.
12788    if let Ok(forced) = std::env::var("AUTUMN_SERVE_FORCE_UNIX_SOCKET")
12789        && !forced.is_empty()
12790    {
12791        config.server.unix_socket = Some(forced);
12792    }
12793
12794    // #1605: tighten every TTL-native subsystem knob to its `[retention]`
12795    // window before anything is built from the config, so the cap flows into
12796    // the idempotency layer's TTL, the session cookie's Max-Age and Redis
12797    // TTL, and `autumn_job_tracking.expires_at` without each of those sites
12798    // having to know the policy exists. A pure `min`, so a config with no
12799    // `[retention]` section is untouched.
12800    config.apply_retention_caps();
12801
12802    // Install the `[metrics]` cardinality caps before anything can record
12803    // through the call-site facade. The registry is process-global and its
12804    // caps are read at each decision rather than baked in at registration, so
12805    // this must land before the first `metrics::counter(...)` call — otherwise
12806    // an early instrument would be admitted (or refused) under the defaults
12807    // and, for `max_labels_per_series`, would canonicalize its label set to a
12808    // different series key than every later sample.
12809    //
12810    // It lives in `load_config_only`, the prefix EVERY mode shares — the
12811    // telemetry-initializing path calls it too — so trunk's invariant ("no
12812    // path boots with the defaults silently in force") is unchanged, and the
12813    // one-shot dump modes gain it as well. Setting caps touches no collector
12814    // and opens nothing, so it does not violate what those modes promise.
12815    crate::metrics::set_limits(config.metrics.limits());
12816
12817    Ok(config)
12818}
12819
12820/// Register the embedded `static/` tree (if any) as the process-wide asset
12821/// source. Called by each `run` path before the router is built so `/static/*`
12822/// serves from the binary and `asset_url()` resolves against the embedded
12823/// manifest.
12824#[cfg(feature = "embed-assets")]
12825fn register_embedded_static_dir(embedded_static: Option<crate::assets::EmbeddedStaticDir>) {
12826    if let Some(dir) = embedded_static {
12827        crate::assets::register_embedded_static(dir);
12828    }
12829}
12830
12831/// Prefer an embedded locale bundle over disk auto-loading when no explicit
12832/// bundle was provided. Returns `explicit` unchanged when it is `Some` or when
12833/// no embedded locales were registered.
12834#[cfg(all(feature = "embed-assets", feature = "i18n"))]
12835fn embedded_i18n_bundle(
12836    explicit: Option<Arc<crate::i18n::Bundle>>,
12837    embedded_locales: Option<&'static include_dir::Dir<'static>>,
12838    config: &AutumnConfig,
12839) -> Option<Arc<crate::i18n::Bundle>> {
12840    explicit.or_else(|| {
12841        embedded_locales.map(|dir| {
12842            Arc::new(
12843                crate::i18n::Bundle::load_from_embedded(dir, &config.i18n)
12844                    .unwrap_or_else(|e| panic!("embedded_locales: {e}")),
12845            )
12846        })
12847    })
12848}
12849
12850/// Excludes every `#[static_get]` route from locale-prefix routing (issue
12851/// #1251).
12852///
12853/// `#[static_get]` pre-rendering (`autumn build`, and ISR re-renders) is not
12854/// locale-aware: it requests each `StaticRouteMeta::path` once and writes the
12855/// single response it gets back to disk. Locale-prefixing that same path
12856/// would replace its content with a bare-path 308 redirect (the locale
12857/// segment is only known once nested under `/{locale}`), and
12858/// `render_static_routes` treats any non-2xx response as a build failure —
12859/// so without this exclusion, enabling `locale_prefix_enabled` breaks
12860/// `autumn build` for every app with a static route.
12861///
12862/// Static routes therefore keep serving at their single, unprefixed path
12863/// (matching pre-#1251 behavior) even when locale-prefix routing is on for
12864/// the rest of the app. Full per-locale static generation is a natural
12865/// follow-up, not attempted here.
12866///
12867/// Populates [`I18nConfig::locale_prefix_exclude_exact`](crate::i18n::I18nConfig::locale_prefix_exclude_exact),
12868/// not [`I18nConfig::locale_prefix_exclude`](crate::i18n::I18nConfig::locale_prefix_exclude):
12869/// a static route is a single literal path, not a namespace, so it must be
12870/// excluded by exact match — otherwise a static `/posts` would, as a
12871/// *prefix*, also swallow an unrelated dynamic sibling like `/posts/{slug}`
12872/// (Codex review).
12873#[cfg(feature = "i18n")]
12874fn exclude_static_routes_from_locale_prefix(
12875    config: &mut AutumnConfig,
12876    static_metas: &[crate::static_gen::StaticRouteMeta],
12877) {
12878    if config.i18n.locale_prefix_enabled {
12879        config
12880            .i18n
12881            .locale_prefix_exclude_exact
12882            .extend(static_metas.iter().map(|meta| meta.path.to_owned()));
12883    }
12884}
12885
12886#[cfg(not(feature = "i18n"))]
12887const fn exclude_static_routes_from_locale_prefix(
12888    _config: &mut AutumnConfig,
12889    _static_metas: &[crate::static_gen::StaticRouteMeta],
12890) {
12891}
12892
12893/// Derives the sitemap's locale-prefix config (issue #1251) from
12894/// `[i18n] locale_prefix_enabled`. `None` when the feature is off, disabled,
12895/// or the `i18n` cargo feature isn't compiled in — the sitemap then lists a
12896/// single unprefixed URL per static path, exactly as before this feature.
12897#[cfg(feature = "i18n")]
12898fn sitemap_locale_config(config: &AutumnConfig) -> Option<crate::seo::SitemapLocaleConfig<'_>> {
12899    config
12900        .i18n
12901        .locale_prefix_enabled
12902        .then_some(crate::seo::SitemapLocaleConfig {
12903            supported_locales: &config.i18n.supported_locales,
12904            exclude_prefixes: &config.i18n.locale_prefix_exclude,
12905            exclude_exact: &config.i18n.locale_prefix_exclude_exact,
12906        })
12907}
12908
12909#[cfg(not(feature = "i18n"))]
12910const fn sitemap_locale_config(
12911    _config: &AutumnConfig,
12912) -> Option<crate::seo::SitemapLocaleConfig<'_>> {
12913    None
12914}
12915
12916#[cfg(feature = "i18n")]
12917fn resolve_i18n_bundle(
12918    explicit_bundle: Option<Arc<crate::i18n::Bundle>>,
12919    auto_load: bool,
12920    config: &AutumnConfig,
12921    env: &dyn crate::config::Env,
12922) -> Option<Arc<crate::i18n::Bundle>> {
12923    if explicit_bundle.is_some() {
12924        return explicit_bundle;
12925    }
12926    if !auto_load {
12927        return None;
12928    }
12929
12930    let dir = project_dir(&config.i18n.dir, env);
12931    Some(Arc::new(
12932        crate::i18n::Bundle::load_from_dir(&dir, &config.i18n)
12933            .unwrap_or_else(|e| panic!("i18n_auto: {e}")),
12934    ))
12935}
12936
12937#[cfg(feature = "i18n")]
12938fn install_i18n_bundle_layer(
12939    mut custom_layers: Vec<CustomLayerRegistration>,
12940    state: &AppState,
12941    bundle: Option<Arc<crate::i18n::Bundle>>,
12942    i18n: &crate::i18n::I18nConfig,
12943) -> Vec<CustomLayerRegistration> {
12944    // #1384: install the resolution defaults from config first, before the
12945    // no-bundle early return. `locale_prefix_enabled` works without
12946    // `.i18n()`/`.i18n_auto()` — the router builds its nests straight from
12947    // `I18nConfig` — and in that shape no `Bundle` exists, but column decoding
12948    // still needs the app's default locale. Without this, a `default_locale = "fr"`
12949    // app attributed every legacy plain-text value to the last-resort "en", so a
12950    // `/fr/...` request rendered upgraded content as empty and a later write could
12951    // persist it under the wrong locale. A bundle, when present, re-installs the
12952    // same values below: it derives them from this same `I18nConfig`.
12953    crate::i18n::install_locale_defaults(&i18n.default_locale, i18n.resolved_fallback_chain());
12954
12955    let Some(bundle) = bundle else {
12956        return custom_layers;
12957    };
12958
12959    tracing::info!(
12960        locales = ?bundle.locales(),
12961        default = bundle.default_locale(),
12962        "i18n bundle loaded"
12963    );
12964    // #1384: content resolution outside a request (a job worker, a scheduled
12965    // task, a CLI command) has no locale scope to read a chain from. Publish
12966    // the bundle's resolved chain — the same one UI strings walk — as the
12967    // process default so `Translated::resolve` behaves identically there.
12968    crate::i18n::install_locale_defaults(bundle.default_locale(), bundle.fallback_chain().to_vec());
12969    state.insert_extension::<Arc<crate::i18n::Bundle>>(bundle.clone());
12970    // Use the existing IntoAppLayer plumbing so the Extension is visible to
12971    // every request. axum::Extension<T> is itself a tower::Layer when T:
12972    // Clone + Send + Sync + 'static.
12973    let ambient_layer = crate::i18n::AmbientLocaleLayer::new(&bundle);
12974    let ext_layer = axum::Extension(bundle);
12975    custom_layers.push(CustomLayerRegistration {
12976        type_id: TypeId::of::<axum::Extension<Arc<crate::i18n::Bundle>>>(),
12977        type_name: std::any::type_name::<axum::Extension<Arc<crate::i18n::Bundle>>>(),
12978        layer: tower::util::BoxCloneSyncServiceLayer::new(ext_layer),
12979    });
12980    // #1384: publish the negotiated locale as the ambient one for the whole
12981    // handler, so a `#[translatable]` field resolves itself with no locale
12982    // argument in the signature. Registered AFTER the bundle Extension —
12983    // registration order is outermost-first, so this layer runs INSIDE it and
12984    // its `Locale` extraction can see the bundle it needs to negotiate against.
12985    custom_layers.push(CustomLayerRegistration {
12986        type_id: TypeId::of::<crate::i18n::AmbientLocaleLayer>(),
12987        type_name: std::any::type_name::<crate::i18n::AmbientLocaleLayer>(),
12988        layer: tower::util::BoxCloneSyncServiceLayer::new(ambient_layer),
12989    });
12990    custom_layers
12991}
12992
12993#[cfg(feature = "db")]
12994struct DatabaseBootstrap {
12995    topology: Option<crate::db::DatabaseTopology>,
12996    shards: Option<crate::sharding::ShardSet>,
12997    replica_readiness: Option<crate::migrate::ReplicaMigrationReadiness>,
12998    replica_migration_check: Option<(String, String)>,
12999}
13000
13001/// Build the `ShardSet` for a sharded app (or `None` when no `[[database.shards]]`
13002/// are configured). Resolves the shard router first: an explicit
13003/// `with_shard_router` wins; otherwise `directory_routing_enabled` opts into the
13004/// control-DB directory router (bound to the just-built control primary pool);
13005/// otherwise the hash router. The directory flag is documented as having no
13006/// effect without shards, so a shardless profile that leaves it enabled must not
13007/// fail startup — hence the early `None` return.
13008///
13009/// `spawn_directory_listener` gates the directory-router cache-invalidation
13010/// listener: it opens control-DB connections, so it is spawned only at real
13011/// runtime, never during a static build (`autumn build`) which must not touch
13012/// the database.
13013#[cfg(feature = "db")]
13014async fn resolve_shard_set(
13015    config: &AutumnConfig,
13016    shard_router: Option<Arc<dyn crate::sharding::ShardRouter>>,
13017    shard_provider: Option<ShardProviderFactory>,
13018    directory_routing_enabled: bool,
13019    spawn_directory_listener: bool,
13020    topology: Option<&crate::db::DatabaseTopology>,
13021) -> Result<Option<crate::sharding::ShardSet>, String> {
13022    if !config.database.has_shards() {
13023        return Ok(None);
13024    }
13025    let router: Arc<dyn crate::sharding::ShardRouter> = match shard_router {
13026        Some(explicit) => explicit,
13027        None if directory_routing_enabled => {
13028            let control_primary = topology
13029                .map(crate::db::DatabaseTopology::primary)
13030                .ok_or_else(|| {
13031                    "directory_shard_router is enabled but no control database is configured. \
13032                     The directory router needs a control `database.primary_url`/`url` to read \
13033                     the tenant→shard directory. Set one, or disable directory routing to use \
13034                     the hash router."
13035                        .to_owned()
13036                })?;
13037            // Directory routing resolves the tenant→shard key by checking out a
13038            // *second* control connection during extraction. A handler that
13039            // already holds `Db` (or another control checkout) before extracting
13040            // `ShardedDb` / a sharded repository would then deadlock on a control
13041            // pool sized to 1 — the first checkout cannot be released until the
13042            // handler runs. Require at least 2 control connections so these
13043            // mixed control+tenant handlers always make progress.
13044            let control_max = control_primary.status().max_size;
13045            if control_max < 2 {
13046                return Err(format!(
13047                    "directory_shard_router requires a control database pool of at least 2 \
13048                     connections, but the configured maximum is {control_max}. Directory \
13049                     routing checks out a second control connection during extraction to \
13050                     resolve the tenant→shard key, which deadlocks a pool sized to 1 when a \
13051                     handler already holds a control connection (e.g. `Db` + `ShardedDb`). \
13052                     Increase the control pool size (database.pool.max_size), or disable \
13053                     directory routing to use the hash router."
13054                ));
13055            }
13056            // Bound directory lookups with the configured database statement
13057            // timeout (capped to Postgres' i32 millisecond range).
13058            let timeout_ms = config.database.statement_timeout.map_or(0, |d| {
13059                u64::try_from(d.as_millis())
13060                    .unwrap_or(i32::MAX as u64)
13061                    .min(i32::MAX as u64)
13062            });
13063            let dir_router = Arc::new(
13064                crate::sharding::DirectoryShardRouter::new(control_primary.clone())
13065                    .with_statement_timeout_ms(timeout_ms),
13066            );
13067            // Spawn the cache-invalidation listener on the control DB so a re-pin
13068            // (e.g. during a slot move) evicts cached tenant→shard mappings fleet-
13069            // wide the moment it commits (LISTEN/NOTIFY) rather than waiting out
13070            // the TTL. Skipped during a static build (no DB access); needs the
13071            // control URL, without one we silently fall back to TTL-only refresh.
13072            if spawn_directory_listener {
13073                // Prefer the provider-resolved control URL carried on the
13074                // topology (managed Postgres has no `database.primary_url` in
13075                // config); fall back to the configured URL. Without this a
13076                // managed control DB would get no LISTEN/NOTIFY task (absent
13077                // URL) or listen on a stale pre-provider URL.
13078                if let Some(control_url) = topology
13079                    .and_then(crate::db::DatabaseTopology::migration_url)
13080                    .or_else(|| config.database.effective_primary_url())
13081                {
13082                    // Detach: the listener runs for the life of the process;
13083                    // dropping the JoinHandle leaves the task running rather than
13084                    // aborting it.
13085                    #[cfg(feature = "postgres")]
13086                    drop(
13087                        crate::sharding::DirectoryShardRouter::spawn_invalidation_listener(
13088                            Arc::clone(&dir_router),
13089                            control_url.to_owned(),
13090                            crate::sharding::DEFAULT_DIRECTORY_INVALIDATION_SWEEP_INTERVAL,
13091                        ),
13092                    );
13093                    // LISTEN needs `postgres`. Without it, no listener starts.
13094                    #[cfg(not(feature = "postgres"))]
13095                    let _ = control_url;
13096                } else {
13097                    // Directory routing is active but there is no control URL for
13098                    // a dedicated LISTEN connection — a custom
13099                    // `DatabasePoolProvider` supplied the control pool without
13100                    // `database.primary_url`/`url`. The router still serves lookups
13101                    // from that pool, but re-pins are not invalidated fleet-wide on
13102                    // commit; they take effect only after the cache TTL expires.
13103                    // Warn rather than fall back silently, so an operator relying
13104                    // on the directory for slot moves configures a control URL, or
13105                    // accepts TTL-only refresh, deliberately.
13106                    tracing::warn!(
13107                        "directory shard routing is enabled but no control database URL is \
13108                         configured (database.primary_url/url is unset, e.g. a custom \
13109                         DatabasePoolProvider supplied the control pool); the cache-\
13110                         invalidation LISTEN/NOTIFY task cannot be started, so directory \
13111                         re-pins will only take effect after the cache TTL expires rather \
13112                         than fleet-wide on commit"
13113                    );
13114                }
13115            }
13116            dir_router
13117        }
13118        None => Arc::new(crate::sharding::HashShardRouter),
13119    };
13120    let set = match shard_provider {
13121        Some(factory) => {
13122            let topologies = factory(config.database.clone())
13123                .await
13124                .map_err(|e| format!("Failed to create shard pools: {e}"))?;
13125            // A custom shard provider established shard pools without going
13126            // through the built-in `create_shard_topology` factory, which
13127            // validates `database.statement_timeout` internally, so enforce the
13128            // same fail-closed guard here — but only now that pools were really
13129            // established. `resolve_shard_set` already returned early for a
13130            // shardless profile, so this Some-gated check never rejects a
13131            // no-database path. A shard set establishing SQLite pools under a
13132            // nonzero timeout still fails closed.
13133            #[cfg(feature = "sqlite")]
13134            {
13135                // Scope the lock-wait wording to a `cache=shared` shard when
13136                // any shard is one (issue #2881). Each returned topology pairs
13137                // positionally with its configured shard; prefer the target
13138                // the provider actually resolved, as the control-pool guard
13139                // does.
13140                let target = topologies
13141                    .iter()
13142                    .zip(&config.database.shards)
13143                    .map(|(topology, shard)| {
13144                        topology
13145                            .migration_url()
13146                            .unwrap_or(shard.primary_url.as_str())
13147                    })
13148                    .find(|url| crate::db::sqlite_target_is_shared_cache(url))
13149                    .unwrap_or_default();
13150                crate::db::reject_sqlite_unsupported_timeouts(&config.database, target)
13151                    .map_err(|e| format!("Failed to create shard pools: {e}"))?;
13152            }
13153            crate::sharding::build_shard_set(&config.database, topologies, router)
13154        }
13155        None => crate::sharding::create_shard_set(&config.database, router)
13156            .map(|set| set.expect("has_shards() checked above")),
13157    }
13158    .map_err(|e| format!("Failed to configure shards: {e}"))?;
13159    Ok(Some(set))
13160}
13161
13162#[cfg(feature = "db")]
13163// The `sqlite` feature adds a small fail-fast startup-migration guard block
13164// (issue #1614) that pushes this orchestration fn just over the line limit.
13165#[allow(clippy::too_many_lines)]
13166async fn setup_database(
13167    config: &AutumnConfig,
13168    migrations: Vec<(&'static str, crate::migrate::EmbeddedMigrations)>,
13169    pool_provider: Option<PoolProviderFactory>,
13170    shard_provider: Option<ShardProviderFactory>,
13171    shard_router: Option<Arc<dyn crate::sharding::ShardRouter>>,
13172    directory_shard_router: bool,
13173    hook_queue_migration_mode: RepositoryCommitHookQueueMigrationMode,
13174) -> Result<DatabaseBootstrap, String> {
13175    // #1628: declare replication BEFORE any pool exists, so every pooled SQLite
13176    // connection is created with `wal_autocheckpoint = 0` and the replicator is
13177    // the only component that ever checkpoints. The flag latches on and is never
13178    // cleared — see `set_sqlite_replication_active`.
13179    if config.replication.as_ref().is_some_and(|r| r.enabled) {
13180        crate::db::set_sqlite_replication_active();
13181    }
13182    // Shared, not copied: `EmbeddedMigrations` cannot be cloned, and a SQLite
13183    // fleet (ADR 0019) migrates each tenant database from the same list long
13184    // after boot.
13185    let migrations = Arc::new(migrations_with_repository_framework_migrations(
13186        migrations,
13187        crate::repository_commit_hooks::has_repository_commit_hook_descriptors(),
13188        crate::version_history::has_versioned_repository_descriptors(),
13189        crate::derivation::has_derivation_descriptors(),
13190        hook_queue_migration_mode,
13191    ));
13192    // Directory routing is only actually active when the app did NOT supply an
13193    // explicit shard router: an explicit `with_shard_router(...)` takes
13194    // precedence over `directory_shard_router` in `resolve_shard_set`, so in
13195    // that case the `DirectoryShardRouter` is never constructed and the
13196    // directory table is never consulted. Gate the migration on the same
13197    // condition so an explicit-router app doesn't create `_autumn_shard_directory`
13198    // (or warn about a pending directory migration) for a table it won't use.
13199    let use_directory_router = shard_router.is_none()
13200        && (directory_shard_router || config.database.directory_shard_router);
13201    // A fleet routes by its mode and path template, never by a router.
13202    #[cfg(feature = "sqlite")]
13203    if config.database.fleet.is_some() && shard_router.is_some() {
13204        tracing::warn!(
13205            "with_shard_router is ignored: database.fleet routes each key to its own \
13206             database by fleet mode (ADR 0019)"
13207        );
13208    }
13209    // The tenant→shard directory table is a CONTROL-plane table: create it at
13210    // startup only when directory routing is active (and shards exist), and
13211    // only on the control target — not via the shared list above, which is also
13212    // applied to every shard. Like the other runtime framework migrations, it is
13213    // suppressed during a static build (`autumn build`, AUTUMN_BUILD_STATIC=1):
13214    // the build only renders assets and must not touch the database, so it must
13215    // not create `_autumn_shard_directory`.
13216    let directory_migration_required = directory_migration_is_required(
13217        use_directory_router,
13218        config.database.has_shards(),
13219        hook_queue_migration_mode,
13220    );
13221    let shard_map_migration_required =
13222        shard_map_migration_is_required(config.database.has_shards(), hook_queue_migration_mode);
13223    let check_replica_migrations = !migrations.is_empty();
13224    #[cfg(feature = "sqlite")]
13225    let custom_pool_provider = pool_provider.is_some();
13226    let topology = match pool_provider {
13227        Some(factory) => factory(config.database.clone()).await,
13228        None => crate::db::create_topology(&config.database),
13229    }
13230    .map_err(|e| format!("Failed to create database pool: {e}"))?;
13231    // Fail-closed statement-timeout guard, enforced only once a control pool has
13232    // actually been established — the provider, built-in or custom, returned
13233    // `Some(..)`. The built-in `create_topology`/`create_shard_topology` factories
13234    // validate `database.statement_timeout` internally, but a custom
13235    // `with_pool_provider` provider can build its own SQLite pool without them:
13236    // the default `DatabasePoolProvider::create_topology` only delegates to
13237    // `create_pool`, and both factories are overridable, so a custom provider
13238    // could otherwise discard the timeout and break the fail-closed guarantee.
13239    // Under the `sqlite` feature `RuntimeBackend` is always SQLite, so an
13240    // established pool plus a nonzero timeout is exactly the fail-closed
13241    // condition. A provider returning `Ok(None)` opts into the supported
13242    // no-database mode — no pool or statement needs a timeout — so it must still
13243    // boot, matching the built-in path. Gating on `topology.is_some()` preserves
13244    // that opt-out. The check is idempotent with the built-in factories' own, and
13245    // `resolve_shard_set` applies the same Some-gated guard for shards.
13246    #[cfg(feature = "sqlite")]
13247    if let Some(topology) = topology.as_ref() {
13248        // Prefer the provider-resolved target (a custom provider may have no
13249        // static URL, or a different one) so the message describes the pool
13250        // that was actually built, as the migration path below does.
13251        let target = topology
13252            .migration_url()
13253            .or_else(|| config.database.effective_primary_url())
13254            .unwrap_or_default();
13255        crate::db::reject_sqlite_unsupported_timeouts(&config.database, target)
13256            .map_err(|e| format!("Failed to create database pool: {e}"))?;
13257    }
13258
13259    // Spawn the directory invalidation listener only at real runtime — a static
13260    // build must not open control-DB connections.
13261    let runtime_boot = hook_queue_migration_mode == RepositoryCommitHookQueueMigrationMode::Runtime;
13262    let shards = match resolve_shard_set(
13263        config,
13264        shard_router,
13265        shard_provider,
13266        use_directory_router,
13267        runtime_boot,
13268        topology.as_ref(),
13269    )
13270    .await
13271    {
13272        Ok(shards) => shards,
13273        Err(e) => {
13274            // The (managed) control topology is already up at this point, so a
13275            // later setup failure — directory control-pool sizing, shard pool
13276            // construction — must stop the managed Postgres child before the
13277            // caller's `process::exit` (which skips `on_shutdown`/`Drop`).
13278            // No-op when no managed cluster was started.
13279            #[cfg(feature = "managed-pg")]
13280            crate::managed_pg::emergency_stop_async().await;
13281            return Err(e);
13282        }
13283    };
13284
13285    // Skip migrations when the provider opted out of a database by returning
13286    // `Ok(None)`, even with `database.url` configured. Custom providers signal
13287    // "this app runs without a DB" that way, and migrating against the URL anyway
13288    // would defeat the opt-out.
13289    //
13290    // A provider may also resolve its primary URL at runtime (managed Postgres)
13291    // and carry it on the topology. Prefer that URL, so migrations target the pool
13292    // actually built rather than a stale or absent configured one.
13293    let provider_migration_url = topology
13294        .as_ref()
13295        .and_then(|t| t.migration_url())
13296        .map(str::to_owned);
13297
13298    // SQLite sharding guard (#1614, PR3). The SQLite startup-migration path applies
13299    // registered migrations to a `sqlite://` control target — `run_startup_migrations`
13300    // routes them through `crate::migrate::auto_migrate_sqlite` — so registered
13301    // migrations no longer fail fast here. Sharding is what remains unsupported:
13302    // the directory/shard-map control migrations and per-shard fan-out are
13303    // Postgres-specific. So a sqlite control target with sharding enabled fails
13304    // fast, as does any `sqlite:` shard `primary_url`, because the shard loop
13305    // routes each shard through the Postgres-only harness. Empty when unsharded or
13306    // when the shard loop will not run. See `sqlite_sharding_unsupported_guard`.
13307    #[cfg(feature = "sqlite")]
13308    let sqlite_guard_shard_urls: Vec<&str> = if shards.is_some() {
13309        config
13310            .database
13311            .shards
13312            .iter()
13313            .map(|shard| shard.primary_url.as_str())
13314            .collect()
13315    } else {
13316        Vec::new()
13317    };
13318    #[cfg(feature = "sqlite")]
13319    #[allow(clippy::question_mark)] // managed-pg child must be stopped before returning
13320    if let Err(e) = sqlite_sharding_unsupported_guard(
13321        if topology.is_some() {
13322            provider_migration_url
13323                .as_deref()
13324                .or_else(|| config.database.effective_primary_url())
13325        } else {
13326            None
13327        },
13328        directory_migration_required
13329            || shard_map_migration_required
13330            || config.database.has_shards(),
13331        &sqlite_guard_shard_urls,
13332    ) {
13333        #[cfg(feature = "managed-pg")]
13334        crate::managed_pg::emergency_stop_async().await;
13335        return Err(e);
13336    }
13337
13338    run_startup_migrations(
13339        config,
13340        topology.is_some(),
13341        shards.is_some(),
13342        provider_migration_url,
13343        Arc::clone(&migrations),
13344        directory_migration_required,
13345        shard_map_migration_required,
13346    )
13347    .await;
13348
13349    // A SQLite fleet (ADR 0019) becomes the shard set: tenant data routes to
13350    // per-tenant / per-slot files, each migrated with the non-control sets when
13351    // it opens. Built at runtime only — a static build must not create the
13352    // fleet directory.
13353    #[cfg(feature = "sqlite")]
13354    let shards = match (runtime_boot, topology.as_ref()) {
13355        (true, Some(topology)) => {
13356            if config.database.fleet.is_some() {
13357                fleet_control_target_identified(custom_pool_provider, topology.migration_url())?;
13358            }
13359            crate::db::fleet::build_for_app(config, topology.migration_url(), &migrations, |set| {
13360                !migration_set_is_control_framework(set)
13361            })?
13362            .map_or(shards, |fleet| {
13363                tracing::info!(
13364                    mode = %fleet.mode(),
13365                    root = %fleet.root().display(),
13366                    max_open = fleet.max_open(),
13367                    "SQLite database fleet ready"
13368                );
13369                if config
13370                    .database
13371                    .fleet
13372                    .as_ref()
13373                    .is_some_and(|f| f.restore_missing)
13374                    && !config.replication.as_ref().is_some_and(|r| r.enabled)
13375                {
13376                    tracing::warn!(
13377                        "database.fleet.restore_missing is set but [replication] is off; there is \
13378                         no replica to restore a missing database from"
13379                    );
13380                }
13381                Some(crate::sharding::ShardSet::from_fleet(
13382                    fleet,
13383                    topology.primary().clone(),
13384                ))
13385            })
13386        }
13387        _ => shards,
13388    };
13389
13390    // Derivations (#1769): the state table exists by now, so reconcile each
13391    // declared `#[derivation]` against it and repair whatever changed. A
13392    // registry collision stops the boot; a database failure only logs, because
13393    // a derivation whose backfill has not run is stale rather than broken and
13394    // `/actuator/derivations` reports exactly that.
13395    // Needs an explicit `if let` rather than `?`, so the managed-pg child is
13396    // stopped before unwinding. `?` would skip the cfg-gated stop call.
13397    #[allow(clippy::question_mark)]
13398    if runtime_boot
13399        && crate::derivation::has_derivation_descriptors()
13400        && let Err(e) = start_derivation_backfill(
13401            topology.as_ref(),
13402            shards.as_ref(),
13403            crate::db::TxTimeouts::from_config(&config.database),
13404        )
13405        .await
13406    {
13407        #[cfg(feature = "managed-pg")]
13408        crate::managed_pg::emergency_stop_async().await;
13409        return Err(e);
13410    }
13411
13412    let (replica_readiness, replica_migration_check) = if topology
13413        .as_ref()
13414        .is_some_and(|topology| check_replica_migrations && topology.replica().is_some())
13415    {
13416        match (
13417            config.database.effective_primary_url(),
13418            config.database.replica_url.as_deref(),
13419        ) {
13420            (Some(primary_url), Some(replica_url)) => {
13421                let primary_url = primary_url.to_owned();
13422                let replica_url = replica_url.to_owned();
13423                let readiness = crate::migrate::check_replica_migration_readiness_blocking(
13424                    primary_url.clone(),
13425                    replica_url.clone(),
13426                )
13427                .await;
13428                (Some(readiness), Some((primary_url, replica_url)))
13429            }
13430            _ => (None, None),
13431        }
13432    } else {
13433        (None, None)
13434    };
13435
13436    if check_replica_migrations && let Some(set) = &shards {
13437        check_shard_replica_migration_parity(config, set).await;
13438    }
13439
13440    // Boot-time guard: compare the current auto-split slot map against the map
13441    // persisted on first boot. Refuses to start if they differ, preventing
13442    // silent data misrouting from topology changes. Inert during static builds,
13443    // when no control DB is configured, and in explicit-slot mode.
13444    #[allow(clippy::question_mark)]
13445    if let Err(e) = Box::pin(enforce_shard_map_guard(
13446        config,
13447        topology.as_ref(),
13448        runtime_boot,
13449    ))
13450    .await
13451    {
13452        // Needs explicit `if let` (not `?`) so the managed-pg child can be stopped
13453        // before unwinding — `?` would skip the cfg-gated emergency stop call.
13454        #[cfg(feature = "managed-pg")]
13455        crate::managed_pg::emergency_stop_async().await;
13456        return Err(e);
13457    }
13458
13459    Ok(DatabaseBootstrap {
13460        topology,
13461        shards,
13462        replica_readiness,
13463        replica_migration_check,
13464    })
13465}
13466
13467/// Batches one boot backfill round runs before it returns its connection.
13468///
13469/// The connection goes back to the pool between rounds. A `SQLite` pool is often
13470/// size 1, so a sweep that held its only connection would stall every request
13471/// for the length of the sweep.
13472#[cfg(feature = "db")]
13473const BOOT_BACKFILL_BATCHES: usize = 8;
13474
13475/// Reconcile the declared derivations on every primary, then repair them in the
13476/// background.
13477///
13478/// Reconciliation runs inline because it is two statements per derivation and
13479/// the answer decides what the backfill has to do. The backfill itself is
13480/// spawned: it sweeps whole parent tables, so blocking the boot on it would
13481/// delay serving traffic the maintained columns are already correct for.
13482///
13483/// A sharded app reconciles and repairs on **every shard primary** as well as on
13484/// the control primary. The state-table migration is applied to shards too, and
13485/// shards are where the tenant rows live, so a shard that never reconciled would
13486/// hold a stale derived column forever.
13487///
13488/// A registry collision is returned to the caller and stops the boot. Two
13489/// derivations on one parent column double count every mutation, which is data
13490/// corruption, so booting on it is worse than not booting.
13491///
13492/// A database failure is logged and skipped instead. The backfill is **not**
13493/// spawned for a target whose reconcile failed: the sweep reads the state the
13494/// reconcile writes, so sweeping after a failed reconcile would work from a
13495/// stale answer.
13496///
13497/// `timeouts` are the app's configured transaction timeouts. This runs at
13498/// boot, outside any request, so the reconcile and the backfill batches get
13499/// them from here (#3057).
13500#[cfg(feature = "db")]
13501async fn start_derivation_backfill(
13502    topology: Option<&crate::db::DatabaseTopology>,
13503    shards: Option<&crate::sharding::ShardSet>,
13504    timeouts: crate::db::TxTimeouts,
13505) -> Result<(), String> {
13506    // No connection needed, so a collision is caught before any data is touched.
13507    crate::derivation::check_registered_derivations()
13508        .map_err(|error| format!("Invalid `#[derivation]` registry: {error}"))?;
13509
13510    let mut targets: Vec<(String, crate::db::Pool<crate::db::RuntimeConnection>)> = Vec::new();
13511    if let Some(topology) = topology {
13512        targets.push(("control".to_owned(), topology.primary().clone()));
13513    }
13514    if let Some(shards) = shards {
13515        for shard in shards.iter() {
13516            targets.push((
13517                format!("shard {}", shard.name()),
13518                shard.primary_pool().clone(),
13519            ));
13520        }
13521    }
13522
13523    for (label, pool) in targets {
13524        let mut conn = match pool.get().await {
13525            Ok(conn) => conn,
13526            Err(error) => {
13527                tracing::warn!(
13528                    %error,
13529                    database = %label,
13530                    "no connection to reconcile derivation definitions"
13531                );
13532                continue;
13533            }
13534        };
13535        match timeouts
13536            .scope(crate::derivation::ensure_derivations(&mut conn))
13537            .await
13538        {
13539            Ok(enqueued) => {
13540                if !enqueued.is_empty() {
13541                    tracing::info!(
13542                        database = %label,
13543                        derivations = ?enqueued,
13544                        "derivation definitions changed; backfill enqueued"
13545                    );
13546                }
13547            }
13548            Err(error) => {
13549                tracing::warn!(
13550                    %error,
13551                    database = %label,
13552                    "could not reconcile derivation definitions; \
13553                     see /actuator/derivations"
13554                );
13555                continue;
13556            }
13557        }
13558        drop(conn);
13559        spawn_derivation_backfill(label, pool, timeouts);
13560    }
13561    Ok(())
13562}
13563
13564/// Sweep one target's enqueued derivations in the background, a few batches per
13565/// pooled connection.
13566///
13567/// The loop is what keeps the connection borrowed briefly. Each round checks out
13568/// a connection, runs [`BOOT_BACKFILL_BATCHES`] batches, returns the connection
13569/// and repeats while the report still lists work. Several replicas doing this
13570/// cooperate: each batch locks the derivation's state row, so they take turns on
13571/// one sweep instead of racing.
13572#[cfg(feature = "db")]
13573fn spawn_derivation_backfill(
13574    label: String,
13575    pool: crate::db::Pool<crate::db::RuntimeConnection>,
13576    timeouts: crate::db::TxTimeouts,
13577) {
13578    tokio::spawn(async move {
13579        let options = crate::derivation::BackfillOptions {
13580            max_batches: Some(BOOT_BACKFILL_BATCHES),
13581            ..crate::derivation::BackfillOptions::default()
13582        };
13583        let mut completed: Vec<String> = Vec::new();
13584        let mut rows_repaired = 0usize;
13585        let mut rounds = 0usize;
13586        loop {
13587            let mut conn = match pool.get().await {
13588                Ok(conn) => conn,
13589                Err(error) => {
13590                    tracing::warn!(
13591                        %error,
13592                        database = %label,
13593                        "no connection to backfill derivations"
13594                    );
13595                    return;
13596                }
13597            };
13598            let report = match timeouts
13599                .scope(crate::derivation::run_backfill(&mut conn, &options))
13600                .await
13601            {
13602                Ok(report) => report,
13603                Err(error) => {
13604                    tracing::warn!(%error, database = %label, "derivation backfill failed");
13605                    return;
13606                }
13607            };
13608            drop(conn);
13609            completed.extend(report.completed);
13610            rows_repaired += report.rows_repaired;
13611            if report.in_progress.is_empty() {
13612                break;
13613            }
13614            rounds += 1;
13615            // A round that left work behind but advanced no checkpoint is
13616            // stuck, not slow: nothing a further round would do differently.
13617            // A round that advanced one is progress, however many parents are
13618            // left (a self-referential derivation sweeps one per batch, so a
13619            // large table takes many rounds), and a checkpoint only moves
13620            // forward, so the sweep terminates on its own.
13621            if report.batches_run == 0 {
13622                tracing::warn!(
13623                    database = %label,
13624                    pending = ?report.in_progress,
13625                    rounds,
13626                    "derivation backfill made no progress; see /actuator/derivations"
13627                );
13628                break;
13629            }
13630        }
13631        if !completed.is_empty() || rows_repaired > 0 {
13632            tracing::info!(
13633                database = %label,
13634                completed = ?completed,
13635                rows_repaired,
13636                "derivation backfill finished"
13637            );
13638        }
13639    });
13640}
13641
13642/// Apply the embedded migration sets control-first, then to each shard in
13643/// declaration order, failing fast on the first apply error: a
13644/// half-migrated fleet that boots is worse than a crashed deploy, and
13645/// already-migrated targets are idempotently skipped on retry.
13646///
13647/// `run_pending_locked` polls with `std::thread::sleep` (up to 60 s under
13648/// contention), so the whole sequence runs off the Tokio worker threads in
13649/// one blocking task that owns the embedded migration sets.
13650/// Apply pending migrations for one target in the `AUTUMN_MIGRATE=1` one-shot,
13651/// returning the number applied — or exiting non-zero on failure.
13652///
13653/// Uses the same locked applier the startup path uses
13654/// ([`run_pending_locked`](crate::migrate::run_pending_locked)). Failure messages
13655/// are REDACTED to a value-free reason plus the target label (`control` /
13656/// `shard:<name>`): the underlying [`MigrationError`](crate::migrate::MigrationError)
13657/// can wrap a driver string that embeds the connection URL, and the deploy path
13658/// must never print a DB URL or secret.
13659#[cfg(feature = "postgres")]
13660fn apply_pending_or_exit(
13661    database_url: &str,
13662    migrations: impl diesel::migration::MigrationSource<diesel::pg::Pg> + Send,
13663    target: &str,
13664    lock_policy: crate::migrate::MigrationLockPolicy,
13665) -> usize {
13666    match crate::migrate::run_pending_locked_with_policy(
13667        database_url,
13668        migrations,
13669        None,
13670        lock_policy,
13671    ) {
13672        Ok(result) => result.applied.len(),
13673        Err(error) => {
13674            let reason = match error {
13675                crate::migrate::MigrationError::Connection(_) => {
13676                    "could not connect to the database"
13677                }
13678                crate::migrate::MigrationError::Migration(_) => "a migration failed to apply",
13679                crate::migrate::MigrationError::LockContention { .. } => {
13680                    "a migration timed out on a table lock on every attempt"
13681                }
13682                _ => "migration error",
13683            };
13684            eprintln!("autumn migrate: {reason} (target {target})");
13685            // `process::exit` skips `on_shutdown`/`Drop`; stop any managed
13686            // Postgres child first so a failure doesn't orphan the data dir/port.
13687            #[cfg(feature = "managed-pg")]
13688            crate::managed_pg::emergency_stop();
13689            std::process::exit(1);
13690        }
13691    }
13692}
13693
13694/// A build without `postgres` cannot migrate a Postgres target. It prints an
13695/// error and exits.
13696#[cfg(all(feature = "db", not(feature = "postgres")))]
13697fn apply_pending_or_exit(
13698    _database_url: &str,
13699    _migrations: impl diesel::migration::MigrationSource<crate::db::RuntimeBackend> + Send,
13700    target: &str,
13701    _lock_policy: crate::migrate::MigrationLockPolicy,
13702) -> usize {
13703    eprintln!(
13704        "autumn migrate: {} (target {target})",
13705        crate::migrate::NO_POSTGRES_MSG
13706    );
13707    std::process::exit(1);
13708}
13709
13710/// Apply pending migrations for one `SQLite` target in the `AUTUMN_MIGRATE=1`
13711/// one-shot, returning the number applied — or exiting non-zero on failure
13712/// (issue #1614, PR3).
13713///
13714/// The `SQLite` counterpart to [`apply_pending_or_exit`]: it uses the unlocked
13715/// [`run_pending_sqlite`](crate::migrate::run_pending_sqlite) harness (`SQLite`
13716/// is single-writer, so there is no advisory lock and no cross-process race to
13717/// serialize) and REDACTS failure messages to a value-free reason plus the
13718/// target label, exactly like the Postgres path, so a driver string embedding
13719/// the database path is never printed.
13720#[cfg(feature = "sqlite")]
13721fn apply_pending_sqlite_or_exit(
13722    database_url: &str,
13723    migrations: impl diesel::migration::MigrationSource<diesel::sqlite::Sqlite> + Send,
13724    target: &str,
13725) -> usize {
13726    // Reject ANY in-memory target (private OR shared-cache) with registered
13727    // migrations up front, BEFORE `run_pending_sqlite` (whose `Migration` error
13728    // would be redacted to a value-free reason below, hiding the guidance). The
13729    // migrated schema never survives to the runtime pool — a private `:memory:`
13730    // connection is its own empty database, and a shared in-memory database is
13731    // destroyed when its last connection closes (issue #1614 follow-up).
13732    if let Some(err) = crate::migrate::reject_in_memory_migrations(database_url, &migrations) {
13733        eprintln!("autumn migrate: {err} (target {target})");
13734        std::process::exit(1);
13735    }
13736    match crate::migrate::run_pending_sqlite(database_url, migrations) {
13737        Ok(result) => result.applied.len(),
13738        Err(error) => {
13739            let reason = match error {
13740                crate::migrate::MigrationError::Connection(_) => {
13741                    "could not connect to the database"
13742                }
13743                crate::migrate::MigrationError::Migration(_) => "a migration failed to apply",
13744                _ => "migration error",
13745            };
13746            eprintln!("autumn migrate: {reason} (target {target})");
13747            std::process::exit(1);
13748        }
13749    }
13750}
13751
13752/// The per-process fleet work [`start_fleet_runtime`] started.
13753#[cfg(feature = "sqlite")]
13754struct FleetRuntime {
13755    /// The idle sweeper. It closes every database when its token fires.
13756    maintenance: tokio::task::JoinHandle<()>,
13757    /// The pending-work sweeper loop, when one runs.
13758    sweeper: Option<tokio::task::JoinHandle<()>>,
13759    /// Slot ownership (issue #3233), when configured.
13760    ownership: Option<FleetOwnershipRuntime>,
13761}
13762
13763/// The ownership loop of a fleet. It renews while the databases close, then
13764/// [`finish`](Self::finish) releases the ranges.
13765#[cfg(feature = "sqlite")]
13766struct FleetOwnershipRuntime {
13767    fleet: crate::db::fleet::DatabaseFleet,
13768    stop: tokio_util::sync::CancellationToken,
13769    handle: tokio::task::JoinHandle<()>,
13770}
13771
13772#[cfg(feature = "sqlite")]
13773impl FleetOwnershipRuntime {
13774    /// Stop renewing, then release the ranges so the next owner can claim at
13775    /// once. Call it after the fleet closed its databases. Another process
13776    /// on the volume, or unshipped frames, keep the ranges: they lapse.
13777    async fn finish(self) {
13778        self.stop.cancel();
13779        let _ = self.handle.await;
13780        if self.fleet.release_ownership().await {
13781            tracing::info!("shutdown: fleet slot ranges released");
13782        }
13783    }
13784
13785    /// Stop renewing without a release: the fleet did not close within the
13786    /// shutdown budget, so databases may still be open. The ranges lapse.
13787    /// The loop ends within a renew interval; the caller need not wait.
13788    fn abandon(self) -> tokio::task::JoinHandle<()> {
13789        self.stop.cancel();
13790        self.handle
13791    }
13792}
13793
13794/// Claim this host's slot ranges and install them on the fleet (issue #3233):
13795/// the ownership table on the control database, one first claim, the renew
13796/// loop and the `fleet:ownership` indicator. `None` without
13797/// `[database.fleet.ownership]`.
13798#[cfg(feature = "sqlite")]
13799async fn start_fleet_ownership(
13800    state: &AppState,
13801    fleet: &crate::db::fleet::DatabaseFleet,
13802) -> Result<Option<FleetOwnershipRuntime>, String> {
13803    use crate::db::fleet_ownership::{
13804        FleetOwnership, OWNERSHIP_INDICATOR, OwnerId, SlotRange, SqlSlotOwnership,
13805    };
13806    let Some(config) = state.config().database.fleet.and_then(|f| f.ownership) else {
13807        return Ok(None);
13808    };
13809    let control = state
13810        .pool()
13811        .cloned()
13812        .ok_or("fleet ownership needs a control database")?;
13813    let store = SqlSlotOwnership::new(control, state.clock_arc());
13814    store.ensure_schema().await.map_err(|e| e.to_string())?;
13815    let ranges = config
13816        .ranges()?
13817        .into_iter()
13818        .map(|(start, end)| SlotRange::new(start, end))
13819        .collect::<Result<Vec<_>, _>>()
13820        .map_err(|e| e.to_string())?;
13821    let owner = OwnerId::new(config.owner.as_str()).map_err(|e| e.to_string())?;
13822    let ownership = Arc::new(
13823        FleetOwnership::new(
13824            Arc::new(store),
13825            owner,
13826            ranges,
13827            std::time::Duration::from_secs(config.lease_secs),
13828        )
13829        .map_err(|e| e.to_string())?,
13830    );
13831    // Claim before the first open. A range another host holds waits for its
13832    // lease; the loop claims it then.
13833    let report = fleet
13834        .start_ownership(Arc::clone(&ownership))
13835        .await
13836        .map_err(|e| e.to_string())?;
13837    tracing::info!(
13838        owner = %ownership.owner(),
13839        claimed = report.claimed.len(),
13840        waiting = report.waiting.len(),
13841        failed = report.failed.len(),
13842        "fleet slot ownership installed"
13843    );
13844    if let Err(e) = state.health_indicator_registry.register(
13845        OWNERSHIP_INDICATOR,
13846        crate::actuator::IndicatorGroup::HealthOnly,
13847        ownership.indicator(),
13848    ) {
13849        tracing::warn!("{e}");
13850    }
13851    let stop = tokio_util::sync::CancellationToken::new();
13852    let handle = fleet.spawn_ownership(stop.clone());
13853    Ok(Some(FleetOwnershipRuntime {
13854        fleet: fleet.clone(),
13855        stop,
13856        handle,
13857    }))
13858}
13859
13860/// Start the per-process work of a `[database.fleet]` (ADR 0019): a commit-hook
13861/// worker for each database while it is open, the idle sweeper, which closes
13862/// every database when `shutdown` fires, and the pending-work sweeper
13863/// (issue 3234). `None` when the app has no fleet.
13864///
13865/// The hook worker runs in every role, `web` included. A fleet database's
13866/// queue can only be drained by a process that has the database open, and a
13867/// `worker` process does not discover fleet databases, so gating this on the
13868/// role would leave a `web` process's after-commit hooks pending forever.
13869/// Rows are claimed one at a time under `BEGIN IMMEDIATE` with a lease, so two
13870/// processes draining one database is safe.
13871///
13872/// The pending-work sweeper is installed in every role, so enqueue code marks
13873/// its database. A failed install stops the process. Its loop runs only with
13874/// `sweep` (a server, not a one-off task). A role that runs workers opens
13875/// closed databases and relays their outbox; a `web` role drains the hooks of
13876/// the databases it has open.
13877#[cfg(feature = "sqlite")]
13878async fn start_fleet_runtime(
13879    state: &AppState,
13880    shutdown: tokio_util::sync::CancellationToken,
13881    sweep: Option<tokio_util::sync::CancellationToken>,
13882) -> Option<FleetRuntime> {
13883    let fleet = state.shards()?.fleet()?.clone();
13884    // Fatal: a host that cannot tell which slots it owns must not serve.
13885    let ownership = match start_fleet_ownership(state, &fleet).await {
13886        Ok(ownership) => ownership,
13887        Err(error) => {
13888            tracing::error!(%error, "SQLite fleet slot ownership could not start");
13889            eprintln!("SQLite fleet slot ownership could not start: {error}");
13890            exit_stop_managed_pg();
13891            std::process::exit(1);
13892        }
13893    };
13894    fleet.add_lifecycle(Arc::new(FleetCommitHookWorkers {
13895        #[cfg(feature = "ws")]
13896        channels: state.channels().clone(),
13897    }));
13898    let options = crate::db::fleet_sweeper::FleetSweeperOptions {
13899        relay: state.role().runs_workers(),
13900        ..crate::db::fleet_sweeper::FleetSweeperOptions::default()
13901    };
13902    // Fatal: without the sweeper, enqueue code marks nothing, and work in a
13903    // database that closes waits for its next open, silently.
13904    let sweeper = match crate::db::fleet_sweeper::FleetSweeper::install_with(state, options).await {
13905        Ok(sweeper) => sweeper,
13906        Err(error) => {
13907            tracing::error!(%error, "the SQLite fleet sweeper could not start");
13908            eprintln!("the SQLite fleet sweeper could not start: {error}");
13909            exit_stop_managed_pg();
13910            std::process::exit(1);
13911        }
13912    };
13913    let sweeper = sweeper
13914        .zip(sweep)
13915        .map(|(sweeper, token)| sweeper.spawn(token));
13916    Some(FleetRuntime {
13917        maintenance: fleet.spawn_maintenance(shutdown),
13918        sweeper,
13919        ownership,
13920    })
13921}
13922
13923/// Replicate every database of the app's fleet next to the control database
13924/// (ADR 0019 §5): installs the lifecycle hooks, registers the
13925/// `replication:fleet` indicator and starts the loop thread. A no-op without a
13926/// fleet.
13927#[cfg(feature = "sqlite")]
13928fn start_fleet_replication(
13929    state: &AppState,
13930    config: &AutumnConfig,
13931    runtime: &crate::replication::ReplicationRuntime,
13932    lag_alert_after: std::time::Duration,
13933) -> Result<(), String> {
13934    let Some(fleet) = state.shards().and_then(crate::sharding::ShardSet::fleet) else {
13935        return Ok(());
13936    };
13937    let restore_missing = config
13938        .database
13939        .fleet
13940        .as_ref()
13941        .is_some_and(|f| f.restore_missing);
13942    if restore_missing
13943        && config
13944            .database
13945            .fleet
13946            .as_ref()
13947            .is_some_and(|f| f.ownership.is_none())
13948    {
13949        tracing::warn!(
13950            "database.fleet.restore_missing is on without [database.fleet.ownership]: nothing \
13951             fences the previous owner, so two hosts can write one database and diverge"
13952        );
13953    }
13954    let replication = Arc::new(
13955        crate::db::fleet_replication::FleetReplication::new(
13956            Arc::clone(runtime.replicator.destination()),
13957            runtime.settings.clone(),
13958            lag_alert_after,
13959            restore_missing,
13960            state.clock_arc(),
13961        )
13962        // The server path sets a tighter cutoff from the budget left.
13963        .with_final_flush(std::time::Duration::from_secs(
13964            config.server.shutdown_timeout_secs,
13965        )),
13966    );
13967    fleet
13968        .install_replication(Arc::clone(&replication))
13969        .map_err(|_| "the fleet already replicates".to_owned())?;
13970    if let Err(e) = state.health_indicator_registry.register(
13971        crate::db::fleet_replication::FLEET_REPLICATION_INDICATOR,
13972        crate::actuator::IndicatorGroup::HealthOnly,
13973        replication.indicator(),
13974    ) {
13975        tracing::warn!("{e}");
13976    }
13977    replication
13978        .spawn_loop()
13979        .map_err(|e| format!("could not start the fleet replication thread: {e}"))?;
13980    tracing::info!(
13981        prefix = %format!("{}/fleet/", runtime.settings.root),
13982        restore_missing,
13983        "SQLite fleet replication is enabled"
13984    );
13985    Ok(())
13986}
13987
13988/// Runs the durable commit-hook worker of each open fleet database, and stops
13989/// it when the database closes. Hooks enqueued on a database that is closed
13990/// are drained the next time it opens.
13991#[cfg(feature = "sqlite")]
13992struct FleetCommitHookWorkers {
13993    #[cfg(feature = "ws")]
13994    channels: crate::channels::Channels,
13995}
13996
13997#[cfg(feature = "sqlite")]
13998impl crate::db::fleet::FleetLifecycle for FleetCommitHookWorkers {
13999    fn on_open(&self, db: &crate::db::fleet::FleetDatabase) -> Result<(), String> {
14000        // A stop token apart from `db.closed()`: the fleet stops the worker
14001        // and waits for it before it closes the pool.
14002        let stop = tokio_util::sync::CancellationToken::new();
14003        #[cfg(feature = "ws")]
14004        let handle = crate::repository_commit_hooks::start_repository_commit_hook_worker(
14005            db.pool().clone(),
14006            Some(self.channels.clone()),
14007            stop.clone(),
14008        );
14009        #[cfg(not(feature = "ws"))]
14010        let handle = crate::repository_commit_hooks::start_repository_commit_hook_worker(
14011            db.pool().clone(),
14012            stop.clone(),
14013        );
14014        if let Some(handle) = handle {
14015            db.register_worker(stop, handle);
14016        }
14017        Ok(())
14018    }
14019
14020    fn on_close(&self, db: &crate::db::fleet::FleetDatabase) {
14021        crate::repository_commit_hooks::forget_sqlite_repository_commit_hook_kick(db.pool());
14022    }
14023}
14024
14025/// A fleet must know which file the control pool opens, to refuse one inside
14026/// its root. The built-in provider opens the configured URL, which
14027/// [`crate::db::fleet::build_for_app`] checks. A custom provider may open
14028/// anything, so it must name its target with
14029/// [`crate::db::DatabaseTopology::with_migration_url`]; without that, boot
14030/// refuses rather than guess.
14031#[cfg(feature = "sqlite")]
14032fn fleet_control_target_identified(
14033    custom_pool_provider: bool,
14034    migration_url: Option<&str>,
14035) -> Result<(), String> {
14036    if !custom_pool_provider {
14037        return Ok(());
14038    }
14039    match migration_url {
14040        None => Err(
14041            "database.fleet with a custom DatabasePoolProvider: the provider's topology must \
14042             name its control database with DatabaseTopology::with_migration_url(...), so the \
14043             fleet can refuse a control file inside database.fleet.root"
14044                .to_owned(),
14045        ),
14046        Some(url)
14047            if crate::config::DatabaseBackend::detect(url)
14048                != Some(crate::config::DatabaseBackend::Sqlite)
14049                || crate::config::sqlite_url_file(url).is_none() =>
14050        {
14051            Err(format!(
14052                "database.fleet with a custom DatabasePoolProvider: the provider names its control \
14053                 database as {}, which is not a file-backed sqlite: database; a fleet keeps \
14054                 framework state (sessions, jobs, flags) there, so it must survive a restart",
14055                crate::db_url::redact_target(url)
14056            ))
14057        }
14058        Some(_) => Ok(()),
14059    }
14060}
14061
14062/// Migrate every database of a `[database.fleet]` (ADR 0019) for the
14063/// `AUTUMN_MIGRATE=1` one-shot, returning the number of migrations applied, or
14064/// exiting non-zero when any database fails (each failure is printed, redacted).
14065#[cfg(feature = "sqlite")]
14066async fn migrate_fleet_or_exit(
14067    config: &AutumnConfig,
14068    migrations: Arc<Vec<(&'static str, crate::migrate::EmbeddedMigrations)>>,
14069) -> usize {
14070    let fleet = match crate::db::fleet::build_for_app(config, None, &migrations, |set| {
14071        !migration_set_is_control_framework(set)
14072    }) {
14073        Ok(Some(fleet)) => fleet,
14074        Ok(None) => return 0,
14075        Err(error) => {
14076            eprintln!("autumn migrate: {error}");
14077            std::process::exit(1);
14078        }
14079    };
14080    match fleet
14081        .migrate_all(crate::db::fleet::DEFAULT_FLEET_CONCURRENCY)
14082        .await
14083    {
14084        Ok(report) if report.failed.is_empty() => {
14085            eprintln!(
14086                "autumn migrate: fleet: {} database(s), {} migration(s) applied",
14087                report.databases, report.applied
14088            );
14089            report.applied
14090        }
14091        Ok(report) => {
14092            for (key, error) in &report.failed {
14093                eprintln!("autumn migrate: fleet database {key}: {error}");
14094            }
14095            eprintln!(
14096                "autumn migrate: fleet: {} of {} database(s) failed; the others are migrated",
14097                report.failed.len(),
14098                report.databases
14099            );
14100            std::process::exit(1);
14101        }
14102        Err(error) => {
14103            eprintln!("autumn migrate: fleet: {error}");
14104            std::process::exit(1);
14105        }
14106    }
14107}
14108
14109/// Guard the startup-migration path against a **sharded** `SQLite` deployment.
14110///
14111/// PR3 (#1614) wired a working `SQLite` startup-migration path: registered
14112/// migrations now apply to a `sqlite://` control target through diesel's
14113/// `MigrationHarness` on a `SqliteConnection`, with no advisory lock (see
14114/// [`crate::migrate::run_pending_sqlite`] / [`crate::migrate::auto_migrate_sqlite`],
14115/// routed from [`run_startup_migrations`]). So registered migrations are no
14116/// longer rejected here — a `sqlite://` control target with `.migrations(...)`
14117/// boots and applies its schema.
14118///
14119/// What remains unsupported is **sharding on `SQLite`**: the shard-directory and
14120/// shard-map control tables and the per-shard fan-out are Postgres/sharding
14121/// primitives (advisory locks, Postgres DDL), and a single-node `SQLite`
14122/// deployment has no shards. Two situations are therefore rejected here with an
14123/// actionable message rather than attempting to create Postgres-shaped sharding
14124/// tables on `SQLite`:
14125///
14126///   * a `SQLite` **control** target for which the sharding control migrations
14127///     are required or shards are configured (`control_sharding_required` —
14128///     `directory_migration_required || shard_map_migration_required ||
14129///     config.database.has_shards()`); and
14130///   * any `SQLite` **shard** `primary_url`: the shard loop in
14131///     [`run_startup_migrations`] migrates every shard through the Postgres-only
14132///     harness, so a `sqlite:` shard is sharding-on-`SQLite` regardless of the
14133///     control backend. `shard_urls` is empty when the app is unsharded or the
14134///     shard loop won't run, so the Postgres path is unaffected.
14135///
14136/// Uses the same [`crate::config::DatabaseBackend::detect`] predicate as
14137/// `db::build_pool`, so the gate and the pool routing agree on what "is a
14138/// `SQLite` URL" means.
14139#[cfg(feature = "sqlite")]
14140fn sqlite_sharding_unsupported_guard(
14141    control_url: Option<&str>,
14142    control_sharding_required: bool,
14143    shard_urls: &[&str],
14144) -> Result<(), String> {
14145    fn is_sqlite(url: &str) -> bool {
14146        crate::config::DatabaseBackend::detect(url) == Some(crate::config::DatabaseBackend::Sqlite)
14147    }
14148    if control_url.is_some_and(is_sqlite) && control_sharding_required {
14149        return Err(
14150            "SQLite deployments do not support sharding. The configured sqlite:// control target \
14151             has sharding enabled (shards and/or the directory/shard-map control migrations), \
14152             which is a Postgres-only capability \u{2014} remove the shard configuration to run \
14153             on SQLite, or use a Postgres control database. Tracking: #1614."
14154                .to_owned(),
14155        );
14156    }
14157    if shard_urls.iter().copied().any(is_sqlite) {
14158        return Err(
14159            "SQLite deployments do not support sharding. A configured shard targets a SQLite \
14160             database, and per-shard migration/fan-out is a Postgres-only capability \u{2014} \
14161             remove the SQLite shard configuration to run on SQLite, or use Postgres shard \
14162             targets. Tracking: #1614."
14163                .to_owned(),
14164        );
14165    }
14166    Ok(())
14167}
14168
14169#[cfg(all(test, feature = "sqlite"))]
14170mod sqlite_sharding_unsupported_guard_tests {
14171    use super::sqlite_sharding_unsupported_guard;
14172
14173    #[test]
14174    fn sqlite_control_target_with_sharding_fails_fast() {
14175        // PR3: a sqlite:// control URL with sharding enabled must fail fast with
14176        // an actionable, sharding-named boot error — sharding is Postgres-only.
14177        for url in [
14178            "sqlite:///var/lib/app.db",
14179            "sqlite://./relative.db",
14180            "sqlite::memory:",
14181        ] {
14182            let err = sqlite_sharding_unsupported_guard(Some(url), true, &[])
14183                .expect_err("sqlite control target + sharding must be rejected");
14184            assert!(
14185                err.contains("do not support sharding"),
14186                "message must name the sharding situation clearly: {err}"
14187            );
14188            assert!(
14189                err.contains("#1614"),
14190                "message must point at the tracking issue: {err}"
14191            );
14192        }
14193    }
14194
14195    #[test]
14196    fn sqlite_control_target_without_sharding_boots() {
14197        // PR3 behavior: a sqlite target with registered (non-sharding)
14198        // migrations now boots — they are applied by `auto_migrate_sqlite`.
14199        for url in [
14200            "sqlite:///var/lib/app.db",
14201            "sqlite://./relative.db",
14202            "sqlite::memory:",
14203        ] {
14204            assert!(
14205                sqlite_sharding_unsupported_guard(Some(url), false, &[]).is_ok(),
14206                "sqlite target without sharding must boot (migrations now applied): {url}"
14207            );
14208        }
14209    }
14210
14211    #[test]
14212    fn postgres_target_is_unchanged() {
14213        // The default Postgres path is untouched, sharding required or not.
14214        for url in [
14215            "postgres://u@h/db",
14216            "postgresql://user:pass@db:5432/app",
14217            "host=db user=app sslmode=require",
14218        ] {
14219            assert!(
14220                sqlite_sharding_unsupported_guard(Some(url), true, &[]).is_ok(),
14221                "postgres target must never be gated: {url}"
14222            );
14223            assert!(
14224                sqlite_sharding_unsupported_guard(Some(url), false, &[]).is_ok(),
14225                "postgres target must never be gated: {url}"
14226            );
14227        }
14228    }
14229
14230    #[test]
14231    fn absent_control_url_boots() {
14232        // No control URL (no-DB / opt-out provider): nothing to gate.
14233        assert!(sqlite_sharding_unsupported_guard(None, true, &[]).is_ok());
14234        assert!(sqlite_sharding_unsupported_guard(None, false, &[]).is_ok());
14235    }
14236
14237    #[test]
14238    fn sqlite_shard_target_fails_fast() {
14239        // A `sqlite:` shard `primary_url` is sharding-on-SQLite regardless of the
14240        // control backend and regardless of migrations — the shard loop routes it
14241        // through the Postgres-only harness. It must be rejected with the
14242        // actionable sharding error.
14243        for shards in [
14244            &["sqlite:///var/lib/shard0.db"][..],
14245            &["sqlite:///var/lib/shard0.db", "postgres://u@h/shard1"][..],
14246        ] {
14247            let err = sqlite_sharding_unsupported_guard(None, false, shards)
14248                .expect_err("a sqlite shard target must be rejected");
14249            assert!(
14250                err.contains("do not support sharding") && err.contains("shard"),
14251                "message must name the SQLite shard situation clearly: {err}"
14252            );
14253            assert!(
14254                err.contains("#1614"),
14255                "message must point at the tracking issue: {err}"
14256            );
14257        }
14258    }
14259
14260    #[test]
14261    fn postgres_shard_targets_are_unchanged() {
14262        // All-Postgres shards are never gated, sharding required or not.
14263        assert!(
14264            sqlite_sharding_unsupported_guard(
14265                Some("postgres://u@h/control"),
14266                true,
14267                &["postgres://u@h/shard0", "postgres://u@h/shard1"],
14268            )
14269            .is_ok(),
14270            "all-postgres shard targets must never be gated"
14271        );
14272    }
14273
14274    #[test]
14275    fn migrate_only_mode_reuses_the_boot_guard_for_sqlite_targets() {
14276        // `run_migrate_only_mode` (the `AUTUMN_MIGRATE=1` one-shot) applies the
14277        // SAME guard as normal boot BEFORE its migration loop, so the boot and
14278        // migrate-only paths cannot drift. A sqlite control/shard target with
14279        // sharding still fails fast; a plain sqlite control target now proceeds
14280        // (its migrations are applied by the sqlite apply path).
14281
14282        // A sqlite CONTROL migrate target with sharding → actionable error.
14283        let err = sqlite_sharding_unsupported_guard(Some("sqlite:///var/lib/app.db"), true, &[])
14284            .expect_err("sqlite migrate control target with sharding must be rejected");
14285        assert!(
14286            err.contains("do not support sharding") && err.contains("#1614"),
14287            "migrate-only sqlite control error must be the actionable sharding message: {err}"
14288        );
14289
14290        // A sqlite SHARD migrate target → actionable error.
14291        let shard_err = sqlite_sharding_unsupported_guard(
14292            Some("postgres://u@h/control"),
14293            true,
14294            &["sqlite:///var/lib/shard0.db"],
14295        )
14296        .expect_err("sqlite migrate shard target must be rejected");
14297        assert!(
14298            shard_err.contains("do not support sharding") && shard_err.contains("shard"),
14299            "migrate-only sqlite shard error must be the actionable sharding message: {shard_err}"
14300        );
14301
14302        // An all-Postgres migrate configuration (control + shards) is never gated.
14303        assert!(
14304            sqlite_sharding_unsupported_guard(
14305                Some("postgres://u@h/control"),
14306                true,
14307                &["postgres://u@h/shard0"],
14308            )
14309            .is_ok(),
14310            "an all-postgres migrate configuration must proceed unchanged"
14311        );
14312
14313        // A plain sqlite migrate control target (no sharding) proceeds — its
14314        // migrations are applied by the sqlite apply path.
14315        assert!(
14316            sqlite_sharding_unsupported_guard(Some("sqlite:///var/lib/app.db"), false, &[]).is_ok(),
14317            "sqlite control target without sharding must never be gated"
14318        );
14319    }
14320}
14321
14322/// Combine the app's registered migration sets with the two standalone
14323/// shard-directory / shard-map control migrations — applied straight from
14324/// their own `const`s rather than through `migrations` — into one input for
14325/// [`crate::migrate::compute_migration_disambiguation`]. Without this, a
14326/// plugin claiming one of those fixed, framework-owned versions under a
14327/// different name would skip past collision detection entirely, since
14328/// neither standalone set is otherwise part of the registered `migrations`
14329/// this function is given.
14330///
14331/// The two standalone sets are included only when `shards_configured` is
14332/// true. An unsharded app (including every `sqlite` app, which rejects
14333/// sharding outright) never applies either set, so including them
14334/// unconditionally would treat their fixed versions as live collision
14335/// participants for an app that will never actually record them — risking
14336/// an already-applied, unrelated migration being reassigned a new
14337/// substitute the moment this framework version is adopted, purely because
14338/// of a coincidental version match against a migration that was never a
14339/// real collision for that app. `shards_configured` (`database.has_shards()`)
14340/// is a conservative superset of the precise runtime conditions
14341/// `directory_migration_required`/`shard_map_migration_required` gate the
14342/// actual apply on (both imply shards are configured; the converse doesn't
14343/// hold for e.g. an explicit custom shard router) — deliberately so: BOTH
14344/// call sites (`run_startup_migrations` and the `autumn migrate` CLI path,
14345/// which cannot cheaply reconstruct the precise runtime flags) must use the
14346/// IDENTICAL condition, or the two paths could reach different
14347/// disambiguation decisions for the same migration depending on which one
14348/// happens to run first — the exact hazard this whole mechanism exists to
14349/// avoid. The residual narrow case (a sharded app using an explicit custom
14350/// router) may see a harmless, unnecessary disambiguation entry for a shard
14351/// version it will never apply; that is safe, just imprecise.
14352#[cfg(feature = "db")]
14353fn migration_sets_for_disambiguation<'a>(
14354    migrations: &'a [(&'static str, crate::migrate::EmbeddedMigrations)],
14355    shards_configured: bool,
14356) -> Vec<(&'a str, &'a crate::migrate::EmbeddedMigrations)> {
14357    let owned = migrations.iter().map(|(name, set)| (*name, set));
14358    if shards_configured {
14359        owned
14360            .chain([
14361                (
14362                    "shard-directory",
14363                    &crate::sharding::SHARD_DIRECTORY_MIGRATIONS,
14364                ),
14365                ("shard-map", &crate::sharding::SHARD_MAP_MIGRATIONS),
14366            ])
14367            .collect()
14368    } else {
14369        owned.collect()
14370    }
14371}
14372
14373#[cfg(feature = "db")]
14374#[allow(
14375    clippy::too_many_arguments,
14376    clippy::fn_params_excessive_bools,
14377    clippy::too_many_lines
14378)]
14379async fn run_startup_migrations(
14380    config: &AutumnConfig,
14381    control_configured: bool,
14382    shards_configured: bool,
14383    provider_migration_url: Option<String>,
14384    migrations: Arc<Vec<(&'static str, crate::migrate::EmbeddedMigrations)>>,
14385    directory_migration_required: bool,
14386    shard_map_migration_required: bool,
14387) {
14388    let control_url = if control_configured {
14389        // Prefer a provider-resolved URL (e.g. managed Postgres, whose socket
14390        // URL isn't in config) carried on the topology: the runtime pool is
14391        // built from it, so embedded startup migrations must target it — even if
14392        // a stale `database.url`/`primary_url` is still configured (an existing
14393        // app adopting the provider). Fall back to the configured URL otherwise.
14394        provider_migration_url
14395            .or_else(|| config.database.effective_primary_url().map(str::to_owned))
14396    } else {
14397        None
14398    };
14399    let shard_targets: Vec<(String, String)> = if shards_configured {
14400        config
14401            .database
14402            .shards
14403            .iter()
14404            .map(|shard| (format!("shard:{}", shard.name), shard.primary_url.clone()))
14405            .collect()
14406    } else {
14407        Vec::new()
14408    };
14409    let profile = config.profile.clone();
14410    let auto_migrate = config.database.auto_migrate;
14411    let auto_in_prod = config.database.auto_migrate_in_production;
14412    let lock_policy = crate::migrate::MigrationLockPolicy::from_config(&config.database);
14413    // Computed once, on the FINAL registered set (after `setup_database`'s own
14414    // fold added any shard-required sets), so a version collision between ANY
14415    // two registered sources is resolved automatically rather than causing
14416    // one migration to be silently skipped — see
14417    // `compute_migration_disambiguation` and `migration_sets_for_disambiguation`.
14418    let disambiguation_sets =
14419        migration_sets_for_disambiguation(&migrations, config.database.has_shards());
14420    let disambiguated = crate::migrate::compute_migration_disambiguation(&disambiguation_sets);
14421    #[cfg(feature = "sqlite")]
14422    let sqlite_history_sets = crate::migrate::sqlite_collision_pairs(&disambiguation_sets);
14423    let migration_result = crate::time::spawn_blocking(move || {
14424        // SQLite single-writer startup-migration path (#1614, PR3): apply the
14425        // registered migrations to a `sqlite://` control target with no advisory
14426        // lock. Sharding — directory, shard-map, per-shard fan-out — is
14427        // Postgres-only and is rejected upstream in `setup_database` by
14428        // `sqlite_sharding_unsupported_guard`, so nothing shard- or
14429        // directory-related runs here, and the framework migrations below are
14430        // skipped for a SQLite control target.
14431        #[cfg(feature = "sqlite")]
14432        if let Some(url) = control_url.as_deref()
14433            && crate::config::DatabaseBackend::detect(url)
14434                == Some(crate::config::DatabaseBackend::Sqlite)
14435        {
14436            // Only when this boot applies (the same decision `auto_migrate_sqlite`
14437            // makes): a migration this database already ran under a version the
14438            // map now gives a substitute keeps its record, moved to that
14439            // substitute, rather than running twice. A report-only boot touches
14440            // nothing.
14441            if crate::migrate::should_auto_apply(profile.as_deref(), auto_migrate, auto_in_prod)
14442                && let Err(error) = crate::migrate::adopt_sqlite_collision_history(
14443                    url,
14444                    &sqlite_history_sets,
14445                    &disambiguated,
14446                )
14447            {
14448                tracing::error!(
14449                    error = %error,
14450                    target = "control",
14451                    "Could not move an already-applied migration's version record"
14452                );
14453                std::process::exit(1);
14454            }
14455            for (_, mig) in migrations.iter() {
14456                crate::migrate::auto_migrate_sqlite(
14457                    url,
14458                    profile.as_deref(),
14459                    auto_migrate,
14460                    auto_in_prod,
14461                    crate::migrate::DisambiguatedMigrations::new(mig, &disambiguated),
14462                    "control",
14463                );
14464            }
14465            return;
14466        }
14467
14468        if let Some(url) = control_url {
14469            for (_, mig) in migrations.iter() {
14470                crate::migrate::auto_migrate(
14471                    &url,
14472                    profile.as_deref(),
14473                    auto_migrate,
14474                    auto_in_prod,
14475                    crate::migrate::DisambiguatedMigrations::new(mig, &disambiguated),
14476                    "control",
14477                    lock_policy,
14478                );
14479            }
14480            // The shard directory table lives on the control plane only, so it
14481            // is applied here and never to the per-shard targets below.
14482            if directory_migration_required {
14483                crate::migrate::auto_migrate(
14484                    &url,
14485                    profile.as_deref(),
14486                    auto_migrate,
14487                    auto_in_prod,
14488                    crate::migrate::DisambiguatedMigrations::new(
14489                        &crate::sharding::SHARD_DIRECTORY_MIGRATIONS,
14490                        &disambiguated,
14491                    ),
14492                    "control",
14493                    lock_policy,
14494                );
14495            }
14496            // The shard-map guard table also lives on the control plane only. It
14497            // follows the same resolved, profile-agnostic auto-migrate decision as
14498            // the app migrations above (#1903): default-on in dev, opt-in in prod
14499            // via `auto_migrate`/`auto_migrate_in_production`, over the same
14500            // advisory-locked apply path. Under a report-only decision the missing
14501            // table is reported rather than force-applied, so a DB-free or offline
14502            // startup never fails fatally on an unreachable control target.
14503            if shard_map_migration_required {
14504                crate::migrate::auto_migrate(
14505                    &url,
14506                    profile.as_deref(),
14507                    auto_migrate,
14508                    auto_in_prod,
14509                    crate::migrate::DisambiguatedMigrations::new(
14510                        &crate::sharding::SHARD_MAP_MIGRATIONS,
14511                        &disambiguated,
14512                    ),
14513                    "control",
14514                    lock_policy,
14515                );
14516            }
14517        }
14518        // Shards hold tenant data, not the control-plane schema. If the app
14519        // registered the full control `FRAMEWORK_MIGRATIONS` set (as some
14520        // examples do), skip it for shard targets — otherwise startup would
14521        // create the control tables on every shard and (with auto-migrate off)
14522        // keep reporting them as pending, even though `autumn migrate --shard`
14523        // applies only the shard-required framework migrations.
14524        for (target, url) in &shard_targets {
14525            for (_, mig) in migrations
14526                .iter()
14527                .filter(|(_, mig)| !migration_set_is_control_framework(mig))
14528            {
14529                crate::migrate::auto_migrate(
14530                    url,
14531                    profile.as_deref(),
14532                    auto_migrate,
14533                    auto_in_prod,
14534                    crate::migrate::DisambiguatedMigrations::new(mig, &disambiguated),
14535                    target,
14536                    lock_policy,
14537                );
14538            }
14539        }
14540    })
14541    .await;
14542    if let Err(e) = migration_result {
14543        tracing::error!(error = %e, "Migration task panicked");
14544        // Same orphan hazard as a migration failure: `process::exit` skips
14545        // `on_shutdown`, so stop any managed Postgres before bailing. We are back
14546        // on the Tokio runtime here (after the `spawn_blocking` await), so use the
14547        // async stop — the sync `emergency_stop` would panic nesting a runtime.
14548        #[cfg(feature = "managed-pg")]
14549        crate::managed_pg::emergency_stop_async().await;
14550        std::process::exit(1);
14551    }
14552}
14553
14554/// Per-shard replica migration parity feeds each shard's runtime state
14555/// (the analogue of `ProbeState`'s control-replica dependency), which
14556/// gates that shard's replica reads per its `replica_fallback`.
14557#[cfg(feature = "db")]
14558async fn check_shard_replica_migration_parity(
14559    config: &AutumnConfig,
14560    set: &crate::sharding::ShardSet,
14561) {
14562    for (shard_config, shard) in config.database.shards.iter().zip(set.iter()) {
14563        let Some(replica_url) = shard_config.replica_url.as_deref() else {
14564            continue;
14565        };
14566        // Remember the URLs so the per-shard health indicator can re-run
14567        // the parity comparison on later readiness probes, and claim the
14568        // recheck throttle slot for the check that runs right here.
14569        shard
14570            .runtime()
14571            .configure_migration_check(shard_config.primary_url.clone(), replica_url.to_owned());
14572        let _ = shard.runtime().parity_check_due();
14573        let readiness = crate::migrate::check_replica_migration_readiness_blocking(
14574            shard_config.primary_url.clone(),
14575            replica_url.to_owned(),
14576        )
14577        .await;
14578        if readiness.is_ready() {
14579            shard.runtime().mark_replica_migrations_ready();
14580        } else if let Some(detail) = readiness.detail() {
14581            tracing::warn!(
14582                shard = %shard.name(),
14583                detail = %detail,
14584                "shard replica migrations are not ready"
14585            );
14586            shard.runtime().mark_replica_migrations_unready(detail);
14587        }
14588    }
14589}
14590
14591#[cfg(feature = "db")]
14592const REPOSITORY_COMMIT_HOOK_QUEUE_MIGRATION: &str =
14593    "20260515000000_create_repository_commit_hook_queue";
14594
14595#[cfg(feature = "db")]
14596const VERSION_HISTORY_MIGRATION: &str = "20260526000000_create_version_history";
14597
14598#[cfg(feature = "db")]
14599const DERIVATION_MIGRATION: &str = "20260907101530_create_derivations";
14600
14601/// Whether startup should create the control-plane `_autumn_shard_directory`
14602/// table. It is required only when directory routing is enabled AND shards are
14603/// configured AND we are in a real runtime boot — never during a static build
14604/// (`autumn build`, `AUTUMN_BUILD_STATIC=1`), which renders assets and must not
14605/// touch the database, mirroring how the other runtime framework migrations are
14606/// suppressed in [`migrations_with_repository_framework_migrations`].
14607#[cfg(feature = "db")]
14608const fn directory_migration_is_required(
14609    directory_routing_enabled: bool,
14610    has_shards: bool,
14611    mode: RepositoryCommitHookQueueMigrationMode,
14612) -> bool {
14613    directory_routing_enabled
14614        && has_shards
14615        && matches!(mode, RepositoryCommitHookQueueMigrationMode::Runtime)
14616}
14617
14618/// Whether startup should create the control-plane `_autumn_shard_map` table.
14619/// Required whenever shards are configured and we are in a real runtime boot —
14620/// never during a static build (`autumn build`, `AUTUMN_BUILD_STATIC=1`).
14621/// The guard itself is further gated to auto-split mode inside
14622/// `enforce_shard_map_guard`; the table is always created when shards are
14623/// present so an app can switch from explicit to auto-split later without a
14624/// manual migration.
14625#[cfg(feature = "db")]
14626const fn shard_map_migration_is_required(
14627    has_shards: bool,
14628    mode: RepositoryCommitHookQueueMigrationMode,
14629) -> bool {
14630    has_shards && matches!(mode, RepositoryCommitHookQueueMigrationMode::Runtime)
14631}
14632
14633/// Row type for reading `_autumn_shard_map`.
14634#[cfg(feature = "postgres")]
14635#[derive(diesel::QueryableByName)]
14636struct ShardMapRow {
14637    #[diesel(sql_type = diesel::sql_types::Text)]
14638    shard_name: String,
14639    #[diesel(sql_type = diesel::sql_types::Text)]
14640    slots: String,
14641}
14642
14643/// Check and persist the shard slot map in `_autumn_shard_map`.
14644///
14645/// This is the DB-backed core of the boot-time guard: it reads existing rows,
14646/// delegates to the pure [`crate::config::check_stored_slot_map`] for the
14647/// comparison, and persists the map on first boot (no rows yet). Factored out
14648/// of `enforce_shard_map_guard` so integration tests can drive it directly
14649/// without a full `AutumnConfig`.
14650///
14651/// # Errors
14652///
14653/// Returns a `String` error when the computed auto-split map differs from the
14654/// stored map, indicating a topology change that would silently misroute data.
14655#[cfg(feature = "postgres")]
14656pub async fn run_shard_map_guard(
14657    control_pool: &deadpool::managed::Pool<
14658        diesel_async::pooled_connection::AsyncDieselConnectionManager<
14659            diesel_async::AsyncPgConnection,
14660        >,
14661    >,
14662    computed: &[crate::config::ShardSlotAssignment],
14663    auto_split: bool,
14664) -> Result<(), String> {
14665    use diesel_async::RunQueryDsl as _;
14666
14667    if !auto_split {
14668        return Ok(());
14669    }
14670
14671    let mut conn = match control_pool.get().await {
14672        Ok(conn) => conn,
14673        Err(e) => {
14674            return Err(format!(
14675                "shard-map guard could not acquire a control connection: {e} — \
14676                 ensure the control database is reachable to enforce topology \
14677                 change detection"
14678            ));
14679        }
14680    };
14681
14682    let rows: Vec<ShardMapRow> = match diesel::sql_query(
14683        "SELECT shard_name, slots FROM _autumn_shard_map ORDER BY shard_name",
14684    )
14685    .load::<ShardMapRow>(&mut conn)
14686    .await
14687    {
14688        Ok(rows) => rows,
14689        Err(e) => {
14690            return Err(format!(
14691                "shard-map guard could not read _autumn_shard_map: {e} — \
14692                 run `autumn migrate` to create the control schema before \
14693                 starting with auto-split shards"
14694            ));
14695        }
14696    };
14697
14698    let stored: Vec<crate::config::ShardSlotAssignment> = rows
14699        .into_iter()
14700        .map(|r| crate::config::ShardSlotAssignment {
14701            name: r.shard_name,
14702            ranges: r.slots,
14703        })
14704        .collect();
14705    let stored_opt = if stored.is_empty() {
14706        None
14707    } else {
14708        Some(stored.as_slice())
14709    };
14710
14711    crate::config::check_stored_slot_map(auto_split, computed, stored_opt)?;
14712
14713    // First boot: persist the current map so future boots can compare against it.
14714    // Wrapped in a transaction so a mid-loop failure leaves no partial rows —
14715    // partial rows would cause a spurious mismatch error on the next boot attempt.
14716    if stored.is_empty() {
14717        use diesel_async::AsyncConnection as _;
14718        let assignments: Vec<_> = computed.to_vec();
14719        conn.transaction::<(), diesel::result::Error, _>(async move |conn| {
14720            for assignment in &assignments {
14721                diesel::sql_query(
14722                    "INSERT INTO _autumn_shard_map (shard_name, slots) VALUES ($1, $2) \
14723                     ON CONFLICT (shard_name) DO UPDATE \
14724                     SET slots = EXCLUDED.slots, updated_at = NOW()",
14725                )
14726                .bind::<diesel::sql_types::Text, _>(&assignment.name)
14727                .bind::<diesel::sql_types::Text, _>(&assignment.ranges)
14728                .execute(conn)
14729                .await?;
14730            }
14731            Ok(())
14732        })
14733        .await
14734        .map_err(|e| format!("shard-map guard could not persist map: {e}"))?;
14735    }
14736
14737    Ok(())
14738}
14739
14740/// Boot-time shard-map guard: compare the auto-split slot map against the
14741/// persisted map and refuse to start if they differ.
14742///
14743/// No-op when:
14744/// - not a runtime boot (static build),
14745/// - no shards configured,
14746/// - no control database topology, or
14747/// - the slot map uses explicit `slots` declarations (auto-split is inactive).
14748// Sharding (the shard-map guard, auto-split, and control-DB shard map) is a
14749// Postgres-only feature: `run_shard_map_guard` drives Postgres `sql_query`
14750// against a `Pool<AsyncPgConnection>` control pool. Under the `sqlite` feature
14751// the runtime topology's pool is a SQLite pool that cannot feed it, and SQLite
14752// deployments are single-node/unsharded, so the guard is a no-op.
14753#[cfg(all(feature = "db", feature = "sqlite"))]
14754#[allow(clippy::unused_async)]
14755async fn enforce_shard_map_guard(
14756    config: &AutumnConfig,
14757    topology: Option<&crate::db::DatabaseTopology>,
14758    runtime_boot: bool,
14759) -> Result<(), String> {
14760    let _ = (config, topology, runtime_boot);
14761    Ok(())
14762}
14763
14764#[cfg(all(feature = "db", not(feature = "sqlite")))]
14765async fn enforce_shard_map_guard(
14766    config: &AutumnConfig,
14767    topology: Option<&crate::db::DatabaseTopology>,
14768    runtime_boot: bool,
14769) -> Result<(), String> {
14770    if !runtime_boot || !config.database.has_shards() {
14771        return Ok(());
14772    }
14773    let Some(topology) = topology else {
14774        return Ok(());
14775    };
14776    if !config.database.shards_auto_split() {
14777        return Ok(());
14778    }
14779    let computed = config
14780        .database
14781        .resolved_shard_assignments()
14782        .map_err(|e| format!("shard-map guard: {e}"))?;
14783    run_shard_map_guard(topology.primary(), &computed, true).await
14784}
14785
14786#[cfg(feature = "db")]
14787#[derive(Clone, Copy, Debug, Eq, PartialEq)]
14788enum RepositoryCommitHookQueueMigrationMode {
14789    Runtime,
14790    StaticBuild,
14791}
14792
14793#[cfg(feature = "db")]
14794fn migrations_with_repository_framework_migrations(
14795    mut migrations: Vec<(&'static str, crate::migrate::EmbeddedMigrations)>,
14796    hook_queue_required: bool,
14797    version_history_required: bool,
14798    derivations_required: bool,
14799    mode: RepositoryCommitHookQueueMigrationMode,
14800) -> Vec<(&'static str, crate::migrate::EmbeddedMigrations)> {
14801    if hook_queue_required
14802        && mode == RepositoryCommitHookQueueMigrationMode::Runtime
14803        && !shard_applied_sets_include(&migrations, REPOSITORY_COMMIT_HOOK_QUEUE_MIGRATION)
14804    {
14805        migrations.push((
14806            "repository-commit-hooks",
14807            crate::repository_commit_hooks::REPOSITORY_COMMIT_HOOK_MIGRATIONS,
14808        ));
14809    }
14810    if version_history_required
14811        && mode == RepositoryCommitHookQueueMigrationMode::Runtime
14812        && !shard_applied_sets_include(&migrations, VERSION_HISTORY_MIGRATION)
14813    {
14814        migrations.push((
14815            "version-history",
14816            crate::version_history::VERSION_HISTORY_MIGRATIONS,
14817        ));
14818    }
14819    // The derivation state table follows the same rule as the two above: it is a
14820    // shard-applied set, it is appended only when the binary actually links a
14821    // `#[derivation]`, and never during a static build, which renders assets
14822    // and must not touch the database.
14823    if derivations_required
14824        && mode == RepositoryCommitHookQueueMigrationMode::Runtime
14825        && !shard_applied_sets_include(&migrations, DERIVATION_MIGRATION)
14826    {
14827        migrations.push(("derivations", crate::derivation::DERIVATION_MIGRATIONS));
14828    }
14829    migrations
14830}
14831
14832/// Whether `migration_name` is already present in a set that shard targets will
14833/// actually apply — i.e. a *non*-control-framework set.
14834///
14835/// The full control [`FRAMEWORK_MIGRATIONS`](crate::migrate::FRAMEWORK_MIGRATIONS)
14836/// set is deliberately excluded: `run_startup_migrations` strips it from shard
14837/// targets, so a migration present *only* inside it never reaches the shards. If
14838/// de-duplication counted it, a sharded app that registers `FRAMEWORK_MIGRATIONS`
14839/// (and uses commit hooks / versioning) would skip appending the standalone
14840/// shard-required set yet have the control set filtered out on shards — leaving
14841/// shards without `_autumn_repository_commit_hook_queue` / `_autumn_version_history`.
14842/// Matching only shard-applied sets ensures the standalone set is appended
14843/// whenever the shards would otherwise be missing it. Re-applying it to the
14844/// control target is harmless: it shares the migration version already recorded
14845/// by the control framework set, so Diesel skips it there.
14846#[cfg(feature = "db")]
14847fn shard_applied_sets_include(
14848    migrations: &[(&'static str, crate::migrate::EmbeddedMigrations)],
14849    migration_name: &str,
14850) -> bool {
14851    use crate::db::RuntimeBackend;
14852    use diesel::migration::{Migration, MigrationSource as _};
14853
14854    migrations
14855        .iter()
14856        .filter(|(_, set)| !migration_set_is_control_framework(set))
14857        .any(|(_, source)| {
14858            let Ok(source_migrations): Result<Vec<Box<dyn Migration<RuntimeBackend>>>, _> =
14859                source.migrations()
14860            else {
14861                return false;
14862            };
14863
14864            source_migrations
14865                .iter()
14866                .any(|migration| migration.name().to_string() == migration_name)
14867        })
14868}
14869
14870/// Whether a migration set is the control-plane
14871/// [`FRAMEWORK_MIGRATIONS`](crate::migrate::FRAMEWORK_MIGRATIONS), so it can be
14872/// skipped on shard targets.
14873///
14874/// Identified by containing a *control-only* migration — one in
14875/// `FRAMEWORK_MIGRATIONS` but not in the shard-required version-history /
14876/// commit-hook sets. Those two sets' migrations are duplicated into the control
14877/// `migrations/` directory, so a plain name overlap would also (wrongly) match
14878/// the standalone `VERSION_HISTORY_MIGRATIONS` / `REPOSITORY_COMMIT_HOOK_MIGRATIONS`
14879/// sets and strip them from shards.
14880#[cfg(feature = "db")]
14881fn migration_set_is_control_framework(set: &crate::migrate::EmbeddedMigrations) -> bool {
14882    use crate::db::RuntimeBackend;
14883    use diesel::migration::{Migration, MigrationSource as _};
14884
14885    fn names(set: &crate::migrate::EmbeddedMigrations) -> std::collections::HashSet<String> {
14886        let migrations: Vec<Box<dyn Migration<RuntimeBackend>>> =
14887            set.migrations().unwrap_or_default();
14888        migrations.iter().map(|m| m.name().to_string()).collect()
14889    }
14890
14891    let mut control_only = names(&crate::migrate::FRAMEWORK_MIGRATIONS);
14892    for shard_required in [
14893        &crate::version_history::VERSION_HISTORY_MIGRATIONS,
14894        &crate::repository_commit_hooks::REPOSITORY_COMMIT_HOOK_MIGRATIONS,
14895        &crate::derivation::DERIVATION_MIGRATIONS,
14896    ] {
14897        for name in names(shard_required) {
14898            control_only.remove(&name);
14899        }
14900    }
14901
14902    names(set).iter().any(|name| control_only.contains(name))
14903}
14904
14905#[cfg(feature = "db")]
14906fn apply_replica_migration_readiness(
14907    state: &AppState,
14908    readiness: Option<crate::migrate::ReplicaMigrationReadiness>,
14909) {
14910    let Some(readiness) = readiness else {
14911        return;
14912    };
14913
14914    if readiness.is_ready() {
14915        state.probes().mark_replica_migrations_ready();
14916    } else if let Some(detail) = readiness.detail() {
14917        state.probes().mark_replica_migrations_unready(detail);
14918    }
14919}
14920
14921#[cfg(feature = "db")]
14922fn configure_replica_migration_check(state: &AppState, check: Option<(String, String)>) {
14923    let Some((primary_url, replica_url)) = check else {
14924        return;
14925    };
14926
14927    state
14928        .probes()
14929        .configure_replica_migration_check(primary_url, replica_url);
14930}
14931
14932/// Refuse to start when a `#[repository(api = ...)]`-mounted route
14933/// has no paired `policy = ...` argument in `prod` profile builds.
14934///
14935/// The issue text spells out the rationale: silently shipping
14936/// auto-generated CRUD endpoints with no record-level authz is a
14937/// security regression. The escape hatch is
14938/// `[security] allow_unauthorized_repository_api = true`.
14939/// Pure offender-collection logic for
14940/// [`validate_repository_api_policies`].
14941///
14942/// Walks both top-level routes and routes registered under
14943/// `.scoped(prefix, layer, routes)` groups, returning every
14944/// `#[repository(api = ...)]`-mounted *mutating* route that has no
14945/// paired `policy = ...` argument. Read-only mounts (GET
14946/// `*_api_list` / `*_api_get`) are intentionally excluded — they
14947/// don't fit the "any authenticated user can write to any record"
14948/// footgun the issue calls out. Read-leak concerns are handled
14949/// separately by `scope = ...`.
14950///
14951/// Returned in (resource type name, api path) form, deduped per
14952/// `(type, path)` pair so a repository with multiple unguarded
14953/// methods only shows up once.
14954fn collect_unguarded_repository_writes(
14955    routes: &[Route],
14956    scoped_groups: &[ScopedGroup],
14957) -> Vec<(String, String)> {
14958    let mut offenders: Vec<(String, String)> = Vec::new();
14959    let mut seen: std::collections::HashSet<(&'static str, &'static str)> =
14960        std::collections::HashSet::new();
14961    let mut record_route = |route: &Route| {
14962        if let Some(meta) = route.repository
14963            && !meta.has_policy
14964            && is_mutating_method(&route.method)
14965            && seen.insert((meta.resource_type_name, meta.api_path))
14966        {
14967            offenders.push((meta.resource_type_name.to_owned(), meta.api_path.to_owned()));
14968        }
14969    };
14970    for route in routes {
14971        record_route(route);
14972    }
14973    for group in scoped_groups {
14974        for route in &group.routes {
14975            record_route(route);
14976        }
14977    }
14978    offenders
14979}
14980
14981/// Format a list of `(type, path)` offenders into the bulleted
14982/// listing the startup tracing emits. Pure so the format string
14983/// can be unit-tested without going through `tracing` machinery.
14984fn format_unguarded_repository_listing(offenders: &[(String, String)]) -> String {
14985    use std::fmt::Write;
14986    let mut s = String::new();
14987    let mut first = true;
14988    for (name, path) in offenders {
14989        if !first {
14990            s.push('\n');
14991        }
14992        first = false;
14993        write!(s, "  - #[repository({name}, api = \"{path}\")]").unwrap();
14994    }
14995    s
14996}
14997
14998/// Fold `.mount_unsubscribe_endpoint()` into the loaded config.
14999///
15000/// The builder flag lives outside the config, so a config that leaves the
15001/// endpoint disabled still mounts it when the app asked for it. That mount
15002/// claims `/_autumn/unsubscribe`, which the `OpenAPI` and MCP collision checks
15003/// must see — an export that ran them against the unmodified config approved a
15004/// mount that startup rejects.
15005///
15006/// Shared because it was already copied at two `run_*` sites before the export
15007/// became the third.
15008#[cfg(feature = "mail")]
15009const fn apply_mail_builder_overrides(config: &mut AutumnConfig, mount_unsubscribe_endpoint: bool) {
15010    if mount_unsubscribe_endpoint {
15011        config.mail.mount_unsubscribe_endpoint = true;
15012    }
15013}
15014
15015/// Run the serving path's whole preflight for a no-boot export.
15016///
15017/// Split out of `run_dump_openapi_mode` for length once it reached nine checks.
15018/// Every one of them calls the router's (or `run()`'s) own function rather than
15019/// re-deriving its rule: a second copy would drift, and a preflight that
15020/// disagreed with the router about what it rejects — in EITHER direction — would
15021/// be worse than none.
15022///
15023/// `mcp_mount_path` is `None` when MCP is not configured, and unused when the
15024/// `mcp` feature is off.
15025/// Everything [`export_preflight`] needs, borrowed from the builder.
15026///
15027/// A context struct rather than nine parameters, following `RouterContext` —
15028/// which exists for the same reason on the serving side.
15029#[cfg(feature = "openapi")]
15030struct ExportPreflight<'a> {
15031    routes: &'a [Route],
15032    scoped_groups: &'a [ScopedGroup],
15033    api_versions: &'a [ApiVersion],
15034    openapi_config: &'a crate::openapi::OpenApiConfig,
15035    merge_routers: &'a [axum::Router<AppState>],
15036    nest_routers: &'a [(String, axum::Router<AppState>)],
15037    declared_routes: &'a [crate::route_listing::RouteInfo],
15038    config: &'a AutumnConfig,
15039    /// `None` when MCP is not configured; unused when the `mcp` feature is off.
15040    mcp_mount_path: Option<&'a str>,
15041}
15042
15043#[cfg(feature = "openapi")]
15044fn export_preflight(ctx: &ExportPreflight<'_>) -> Result<(), String> {
15045    let &ExportPreflight {
15046        routes,
15047        scoped_groups,
15048        api_versions,
15049        openapi_config,
15050        merge_routers,
15051        nest_routers,
15052        declared_routes,
15053        config,
15054        mcp_mount_path,
15055    } = ctx;
15056
15057    // `run()`'s OWN pre-router checks first, in its order: an app with no
15058    // routes, or with an unguarded mutating repository API under a
15059    // production profile, never reaches router construction at all. These hold
15060    // for EVERY role — `run()` performs them before it branches on one.
15061    validate_pre_router_preconditions(routes, scoped_groups, config)?;
15062
15063    // Everything below describes the application router, and a `worker` (or any
15064    // other non-HTTP) role never builds one: `run()` takes the probe-only branch
15065    // and none of these six rules execute. Enforcing them here would REJECT a
15066    // deployment that starts perfectly well — a route legitimately owning
15067    // `/openapi.json` under a worker profile, say — which is the same
15068    // disagreement with the serving path as being too lax, pointing the other
15069    // way, and the more expensive of the two because it blocks correct work.
15070    //
15071    // The document itself is still exported: it is built from the routes and the
15072    // `OpenApiConfig`, neither of which depends on the role, so the contract a
15073    // worker-profile export writes down is the same one the web role serves.
15074    if !config.role.serves_http() {
15075        return Ok(());
15076    }
15077
15078    // What the ROUTER would see for OpenAPI, resolved once and used by every
15079    // mount-sensitive check below, so the three cannot disagree about
15080    // whether the endpoint is mounted.
15081    let mounted_openapi = config.openapi_runtime.enabled.then_some(openapi_config);
15082
15083    let registered_versions: std::collections::HashSet<&str> =
15084        api_versions.iter().map(|av| av.version.as_str()).collect();
15085    let preflight = crate::router::reject_unregistered_api_versions(
15086        routes,
15087        scoped_groups,
15088        &registered_versions,
15089    )
15090    .and_then(|()| {
15091        crate::router::reject_duplicate_user_routes(
15092            routes,
15093            scoped_groups,
15094            merge_routers,
15095            nest_routers,
15096            declared_routes,
15097            config,
15098        )
15099    })
15100    .and_then(|()| {
15101        // The `[openapi]` profile gate decides whether the endpoint is
15102        // MOUNTED. `run()` hands `None` to the router when it is off, so
15103        // neither the path validation nor the collision check runs and an
15104        // application route may legitimately occupy `/openapi.json`.
15105        // Validating unconditionally made the exporter STRICTER than
15106        // startup — rejecting an app that boots fine — which is the same
15107        // class of disagreement as being laxer, just pointing the other
15108        // way. The document itself is still exported: the gate governs
15109        // serving, not whether the contract can be written down.
15110        mounted_openapi.map_or(Ok(()), crate::router::validate_openapi_mount_paths)
15111    })
15112    .and_then(|()| {
15113        crate::router::reject_openapi_path_collisions(
15114            mounted_openapi,
15115            routes,
15116            scoped_groups,
15117            merge_routers,
15118            nest_routers,
15119            config,
15120        )
15121    })
15122    .and_then(|()| {
15123        // The plugin-route half of the router's own mount check. `run()` hands
15124        // the router the SAME gated OpenAPI value and the MCP path.
15125        crate::router::reject_declared_routes_on_context_mounts(
15126            declared_routes,
15127            mounted_openapi,
15128            mcp_mount_path,
15129        )
15130    });
15131
15132    // MCP is part of the same preflight, not a separate concern: an app that
15133    // mounts MCP at a malformed path, or at one a user/OpenAPI route already
15134    // owns, is rejected by `build_router_pre_state` at startup. An export
15135    // that skipped these would certify a router that cannot be built — the
15136    // exact failure the four rules above exist to prevent, one subsystem
15137    // over. Both call the router's own function, so there is still one
15138    // definition per rule.
15139    #[cfg(feature = "mcp")]
15140    let preflight = preflight.and_then(|()| {
15141        let Some(path) = mcp_mount_path else {
15142            return Ok(());
15143        };
15144        crate::router::validate_mcp_mount_path(path).and_then(|()| {
15145            // The SAME gated value: `reject_mcp_path_collisions` reserves
15146            // the OpenAPI paths as claimed GETs, which they are not when the
15147            // endpoint is not mounted.
15148            crate::router::reject_mcp_path_collisions(
15149                path,
15150                routes,
15151                scoped_groups,
15152                config,
15153                mounted_openapi,
15154                merge_routers,
15155                nest_routers,
15156            )
15157        })
15158    });
15159
15160    preflight.map_err(|error| error.to_string())
15161}
15162
15163/// Append every framework-owned scheduled task to the declared list.
15164///
15165/// Two sources, both of which `run()` merges before validating names: the
15166/// `#[repository(..., retention(...))]` sweeps collected from `inventory`, and
15167/// the config-driven `[retention]` sweep. A no-boot export that validated only
15168/// the DECLARED list would miss the collision this check most often catches —
15169/// between a hand-declared `#[scheduled]` fn and one of these generated names.
15170///
15171/// `run()` performs these same two merges inline, at two different points in its
15172/// prologue — the repository sweeps before the config load, the framework one
15173/// after — so it cannot call this without a reordering. What is shared is the
15174/// RULE: each merge here is a single call to the same
15175/// `collect_retention_tasks` / `framework_retention_task` the serving path
15176/// calls, so only the sequencing is restated, not the logic.
15177#[cfg(feature = "openapi")]
15178fn merge_framework_scheduled_tasks(
15179    mut tasks: Vec<crate::task::TaskInfo>,
15180    config: &AutumnConfig,
15181) -> Vec<crate::task::TaskInfo> {
15182    #[cfg(feature = "db")]
15183    tasks.extend(crate::retention::collect_retention_tasks());
15184    tasks.extend(crate::data_retention::framework_retention_tasks(
15185        &config.retention,
15186    ));
15187    tasks
15188}
15189
15190/// Every CONFIG-only precondition the serving path enforces before it boots.
15191///
15192/// Sibling of [`validate_pre_router_preconditions`], which covers the
15193/// route-shaped ones. Both are things `run()` does that
15194/// `build_router_pre_state` does not, so an export that mirrored the router
15195/// faithfully still approved a spec for an app that refuses to start (issue
15196/// #802):
15197///
15198/// * a `web`/`worker` process role on a non-durable jobs backend — the web
15199///   replica would enqueue into an in-memory queue no worker can drain;
15200/// * a duplicate `#[scheduled]` task name, which spawns two loops competing for
15201///   one coordination lock.
15202///
15203/// `tasks` must ALREADY be the fully merged list — hand-declared entries plus
15204/// the `#[repository(..., retention(...))]` sweeps plus the framework's
15205/// `[retention]` sweep — because that is the list whose names actually collide.
15206/// Merging here instead double-counts against the caller's own merge: `run()`
15207/// pushes the framework sweep before reaching this, so synthesising it again
15208/// reported its fixed `autumn-retention-sweep` name as a duplicate and refused
15209/// to start every app with a `[retention]` window. Build the list once, with
15210/// [`merge_framework_scheduled_tasks`].
15211///
15212/// Returns the message to report; the caller decides how to fail, because the
15213/// serving path also has a database pool to stop on the way out and the export
15214/// has none.
15215fn validate_config_preconditions(
15216    config: &AutumnConfig,
15217    tasks: &[crate::task::TaskInfo],
15218) -> Result<(), String> {
15219    if crate::config::split_role_requires_durable_backend(config.role, &config.jobs.backend) {
15220        return Err(format!(
15221            "process role '{role}' requires a durable jobs backend: backend '{backend}' is not \
15222             a recognized durable backend and falls through to the in-process 'local' runtime, \
15223             which cannot be shared across a split web/worker topology. Set jobs.backend = \
15224             \"postgres\" or \"redis\", or run the combined role.",
15225            role = config.role.as_str(),
15226            backend = config.jobs.backend,
15227        ));
15228    }
15229
15230    crate::task::validate_unique_task_names(tasks.iter().map(|task| task.name.as_str()))?;
15231
15232    Ok(())
15233}
15234
15235/// Every route/config precondition the serving path enforces BEFORE it hands
15236/// off to [`crate::router::build_router_pre_state`].
15237///
15238/// That function's own six rules are already shared with the no-boot dump modes
15239/// (issue #802). These are the ones `run()` performed itself, inline and in two
15240/// separate places, so the export preflight did not have them: an app with no
15241/// routes at all, or with a mutating `#[repository(api = ...)]` carrying no
15242/// paired `policy`, could not start yet exported a document `--check` would
15243/// approve. Collecting them here means a rule added to the serving path is in
15244/// the exporter by construction rather than by remembering — which is what the
15245/// one-at-a-time additions of the last three rounds kept failing to be.
15246///
15247/// `Err` carries the message the caller should report; `validate_repository_api_policies`
15248/// exits the process itself when it is fatal, exactly as it does at startup, so
15249/// an export refuses on the same condition a boot would.
15250fn validate_pre_router_preconditions(
15251    routes: &[Route],
15252    scoped_groups: &[ScopedGroup],
15253    config: &AutumnConfig,
15254) -> Result<(), String> {
15255    if routes.is_empty() {
15256        return Err("No routes registered. Did you forget to call .routes()?".to_owned());
15257    }
15258    validate_repository_api_policies(routes, scoped_groups, config);
15259    Ok(())
15260}
15261
15262fn validate_repository_api_policies(
15263    routes: &[Route],
15264    scoped_groups: &[ScopedGroup],
15265    config: &AutumnConfig,
15266) {
15267    let profile = config.profile.as_deref().unwrap_or("default");
15268    let strict =
15269        is_production_profile(profile) && !config.security.allow_unauthorized_repository_api;
15270
15271    let offenders = collect_unguarded_repository_writes(routes, scoped_groups);
15272    if offenders.is_empty() {
15273        return;
15274    }
15275
15276    let listing = format_unguarded_repository_listing(&offenders);
15277
15278    if strict {
15279        tracing::error!(
15280            "refusing to start: the following #[repository(api = ...)] mutating endpoints have no paired `policy = ...` argument:\n{listing}\n\
15281             Add `policy = SomePolicy` to each, or set `[security] allow_unauthorized_repository_api = true` to opt out explicitly."
15282        );
15283        std::process::exit(1);
15284    }
15285    tracing::warn!(
15286        "the following #[repository(api = ...)] mutating endpoints have no paired `policy = ...` argument; \
15287         auto-generated POST/PUT/PATCH/DELETE handlers will accept writes from any authenticated user:\n{listing}\n\
15288         This will become a startup-time error in `prod` profile builds."
15289    );
15290}
15291
15292/// Refuse to start when a `#[repository(policy = X)]`-annotated
15293/// route exists but the corresponding `.policy::<R, _>(X)`
15294/// registration was never actually applied to the live
15295/// [`PolicyRegistry`](crate::authorization::PolicyRegistry).
15296///
15297/// `validate_repository_api_policies` runs *before* the registry is
15298/// populated and only checks the macro-set `has_policy` flag. This
15299/// runs *after* registrations are applied and walks the same routes,
15300/// invoking the macro-emitted `policy_check` probe to confirm the
15301/// policy is really there. Without this, forgetting the
15302/// `.policy::<R, _>(...)` builder call would compile, boot, and
15303/// then 500 on every protected request.
15304/// `(resource_type_name, api_path)` pair identifying a repository
15305/// route that's missing its required runtime registration.
15306type MissingRepositoryRegistration = (String, String);
15307
15308/// Pure offender-collection logic for
15309/// [`validate_repository_policies_registered`].
15310///
15311/// Walks the same routes + scoped groups and invokes the macro-
15312/// emitted `policy_check` / `scope_check` probes against the live
15313/// registry, returning `(missing_policies, missing_scopes)` deduped
15314/// per `(type, path)` pair. Pure so the listing logic can be unit-
15315/// tested without going through the actual `tracing::error!` +
15316/// `std::process::exit(1)` strict path.
15317fn collect_unregistered_repository_handlers(
15318    routes: &[Route],
15319    scoped_groups: &[ScopedGroup],
15320    registry: &crate::authorization::PolicyRegistry,
15321) -> (
15322    Vec<MissingRepositoryRegistration>,
15323    Vec<MissingRepositoryRegistration>,
15324) {
15325    let mut missing_policies: Vec<(String, String)> = Vec::new();
15326    let mut missing_scopes: Vec<(String, String)> = Vec::new();
15327    let mut seen_policies: std::collections::HashSet<(&'static str, &'static str)> =
15328        std::collections::HashSet::new();
15329    let mut seen_scopes: std::collections::HashSet<(&'static str, &'static str)> =
15330        std::collections::HashSet::new();
15331    let mut record_route = |route: &Route| {
15332        if let Some(meta) = route.repository {
15333            if let Some(check) = meta.policy_check
15334                && !check(registry)
15335                && seen_policies.insert((meta.resource_type_name, meta.api_path))
15336            {
15337                missing_policies
15338                    .push((meta.resource_type_name.to_owned(), meta.api_path.to_owned()));
15339            }
15340            if let Some(check) = meta.scope_check
15341                && !check(registry)
15342                && seen_scopes.insert((meta.resource_type_name, meta.api_path))
15343            {
15344                missing_scopes.push((meta.resource_type_name.to_owned(), meta.api_path.to_owned()));
15345            }
15346        }
15347    };
15348    for route in routes {
15349        record_route(route);
15350    }
15351    for group in scoped_groups {
15352        for route in &group.routes {
15353            record_route(route);
15354        }
15355    }
15356    (missing_policies, missing_scopes)
15357}
15358
15359/// Format a `(type, path)` listing for missing-policy startup
15360/// errors. Pure so the format string can be unit-tested.
15361fn format_missing_policy_listing(missing: &[(String, String)]) -> String {
15362    use std::fmt::Write;
15363    let mut s = String::new();
15364    let mut first = true;
15365    for (name, path) in missing {
15366        if !first {
15367            s.push('\n');
15368        }
15369        first = false;
15370        write!(s, "  - #[repository({name}, api = \"{path}\", policy = ...)]: call `.policy::<{name}, _>(...)` on the app builder").unwrap();
15371    }
15372    s
15373}
15374
15375/// Format a `(type, path)` listing for missing-scope startup
15376/// errors. Pure so the format string can be unit-tested.
15377fn format_missing_scope_listing(missing: &[(String, String)]) -> String {
15378    use std::fmt::Write;
15379    let mut s = String::new();
15380    let mut first = true;
15381    for (name, path) in missing {
15382        if !first {
15383            s.push('\n');
15384        }
15385        first = false;
15386        write!(s, "  - #[repository({name}, api = \"{path}\", scope = ...)]: call `.scope::<{name}, _>(...)` on the app builder").unwrap();
15387    }
15388    s
15389}
15390
15391#[allow(clippy::cognitive_complexity)]
15392/// Takes the `PolicyRegistry` rather than the whole `AppState` — which is all
15393/// `collect_unregistered_repository_handlers` ever needed — so the no-boot
15394/// export can run this rule too, against a throwaway registry the deferred
15395/// registrations are replayed onto. Requiring `AppState` would have meant
15396/// building state, which is exactly what an export advertised as opening no
15397/// database must not do (issue #802).
15398fn validate_repository_policies_registered(
15399    routes: &[Route],
15400    scoped_groups: &[ScopedGroup],
15401    registry: &crate::authorization::PolicyRegistry,
15402    config: &AutumnConfig,
15403) {
15404    let profile = config.profile.as_deref().unwrap_or("default");
15405    let strict = is_production_profile(profile);
15406
15407    let (missing_policies, missing_scopes) =
15408        collect_unregistered_repository_handlers(routes, scoped_groups, registry);
15409
15410    if missing_policies.is_empty() && missing_scopes.is_empty() {
15411        return;
15412    }
15413
15414    if !missing_policies.is_empty() {
15415        let listing = format_missing_policy_listing(&missing_policies);
15416
15417        if strict {
15418            tracing::error!(
15419                "refusing to start: the following #[repository] routes declare a `policy = ...` argument, but no policy is registered for the resource type. Without registration, every protected request would fail at runtime with `500 no policy registered`:\n{listing}"
15420            );
15421        } else {
15422            tracing::warn!(
15423                "the following #[repository] routes declare `policy = ...` but no matching `.policy::<R, _>(...)` registration is on the app builder. Protected requests will 500 at runtime:\n{listing}\n\
15424                 This will become a startup-time error in `prod` profile builds."
15425            );
15426        }
15427    }
15428
15429    if !missing_scopes.is_empty() {
15430        let listing = format_missing_scope_listing(&missing_scopes);
15431
15432        if strict {
15433            tracing::error!(
15434                "refusing to start: the following #[repository] routes declare a `scope = ...` argument, but no scope is registered for the resource type. Without registration, every list request would fail at runtime with `500 missing scope registration`:\n{listing}"
15435            );
15436        } else {
15437            tracing::warn!(
15438                "the following #[repository] routes declare `scope = ...` but no matching `.scope::<R, _>(...)` registration is on the app builder. List requests will 500 at runtime:\n{listing}\n\
15439                 This will become a startup-time error in `prod` profile builds."
15440            );
15441        }
15442    }
15443
15444    if strict {
15445        std::process::exit(1);
15446    }
15447}
15448
15449const fn is_mutating_method(method: &http::Method) -> bool {
15450    matches!(
15451        *method,
15452        http::Method::POST | http::Method::PUT | http::Method::PATCH | http::Method::DELETE
15453    )
15454}
15455
15456/// Returns `true` for the framework's accepted production profile
15457/// names. Mirrors the `prod | production` matching used elsewhere
15458/// (`app.rs::run_build_mode`, `migrate.rs::should_auto_apply`,
15459/// etc.) so the repository startup guards don't silently weaken in
15460/// deployments that pick the long-form alias.
15461fn is_production_profile(profile: &str) -> bool {
15462    matches!(profile, "prod" | "production")
15463}
15464
15465#[cfg(test)]
15466mod agent_authority_route_summary_tests {
15467
15468    use super::*;
15469
15470    fn route_with(path: &'static str, mcp_tool: bool) -> Route {
15471        let mut api_doc = crate::openapi::ApiDoc {
15472            method: "GET",
15473            path,
15474            operation_id: "list_items",
15475            mcp_tool,
15476            ..crate::openapi::ApiDoc::default()
15477        };
15478        // The exposure rule is JSON-out gated, so a route with no response
15479        // schema is never a tool no matter what it is tagged with. Give it one.
15480        api_doc.response = Some(crate::openapi::SchemaEntry {
15481            name: "Item",
15482            kind: crate::openapi::SchemaKind::Ref,
15483            identity: None,
15484        });
15485        Route {
15486            method: http::Method::GET,
15487            path,
15488            handler: axum::routing::any(|| async { "" }),
15489            name: "list_items",
15490            api_doc,
15491            repository: None,
15492            idempotency: crate::route::RouteIdempotency::Direct,
15493            timeout: crate::route::RouteTimeout::Inherit,
15494            criticality: crate::admission::Criticality::Default,
15495            seo: crate::seo::SeoRouteDefaults::EMPTY,
15496            api_version: None,
15497            sunset_opt_out: false,
15498        }
15499    }
15500
15501    #[test]
15502    fn a_scoped_route_records_the_path_an_agent_actually_calls() {
15503        // A scoped group's children carry only the child path -- the prefix is
15504        // applied at mount time. Recording `/items` for a tool served at
15505        // `/api/v1/items` would make the manifest wrong about where the agent
15506        // surface is, and a scope rename would produce no drift at all.
15507        let route = route_with("/items", true);
15508        let summary =
15509            agent_authority_route_summary(&route, Some("/api/v1"), AgentExposure::default());
15510        assert_eq!(summary.path, "/api/v1/items");
15511        assert_eq!(summary.method, "GET");
15512        assert!(summary.mcp_tool);
15513
15514        // A top-level route has no prefix and is unchanged.
15515        let top = agent_authority_route_summary(&route, None, AgentExposure::default());
15516        assert_eq!(top.path, "/items");
15517    }
15518
15519    #[test]
15520    fn a_scoped_root_route_joins_the_way_the_openapi_collector_does() {
15521        // Delegated to `join_nested_path` rather than string concatenation, so
15522        // the manifest and the spec cannot disagree about trailing slashes.
15523        let root = route_with("/", true);
15524        let summary = agent_authority_route_summary(&root, Some("/api"), AgentExposure::default());
15525        assert_eq!(
15526            summary.path,
15527            crate::router::join_nested_path("/api", "/"),
15528            "the manifest must join paths exactly as the spec does"
15529        );
15530    }
15531
15532    #[test]
15533    fn the_operation_id_names_the_tool_and_exposure_says_why() {
15534        use crate::agent_authority::manifest::McpExposedBy;
15535
15536        let tagged = agent_authority_route_summary(
15537            &route_with("/items", true),
15538            None,
15539            AgentExposure::default(),
15540        );
15541        assert_eq!(tagged.operation_id, "list_items");
15542        assert_eq!(tagged.exposed_by, Some(McpExposedBy::Attribute));
15543
15544        // Untagged and no hatch: not a tool at all.
15545        let plain = agent_authority_route_summary(
15546            &route_with("/items", false),
15547            None,
15548            AgentExposure::default(),
15549        );
15550        assert!(!plain.mcp_tool);
15551        assert_eq!(plain.exposed_by, None);
15552
15553        // Untagged, but the whole-API hatch sweeps up a read-only verb.
15554        let hatched = agent_authority_route_summary(
15555            &route_with("/items", false),
15556            None,
15557            AgentExposure {
15558                mcp_expose_all: true,
15559                webmcp_expose_all: false,
15560            },
15561        );
15562        assert!(hatched.mcp_tool);
15563        assert_eq!(hatched.exposed_by, Some(McpExposedBy::Hatch));
15564        assert!(
15565            !hatched.webmcp_tool,
15566            "the MCP hatch is not the WebMCP hatch"
15567        );
15568    }
15569
15570    #[test]
15571    fn webmcp_exposure_is_reported_apart_from_mcp() {
15572        use crate::agent_authority::manifest::McpExposedBy;
15573
15574        let mut route = route_with("/items", true);
15575        route.api_doc.webmcp_tool = true;
15576        let both = agent_authority_route_summary(&route, None, AgentExposure::default());
15577        assert!(both.mcp_tool && both.webmcp_tool);
15578        assert_eq!(both.webmcp_exposed_by, Some(McpExposedBy::Attribute));
15579
15580        let web_hatch = agent_authority_route_summary(
15581            &route_with("/items", false),
15582            None,
15583            AgentExposure {
15584                mcp_expose_all: false,
15585                webmcp_expose_all: true,
15586            },
15587        );
15588        assert!(!web_hatch.mcp_tool);
15589        assert_eq!(web_hatch.webmcp_exposed_by, Some(McpExposedBy::Hatch));
15590
15591        let mut excluded = route_with("/items", false);
15592        excluded.api_doc.webmcp_exclude = true;
15593        let excluded = agent_authority_route_summary(
15594            &excluded,
15595            None,
15596            AgentExposure {
15597                mcp_expose_all: false,
15598                webmcp_expose_all: true,
15599            },
15600        );
15601        assert!(!excluded.webmcp_tool);
15602    }
15603}
15604
15605#[cfg(all(test, feature = "sqlite"))]
15606mod fleet_boot_tests {
15607    use super::*;
15608
15609    async fn table_exists(db: &crate::db::fleet::FleetDatabase, table: &str) -> bool {
15610        use diesel_async::RunQueryDsl as _;
15611        #[derive(diesel::QueryableByName)]
15612        struct Count {
15613            #[diesel(sql_type = diesel::sql_types::BigInt)]
15614            n: i64,
15615        }
15616        let mut conn = db.pool().get().await.unwrap();
15617        diesel::sql_query(
15618            "SELECT COUNT(*) AS n FROM sqlite_master WHERE type = 'table' AND name = ?",
15619        )
15620        .bind::<diesel::sql_types::Text, _>(table)
15621        .get_result::<Count>(&mut *conn)
15622        .await
15623        .unwrap()
15624        .n == 1
15625    }
15626
15627    /// A symlinked fleet root that resolves to the control database's
15628    /// directory passes the lexical config check, so boot compares the
15629    /// canonical paths: tenant `control` must never open the control database.
15630    #[cfg(unix)]
15631    #[test]
15632    #[allow(
15633        clippy::literal_string_with_formatting_args,
15634        reason = "fleet path templates use {placeholders}"
15635    )]
15636    fn boot_refuses_a_fleet_root_that_aliases_the_control_database() {
15637        let tmp = tempfile::tempdir().unwrap();
15638        let data = tmp.path().join("data");
15639        std::fs::create_dir(&data).unwrap();
15640        let link = tmp.path().join("fleet-link");
15641        std::os::unix::fs::symlink(&data, &link).unwrap();
15642        let mut config = AutumnConfig {
15643            profile: Some("dev".into()),
15644            ..AutumnConfig::default()
15645        };
15646        config.database.url = Some(format!("sqlite://{}", data.join("control.db").display()));
15647        config.database.fleet = Some(crate::config::DatabaseFleetConfig {
15648            mode: crate::fleet_layout::FleetMode::Tenant,
15649            root: link.display().to_string(),
15650            path: Some("{tenant}.db".to_owned()),
15651            max_open: 4,
15652            pool_size: 1,
15653            create_on_demand: None,
15654            idle_close_secs: 0,
15655            restore_missing: false,
15656            ownership: None,
15657        });
15658        config
15659            .database
15660            .validate()
15661            .expect("lexically, the paths differ");
15662        let Err(err) =
15663            crate::db::fleet::build_for_app(&config, None, &Arc::new(Vec::new()), |_| true)
15664        else {
15665            panic!("a fleet root aliasing the control database must be refused");
15666        };
15667        assert!(err.contains("control database"), "{err}");
15668    }
15669
15670    /// A custom pool provider can resolve the control database at runtime
15671    /// (the topology's `migration_url`). That target is checked too: config
15672    /// pointing outside the root does not let a resolved control file inside
15673    /// it through.
15674    #[test]
15675    #[allow(
15676        clippy::literal_string_with_formatting_args,
15677        reason = "fleet path templates use {placeholders}"
15678    )]
15679    fn boot_refuses_a_provider_resolved_control_database_inside_the_root() {
15680        let tmp = tempfile::tempdir().unwrap();
15681        let root = tmp.path().join("fleet");
15682        std::fs::create_dir(&root).unwrap();
15683        let mut config = AutumnConfig {
15684            profile: Some("dev".into()),
15685            ..AutumnConfig::default()
15686        };
15687        config.database.url = Some(format!(
15688            "sqlite://{}",
15689            tmp.path().join("control.db").display()
15690        ));
15691        config.database.fleet = Some(crate::config::DatabaseFleetConfig {
15692            mode: crate::fleet_layout::FleetMode::Tenant,
15693            root: root.display().to_string(),
15694            path: Some("{tenant}.db".to_owned()),
15695            max_open: 4,
15696            pool_size: 1,
15697            create_on_demand: None,
15698            idle_close_secs: 0,
15699            restore_missing: false,
15700            ownership: None,
15701        });
15702        let resolved = format!("sqlite://{}", root.join("control.db").display());
15703        let Err(err) = crate::db::fleet::build_for_app(
15704            &config,
15705            Some(&resolved),
15706            &Arc::new(Vec::new()),
15707            |_| true,
15708        ) else {
15709            panic!("a resolved control database inside the root must be refused");
15710        };
15711        assert!(err.contains("control database"), "{err}");
15712        crate::db::fleet::build_for_app(&config, None, &Arc::new(Vec::new()), |_| true)
15713            .expect("the configured control database is outside the root");
15714    }
15715
15716    /// A custom pool provider must say which file its control pool opens
15717    /// (`DatabaseTopology::with_migration_url`), or the fleet cannot rule out
15718    /// that it lies inside the root: boot refuses rather than guess.
15719    #[test]
15720    fn a_fleet_requires_a_custom_provider_to_name_its_control_database() {
15721        let err = fleet_control_target_identified(true, None).unwrap_err();
15722        assert!(err.contains("with_migration_url"), "{err}");
15723        fleet_control_target_identified(true, Some("sqlite:///srv/control.db"))
15724            .expect("named: checked against the root");
15725        // Named, but not a file: framework state would vanish on restart.
15726        for target in [
15727            "sqlite::memory:",
15728            "file::memory:?cache=shared",
15729            "postgres://db/app",
15730        ] {
15731            let err = fleet_control_target_identified(true, Some(target)).unwrap_err();
15732            assert!(err.contains("file-backed"), "{target}: {err}");
15733        }
15734        fleet_control_target_identified(false, None)
15735            .expect("the built-in provider opens the configured url, already checked");
15736    }
15737
15738    /// The boot path builds the fleet from the same folded list it migrates
15739    /// the control database with, minus the control-plane set: tenant files
15740    /// get the shard framework tables, never `api_tokens` or the job queue.
15741    #[tokio::test]
15742    async fn the_boot_fleet_gets_the_shard_sets_and_never_the_control_set() {
15743        let tmp = tempfile::tempdir().unwrap();
15744        let mut config = AutumnConfig {
15745            profile: Some("dev".into()),
15746            ..AutumnConfig::default()
15747        };
15748        config.database.url = Some(format!(
15749            "sqlite://{}",
15750            tmp.path().join("control.db").display()
15751        ));
15752        config.database.fleet = Some(crate::config::DatabaseFleetConfig {
15753            mode: crate::fleet_layout::FleetMode::Slot,
15754            root: tmp.path().join("fleet").display().to_string(),
15755            path: None,
15756            max_open: 4,
15757            pool_size: 1,
15758            create_on_demand: None,
15759            idle_close_secs: 0,
15760            restore_missing: false,
15761            ownership: None,
15762        });
15763        let all = Arc::new(vec![
15764            ("framework", crate::migrate::FRAMEWORK_MIGRATIONS),
15765            (
15766                "version-history",
15767                crate::version_history::VERSION_HISTORY_MIGRATIONS,
15768            ),
15769        ]);
15770        let fleet = crate::db::fleet::build_for_app(&config, None, &all, |set| {
15771            !migration_set_is_control_framework(set)
15772        })
15773        .unwrap()
15774        .expect("a fleet is configured");
15775        let db = fleet.open_for("acme").await.unwrap();
15776        assert!(table_exists(&db, "_autumn_version_history").await);
15777        assert!(!table_exists(&db, "api_tokens").await);
15778        assert!(!table_exists(&db, "autumn_jobs").await);
15779
15780        config.database.fleet = None;
15781        assert!(
15782            crate::db::fleet::build_for_app(&config, None, &Arc::new(Vec::new()), |_| true)
15783                .unwrap()
15784                .is_none()
15785        );
15786    }
15787
15788    /// A control database plus a slot fleet under `dir`, the way `run()` sees
15789    /// them: the config, the built fleet, and an `AppState` carrying it.
15790    #[allow(
15791        clippy::literal_string_with_formatting_args,
15792        reason = "fleet path templates use {placeholders}"
15793    )]
15794    fn booted(dir: &std::path::Path) -> (AutumnConfig, crate::db::fleet::DatabaseFleet, AppState) {
15795        let mut config = AutumnConfig {
15796            profile: Some("dev".into()),
15797            ..AutumnConfig::default()
15798        };
15799        config.database.url = Some(format!("sqlite://{}", dir.join("control.db").display()));
15800        config.database.fleet = Some(crate::config::DatabaseFleetConfig {
15801            mode: crate::fleet_layout::FleetMode::Slot,
15802            root: dir.join("fleet").display().to_string(),
15803            path: None,
15804            max_open: 4,
15805            pool_size: 1,
15806            create_on_demand: None,
15807            idle_close_secs: 0,
15808            restore_missing: false,
15809            ownership: None,
15810        });
15811        let all = Arc::new(vec![("framework", crate::migrate::FRAMEWORK_MIGRATIONS)]);
15812        let fleet = crate::db::fleet::build_for_app(&config, None, &all, |set| {
15813            !migration_set_is_control_framework(set)
15814        })
15815        .unwrap()
15816        .expect("a fleet is configured");
15817        let control = crate::db::create_pool(&config.database)
15818            .unwrap()
15819            .expect("a control url is configured");
15820        let state = AppState::for_test().with_pool(control.clone()).with_shards(
15821            crate::sharding::ShardSet::from_fleet(fleet.clone(), control),
15822        );
15823        (config, fleet, state)
15824    }
15825
15826    /// `run()`'s fleet runtime: every open database gets a commit-hook worker,
15827    /// the pending-work sweeper runs until its token fires, and cancelling the
15828    /// token closes every database before the task ends.
15829    #[tokio::test]
15830    async fn the_fleet_runtime_closes_every_database_on_shutdown() {
15831        let tmp = tempfile::tempdir().unwrap();
15832        let (_config, fleet, state) = booted(tmp.path());
15833        let (shutdown, sweep) = (
15834            tokio_util::sync::CancellationToken::new(),
15835            tokio_util::sync::CancellationToken::new(),
15836        );
15837        let runtime = start_fleet_runtime(&state, shutdown.clone(), Some(sweep.clone()))
15838            .await
15839            .expect("the app has a fleet");
15840        let names: Vec<String> = state
15841            .health_indicator_registry()
15842            .run_all()
15843            .await
15844            .into_iter()
15845            .map(|result| result.name)
15846            .collect();
15847        assert!(
15848            names.iter().any(|name| name == "fleet:sweeper"),
15849            "{names:?}"
15850        );
15851        let db = fleet.open_for("acme").await.unwrap();
15852        // The worker runs against the database's own pool.
15853        assert!(!db.closed().is_cancelled());
15854        drop(db);
15855        assert_eq!(fleet.stats().open, 1);
15856        sweep.cancel();
15857        runtime
15858            .sweeper
15859            .expect("a server runs the sweeper loop")
15860            .await
15861            .unwrap();
15862        shutdown.cancel();
15863        runtime.maintenance.await.unwrap();
15864        assert_eq!(fleet.stats().open, 0, "shutdown closed every database");
15865
15866        assert!(
15867            start_fleet_runtime(&AppState::for_test(), shutdown, None)
15868                .await
15869                .is_none(),
15870            "no fleet, no runtime"
15871        );
15872    }
15873
15874    /// `[database.fleet.ownership]` (issue #3233): boot claims the ranges on
15875    /// the control database before the first open, registers
15876    /// `fleet:ownership`, opens under the epoch, and releases at shutdown.
15877    #[tokio::test]
15878    async fn the_fleet_runtime_claims_and_releases_slot_ownership() {
15879        use crate::db::fleet_ownership::{SlotOwnershipStore as _, SqlSlotOwnership};
15880        let tmp = tempfile::tempdir().unwrap();
15881        let (mut config, fleet, state) = booted(tmp.path());
15882        if let Some(fleet_config) = config.database.fleet.as_mut() {
15883            fleet_config.ownership = Some(crate::config::FleetOwnershipConfig {
15884                owner: "host-a".to_owned(),
15885                slots: vec![crate::config::SlotSpec::Range("0-16383".to_owned())],
15886                lease_secs: 30,
15887            });
15888        }
15889        state.insert_extension(config);
15890        let shutdown = tokio_util::sync::CancellationToken::new();
15891        let runtime = start_fleet_runtime(&state, shutdown.clone(), None)
15892            .await
15893            .expect("the app has a fleet");
15894        assert!(fleet.ownership().is_some());
15895        assert!(
15896            state
15897                .health_indicator_registry()
15898                .contains(crate::db::fleet_ownership::OWNERSHIP_INDICATOR)
15899        );
15900        let db = fleet.open_for("acme").await.unwrap();
15901        assert_eq!(db.epoch().map(crate::lock::FencingToken::get), Some(1));
15902        drop(db);
15903
15904        shutdown.cancel();
15905        runtime.maintenance.await.unwrap();
15906        runtime.ownership.expect("ownership runs").finish().await;
15907        let store = SqlSlotOwnership::new(state.pool().cloned().unwrap(), state.clock_arc());
15908        let rows = store.rows().await.unwrap();
15909        assert_eq!(rows.len(), 1);
15910        assert!(
15911            rows[0].lease_expires_ms <= state.clock().now().timestamp_millis(),
15912            "released at shutdown"
15913        );
15914    }
15915
15916    /// The fleet did not close within the shutdown budget: renewals stop, but
15917    /// nothing is released while databases may still be open.
15918    #[tokio::test]
15919    async fn a_fleet_that_did_not_close_stops_renewing_without_a_release() {
15920        use crate::db::fleet_ownership::{SlotOwnershipStore as _, SqlSlotOwnership};
15921        let tmp = tempfile::tempdir().unwrap();
15922        let (mut config, _fleet, state) = booted(tmp.path());
15923        if let Some(fleet_config) = config.database.fleet.as_mut() {
15924            fleet_config.ownership = Some(crate::config::FleetOwnershipConfig {
15925                owner: "host-a".to_owned(),
15926                slots: vec![crate::config::SlotSpec::Range("0-16383".to_owned())],
15927                lease_secs: 30,
15928            });
15929        }
15930        state.insert_extension(config);
15931        let shutdown = tokio_util::sync::CancellationToken::new();
15932        let runtime = start_fleet_runtime(&state, shutdown.clone(), None)
15933            .await
15934            .expect("the app has a fleet");
15935        let renewals = runtime.ownership.expect("ownership runs").abandon();
15936        tokio::time::timeout(std::time::Duration::from_secs(5), renewals)
15937            .await
15938            .expect("the renew loop stops")
15939            .unwrap();
15940        let store = SqlSlotOwnership::new(state.pool().cloned().unwrap(), state.clock_arc());
15941        let rows = store.rows().await.unwrap();
15942        assert!(
15943            rows[0].lease_expires_ms > state.clock().now().timestamp_millis(),
15944            "not released: the leases lapse"
15945        );
15946        shutdown.cancel();
15947        runtime.maintenance.await.unwrap();
15948    }
15949
15950    /// `[replication]` on a fleet app: the fleet replicates next to the control
15951    /// database, reports `replication:fleet`, and refuses a second install.
15952    #[tokio::test]
15953    async fn fleet_replication_installs_beside_the_control_database() {
15954        let tmp = tempfile::tempdir().unwrap();
15955        let (config, fleet, state) = booted(tmp.path());
15956        let replicas = tmp.path().join("replicas");
15957        std::fs::create_dir(&replicas).unwrap();
15958        let replication_config = crate::config::ReplicationConfig {
15959            enabled: true,
15960            path: Some(replicas.display().to_string()),
15961            ..crate::config::ReplicationConfig::default()
15962        };
15963        let control_url = config.database.url.clone().unwrap();
15964        let runtime = crate::time::spawn_blocking(move || {
15965            crate::replication::build(
15966                &replication_config,
15967                &control_url,
15968                "dev",
15969                None,
15970                Arc::new(crate::time::SystemClock),
15971            )
15972        })
15973        .await
15974        .unwrap()
15975        .expect("a file destination builds");
15976        let lag = std::time::Duration::from_secs(30);
15977
15978        start_fleet_replication(&state, &config, &runtime, lag).unwrap();
15979        assert!(fleet.replicating());
15980        assert!(
15981            state
15982                .health_indicator_registry()
15983                .contains(crate::db::fleet_replication::FLEET_REPLICATION_INDICATOR)
15984        );
15985        let again = start_fleet_replication(&state, &config, &runtime, lag).unwrap_err();
15986        assert!(again.contains("already replicates"), "{again}");
15987        if let Some(replication) = fleet.replication() {
15988            replication.stop();
15989        }
15990
15991        start_fleet_replication(&AppState::for_test(), &config, &runtime, lag)
15992            .expect("no fleet, nothing to replicate");
15993    }
15994
15995    /// `AUTUMN_MIGRATE=1` on a fleet app migrates every database on disk and
15996    /// reports the count; an app without a fleet migrates nothing.
15997    #[tokio::test]
15998    async fn migrate_mode_migrates_every_fleet_database() {
15999        let tmp = tempfile::tempdir().unwrap();
16000        let (mut config, fleet, _state) = booted(tmp.path());
16001        for tenant in ["acme", "globex"] {
16002            fleet.open_for(tenant).await.unwrap();
16003        }
16004        fleet.close_all().await;
16005        let all = Arc::new(vec![("framework", crate::migrate::FRAMEWORK_MIGRATIONS)]);
16006        // Created databases are migrated already: nothing is pending.
16007        assert_eq!(migrate_fleet_or_exit(&config, Arc::clone(&all)).await, 0);
16008        config.database.fleet = None;
16009        assert_eq!(migrate_fleet_or_exit(&config, all).await, 0);
16010    }
16011}
16012
16013#[cfg(test)]
16014mod validate_repository_api_policies_tests {
16015    use super::*;
16016    use crate::RepositoryApiMeta;
16017
16018    fn build_route(
16019        method: http::Method,
16020        path: &'static str,
16021        meta: Option<RepositoryApiMeta>,
16022    ) -> Route {
16023        Route {
16024            method,
16025            path,
16026            handler: axum::routing::any(|| async { "" }),
16027            name: "test_route",
16028            api_doc: crate::openapi::ApiDoc::default(),
16029            repository: meta,
16030            idempotency: crate::route::RouteIdempotency::Direct,
16031            timeout: crate::route::RouteTimeout::Inherit,
16032            criticality: crate::admission::Criticality::Default,
16033            seo: crate::seo::SeoRouteDefaults::EMPTY,
16034            api_version: None,
16035            sunset_opt_out: false,
16036        }
16037    }
16038
16039    fn unguarded(path: &'static str, type_name: &'static str) -> RepositoryApiMeta {
16040        RepositoryApiMeta {
16041            resource_type_name: type_name,
16042            api_path: path,
16043            has_policy: false,
16044            policy_check: None,
16045            scope_check: None,
16046        }
16047    }
16048
16049    /// Tests in this module historically used a duplicated copy of
16050    /// the offender-collection logic. Now they call the production
16051    /// helper directly so coverage tracks the real code path.
16052    fn collect_offenders(routes: &[Route]) -> Vec<(String, String)> {
16053        collect_unguarded_repository_writes(routes, &[])
16054    }
16055
16056    #[test]
16057    fn read_only_mount_without_policy_is_not_an_offender() {
16058        let routes = vec![
16059            build_route(
16060                http::Method::GET,
16061                "/api/posts",
16062                Some(unguarded("/api/posts", "Post")),
16063            ),
16064            build_route(
16065                http::Method::GET,
16066                "/api/posts/{id}",
16067                Some(unguarded("/api/posts", "Post")),
16068            ),
16069        ];
16070        let offenders = collect_offenders(&routes);
16071        assert!(
16072            offenders.is_empty(),
16073            "read-only mounts should not trigger the unauthorized-repo guard"
16074        );
16075    }
16076
16077    #[test]
16078    fn write_mount_without_policy_is_an_offender() {
16079        let routes = vec![build_route(
16080            http::Method::POST,
16081            "/api/posts",
16082            Some(unguarded("/api/posts", "Post")),
16083        )];
16084        let offenders = collect_offenders(&routes);
16085        assert_eq!(offenders.len(), 1);
16086        assert_eq!(offenders[0].0, "Post");
16087        assert_eq!(offenders[0].1, "/api/posts");
16088    }
16089
16090    #[test]
16091    fn mixed_mount_only_dedups_one_offender_per_repository() {
16092        let routes = vec![
16093            build_route(
16094                http::Method::GET,
16095                "/api/posts",
16096                Some(unguarded("/api/posts", "Post")),
16097            ),
16098            build_route(
16099                http::Method::POST,
16100                "/api/posts",
16101                Some(unguarded("/api/posts", "Post")),
16102            ),
16103            build_route(
16104                http::Method::PUT,
16105                "/api/posts/{id}",
16106                Some(unguarded("/api/posts", "Post")),
16107            ),
16108            build_route(
16109                http::Method::DELETE,
16110                "/api/posts/{id}",
16111                Some(unguarded("/api/posts", "Post")),
16112            ),
16113        ];
16114        let offenders = collect_offenders(&routes);
16115        assert_eq!(offenders.len(), 1);
16116    }
16117
16118    #[test]
16119    fn is_mutating_method_classifies_methods() {
16120        assert!(is_mutating_method(&http::Method::POST));
16121        assert!(is_mutating_method(&http::Method::PUT));
16122        assert!(is_mutating_method(&http::Method::PATCH));
16123        assert!(is_mutating_method(&http::Method::DELETE));
16124        assert!(!is_mutating_method(&http::Method::GET));
16125        assert!(!is_mutating_method(&http::Method::HEAD));
16126        assert!(!is_mutating_method(&http::Method::OPTIONS));
16127    }
16128
16129    // ── registry-aware validation (post-registration) ─────────────
16130
16131    use crate::authorization::{Policy, PolicyRegistry};
16132
16133    #[derive(Debug, Clone, PartialEq)]
16134    struct TestPost;
16135
16136    #[derive(Default)]
16137    struct TestPostPolicy;
16138    impl Policy<TestPost> for TestPostPolicy {}
16139
16140    fn guarded_with_check(path: &'static str, type_name: &'static str) -> RepositoryApiMeta {
16141        RepositoryApiMeta {
16142            resource_type_name: type_name,
16143            api_path: path,
16144            has_policy: true,
16145            policy_check: Some(|registry: &PolicyRegistry| registry.has_policy::<TestPost>()),
16146            scope_check: None,
16147        }
16148    }
16149
16150    fn collect_missing(routes: &[Route], registry: &PolicyRegistry) -> Vec<(String, String)> {
16151        let (missing_policies, _) = collect_unregistered_repository_handlers(routes, &[], registry);
16152        missing_policies
16153    }
16154
16155    #[test]
16156    fn registry_check_flags_routes_missing_their_policy_registration() {
16157        // Macro emits `policy = X` but no `.policy::<TestPost, _>(...)`
16158        // call on the builder — registry has nothing.
16159        let registry = PolicyRegistry::default();
16160        let routes = vec![build_route(
16161            http::Method::POST,
16162            "/api/posts",
16163            Some(guarded_with_check("/api/posts", "TestPost")),
16164        )];
16165        let missing = collect_missing(&routes, &registry);
16166        assert_eq!(missing.len(), 1);
16167        assert_eq!(missing[0].0, "TestPost");
16168        assert_eq!(missing[0].1, "/api/posts");
16169    }
16170
16171    #[test]
16172    fn registry_check_passes_when_policy_is_registered() {
16173        let registry = PolicyRegistry::default();
16174        registry.register_policy::<TestPost, _>(TestPostPolicy);
16175        let routes = vec![build_route(
16176            http::Method::POST,
16177            "/api/posts",
16178            Some(guarded_with_check("/api/posts", "TestPost")),
16179        )];
16180        let missing = collect_missing(&routes, &registry);
16181        assert!(missing.is_empty(), "policy is registered, no offenders");
16182    }
16183
16184    #[test]
16185    fn registry_check_skips_routes_without_policy_check_fn() {
16186        // Routes mounted without `policy = ...` carry
16187        // `policy_check: None` and are not subject to this check —
16188        // they're handled by `validate_repository_api_policies` which
16189        // looks at `has_policy` instead.
16190        let registry = PolicyRegistry::default();
16191        let routes = vec![build_route(
16192            http::Method::POST,
16193            "/api/posts",
16194            Some(unguarded("/api/posts", "TestPost")),
16195        )];
16196        let missing = collect_missing(&routes, &registry);
16197        assert!(missing.is_empty());
16198    }
16199
16200    #[test]
16201    fn registry_check_dedups_one_offender_per_repository() {
16202        let registry = PolicyRegistry::default();
16203        let routes = vec![
16204            build_route(
16205                http::Method::GET,
16206                "/api/posts",
16207                Some(guarded_with_check("/api/posts", "TestPost")),
16208            ),
16209            build_route(
16210                http::Method::POST,
16211                "/api/posts",
16212                Some(guarded_with_check("/api/posts", "TestPost")),
16213            ),
16214            build_route(
16215                http::Method::DELETE,
16216                "/api/posts/{id}",
16217                Some(guarded_with_check("/api/posts", "TestPost")),
16218            ),
16219        ];
16220        let missing = collect_missing(&routes, &registry);
16221        assert_eq!(missing.len(), 1);
16222    }
16223
16224    // ── Scope registration validation ─────────────────────────────
16225
16226    use crate::authorization::{BoxFuture, PolicyContext, Scope};
16227
16228    #[derive(Default)]
16229    struct TestPostScope;
16230    impl Scope<TestPost> for TestPostScope {
16231        fn list<'a>(
16232            &'a self,
16233            _ctx: &'a PolicyContext,
16234            _conn: &'a mut crate::db::RuntimeConnection,
16235        ) -> BoxFuture<'a, crate::AutumnResult<Vec<TestPost>>> {
16236            Box::pin(async { Ok(Vec::new()) })
16237        }
16238    }
16239
16240    fn scope_only_meta(path: &'static str, type_name: &'static str) -> RepositoryApiMeta {
16241        RepositoryApiMeta {
16242            resource_type_name: type_name,
16243            api_path: path,
16244            has_policy: false,
16245            policy_check: None,
16246            scope_check: Some(|registry: &PolicyRegistry| registry.scope::<TestPost>().is_some()),
16247        }
16248    }
16249
16250    fn collect_missing_scopes(
16251        routes: &[Route],
16252        registry: &PolicyRegistry,
16253    ) -> Vec<(String, String)> {
16254        let (_, missing_scopes) = collect_unregistered_repository_handlers(routes, &[], registry);
16255        missing_scopes
16256    }
16257
16258    #[test]
16259    fn scope_check_flags_unregistered_scope() {
16260        let registry = PolicyRegistry::default();
16261        let routes = vec![build_route(
16262            http::Method::GET,
16263            "/api/posts",
16264            Some(scope_only_meta("/api/posts", "TestPost")),
16265        )];
16266        let missing = collect_missing_scopes(&routes, &registry);
16267        assert_eq!(missing.len(), 1);
16268        assert_eq!(missing[0].0, "TestPost");
16269    }
16270
16271    #[test]
16272    fn scope_check_passes_when_scope_is_registered() {
16273        let registry = PolicyRegistry::default();
16274        registry.register_scope::<TestPost, _>(TestPostScope);
16275        let routes = vec![build_route(
16276            http::Method::GET,
16277            "/api/posts",
16278            Some(scope_only_meta("/api/posts", "TestPost")),
16279        )];
16280        let missing = collect_missing_scopes(&routes, &registry);
16281        assert!(missing.is_empty());
16282    }
16283
16284    #[test]
16285    fn scope_check_skips_routes_without_scope_check_fn() {
16286        let registry = PolicyRegistry::default();
16287        let routes = vec![build_route(
16288            http::Method::POST,
16289            "/api/posts",
16290            Some(unguarded("/api/posts", "TestPost")),
16291        )];
16292        let missing = collect_missing_scopes(&routes, &registry);
16293        assert!(missing.is_empty());
16294    }
16295
16296    // ── prod / production profile parity ────────────────────────
16297
16298    #[test]
16299    fn purge_guard_uses_the_resolved_profile_and_the_force_flag() {
16300        use FrameworkRetentionMode::{Purge, Report};
16301        assert!(framework_purge_refusal(Purge, Some("prod"), None).is_some());
16302        assert!(framework_purge_refusal(Purge, None, None).is_some());
16303        assert!(framework_purge_refusal(Purge, Some("prod"), Some("0")).is_some());
16304        assert!(framework_purge_refusal(Purge, Some("prod"), Some("1")).is_none());
16305        assert!(framework_purge_refusal(Purge, Some("dev"), None).is_none());
16306        assert!(framework_purge_refusal(Report, Some("prod"), None).is_none());
16307    }
16308
16309    #[test]
16310    fn is_production_profile_matches_both_aliases() {
16311        assert!(is_production_profile("prod"));
16312        assert!(is_production_profile("production"));
16313        assert!(!is_production_profile("dev"));
16314        assert!(!is_production_profile("staging"));
16315        assert!(!is_production_profile("test"));
16316        assert!(!is_production_profile("default"));
16317        // Case-sensitive (matches the framework's elsewhere
16318        // matching pattern in app.rs::run_build_mode and
16319        // migrate.rs).
16320        assert!(!is_production_profile("Prod"));
16321        assert!(!is_production_profile("Production"));
16322    }
16323
16324    // ── Formatter helpers ─────────────────────────────────────────
16325
16326    #[test]
16327    fn format_unguarded_listing_renders_one_bullet_per_offender() {
16328        let offenders = vec![
16329            ("Post".to_owned(), "/api/posts".to_owned()),
16330            ("Comment".to_owned(), "/api/comments".to_owned()),
16331        ];
16332        let listing = format_unguarded_repository_listing(&offenders);
16333        assert!(listing.contains("Post"));
16334        assert!(listing.contains("/api/posts"));
16335        assert!(listing.contains("Comment"));
16336        assert!(listing.contains("/api/comments"));
16337        assert_eq!(listing.matches("\n  - ").count() + 1, 2);
16338    }
16339
16340    #[test]
16341    fn format_unguarded_listing_empty_input_yields_empty_string() {
16342        let listing = format_unguarded_repository_listing(&[]);
16343        assert!(listing.is_empty());
16344    }
16345
16346    #[test]
16347    fn format_missing_policy_listing_includes_policy_call_hint() {
16348        let missing = vec![("Post".to_owned(), "/api/posts".to_owned())];
16349        let listing = format_missing_policy_listing(&missing);
16350        assert!(listing.contains("Post"));
16351        assert!(listing.contains("/api/posts"));
16352        assert!(listing.contains(".policy::<Post, _>"));
16353        assert!(listing.contains("policy = ..."));
16354    }
16355
16356    #[test]
16357    fn format_missing_scope_listing_includes_scope_call_hint() {
16358        let missing = vec![("Post".to_owned(), "/api/posts".to_owned())];
16359        let listing = format_missing_scope_listing(&missing);
16360        assert!(listing.contains("Post"));
16361        assert!(listing.contains("/api/posts"));
16362        assert!(listing.contains(".scope::<Post, _>"));
16363        assert!(listing.contains("scope = ..."));
16364    }
16365
16366    // ── Scoped-groups path coverage ──────────────────────────────
16367
16368    #[test]
16369    fn collect_unguarded_walks_scoped_groups() {
16370        // The scoped-group path catches `#[repository(api = ...)]`
16371        // mounts that live inside `.scoped(prefix, layer, routes)`.
16372        // Without walking them, the prod-mode guard would silently
16373        // miss those routes.
16374        let group_route = build_route(
16375            http::Method::POST,
16376            "/api/posts",
16377            Some(unguarded("/api/posts", "Post")),
16378        );
16379        let group = ScopedGroup {
16380            prefix: "/scoped".to_owned(),
16381            routes: vec![group_route],
16382            source: crate::route_listing::RouteSource::User,
16383            layer_guarded: false,
16384            apply_layer: Box::new(|r| r),
16385        };
16386        let offenders = collect_unguarded_repository_writes(&[], std::slice::from_ref(&group));
16387        assert_eq!(offenders.len(), 1);
16388        assert_eq!(offenders[0].0, "Post");
16389    }
16390
16391    #[test]
16392    fn scoped_guarded_marks_the_group_and_scoped_does_not() {
16393        let app = app()
16394            .scoped(
16395                "/open",
16396                tower::layer::util::Identity::new(),
16397                vec![build_route(http::Method::GET, "/a", None)],
16398            )
16399            .scoped_guarded(
16400                "/api",
16401                tower::layer::util::Identity::new(),
16402                vec![build_route(http::Method::GET, "/b", None)],
16403            );
16404        let flags: Vec<(&str, bool)> = app
16405            .scoped_groups
16406            .iter()
16407            .map(|g| (g.prefix.as_str(), g.layer_guarded))
16408            .collect();
16409        assert_eq!(flags, vec![("/open", false), ("/api", true)]);
16410    }
16411
16412    #[test]
16413    fn collect_unregistered_walks_scoped_groups() {
16414        let group_route = build_route(
16415            http::Method::POST,
16416            "/api/posts",
16417            Some(guarded_with_check("/api/posts", "TestPost")),
16418        );
16419        let group = ScopedGroup {
16420            prefix: "/scoped".to_owned(),
16421            routes: vec![group_route],
16422            source: crate::route_listing::RouteSource::User,
16423            layer_guarded: false,
16424            apply_layer: Box::new(|r| r),
16425        };
16426        let registry = PolicyRegistry::default();
16427        let (missing, _) =
16428            collect_unregistered_repository_handlers(&[], std::slice::from_ref(&group), &registry);
16429        assert_eq!(missing.len(), 1);
16430        assert_eq!(missing[0].0, "TestPost");
16431    }
16432}
16433
16434/// Publish the builder's story gallery (if any) as the [`StoryRegistry`]
16435/// (`crate::stories::StoryRegistry`) `AppState` extension read by the
16436/// `/_stories` handlers. Shared by the `run()` and build/SSG
16437/// state-construction paths so the two stay in lockstep.
16438#[cfg(feature = "maud")]
16439fn install_story_registry(state: &AppState, story_gallery: Option<crate::stories::StoryGallery>) {
16440    if let Some(gallery) = story_gallery {
16441        state.insert_extension(gallery.into_registry());
16442    }
16443}
16444
16445fn build_state(
16446    config: &AutumnConfig,
16447    #[cfg(feature = "db")] database_topology: Option<&crate::db::DatabaseTopology>,
16448    #[cfg(feature = "db")] shards: Option<crate::sharding::ShardSet>,
16449    #[cfg(feature = "ws")] channels_backend: Option<Arc<dyn crate::channels::ChannelsBackend>>,
16450) -> AppState {
16451    let probes = crate::probe::ProbeState::pending_startup();
16452    #[cfg(feature = "ws")]
16453    let channels = channels_backend.map_or_else(
16454        || {
16455            crate::channels::Channels::from_config(
16456                &config.channels,
16457                probes.shutdown_signal().child_token(),
16458            )
16459            .unwrap_or_else(|error| {
16460                tracing::error!(error = %error, "Failed to configure channels backend");
16461                std::process::exit(1);
16462            })
16463        },
16464        crate::channels::Channels::with_shared_backend,
16465    );
16466
16467    // One tracker, shared with the collaboration hub: a hub built on a second
16468    // `Presence` would report a participant list that disagrees with
16469    // `state.presence()` (#1806).
16470    #[cfg(feature = "presence")]
16471    let presence = crate::presence::Presence::new(channels.clone());
16472
16473    let state = AppState {
16474        extensions: std::sync::Arc::new(std::sync::RwLock::new(std::collections::HashMap::new())),
16475        #[cfg(feature = "db")]
16476        pool: database_topology.map(|topology| topology.primary().clone()),
16477        #[cfg(feature = "db")]
16478        replica_pool: database_topology.and_then(|topology| topology.replica().cloned()),
16479        #[cfg(feature = "db")]
16480        shards,
16481        #[cfg(all(feature = "db", feature = "reporting"))]
16482        db_capture_gap: database_topology
16483            .and_then(|topology| topology.capture_gap().map(std::sync::Arc::from)),
16484        profile: config.profile.as_deref().map(Arc::from),
16485        role: config.role,
16486        started_at: crate::time::monotonic_now(),
16487        health_detailed: config.health.detailed,
16488        probes,
16489        metrics: crate::middleware::MetricsCollector::new(),
16490        log_levels: crate::actuator::LogLevels::new(&config.log.level),
16491        task_registry: crate::actuator::TaskRegistry::new(),
16492        job_registry: crate::actuator::JobRegistry::new(),
16493        config_props: crate::actuator::ConfigProperties::from_config(config),
16494        metrics_source_registry: crate::actuator::MetricsSourceRegistry::new(),
16495        health_indicator_registry: crate::actuator::HealthIndicatorRegistry::new(),
16496        #[cfg(all(feature = "collab", feature = "presence"))]
16497        collab: crate::collab::CollabHub::new(channels.clone(), presence.clone()),
16498        #[cfg(feature = "presence")]
16499        presence,
16500        #[cfg(feature = "ws")]
16501        channels,
16502        policy_registry: crate::authorization::PolicyRegistry::default(),
16503        forbidden_response: config.security.forbidden_response,
16504        auth_session_key: Arc::from(config.auth.session_key.as_str()),
16505        shared_cache: None,
16506        clock: std::sync::Arc::new(crate::time::SystemClock),
16507        entropy: std::sync::Arc::new(crate::entropy::OsEntropy),
16508        app_id: AppState::next_app_id(),
16509    };
16510    state.apply_health_config(&config.health);
16511    #[cfg(feature = "db")]
16512    if state.replica_pool.is_some() {
16513        state
16514            .probes()
16515            .configure_replica_dependency(config.database.replica_fallback);
16516        state.probes().configure_replica_max_lag(
16517            config
16518                .database
16519                .replica_max_lag_ms
16520                .filter(|ms| *ms > 0)
16521                .map(std::time::Duration::from_millis),
16522        );
16523    }
16524    // Surface every shard in /ready and /actuator/health as a
16525    // `db:shard:<name>` component (replica readiness refresh + pool stats).
16526    #[cfg(feature = "db")]
16527    if let Some(set) = state.shards() {
16528        crate::sharding::register_shard_health_indicators(
16529            set,
16530            &state.health_indicator_registry,
16531            config.health.ping_timeout(),
16532        );
16533    }
16534    state.insert_extension(config.clone());
16535    state.insert_extension(crate::step_up::StepUpGlobalConfig {
16536        default_max_age_secs: config.auth.step_up.default_max_age_secs,
16537    });
16538    #[cfg(feature = "http-client")]
16539    state.insert_extension(crate::http_client::SharedReqwestClient {
16540        client: crate::http_client::Client::build_inner(&config.http.client),
16541        timeout_secs: config.http.client.timeout_secs,
16542    });
16543    #[cfg(feature = "http-client")]
16544    crate::http_client::install_shared_throttle(&state, &config.http.client);
16545    state
16546}
16547
16548/// Build the route listing string for the transparency log.
16549fn format_route_lines(
16550    routes: &[Route],
16551    scoped_groups: &[ScopedGroup],
16552    config: &AutumnConfig,
16553) -> String {
16554    use std::fmt::Write as _;
16555
16556    let mut out = String::new();
16557    for route in routes {
16558        let _ = write!(
16559            out,
16560            "\n    {} {:<8} -> {}",
16561            route.path, route.method, route.name
16562        );
16563    }
16564    for group in scoped_groups {
16565        for route in &group.routes {
16566            let _ = write!(
16567                out,
16568                "\n    {}{} {:<8} -> {} (scoped)",
16569                group.prefix, route.path, route.method, route.name
16570            );
16571        }
16572    }
16573    let mut probe_paths = std::collections::HashSet::new();
16574    for (path, name) in [
16575        (config.health.live_path.as_str(), "live"),
16576        (config.health.ready_path.as_str(), "ready"),
16577        (config.health.startup_path.as_str(), "startup"),
16578        (config.health.path.as_str(), "health"),
16579    ] {
16580        if probe_paths.insert(path) {
16581            let _ = write!(out, "\n    {} {:<8} -> {}", path, "GET", name);
16582        }
16583    }
16584    let _ = write!(
16585        out,
16586        "\n    {} {:<8} -> actuator",
16587        crate::actuator::actuator_route_glob(&config.actuator.prefix),
16588        "GET"
16589    );
16590    #[cfg(feature = "htmx")]
16591    {
16592        out.push_str("\n    /static/js/htmx.min.js GET -> htmx");
16593        out.push_str("\n    /static/js/autumn-htmx-csrf.js GET -> htmx csrf");
16594    }
16595    out
16596}
16597
16598/// Build the scheduled task listing string. Returns `None` if there are no tasks.
16599fn format_task_lines(tasks: &[crate::task::TaskInfo]) -> Option<String> {
16600    use std::fmt::Write as _;
16601
16602    if tasks.is_empty() {
16603        return None;
16604    }
16605
16606    let mut out = String::new();
16607    for task in tasks {
16608        let schedule = task.schedule.to_string();
16609        let _ = write!(out, "\n    {} ({schedule})", task.name);
16610    }
16611    Some(out)
16612}
16613
16614/// Build the active middleware listing string.
16615fn format_middleware_list(config: &AutumnConfig) -> String {
16616    let mut items = vec![
16617        "RequestId",
16618        "SecurityHeaders",
16619        "Session (in-memory)",
16620        "ErrorPages",
16621    ];
16622    if !config.cors.allowed_origins.is_empty() {
16623        items.push("CORS");
16624    }
16625    if config.security.csrf.enabled {
16626        items.push("CSRF");
16627    }
16628    items.push("Metrics");
16629    items.join(", ")
16630}
16631
16632/// Mask a database URL for safe logging in the startup summary.
16633///
16634/// Delegates to the shared redactor ([`crate::db_url::redact_target`]), which
16635/// also covers the shapes this function's own `Url::password()` check never saw
16636/// — a `?password=` query parameter, a libpq keyword/value string, a `SQLite`
16637/// target with userinfo.
16638fn mask_database_url(url: &str, pool_size: usize) -> String {
16639    format!(
16640        "{} (pool_size={pool_size})",
16641        crate::db_url::redact_target(url)
16642    )
16643}
16644
16645/// Build the configuration summary string.
16646fn format_config_summary(config: &AutumnConfig) -> String {
16647    let profile = config.profile.as_deref().unwrap_or("none");
16648    let db_status = config.database.effective_primary_url().map_or_else(
16649        || "not configured".to_owned(),
16650        |url| {
16651            let primary = mask_database_url(url, config.database.effective_primary_pool_size());
16652            if config.database.replica_url.is_some() {
16653                format!(
16654                    "primary={primary}, replica=configured (pool_size={})",
16655                    config.database.effective_replica_pool_size()
16656                )
16657            } else {
16658                primary
16659            }
16660        },
16661    );
16662    let telemetry_status = if config.telemetry.enabled {
16663        let endpoint = config
16664            .telemetry
16665            .otlp_endpoint
16666            .as_deref()
16667            .unwrap_or("<missing endpoint>");
16668        format!("{:?} -> {endpoint}", config.telemetry.protocol)
16669    } else {
16670        "disabled".to_owned()
16671    };
16672    format!(
16673        "\
16674        \n    profile:    {profile}\
16675        \n    server:     {}:{}\
16676        \n    database:   {db_status}\
16677        \n    log_level:  {}\
16678        \n    log_format: {:?}\
16679        \n    telemetry:  {telemetry_status}\
16680        \n    health:     {} (detailed={})\
16681        \n    actuator:   sensitive={}\
16682        \n    shutdown:   prestop={}s drain={}s",
16683        config.server.host,
16684        config.server.port,
16685        config.log.level,
16686        config.log.format,
16687        config.health.path,
16688        config.health.detailed,
16689        config.actuator.sensitive,
16690        config.server.prestop_grace_secs,
16691        config.server.shutdown_timeout_secs,
16692    )
16693}
16694
16695/// Resolve a project-relative subdirectory (e.g. `"dist"` or `"static"`)
16696/// against `AUTUMN_MANIFEST_DIR` if set, otherwise use it as-is.
16697pub(crate) fn project_dir(subdir: &str, env: &dyn crate::config::Env) -> std::path::PathBuf {
16698    env.var("AUTUMN_MANIFEST_DIR").map_or_else(
16699        |_| std::path::PathBuf::from(subdir),
16700        |d| std::path::PathBuf::from(d).join(subdir),
16701    )
16702}
16703
16704/// Serve in-place upgrades for the lifetime of the process (issue #1674).
16705///
16706/// Each `SIGUSR2` attempts one handover. A failed attempt is logged and the
16707/// current build carries on serving — including its live state, which is
16708/// unfrozen again — so a later signal (with a fixed binary) can retry.
16709#[cfg(unix)]
16710async fn watch_for_in_place_upgrade(
16711    config: &crate::config::UpgradeConfig,
16712    signal: Option<tokio::signal::unix::Signal>,
16713    socket: Option<crate::upgrade::HandoffSocket>,
16714    state: AppState,
16715    cutover: tokio_util::sync::CancellationToken,
16716) {
16717    // Registered at the top of `run()` so the signal is never fatal; `None`
16718    // means the handler could not be installed at all, which was reported then.
16719    let Some(mut signal) = signal else {
16720        return;
16721    };
16722    // Clamped: a zero-second budget would abandon every successor before it
16723    // could possibly have finished booting.
16724    let ready_timeout = std::time::Duration::from_secs(config.ready_timeout_secs.max(1));
16725
16726    while signal.recv().await.is_some() {
16727        if !config.enabled {
16728            tracing::warn!(
16729                "SIGUSR2 received but in-place upgrade is disabled \
16730                 ([server.upgrade] enabled = false); ignoring"
16731            );
16732            continue;
16733        }
16734        // Documented as incompatible, and refused rather than attempted: the
16735        // successor would start a second postmaster over the same data
16736        // directory, and this process's drain would stop the cluster under it.
16737        #[cfg(feature = "managed-pg")]
16738        if crate::managed_pg::is_supervising() {
16739            let error = crate::upgrade::UpgradeError::Unsupported(
16740                "it supervises a managed Postgres cluster, which cannot be handed over with \
16741                 the socket — the successor would start a second postmaster over the same \
16742                 data directory",
16743            );
16744            tracing::error!(error = %error, "in-place upgrade refused");
16745            continue;
16746        }
16747        let Some(socket) = socket.as_ref() else {
16748            let error = crate::upgrade::UpgradeError::UnsupportedListener(
16749                "the server is not bound to a plain TCP listener",
16750            );
16751            tracing::error!(error = %error, "in-place upgrade refused");
16752            continue;
16753        };
16754        tracing::info!("SIGUSR2 received, starting an in-place upgrade");
16755        let plan = crate::upgrade::UpgradePlan {
16756            socket,
16757            registry: state.extension::<crate::upgrade::LiveStateRegistry>(),
16758            ready_timeout,
16759            clock: state.clock_arc(),
16760            entropy: state.entropy_arc(),
16761        };
16762        match crate::upgrade::upgrade_in_place(plan).await {
16763            Ok(handover) => {
16764                tracing::info!(
16765                    successor_pid = handover.successor_pid,
16766                    generation = handover.generation,
16767                    elapsed_ms = u64::try_from(handover.elapsed.as_millis()).unwrap_or(u64::MAX),
16768                    "in-place upgrade complete: the successor is serving on this socket, \
16769                     draining this build"
16770                );
16771                cutover.cancel();
16772                return;
16773            }
16774            Err(e) => {
16775                tracing::error!(
16776                    error = %e,
16777                    "in-place upgrade abandoned; this build is still serving and its live \
16778                     state is writable again"
16779                );
16780            }
16781        }
16782    }
16783}
16784
16785enum DrainCause {
16786    /// `SIGTERM`/Ctrl-C, or a canary rollback flag: the process is going away
16787    /// and its address goes with it.
16788    Signal,
16789    /// An in-place upgrade (#1674): a successor is already accepting on this
16790    /// process's listening socket, so the address stays up throughout.
16791    UpgradeCutover,
16792}
16793
16794/// Environment variable naming a file whose appearance drains this app.
16795///
16796/// Windows has no `SIGTERM`, so a parent process that supervises an Autumn app
16797/// (today: `autumn dev`) could only stop it with `TerminateProcess` — which
16798/// skips `on_shutdown` hooks entirely, orphaning a managed Postgres child on
16799/// every hot reload (issue #1616). Setting this variable gives such a parent a
16800/// portable way to request the *same* graceful drain a signal triggers: create
16801/// the file, and the app runs its normal shutdown sequence.
16802///
16803/// Opt-in: unset (or empty), nothing watches anything. The file's contents are
16804/// never read — its existence is the whole signal — so a parent can create it
16805/// with a plain zero-byte `File::create`. It must be a regular file; a directory
16806/// at that path is ignored.
16807///
16808/// **Honored on non-Unix targets only.** On Unix `SIGTERM` already does this
16809/// job, and arming a file-triggered drain there would put a production
16810/// deployment one operator-configured path away from being drainable by anything
16811/// that can create a file. The variable is accepted and ignored on Unix.
16812pub const SHUTDOWN_SIGNAL_FILE_ENV: &str = "AUTUMN_SHUTDOWN_SIGNAL_FILE";
16813
16814/// Resolve the cooperative-shutdown file from a raw environment value.
16815///
16816/// Split out from the env read so it is testable without mutating process-global
16817/// state. A blank value is treated as unset: an exported-but-empty variable must
16818/// not resolve to the current directory, where an unrelated file would drain the
16819/// app.
16820#[cfg_attr(
16821    unix,
16822    allow(
16823        dead_code,
16824        reason = "the non-Unix drain arm, compiled and tested on every platform"
16825    )
16826)]
16827fn external_shutdown_path_from(value: Option<&std::ffi::OsStr>) -> Option<std::path::PathBuf> {
16828    let trimmed = value?.to_string_lossy().trim().to_string();
16829    if trimmed.is_empty() {
16830        return None;
16831    }
16832    // Trim before building the path, not just before testing for emptiness: a
16833    // value with stray whitespace would otherwise resolve to a sibling path the
16834    // parent never writes, and the failure would be silent — every reload
16835    // stalling the full budget and hard-killing, which is the bug this exists
16836    // to fix.
16837    Some(std::path::PathBuf::from(trimmed))
16838}
16839
16840/// Resolve when the cooperative-shutdown file at `path` exists.
16841///
16842/// Never resolves when `path` is `None` (the feature is not configured), so an
16843/// app that does not opt in is unaffected. Resolves immediately when the file is
16844/// already present at boot: a supervising parent removes a stale file before
16845/// spawning, so a file that survives into a fresh process means a stop was
16846/// requested and unhandled — draining is the safe direction.
16847#[cfg_attr(
16848    unix,
16849    allow(
16850        dead_code,
16851        reason = "the non-Unix drain arm, compiled and tested on every platform"
16852    )
16853)]
16854async fn external_shutdown_signal(path: Option<std::path::PathBuf>) {
16855    let Some(path) = path else {
16856        std::future::pending::<()>().await;
16857        return;
16858    };
16859    let interval = std::time::Duration::from_millis(100);
16860    loop {
16861        // `is_file`, not "exists": `metadata` succeeds on a directory, so a
16862        // variable pointed at one would drain the app on every boot — bind,
16863        // drain, exit 0 — which a supervisor reads as a healthy restart loop
16864        // rather than the misconfiguration it is.
16865        if tokio::fs::metadata(&path).await.is_ok_and(|m| m.is_file()) {
16866            return;
16867        }
16868        tokio::time::sleep(interval).await;
16869    }
16870}
16871
16872/// Wait for a shutdown signal (Ctrl+C, SIGTERM on Unix, or a canary rollback
16873/// flag file written by a controller).
16874///
16875/// Returns the reason the drain began. Also resolves on an in-place
16876/// upgrade cutover (#1674), whose drain skips the readiness flip and the
16877/// prestop grace because the address never goes away.
16878///
16879/// Returns when any signal is received. Axum's `with_graceful_shutdown`
16880/// then stops accepting new connections and drains in-flight requests.
16881///
16882/// The canary rollback arm lets a progressive-delivery controller drain and
16883/// retire a bad canary replica without sending `SIGTERM` by hand: it writes
16884/// [`crate::canary::CANARY_ROLLBACK_FLAG_FILE`] and Autumn runs the identical
16885/// graceful-shutdown sequence (ready → 503, prestop grace, drain, clean exit).
16886async fn shutdown_signal(upgrade_cutover: tokio_util::sync::CancellationToken) -> DrainCause {
16887    let ctrl_c = async {
16888        tokio::signal::ctrl_c()
16889            .await
16890            .expect("Failed to install Ctrl+C handler");
16891        tracing::info!("Received Ctrl+C, starting graceful shutdown");
16892    };
16893
16894    #[cfg(unix)]
16895    let terminate = async {
16896        tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
16897            .expect("Failed to install SIGTERM handler")
16898            .recv()
16899            .await;
16900        tracing::info!("Received SIGTERM, starting graceful shutdown");
16901    };
16902
16903    // Windows has no `SIGTERM`. What a supervisor, a service host, or the OS
16904    // itself actually delivers is a console control event, and every one of them
16905    // means the same thing `SIGTERM` means: this process is going away, drain
16906    // now. Handling them is what makes a supervisor-stopped Windows server run
16907    // its readiness flip, prestop grace, in-flight drain and `on_shutdown` hooks
16908    // instead of dying mid-request (#1639).
16909    //
16910    // How much time each event actually buys differs, and the difference
16911    // matters enough to be exact about:
16912    //
16913    // * `CTRL_C` and `CTRL_BREAK` do not terminate the process at all. The drain
16914    //   runs to completion.
16915    // * `CTRL_CLOSE`, `CTRL_LOGOFF` and `CTRL_SHUTDOWN` are told-not-asked: the
16916    //   OS-imposed budget is how long the *handler routine* may run, and tokio's
16917    //   handler returns immediately (which is what lets this future wake), so
16918    //   the drain then races process termination. It is best-effort, and on a
16919    //   long budget it will lose.
16920    //
16921    // That is why a supervisor should stop an Autumn app through
16922    // `autumn serve stop` or the Service Control Manager, both of which wait for
16923    // the app's own recorded budget. `docs/guide/daemon.md` says so where an
16924    // operator will read it. These arms are still worth having: they turn the
16925    // common console cases into a real drain, and cost nothing in the rest.
16926    #[cfg(windows)]
16927    let terminate = async {
16928        use tokio::signal::windows;
16929        let mut close = windows::ctrl_close().expect("Failed to install CTRL_CLOSE handler");
16930        let mut logoff = windows::ctrl_logoff().expect("Failed to install CTRL_LOGOFF handler");
16931        let mut shutdown =
16932            windows::ctrl_shutdown().expect("Failed to install CTRL_SHUTDOWN handler");
16933        let mut brk = windows::ctrl_break().expect("Failed to install CTRL_BREAK handler");
16934        let event = tokio::select! {
16935            _ = close.recv() => "CTRL_CLOSE",
16936            _ = logoff.recv() => "CTRL_LOGOFF",
16937            _ = shutdown.recv() => "CTRL_SHUTDOWN",
16938            _ = brk.recv() => "CTRL_BREAK",
16939        };
16940        tracing::info!(
16941            event,
16942            "Received a console control event, starting graceful shutdown"
16943        );
16944    };
16945
16946    #[cfg(not(any(unix, windows)))]
16947    let terminate = std::future::pending::<()>();
16948
16949    let canary_rollback = async {
16950        canary_rollback_signal(std::path::Path::new(
16951            crate::canary::CANARY_ROLLBACK_FLAG_FILE,
16952        ))
16953        .await;
16954        tracing::info!("Canary rollback signalled, starting graceful shutdown");
16955    };
16956
16957    // A supervising parent with no `SIGTERM` to send — `autumn dev` on Windows —
16958    // can request this exact drain by creating a file (#1616).
16959    //
16960    // Deliberately `cfg`-gated to the platforms that need it, rather than armed
16961    // everywhere and left inert by an unset variable. Autumn deploys to Linux,
16962    // where this would be a production drain trigger reachable by anything that
16963    // can create a configured path, with no authentication and no benefit, since
16964    // `SIGTERM` already exists. The helpers stay compiled and unit-tested
16965    // everywhere.
16966    #[cfg(not(unix))]
16967    let external_stop = async {
16968        let path =
16969            external_shutdown_path_from(std::env::var_os(SHUTDOWN_SIGNAL_FILE_ENV).as_deref());
16970        external_shutdown_signal(path.clone()).await;
16971        tracing::warn!(
16972            path = ?path,
16973            "Cooperative shutdown requested via {SHUTDOWN_SIGNAL_FILE_ENV}, starting graceful shutdown"
16974        );
16975    };
16976
16977    #[cfg(unix)]
16978    let external_stop = std::future::pending::<()>();
16979
16980    let upgrade = async {
16981        upgrade_cutover.cancelled().await;
16982        tracing::info!("Successor is serving after an in-place upgrade, draining this build");
16983    };
16984
16985    tokio::select! {
16986        () = ctrl_c => DrainCause::Signal,
16987        () = terminate => DrainCause::Signal,
16988        () = canary_rollback => DrainCause::Signal,
16989        () = external_stop => DrainCause::Signal,
16990        () = upgrade => DrainCause::UpgradeCutover,
16991    }
16992}
16993
16994/// Resolve when the canary rollback flag file is present at `path`.
16995///
16996/// A rollback signal is intentionally **sticky across restarts**: if the flag is
16997/// already present at boot (e.g. a supervisor restarted the process after a
16998/// rollback), this resolves immediately so the replica drains and exits again
16999/// rather than rejoining the canary cohort. The replica keeps draining until a
17000/// controller clears the signal with `autumn canary promote` (or scales the
17001/// replica to zero). At startup the framework also flips `/ready` to draining
17002/// when the flag is present, so a restarted rolled-back replica never serves
17003/// canary traffic.
17004///
17005/// Uses async stat so the 500 ms poll never blocks the executor thread.
17006async fn canary_rollback_signal(path: &std::path::Path) {
17007    let interval = std::time::Duration::from_millis(500);
17008    loop {
17009        if tokio::fs::metadata(path).await.is_ok() {
17010            return;
17011        }
17012        tokio::time::sleep(interval).await;
17013    }
17014}
17015
17016#[cfg(test)]
17017mod tests {
17018    /// `mcp_catalog` and `mcp_instructions` set the MCP runtime. Each one
17019    /// mounts the endpoint at `/mcp`, and a later `mount_mcp` keeps both.
17020    #[cfg(feature = "mcp")]
17021    #[test]
17022    fn mcp_catalog_and_instructions_set_the_runtime() {
17023        use crate::mcp::McpCatalog;
17024
17025        let builder = super::app()
17026            .mcp_catalog(McpCatalog::Progressive)
17027            .mcp_instructions("Todo tools.");
17028        let rt = builder.mcp.as_ref().expect("an MCP runtime");
17029        assert_eq!(rt.mount_path, "/mcp");
17030        assert_eq!(rt.catalog, McpCatalog::Progressive);
17031        assert_eq!(rt.instructions.as_deref(), Some("Todo tools."));
17032
17033        let builder = builder.mount_mcp("/agent");
17034        let rt = builder.mcp.as_ref().expect("an MCP runtime");
17035        assert_eq!(rt.mount_path, "/agent");
17036        assert_eq!(rt.catalog, McpCatalog::Progressive);
17037        assert_eq!(rt.instructions.as_deref(), Some("Todo tools."));
17038
17039        let rt = super::app().mcp_instructions("x").mcp.expect("a runtime");
17040        assert_eq!(rt.catalog, McpCatalog::Full, "the default catalog is full");
17041    }
17042
17043    /// Issue #2347: `on_session_rotation` puts one hook on the shared registry
17044    /// when the state initializers run.
17045    #[test]
17046    fn on_session_rotation_registers_hooks_on_the_state() {
17047        let builder = super::app()
17048            .on_session_rotation(|_| async { Ok(()) })
17049            .on_session_rotation(|_| async { Ok(()) });
17050        let state = crate::state::AppState::test_default();
17051        assert!(state.session_rotation_hooks().is_empty());
17052
17053        super::run_state_initializers(builder.state_initializers, &state);
17054
17055        assert_eq!(state.session_rotation_hooks().len(), 2);
17056    }
17057
17058    /// Regression (#1620): DNS-01 must not silence the multi-replica warning.
17059    ///
17060    /// DNS-01 retires the HTTP-01 token-map hazard — the CA never connects to
17061    /// this host — but it does not distribute certificates. A Postgres-backed
17062    /// (i.e. multi-replica) fleet still has every non-leader replica serving the
17063    /// self-signed placeholder off its own empty on-disk store, so it must still
17064    /// warn, naming the certificate store rather than tokens.
17065    #[cfg(feature = "acme")]
17066    #[test]
17067    fn acme_fleet_warning_covers_the_certificate_store_under_dns01() {
17068        use crate::config::SchedulerBackend;
17069
17070        // Single-replica: silent under either challenge type.
17071        assert!(super::acme_fleet_warning(SchedulerBackend::InProcess, false).is_none());
17072        assert!(super::acme_fleet_warning(SchedulerBackend::InProcess, true).is_none());
17073
17074        // Multi-replica HTTP-01: both hazards named.
17075        let http01 = super::acme_fleet_warning(SchedulerBackend::Postgres, false)
17076            .expect("a distributed backend under HTTP-01 must warn");
17077        assert!(
17078            http01.contains("token"),
17079            "HTTP-01 warning must name the token store: {http01}"
17080        );
17081
17082        // Multi-replica DNS-01: still warns, about the certificate store.
17083        let dns01 = super::acme_fleet_warning(SchedulerBackend::Postgres, true).expect(
17084            "a distributed backend under DNS-01 must still warn: DNS-01 proves domain \
17085                     control but does not share the certificate store",
17086        );
17087        assert!(
17088            dns01.contains("certificate store"),
17089            "DNS-01 warning must name the certificate store as the hazard: {dns01}"
17090        );
17091        assert!(
17092            dns01.contains("cache_dir"),
17093            "DNS-01 warning must name the config key an operator would change: {dns01}"
17094        );
17095        // …and must not repeat the HTTP-01 diagnosis, which does not apply.
17096        assert!(
17097            !dns01.contains("token") && !dns01.contains("404"),
17098            "DNS-01 warning must not blame the HTTP-01 token map: {dns01}"
17099        );
17100    }
17101
17102    /// Issue #1907: `scheduler.backend = "sqlite"` coordinates processes on one
17103    /// host, so it carries the HTTP-01 token hazard but not the certificate
17104    /// store one — every process reads the same `cache_dir`.
17105    #[cfg(feature = "acme")]
17106    #[test]
17107    fn acme_fleet_warning_for_sqlite_names_only_the_token_hazard() {
17108        use crate::config::SchedulerBackend;
17109
17110        let http01 = super::acme_fleet_warning(SchedulerBackend::Sqlite, false)
17111            .expect("multi-process HTTP-01 must warn about the per-process token store");
17112        assert!(
17113            http01.contains("token"),
17114            "the warning must name the token store: {http01}"
17115        );
17116        assert!(
17117            !http01.contains("Run ACME on a single host"),
17118            "a single-host deployment must not be told to move to a single host: {http01}"
17119        );
17120        assert!(
17121            http01.contains("cache_dir"),
17122            "the warning must say the certificate store is already shared: {http01}"
17123        );
17124
17125        // DNS-01 needs no :80 challenge, and the store is already shared, so
17126        // there is nothing left to warn about.
17127        assert!(
17128            super::acme_fleet_warning(SchedulerBackend::Sqlite, true).is_none(),
17129            "DNS-01 on one host clears both hazards"
17130        );
17131    }
17132
17133    /// A clock near the end of representable time must not kill the scheduler.
17134    ///
17135    /// `format_next_task_run_after` runs at the top of every fixed-delay loop.
17136    /// While `now` came from real time its sum with an ordinary delay could not
17137    /// overflow, but an injected clock is unbounded, and chrono's `Add` panics
17138    /// rather than wrapping — so a `FixedClock` near `DateTime::MAX_UTC` would
17139    /// take the task down before its first run, over a string for a log line.
17140    #[test]
17141    fn next_task_run_saturates_instead_of_panicking_at_the_end_of_time() {
17142        let max = chrono::DateTime::<chrono::Utc>::MAX_UTC;
17143
17144        // An ordinary delay that would carry `now` past the representable end.
17145        let rendered = super::format_next_task_run_after(max, std::time::Duration::from_secs(60));
17146        assert_eq!(
17147            rendered,
17148            max.to_rfc3339(),
17149            "a fixed delay past the end of time must clamp, not panic"
17150        );
17151
17152        // A delay too large for `TimeDelta` at all renders as the far future,
17153        // not as `now` — the latter would read as "this task runs immediately".
17154        let absurd = super::format_next_task_run_after(
17155            chrono::Utc::now(),
17156            std::time::Duration::from_secs(u64::MAX),
17157        );
17158        assert_eq!(absurd, max.to_rfc3339());
17159
17160        // The ordinary case is unchanged.
17161        let epoch = chrono::DateTime::from_timestamp(0, 0).unwrap();
17162        assert_eq!(
17163            super::format_next_task_run_after(epoch, std::time::Duration::from_secs(60)),
17164            (epoch + chrono::TimeDelta::seconds(60)).to_rfc3339()
17165        );
17166    }
17167
17168    use super::*;
17169    use axum::body::Body;
17170    use axum::http::{Request, StatusCode};
17171    use std::sync::atomic::{AtomicUsize, Ordering};
17172    use tower::ServiceExt;
17173
17174    // ── omitted-router accounting for `autumn routes audit` ──────────────────
17175
17176    // #1974 item 7: a plugin declares a top-level config section from its
17177    // `build()` via `config_section`, so `server.strict_config` treats that root
17178    // as known-and-opaque. Prove the declaration lands on the builder and that
17179    // the registry is fail-closed (an undeclared root is not registered).
17180    #[test]
17181    fn plugin_declares_config_section_via_build() {
17182        struct DummyMediaPlugin;
17183        impl crate::plugin::Plugin for DummyMediaPlugin {
17184            fn build(self, app: AppBuilder) -> AppBuilder {
17185                app.config_section("media")
17186            }
17187        }
17188
17189        let builder = app().plugin(DummyMediaPlugin);
17190        assert!(
17191            builder.has_config_section("media"),
17192            "a plugin's build() must declare its [media] config section"
17193        );
17194        assert!(
17195            !builder.has_config_section("definitely_not_a_root"),
17196            "only explicitly-declared roots are registered — the seam is fail-closed"
17197        );
17198    }
17199
17200    /// Compute the omitted-router count for a builder using the same inputs
17201    /// `run_dump_routes_mode` feeds `omitted_router_count`: the merge count, the
17202    /// nest prefixes, and the declared routes that prove nest coverage.
17203    fn omitted_for(builder: &AppBuilder) -> usize {
17204        omitted_router_count(
17205            builder.merge_routers.len(),
17206            builder
17207                .nest_routers
17208                .iter()
17209                .map(|(prefix, _)| prefix.as_str()),
17210            &builder.declared_routes,
17211        )
17212    }
17213
17214    /// Regression (#1604): the DOCUMENTED plugin pattern —
17215    /// `app.nest(prefix, router).declare_plugin_routes(routes)`, which the
17216    /// first-party `AdminPlugin` uses — declares route metadata whose paths fall
17217    /// under the nest prefix. That coverage makes the nested raw router
17218    /// enumerable, so it must NOT be counted among the opaque, omitted routers
17219    /// that hard-fail the audit gate. Before the fix, prefix coverage was
17220    /// ignored and the mere presence of the nested raw router pushed
17221    /// `hidden > 0`, false-failing the audit even though every admin route was
17222    /// declared and classified.
17223    #[test]
17224    fn documented_nest_then_declare_is_not_counted_as_omitted() {
17225        let raw =
17226            axum::Router::<AppState>::new().route("/ping", axum::routing::get(|| async { "pong" }));
17227        let declared = vec![crate::route_listing::RouteInfo {
17228            method: "GET".to_owned(),
17229            path: "/admin/ping".to_owned(),
17230            handler: "admin::ping".to_owned(),
17231            ..Default::default()
17232        }];
17233
17234        // The plain documented pattern: nest the raw router, then declare its
17235        // covering route metadata. No dedicated `nest_declared` bookkeeping.
17236        let builder = app().nest("/admin", raw).declare_plugin_routes(declared);
17237
17238        // The raw router is still mounted (serving path is unchanged) …
17239        assert_eq!(builder.nest_routers.len(), 1);
17240        // … and its declared metadata was folded into `declared_routes`.
17241        assert_eq!(builder.declared_routes.len(), 1);
17242
17243        // ⇒ zero omitted routers: the declared route `/admin/ping` falls under
17244        // the `/admin` nest prefix, so the mount is covered and the audit gate
17245        // must NOT fire.
17246        assert_eq!(
17247            omitted_for(&builder),
17248            0,
17249            "a nest whose endpoints are declared is enumerable and must not count as omitted",
17250        );
17251    }
17252
17253    /// The soundness guarantee must survive: a raw router mounted via bare
17254    /// `nest()` or `merge()` without covering declarations is unenumerable and
17255    /// must still count as omitted so `autumn routes audit` fails closed.
17256    #[test]
17257    fn undeclared_nest_and_merge_still_count_as_omitted() {
17258        let raw_nest =
17259            axum::Router::<AppState>::new().route("/x", axum::routing::get(|| async { "x" }));
17260        let raw_merge =
17261            axum::Router::<AppState>::new().route("/y", axum::routing::get(|| async { "y" }));
17262
17263        let builder = app().nest("/v2", raw_nest).merge(raw_merge);
17264
17265        assert_eq!(builder.nest_routers.len(), 1);
17266        assert_eq!(builder.merge_routers.len(), 1);
17267        // Nothing was declared, so nothing covers the nest.
17268        assert!(builder.declared_routes.is_empty());
17269
17270        assert_eq!(
17271            omitted_for(&builder),
17272            2,
17273            "an undeclared nest and a merge are both opaque and must be reported",
17274        );
17275    }
17276
17277    /// An undeclared `merge()` is rootless — it cannot be prefix-matched — so it
17278    /// stays omitted even when unrelated declared routes exist. Guards against a
17279    /// declaration for one mount silently covering an unrelated raw `merge()`.
17280    #[test]
17281    fn declared_routes_do_not_cover_a_rootless_merge() {
17282        let raw_merge =
17283            axum::Router::<AppState>::new().route("/y", axum::routing::get(|| async { "y" }));
17284
17285        let builder =
17286            app()
17287                .merge(raw_merge)
17288                .declare_plugin_routes(vec![crate::route_listing::RouteInfo {
17289                    method: "GET".to_owned(),
17290                    path: "/admin/ok".to_owned(),
17291                    handler: "admin::ok".to_owned(),
17292                    ..Default::default()
17293                }]);
17294
17295        assert_eq!(
17296            omitted_for(&builder),
17297            1,
17298            "a merge has no prefix to match declarations against and must always count",
17299        );
17300    }
17301
17302    /// A declared nest alongside an *undeclared* nest: only the undeclared one
17303    /// is omitted. Prefix-matching must cover the `/admin` mount (a declared
17304    /// route falls under it) without spilling onto the unrelated `/raw` mount
17305    /// (no declared route falls under it).
17306    #[test]
17307    fn mixed_declared_and_undeclared_nests_count_only_the_undeclared() {
17308        let declared_raw =
17309            axum::Router::<AppState>::new().route("/ok", axum::routing::get(|| async { "ok" }));
17310        let undeclared_raw = axum::Router::<AppState>::new()
17311            .route("/opaque", axum::routing::get(|| async { "opaque" }));
17312
17313        let builder = app()
17314            .nest("/admin", declared_raw)
17315            .declare_plugin_routes(vec![crate::route_listing::RouteInfo {
17316                method: "GET".to_owned(),
17317                path: "/admin/ok".to_owned(),
17318                handler: "admin::ok".to_owned(),
17319                ..Default::default()
17320            }])
17321            .nest("/raw", undeclared_raw);
17322
17323        assert_eq!(builder.nest_routers.len(), 2);
17324        assert_eq!(builder.declared_routes.len(), 1);
17325        assert_eq!(
17326            omitted_for(&builder),
17327            1,
17328            "only the bare nest() is omitted; the declared mount is covered",
17329        );
17330    }
17331
17332    /// A declared route whose path merely *shares a leading substring* with a
17333    /// nest prefix (`/administrators` vs `/admin`) must NOT cover the nest:
17334    /// prefix-matching honours path-segment boundaries, so this bare nest still
17335    /// counts as omitted and the audit fails closed.
17336    #[test]
17337    fn prefix_match_respects_path_segment_boundaries() {
17338        let raw = axum::Router::<AppState>::new().route("/x", axum::routing::get(|| async { "x" }));
17339
17340        let builder = app().nest("/admin", raw).declare_plugin_routes(vec![
17341            crate::route_listing::RouteInfo {
17342                method: "GET".to_owned(),
17343                path: "/administrators".to_owned(),
17344                handler: "other::index".to_owned(),
17345                ..Default::default()
17346            },
17347        ]);
17348
17349        assert_eq!(
17350            omitted_for(&builder),
17351            1,
17352            "`/administrators` is not under the `/admin` nest prefix; the nest stays omitted",
17353        );
17354    }
17355
17356    #[test]
17357    fn is_dump_jobs_mode_only_true_for_exactly_one() {
17358        // `autumn jobs manifest` sets AUTUMN_DUMP_JOBS=1 to select the manifest
17359        // dump path in `run()`. Any other value (or an unset var) must fall
17360        // through to the normal boot path.
17361        temp_env::with_var("AUTUMN_DUMP_JOBS", Some("1"), || {
17362            assert!(is_dump_jobs_mode(), "`1` must select the jobs-dump path");
17363        });
17364        temp_env::with_var("AUTUMN_DUMP_JOBS", Some("0"), || {
17365            assert!(!is_dump_jobs_mode(), "`0` must not select the dump path");
17366        });
17367        temp_env::with_var("AUTUMN_DUMP_JOBS", Some("true"), || {
17368            assert!(
17369                !is_dump_jobs_mode(),
17370                "only the literal `1` enables the mode"
17371            );
17372        });
17373        temp_env::with_var("AUTUMN_DUMP_JOBS", None::<&str>, || {
17374            assert!(!is_dump_jobs_mode(), "unset must not select the dump path");
17375        });
17376    }
17377
17378    #[test]
17379    fn is_retention_dry_run_mode_only_true_for_exactly_one() {
17380        // `autumn retention --dry-run` sets AUTUMN_RETENTION_DRY_RUN=1 to
17381        // select the dry-run path in `run()`. Any other value (or an unset
17382        // var) must fall through to the normal boot path (issue #1342).
17383        temp_env::with_var("AUTUMN_RETENTION_DRY_RUN", Some("1"), || {
17384            assert!(
17385                is_retention_dry_run_mode(),
17386                "`1` must select the retention dry-run path"
17387            );
17388        });
17389        temp_env::with_var("AUTUMN_RETENTION_DRY_RUN", Some("0"), || {
17390            assert!(
17391                !is_retention_dry_run_mode(),
17392                "`0` must not select the dry-run path"
17393            );
17394        });
17395        temp_env::with_var("AUTUMN_RETENTION_DRY_RUN", Some("true"), || {
17396            assert!(
17397                !is_retention_dry_run_mode(),
17398                "only the literal `1` enables the mode"
17399            );
17400        });
17401        temp_env::with_var("AUTUMN_RETENTION_DRY_RUN", None::<&str>, || {
17402            assert!(
17403                !is_retention_dry_run_mode(),
17404                "unset must not select the dry-run path"
17405            );
17406        });
17407    }
17408
17409    #[cfg(feature = "db")]
17410    #[test]
17411    fn retention_dry_run_model_filter_from_env_trims_and_ignores_blank() {
17412        temp_env::with_var("AUTUMN_RETENTION_MODEL", Some("  Widget  "), || {
17413            assert_eq!(
17414                retention_dry_run_model_filter_from_env().as_deref(),
17415                Some("Widget")
17416            );
17417        });
17418        temp_env::with_var("AUTUMN_RETENTION_MODEL", Some(""), || {
17419            assert_eq!(retention_dry_run_model_filter_from_env(), None);
17420        });
17421        temp_env::with_var("AUTUMN_RETENTION_MODEL", None::<&str>, || {
17422            assert_eq!(retention_dry_run_model_filter_from_env(), None);
17423        });
17424    }
17425
17426    #[cfg(feature = "db")]
17427    #[test]
17428    fn merge_and_validate_task_names_rejects_collision_with_hand_declared_task() {
17429        // `resolve_retention_descriptors` validates collisions only among
17430        // retention-generated task names; it cannot see hand-declared `tasks![...]`
17431        // entries, which real boot merges in through `AppBuilder::build`'s
17432        // `tasks.extend(...)` and `validate_unique_scheduled_task_names`. Without
17433        // this check a dry run could report success for a policy whose generated
17434        // name collides with a hand-declared task, while real boot panics on that
17435        // same collision.
17436        fn dry_run_stub(
17437            _state: AppState,
17438        ) -> std::pin::Pin<
17439            Box<
17440                dyn std::future::Future<
17441                        Output = crate::AutumnResult<crate::retention::RetentionSweepReport>,
17442                    > + Send,
17443            >,
17444        > {
17445            Box::pin(async {
17446                Ok(crate::retention::RetentionSweepReport {
17447                    model: "Widget".to_string(),
17448                    table: "widgets".to_string(),
17449                    rows_swept: 0,
17450                    duration_ms: 0,
17451                    dry_run: true,
17452                })
17453            })
17454        }
17455        fn task_info_stub() -> crate::task::TaskInfo {
17456            crate::task::TaskInfo {
17457                name: "retention-sweep-widgets".to_string(),
17458                schedule: crate::task::Schedule::FixedDelay(std::time::Duration::from_secs(3600)),
17459                coordination: crate::task::TaskCoordination::Fleet,
17460                handler: |_state| Box::pin(async { Ok(()) }),
17461            }
17462        }
17463        let descriptor = crate::retention::RetentionSweepDescriptor {
17464            model_name: "Widget",
17465            table_name: "widgets",
17466            task_info: task_info_stub,
17467            dry_run: dry_run_stub,
17468        };
17469        let hand_declared_collision = crate::task::TaskInfo {
17470            name: "retention-sweep-widgets".to_string(),
17471            schedule: crate::task::Schedule::FixedDelay(std::time::Duration::from_secs(60)),
17472            coordination: crate::task::TaskCoordination::PerReplica,
17473            handler: |_state| Box::pin(async { Ok(()) }),
17474        };
17475
17476        let error = merge_and_validate_task_names(&[&descriptor], vec![hand_declared_collision])
17477            .expect_err("a hand-declared task colliding with a generated task name must error");
17478
17479        assert!(
17480            error.contains("retention-sweep-widgets"),
17481            "the error must name the colliding task: {error}"
17482        );
17483    }
17484
17485    #[cfg(feature = "db")]
17486    #[test]
17487    fn merge_and_validate_task_names_accepts_distinct_names() {
17488        fn dry_run_stub(
17489            _state: AppState,
17490        ) -> std::pin::Pin<
17491            Box<
17492                dyn std::future::Future<
17493                        Output = crate::AutumnResult<crate::retention::RetentionSweepReport>,
17494                    > + Send,
17495            >,
17496        > {
17497            Box::pin(async {
17498                Ok(crate::retention::RetentionSweepReport {
17499                    model: "Widget".to_string(),
17500                    table: "widgets".to_string(),
17501                    rows_swept: 0,
17502                    duration_ms: 0,
17503                    dry_run: true,
17504                })
17505            })
17506        }
17507        fn task_info_stub() -> crate::task::TaskInfo {
17508            crate::task::TaskInfo {
17509                name: "retention-sweep-widgets".to_string(),
17510                schedule: crate::task::Schedule::FixedDelay(std::time::Duration::from_secs(3600)),
17511                coordination: crate::task::TaskCoordination::Fleet,
17512                handler: |_state| Box::pin(async { Ok(()) }),
17513            }
17514        }
17515        let descriptor = crate::retention::RetentionSweepDescriptor {
17516            model_name: "Widget",
17517            table_name: "widgets",
17518            task_info: task_info_stub,
17519            dry_run: dry_run_stub,
17520        };
17521        let unrelated_task = crate::task::TaskInfo {
17522            name: "nightly-report".to_string(),
17523            schedule: crate::task::Schedule::FixedDelay(std::time::Duration::from_secs(60)),
17524            coordination: crate::task::TaskCoordination::PerReplica,
17525            handler: |_state| Box::pin(async { Ok(()) }),
17526        };
17527
17528        assert!(merge_and_validate_task_names(&[&descriptor], vec![unrelated_task]).is_ok());
17529    }
17530
17531    #[test]
17532    fn retention_dry_run_one_shot_dispatches_before_server_start() {
17533        // Mirrors `migrate_only_one_shot_applies_and_exits_without_serving`:
17534        // AUTUMN_RETENTION_DRY_RUN=1 must be handled BEFORE the `let Self {`
17535        // destructure that begins the serving path, so a dry-run never binds
17536        // a port (issue #1342).
17537        let source = include_str!("app.rs").replace("\r\n", "\n");
17538        let run_start = source.find("pub async fn run(self)").expect("run() exists");
17539        let run_end = source
17540            .find("async fn run_build_mode(self)")
17541            .expect("build mode follows run()");
17542        let run_body = &source[run_start..run_end];
17543
17544        let dispatch = run_body
17545            .find("if is_retention_dry_run_mode() {")
17546            .expect("run() dispatches the retention dry-run one-shot");
17547        let server_start = run_body
17548            .find("let Self {")
17549            .expect("run() destructures self to start the server");
17550        assert!(
17551            dispatch < server_start,
17552            "AUTUMN_RETENTION_DRY_RUN must be handled before the server-start path"
17553        );
17554        let dry_run_branch = &run_body[dispatch..server_start];
17555        assert!(
17556            dry_run_branch.contains("self.run_retention_dry_run_mode().await;")
17557                && dry_run_branch.contains("return;"),
17558            "the retention dry-run one-shot must run then return before server start"
17559        );
17560    }
17561
17562    #[test]
17563    fn dump_jobs_manifest_includes_synthesized_durable_listener_default_queue() {
17564        // Regression (#1802, Codex P2): an app that registers a durable listener
17565        // and configures `[jobs.queues]` WITHOUT `default` still drains `default`
17566        // at runtime — `finalize_event_bus` synthesizes a `default`-queue
17567        // `JobInfo` for each durable listener before the runtime starts. The
17568        // `AUTUMN_DUMP_JOBS=1` manifest must reflect that same set through the
17569        // shared `synthesize_durable_listener_jobs` seam, or a topology-aware
17570        // `autumn doctor` would accept a fleet where no tier drains those jobs.
17571        fn listener_handler(
17572            _state: AppState,
17573            _payload: serde_json::Value,
17574        ) -> std::pin::Pin<
17575            Box<dyn std::future::Future<Output = crate::AutumnResult<()>> + Send + 'static>,
17576        > {
17577            Box::pin(async move { Ok(()) })
17578        }
17579        let durable = crate::events::ListenerInfo {
17580            event_name: "UserSignedUp",
17581            listener_name: "app::send_welcome_email".to_string(),
17582            mode: crate::events::DispatchMode::Durable,
17583            job_name: Some("__event_listener::send_welcome_email".to_string()),
17584            max_attempts: 4,
17585            initial_backoff_ms: 250,
17586            handler: listener_handler,
17587        };
17588        let cfg = crate::config::JobQueuesConfig::strict_list(["critical"]);
17589
17590        // The dump path holds no builder-registered jobs, only the durable
17591        // listener; the manifest must still surface `default` (where that
17592        // listener's synthesized job runs) alongside the configured `critical`.
17593        let manifest = dump_jobs_manifest(&cfg, Vec::new(), vec![durable]);
17594        assert_eq!(manifest, "queues = [\"critical\", \"default\"]\n");
17595    }
17596
17597    #[cfg(feature = "db")]
17598    const APP_TEST_MIGRATIONS: crate::migrate::EmbeddedMigrations =
17599        diesel_migrations::embed_migrations!("test_migrations");
17600
17601    /// Shared no-op `MailDeliveryQueue` used by builder tests so the trait
17602    /// impl body is defined once and exercised by at least one test.
17603    #[cfg(feature = "mail")]
17604    struct MailTestNoopQueue;
17605
17606    #[cfg(feature = "mail")]
17607    impl crate::mail::MailDeliveryQueue for MailTestNoopQueue {
17608        fn enqueue<'a>(
17609            &'a self,
17610            _mail: crate::mail::Mail,
17611        ) -> std::pin::Pin<
17612            Box<dyn std::future::Future<Output = Result<(), crate::mail::MailError>> + Send + 'a>,
17613        > {
17614            Box::pin(async { Ok(()) })
17615        }
17616    }
17617
17618    #[cfg(feature = "mail")]
17619    fn test_mail() -> crate::mail::Mail {
17620        crate::mail::Mail::builder()
17621            .to("test@example.com")
17622            .subject("hi")
17623            .text("hello")
17624            .build()
17625            .expect("test mail should build")
17626    }
17627
17628    /// Helper to build a test router with default config and no database.
17629    pub fn test_router(routes: Vec<Route>) -> axum::Router {
17630        let config = AutumnConfig::default();
17631        let state = AppState {
17632            health_detailed: true,
17633            ..AppState::test_default()
17634        };
17635        crate::router::build_router(routes, &config, state)
17636    }
17637
17638    // ── Serve readiness payload (#1639) ────────────────────────────────────
17639    //
17640    // The supervisor learns two things from this file: how long the app will
17641    // drain for, and where it actually bound. The second is what lets
17642    // `autumn serve --daemon` write an address-discovery file on a platform
17643    // with no Unix socket to name in advance.
17644
17645    #[test]
17646    fn a_wildcard_bind_is_published_as_a_dialable_loopback_address() {
17647        // The production smart default binds `0.0.0.0`, and `local_addr()`
17648        // faithfully reports it. Publishing that in `serve.addr` would hand a
17649        // thin client an address nothing can dial while the start reported
17650        // success — the same trap `[cluster] advertise_addr` already rejects.
17651        assert_eq!(dialable_endpoint("0.0.0.0:3000"), "127.0.0.1:3000");
17652        assert_eq!(dialable_endpoint("[::]:3000"), "[::1]:3000");
17653    }
17654
17655    #[test]
17656    fn a_specific_bind_is_published_exactly_as_bound() {
17657        // Rewriting one would be worse than the bug: the supervisor would
17658        // advertise an interface the app is not on.
17659        assert_eq!(dialable_endpoint("192.168.1.10:3000"), "192.168.1.10:3000");
17660        assert_eq!(dialable_endpoint("127.0.0.1:8080"), "127.0.0.1:8080");
17661        assert_eq!(dialable_endpoint("[::1]:8080"), "[::1]:8080");
17662    }
17663
17664    #[test]
17665    fn the_resolved_port_survives_the_rewrite() {
17666        // The port is the half the supervisor could not have known — `port = 0`
17667        // resolves in the kernel — so losing it would defeat the whole report.
17668        assert_eq!(dialable_endpoint("0.0.0.0:49152"), "127.0.0.1:49152");
17669    }
17670
17671    #[test]
17672    fn an_unparseable_address_is_published_unchanged() {
17673        // Better to publish what was bound than to invent something.
17674        assert_eq!(dialable_endpoint("not-an-address"), "not-an-address");
17675    }
17676
17677    #[test]
17678    fn serve_ready_payload_leads_with_the_drain_budget() {
17679        // Line one stays a bare integer so a supervisor that predates the
17680        // address line still reads the budget it always read.
17681        let payload = serve_ready_payload(35, "tcp 127.0.0.1:3000");
17682        assert_eq!(payload.lines().next(), Some("35"));
17683    }
17684
17685    #[test]
17686    fn serve_ready_payload_carries_the_bound_endpoint_on_line_two() {
17687        let payload = serve_ready_payload(35, "tcp 127.0.0.1:3000");
17688        assert_eq!(payload.lines().nth(1), Some("tcp 127.0.0.1:3000"));
17689    }
17690
17691    #[test]
17692    fn serve_ready_payload_survives_a_socket_path_containing_spaces() {
17693        // A Unix socket under a directory with a space is legal, so the format
17694        // has to split on the FIRST separator only, never on every one.
17695        let payload = serve_ready_payload(1, "unix /home/a b/serve.sock");
17696        let line = payload.lines().nth(1).expect("endpoint line");
17697        assert_eq!(line.split_once(' '), Some(("unix", "/home/a b/serve.sock")));
17698    }
17699
17700    #[test]
17701    fn serve_ready_payload_never_emits_a_bare_newline_for_an_unknown_endpoint() {
17702        // An empty endpoint must not leave a blank line a reader could mistake
17703        // for an address. The capability line fills line two.
17704        let payload = serve_ready_payload(7, "");
17705        assert!(payload.lines().all(|l| !l.trim().is_empty()), "{payload:?}");
17706        assert_eq!(
17707            payload.lines().nth(1),
17708            Some(SHUTDOWN_SIGNAL_CAPABILITY_LINE)
17709        );
17710    }
17711
17712    #[test]
17713    fn serve_ready_payload_advertises_the_shutdown_signal_capability() {
17714        for endpoint in ["tcp 127.0.0.1:3000", ""] {
17715            let payload = serve_ready_payload(35, endpoint);
17716            assert!(
17717                serve_ready_advertises_shutdown_signal(&payload),
17718                "{payload:?}"
17719            );
17720        }
17721    }
17722
17723    #[test]
17724    fn the_capability_line_never_displaces_the_endpoint_line() {
17725        let payload = serve_ready_payload(35, "tcp 127.0.0.1:3000");
17726        assert_eq!(payload.lines().nth(1), Some("tcp 127.0.0.1:3000"));
17727    }
17728
17729    #[test]
17730    fn a_budget_only_ready_file_does_not_advertise_the_capability() {
17731        // An app that predates the signal protocol still wrote this much.
17732        for old in ["35", "35\n", "35\ntcp 127.0.0.1:3000", ""] {
17733            assert!(!serve_ready_advertises_shutdown_signal(old), "{old:?}");
17734        }
17735    }
17736
17737    #[test]
17738    fn an_unrelated_capability_is_not_the_shutdown_signal() {
17739        assert!(!serve_ready_advertises_shutdown_signal(
17740            "35\ncapability other"
17741        ));
17742    }
17743
17744    // ── Cooperative external shutdown (#1616) ──────────────────────────────
17745    //
17746    // On Windows there is no SIGTERM: `autumn dev` could only stop the app with
17747    // `TerminateProcess`, which skips `on_shutdown` hooks — so a managed
17748    // Postgres child was orphaned on every hot reload. These cover the runtime
17749    // half of the fix: an opt-in, env-named flag file that drains the app
17750    // through the *same* graceful path a signal takes.
17751
17752    #[test]
17753    fn external_shutdown_path_is_none_when_the_env_var_is_absent() {
17754        assert_eq!(external_shutdown_path_from(None), None);
17755    }
17756
17757    #[test]
17758    fn external_shutdown_path_ignores_an_empty_env_value() {
17759        // An empty value is what an unset-but-exported variable looks like. It
17760        // must not resolve to the current directory, where any stray file would
17761        // drain the app.
17762        assert_eq!(
17763            external_shutdown_path_from(Some(std::ffi::OsStr::new(""))),
17764            None
17765        );
17766        assert_eq!(
17767            external_shutdown_path_from(Some(std::ffi::OsStr::new("   "))),
17768            None
17769        );
17770    }
17771
17772    #[test]
17773    fn external_shutdown_path_trims_surrounding_whitespace() {
17774        // A value that only *looks* blank is rejected above; one with real
17775        // content and stray whitespace must resolve to the path the parent
17776        // actually wrote, not to a sibling with a leading space that will
17777        // never match. Getting this wrong is silent: on Windows every reload
17778        // would stall the full budget and hard-kill, reintroducing the
17779        // orphaned-cluster bug this seam exists to fix.
17780        assert_eq!(
17781            external_shutdown_path_from(Some(std::ffi::OsStr::new("  /tmp/autumn-stop  "))),
17782            Some(std::path::PathBuf::from("/tmp/autumn-stop"))
17783        );
17784    }
17785
17786    #[tokio::test]
17787    async fn external_shutdown_signal_ignores_a_directory_at_the_path() {
17788        // `metadata()` succeeds on a directory. If existence alone were the
17789        // test, pointing the variable at a directory would drain the app on
17790        // every boot — bind, drain, exit 0 — which under a supervisor is a
17791        // restart loop that reports success on every iteration.
17792        let tmp = tempfile::TempDir::new().unwrap();
17793        let path = tmp.path().join("not-a-signal");
17794        std::fs::create_dir(&path).unwrap();
17795
17796        let resolved = tokio::time::timeout(
17797            std::time::Duration::from_millis(400),
17798            external_shutdown_signal(Some(path)),
17799        )
17800        .await;
17801        assert!(resolved.is_err(), "a directory must not count as a signal");
17802    }
17803
17804    #[test]
17805    fn external_shutdown_path_uses_the_env_value_verbatim() {
17806        assert_eq!(
17807            external_shutdown_path_from(Some(std::ffi::OsStr::new("/tmp/autumn-stop"))),
17808            Some(std::path::PathBuf::from("/tmp/autumn-stop"))
17809        );
17810    }
17811
17812    #[tokio::test]
17813    async fn external_shutdown_signal_never_resolves_without_a_path() {
17814        // Not configured must mean "never drains" — not "drains immediately",
17815        // which would take down every app that does not opt in.
17816        let pending = tokio::time::timeout(
17817            std::time::Duration::from_millis(300),
17818            external_shutdown_signal(None),
17819        )
17820        .await;
17821        assert!(
17822            pending.is_err(),
17823            "an unconfigured signal must never resolve"
17824        );
17825    }
17826
17827    #[tokio::test]
17828    async fn external_shutdown_signal_resolves_when_the_file_appears() {
17829        let tmp = tempfile::TempDir::new().unwrap();
17830        let path = tmp.path().join("dev-shutdown.signal");
17831
17832        let writer_path = path.clone();
17833        let writer = tokio::spawn(async move {
17834            tokio::time::sleep(std::time::Duration::from_millis(150)).await;
17835            std::fs::write(&writer_path, b"stop").unwrap();
17836        });
17837
17838        let signalled = tokio::time::timeout(
17839            std::time::Duration::from_secs(5),
17840            external_shutdown_signal(Some(path)),
17841        )
17842        .await;
17843        assert!(signalled.is_ok(), "writing the file must drain the app");
17844        writer.await.unwrap();
17845    }
17846
17847    #[tokio::test]
17848    async fn external_shutdown_signal_resolves_immediately_when_present_at_boot() {
17849        // `autumn dev` removes a stale file before spawning, but a crashed
17850        // parent can leave one behind; resolving immediately is the safe
17851        // direction — the app exits gracefully rather than ignoring a stop.
17852        let tmp = tempfile::TempDir::new().unwrap();
17853        let path = tmp.path().join("dev-shutdown.signal");
17854        std::fs::write(&path, b"stop").unwrap();
17855
17856        let signalled = tokio::time::timeout(
17857            std::time::Duration::from_secs(5),
17858            external_shutdown_signal(Some(path)),
17859        )
17860        .await;
17861        assert!(
17862            signalled.is_ok(),
17863            "a file present at boot must drain the app"
17864        );
17865    }
17866
17867    #[test]
17868    fn external_shutdown_env_var_is_the_name_the_cli_writes() {
17869        // `autumn-cli` mirrors this constant (it cannot depend on autumn-web's
17870        // private items); a rename on either side breaks the dev loop silently,
17871        // so pin the wire name here.
17872        assert_eq!(SHUTDOWN_SIGNAL_FILE_ENV, "AUTUMN_SHUTDOWN_SIGNAL_FILE");
17873    }
17874
17875    #[tokio::test]
17876    async fn canary_rollback_signal_resolves_when_flag_newly_written() {
17877        let tmp = tempfile::TempDir::new().unwrap();
17878        let path = tmp.path().join("canary-rollback.json");
17879
17880        // Flag is absent at boot; writing it after start must resolve the signal.
17881        let writer_path = path.clone();
17882        let writer = tokio::spawn(async move {
17883            tokio::time::sleep(std::time::Duration::from_millis(150)).await;
17884            crate::canary::CanaryState::write_rollback_flag(
17885                &writer_path,
17886                &crate::canary::RollbackSignal::default(),
17887            )
17888            .unwrap();
17889        });
17890
17891        let signalled = tokio::time::timeout(
17892            std::time::Duration::from_secs(5),
17893            canary_rollback_signal(&path),
17894        )
17895        .await;
17896        assert!(signalled.is_ok(), "rollback signal should resolve");
17897        writer.await.unwrap();
17898    }
17899
17900    #[tokio::test]
17901    async fn canary_rollback_signal_resolves_immediately_when_flag_present_at_boot() {
17902        let tmp = tempfile::TempDir::new().unwrap();
17903        let path = tmp.path().join("canary-rollback.json");
17904        // A rollback flag is sticky across restarts: present at boot must trigger
17905        // again so a supervisor restart cannot rejoin a rolled-back replica.
17906        crate::canary::CanaryState::write_rollback_flag(
17907            &path,
17908            &crate::canary::RollbackSignal::default(),
17909        )
17910        .unwrap();
17911
17912        let signalled = tokio::time::timeout(
17913            std::time::Duration::from_secs(5),
17914            canary_rollback_signal(&path),
17915        )
17916        .await;
17917        assert!(
17918            signalled.is_ok(),
17919            "a flag present at boot must trigger rollback (sticky across restarts)"
17920        );
17921    }
17922
17923    // Postgres-only fixture: it configures a distinct `replica_url`, which
17924    // SQLite refuses at both layers — `database_backend_consistency` at config
17925    // time, and `reject_unusable_sqlite_replica` at topology build, which takes
17926    // only a replica naming the same file as its primary. That replica is the
17927    // blocker; the shards some of these fixtures also declare are refused at
17928    // BOOT (`sqlite_sharding_unsupported_guard`) but not by the topology
17929    // builder, so they would not fail here on their own. Gated rather than
17930    // swapped, so the test never asserts a configuration production refuses;
17931    // same treatment `db::`'s replica tests got.
17932    #[cfg(not(feature = "sqlite"))]
17933    #[cfg(feature = "db")]
17934    #[test]
17935    fn build_state_applies_replica_fallback_policy_to_read_routing() {
17936        let mut config = AutumnConfig::default();
17937        config.database.primary_url = Some("postgres://localhost/primary".to_owned());
17938        config.database.primary_pool_size = Some(5);
17939        config.database.replica_url = Some("postgres://localhost/replica".to_owned());
17940        config.database.replica_pool_size = Some(2);
17941        config.database.replica_fallback = crate::config::ReplicaFallback::Primary;
17942        let topology = crate::db::create_topology(&config.database)
17943            .expect("topology should build")
17944            .expect("database should be configured");
17945
17946        let state = build_state(
17947            &config,
17948            Some(&topology),
17949            None,
17950            #[cfg(feature = "ws")]
17951            None,
17952        );
17953        state
17954            .probes()
17955            .mark_replica_unready("replica migrations lag primary");
17956
17957        assert_eq!(state.read_pool().expect("read pool").status().max_size, 5);
17958    }
17959
17960    #[test]
17961    fn build_state_exposes_resolved_process_role() {
17962        use crate::config::ProcessRole;
17963
17964        // Default config resolves to the combined role: existing single-process
17965        // apps see no behavior change and get both HTTP + workers.
17966        let mut config = AutumnConfig::default();
17967        let state = build_state(
17968            &config,
17969            #[cfg(feature = "db")]
17970            None,
17971            #[cfg(feature = "db")]
17972            None,
17973            #[cfg(feature = "ws")]
17974            None,
17975        );
17976        assert_eq!(state.role(), ProcessRole::Combined);
17977        assert!(state.role().serves_http());
17978        assert!(state.role().runs_workers());
17979
17980        // A worker-role config flows through the exact same resolution the
17981        // framework uses to gate the job runtime, reachable via `state.role()`.
17982        config.role = ProcessRole::Worker;
17983        let state = build_state(
17984            &config,
17985            #[cfg(feature = "db")]
17986            None,
17987            #[cfg(feature = "db")]
17988            None,
17989            #[cfg(feature = "ws")]
17990            None,
17991        );
17992        assert_eq!(state.role(), ProcessRole::Worker);
17993        assert!(state.role().runs_workers());
17994        assert!(!state.role().serves_http());
17995    }
17996
17997    // Postgres-only fixture: it configures a distinct `replica_url`, which
17998    // SQLite refuses at both layers — `database_backend_consistency` at config
17999    // time, and `reject_unusable_sqlite_replica` at topology build, which takes
18000    // only a replica naming the same file as its primary. That replica is the
18001    // blocker; the shards some of these fixtures also declare are refused at
18002    // BOOT (`sqlite_sharding_unsupported_guard`) but not by the topology
18003    // builder, so they would not fail here on their own. Gated rather than
18004    // swapped, so the test never asserts a configuration production refuses;
18005    // same treatment `db::`'s replica tests got.
18006    #[cfg(not(feature = "sqlite"))]
18007    #[cfg(feature = "db")]
18008    #[tokio::test]
18009    async fn custom_pool_provider_preserves_configured_replica_topology() {
18010        struct PassthroughPoolProvider;
18011
18012        impl crate::db::DatabasePoolProvider for PassthroughPoolProvider {
18013            async fn create_pool(
18014                &self,
18015                config: &crate::config::DatabaseConfig,
18016            ) -> Result<
18017                Option<
18018                    diesel_async::pooled_connection::deadpool::Pool<crate::db::RuntimeConnection>,
18019                >,
18020                crate::db::PoolError,
18021            > {
18022                crate::db::create_pool(config)
18023            }
18024        }
18025
18026        let mut config = AutumnConfig::default();
18027        config.database.primary_url = Some("postgres://localhost/primary".to_owned());
18028        config.database.primary_pool_size = Some(5);
18029        config.database.replica_url = Some("postgres://localhost/replica".to_owned());
18030        config.database.replica_pool_size = Some(2);
18031        config.database.replica_fallback = crate::config::ReplicaFallback::FailReadiness;
18032        let AppBuilder {
18033            pool_provider_factory,
18034            ..
18035        } = app().with_pool_provider(PassthroughPoolProvider);
18036
18037        let database = setup_database(
18038            &config,
18039            Vec::new(),
18040            pool_provider_factory,
18041            None,
18042            None,
18043            false,
18044            RepositoryCommitHookQueueMigrationMode::Runtime,
18045        )
18046        .await
18047        .expect("custom provider should build database topology");
18048        let topology = database.topology.expect("database should be configured");
18049
18050        assert_eq!(topology.primary().status().max_size, 5);
18051        assert_eq!(
18052            topology
18053                .replica()
18054                .expect("custom provider should create replica pool")
18055                .status()
18056                .max_size,
18057            2
18058        );
18059
18060        let state = build_state(
18061            &config,
18062            Some(&topology),
18063            None,
18064            #[cfg(feature = "ws")]
18065            None,
18066        );
18067        state
18068            .probes()
18069            .mark_replica_connection_unready("replica connection failed");
18070
18071        assert!(state.read_pool().is_none());
18072        let (status, _) = crate::probe::readiness_response(&state).await;
18073        assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE);
18074    }
18075
18076    // Finding 2 (Codex P2), corrected: the fail-closed `statement_timeout` guard
18077    // must fire once a custom provider has actually established a pool. A custom
18078    // provider can build its own SQLite pool without the built-in
18079    // `create_topology`/`create_pool` factories — the default `create_topology`
18080    // only delegates to `create_pool`, and both are overridable — so
18081    // `setup_database` enforces the guard at dispatch. It must run only for an
18082    // established pool (`Some(..)`), not before the provider returns: a provider
18083    // returning `Ok(None)` opts into the supported no-database mode and must still
18084    // boot even with a nonzero `statement_timeout`, matching the built-in path.
18085    // CI's sqlite job runs the named integration targets, not `--lib`, and
18086    // `setup_database` and the shared guard are crate-private, so this boundary is
18087    // reachable only from a unit test.
18088    //
18089    // Case (a): a custom provider that establishes a real in-memory SQLite pool
18090    // (`Ok(Some(..))`) under a nonzero `statement_timeout` must fail closed with
18091    // the actionable error, keeping the original F2 bypass closed.
18092    #[cfg(feature = "sqlite")]
18093    #[tokio::test]
18094    async fn custom_pool_provider_with_established_sqlite_pool_fails_closed_on_statement_timeout() {
18095        // Builds a real in-memory SQLite pool WITHOUT routing the timeout through
18096        // the built-in factory — the exact F2 bypass a custom provider could use.
18097        struct RealSqlitePoolProvider;
18098
18099        impl crate::db::DatabasePoolProvider for RealSqlitePoolProvider {
18100            async fn create_pool(
18101                &self,
18102                config: &crate::config::DatabaseConfig,
18103            ) -> Result<
18104                Option<
18105                    diesel_async::pooled_connection::deadpool::Pool<crate::db::RuntimeConnection>,
18106                >,
18107                crate::db::PoolError,
18108            > {
18109                // Clear `statement_timeout` locally so the built-in `create_pool`
18110                // guard does not fire inside the provider — this provider hands
18111                // back a live SQLite pool that silently dropped the timeout, which
18112                // is exactly the fail-closed condition the dispatch guard closes.
18113                let mut relaxed = config.clone();
18114                relaxed.statement_timeout = None;
18115                crate::db::create_pool(&relaxed)
18116            }
18117        }
18118
18119        let mut config = AutumnConfig::default();
18120        config.database.primary_url = Some("sqlite::memory:".to_owned());
18121        config.database.statement_timeout = Some(std::time::Duration::from_secs(30));
18122        let AppBuilder {
18123            pool_provider_factory,
18124            shard_provider_factory,
18125            ..
18126        } = app().with_pool_provider(RealSqlitePoolProvider);
18127
18128        let Err(err) = setup_database(
18129            &config,
18130            Vec::new(),
18131            pool_provider_factory,
18132            shard_provider_factory,
18133            None,
18134            false,
18135            RepositoryCommitHookQueueMigrationMode::Runtime,
18136        )
18137        .await
18138        else {
18139            panic!(
18140                "sqlite + statement_timeout must fail closed once the provider establishes a pool"
18141            );
18142        };
18143        assert!(
18144            err.contains("database.statement_timeout") && err.contains("SQLite"),
18145            "dispatch guard error must name the config key and SQLite, got: {err}"
18146        );
18147    }
18148
18149    // Case (b) — the exact regression Codex flagged: a custom provider that opts
18150    // into no-database mode (`Ok(None)`) must STILL boot even with a nonzero
18151    // `database.statement_timeout`, because no pool/statement exists to bound. The
18152    // Some-gated guard must not reject this the way the pre-dispatch guard did.
18153    #[cfg(feature = "sqlite")]
18154    #[tokio::test]
18155    async fn custom_pool_provider_no_database_mode_boots_with_statement_timeout() {
18156        struct NoDatabaseProvider;
18157
18158        impl crate::db::DatabasePoolProvider for NoDatabaseProvider {
18159            async fn create_pool(
18160                &self,
18161                _config: &crate::config::DatabaseConfig,
18162            ) -> Result<
18163                Option<
18164                    diesel_async::pooled_connection::deadpool::Pool<crate::db::RuntimeConnection>,
18165                >,
18166                crate::db::PoolError,
18167            > {
18168                // Explicit no-database opt-out.
18169                Ok(None)
18170            }
18171        }
18172
18173        let mut config = AutumnConfig::default();
18174        // A URL may even be configured; the provider's `Ok(None)` still wins.
18175        config.database.primary_url = Some("sqlite::memory:".to_owned());
18176        config.database.statement_timeout = Some(std::time::Duration::from_secs(30));
18177        let AppBuilder {
18178            pool_provider_factory,
18179            shard_provider_factory,
18180            ..
18181        } = app().with_pool_provider(NoDatabaseProvider);
18182
18183        let bootstrap = setup_database(
18184            &config,
18185            Vec::new(),
18186            pool_provider_factory,
18187            shard_provider_factory,
18188            None,
18189            false,
18190            RepositoryCommitHookQueueMigrationMode::Runtime,
18191        )
18192        .await
18193        .expect("no-database provider must boot even with a nonzero statement_timeout");
18194        assert!(
18195            bootstrap.topology.is_none(),
18196            "no-database mode must yield no control topology"
18197        );
18198        assert!(
18199            bootstrap.shards.is_none(),
18200            "no-database mode must yield no shard set"
18201        );
18202    }
18203
18204    // Only the two Postgres-only shard tests use this fixture.
18205    #[cfg(all(feature = "db", not(feature = "sqlite")))]
18206    fn sharded_test_config() -> AutumnConfig {
18207        let mut config = AutumnConfig::default();
18208        config.database.primary_url = Some("postgres://localhost/control".to_owned());
18209        config.database.shards = vec![
18210            crate::config::ShardConfig {
18211                name: "shard0".to_owned(),
18212                primary_url: "postgres://localhost/shard0".to_owned(),
18213                slots: Some(vec![crate::config::SlotSpec::Range("0-8191".to_owned())]),
18214                replica_url: None,
18215                primary_pool_size: Some(3),
18216                replica_pool_size: None,
18217                replica_fallback: None,
18218            },
18219            crate::config::ShardConfig {
18220                name: "shard1".to_owned(),
18221                primary_url: "postgres://localhost/shard1".to_owned(),
18222                slots: Some(vec![crate::config::SlotSpec::Range(
18223                    "8192-16383".to_owned(),
18224                )]),
18225                replica_url: Some("postgres://localhost/shard1_ro".to_owned()),
18226                primary_pool_size: None,
18227                replica_pool_size: Some(2),
18228                replica_fallback: None,
18229            },
18230        ];
18231        config
18232    }
18233
18234    // Postgres-only fixture: it configures a distinct `replica_url`, which
18235    // SQLite refuses at both layers — `database_backend_consistency` at config
18236    // time, and `reject_unusable_sqlite_replica` at topology build, which takes
18237    // only a replica naming the same file as its primary. That replica is the
18238    // blocker; the shards some of these fixtures also declare are refused at
18239    // BOOT (`sqlite_sharding_unsupported_guard`) but not by the topology
18240    // builder, so they would not fail here on their own. Gated rather than
18241    // swapped, so the test never asserts a configuration production refuses;
18242    // same treatment `db::`'s replica tests got.
18243    #[cfg(not(feature = "sqlite"))]
18244    #[cfg(feature = "db")]
18245    #[tokio::test]
18246    async fn setup_database_builds_shard_set_from_config() {
18247        let config = sharded_test_config();
18248
18249        let database = setup_database(
18250            &config,
18251            Vec::new(),
18252            None,
18253            None,
18254            None,
18255            false,
18256            RepositoryCommitHookQueueMigrationMode::Runtime,
18257        )
18258        .await
18259        .expect("sharded config should bootstrap");
18260
18261        assert!(database.topology.is_some(), "control role configured");
18262        let shards = database.shards.expect("shards configured");
18263        assert_eq!(shards.len(), 2);
18264        assert_eq!(
18265            shards
18266                .by_name("shard0")
18267                .expect("shard0")
18268                .primary_pool()
18269                .status()
18270                .max_size,
18271            3
18272        );
18273        assert_eq!(
18274            shards
18275                .by_name("shard1")
18276                .expect("shard1")
18277                .replica_pool()
18278                .expect("shard1 replica")
18279                .status()
18280                .max_size,
18281            2
18282        );
18283
18284        let state = build_state(
18285            &config,
18286            database.topology.as_ref(),
18287            Some(shards),
18288            #[cfg(feature = "ws")]
18289            None,
18290        );
18291        let state_shards = state.shards().expect("state should expose shards");
18292        assert_eq!(state_shards.len(), 2);
18293        // Routing works end-to-end through state-held shards.
18294        let routed = state_shards.route("tenant-1").await.expect("route");
18295        assert!(["shard0", "shard1"].contains(&routed.name()));
18296    }
18297
18298    // Postgres-only fixture: it configures a distinct `replica_url`, which
18299    // SQLite refuses at both layers — `database_backend_consistency` at config
18300    // time, and `reject_unusable_sqlite_replica` at topology build, which takes
18301    // only a replica naming the same file as its primary. That replica is the
18302    // blocker; the shards some of these fixtures also declare are refused at
18303    // BOOT (`sqlite_sharding_unsupported_guard`) but not by the topology
18304    // builder, so they would not fail here on their own. Gated rather than
18305    // swapped, so the test never asserts a configuration production refuses;
18306    // same treatment `db::`'s replica tests got.
18307    #[cfg(not(feature = "sqlite"))]
18308    #[cfg(feature = "db")]
18309    #[tokio::test]
18310    async fn custom_pool_provider_builds_shard_topologies() {
18311        struct CountingProvider(std::sync::Arc<std::sync::atomic::AtomicUsize>);
18312
18313        impl crate::db::DatabasePoolProvider for CountingProvider {
18314            async fn create_pool(
18315                &self,
18316                config: &crate::config::DatabaseConfig,
18317            ) -> Result<
18318                Option<
18319                    diesel_async::pooled_connection::deadpool::Pool<crate::db::RuntimeConnection>,
18320                >,
18321                crate::db::PoolError,
18322            > {
18323                crate::db::create_pool(config)
18324            }
18325
18326            async fn create_shard_topology(
18327                &self,
18328                shard: &crate::config::ShardConfig,
18329                defaults: &crate::config::DatabaseConfig,
18330            ) -> Result<crate::db::DatabaseTopology, crate::db::PoolError> {
18331                self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
18332                crate::db::create_shard_topology(shard, defaults)
18333            }
18334        }
18335
18336        let config = sharded_test_config();
18337        let calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
18338        let AppBuilder {
18339            pool_provider_factory,
18340            shard_provider_factory,
18341            ..
18342        } = app().with_pool_provider(CountingProvider(calls.clone()));
18343
18344        let database = setup_database(
18345            &config,
18346            Vec::new(),
18347            pool_provider_factory,
18348            shard_provider_factory,
18349            None,
18350            false,
18351            RepositoryCommitHookQueueMigrationMode::Runtime,
18352        )
18353        .await
18354        .expect("provider should build shard topologies");
18355
18356        assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 2);
18357        assert_eq!(database.shards.expect("shards").len(), 2);
18358    }
18359
18360    #[cfg(feature = "db")]
18361    #[test]
18362    fn repository_commit_hook_worker_starts_after_job_runtime_initialization() {
18363        let source = include_str!("app.rs").replace("\r\n", "\n");
18364        let server_init = "initialize_job_runtime(
18365            jobs,
18366            &state,
18367            &server_shutdown,";
18368        let server_worker = "start_repository_commit_hook_worker(\n                pool,\n                server_shutdown.child_token(),\n            );";
18369        let task_init = "initialize_job_runtime(jobs, &state, &task_shutdown, &config.jobs, true)";
18370        let task_worker = "start_repository_commit_hook_worker(\n                pool,\n                task_shutdown.child_token(),\n            );";
18371
18372        assert!(
18373            source
18374                .find(server_init)
18375                .expect("normal server path should initialize jobs")
18376                < source
18377                    .find(server_worker)
18378                    .expect("normal server path should start repository hook worker"),
18379            "normal server startup must initialize jobs before repository commit hooks can enqueue them"
18380        );
18381        assert!(
18382            source
18383                .find(task_init)
18384                .expect("task runner path should initialize jobs")
18385                < source
18386                    .find(task_worker)
18387                    .expect("task runner path should start repository hook worker"),
18388            "task runner startup must initialize jobs before repository commit hooks can enqueue them"
18389        );
18390    }
18391
18392    #[cfg(feature = "db")]
18393    #[test]
18394    fn repository_commit_hook_workers_are_gated_on_worker_role() {
18395        // Draining durable after-commit hook rows is background execution, so the
18396        // primary-pool and shard commit-hook worker starts on the normal server
18397        // path must both be guarded by `role.runs_workers()` — a web-role replica
18398        // must not claim/execute hook rows (that work belongs to the worker tier).
18399        let source = include_str!("app.rs").replace("\r\n", "\n");
18400        let primary_gate =
18401            "if role.runs_workers()\n            && let Some(pool) = state.pool().cloned()";
18402        let shard_gate = "if role.runs_workers()\n            && let Some(shards) = state.shards()";
18403        assert!(
18404            source.contains(primary_gate),
18405            "primary-pool commit-hook worker must be gated on role.runs_workers()"
18406        );
18407        assert!(
18408            source.contains(shard_gate),
18409            "shard commit-hook workers must be gated on role.runs_workers()"
18410        );
18411    }
18412
18413    /// The replication loop's final flush must be *triggered* after the drain.
18414    ///
18415    /// Cancelling the token only *wakes* the replication thread; the tick that
18416    /// ships the last committed frames runs immediately after it. So waiting
18417    /// for the thread is not enough on its own — if the token fires at phase 5,
18418    /// with `server_shutdown`, that final tick happens while requests are still
18419    /// draining, and a request that commits during the drain is never
18420    /// replicated even though the wait below succeeds and logs that replication
18421    /// flushed. The token must therefore be independent of `server_shutdown`
18422    /// and cancelled only once the drain has finished.
18423    ///
18424    /// The wait must also not be a blocking join: a blocked join cannot be
18425    /// cancelled, so a stuck upload would hold the process open past the
18426    /// shutdown budget that is supposed to bound it.
18427    ///
18428    /// Source-order test in the house style, because the ordering is a property
18429    /// of this function and nothing smaller: an app-level boot test would need a
18430    /// `SQLite` runtime pool, which only exists in the `sqlite` lane.
18431    #[cfg(feature = "db")]
18432    #[test]
18433    fn the_replication_final_flush_is_waited_for_after_the_drain() {
18434        let source = include_str!("app.rs").replace("\r\n", "\n");
18435        let server_start = source
18436            .find("pub async fn run(self)")
18437            .expect("normal server path should exist");
18438        // Bounded at the next path so the search cannot match this test's own
18439        // source, which necessarily quotes the strings it is looking for.
18440        let build_mode_start = source
18441            .find("async fn run_build_mode(self)")
18442            .expect("static build path should follow server path");
18443        let server_source = &source[server_start..build_mode_start];
18444
18445        let spawn = server_source
18446            .find("Ok(_handle) => replication_done = Some(waiter),")
18447            .expect("the replication thread's completion signal must be kept, not dropped");
18448        let drain = server_source
18449            .find("let server_result = server_task.await")
18450            .expect("the normal server path should await the drain");
18451        let wait = server_source
18452            .find("if let Some(waiter) = replication_done.take()")
18453            .expect("shutdown must wait for the replication thread");
18454        let cancel = server_source
18455            .find("replication_shutdown.cancel();")
18456            .expect("shutdown must cancel the replication token explicitly");
18457
18458        assert!(
18459            spawn < drain && drain < wait && drain < cancel,
18460            "the final flush must be triggered AND awaited AFTER the request drain \
18461             (spawn={spawn}, drain={drain}, wait={wait}, cancel={cancel})"
18462        );
18463        // The token that releases the final tick must not fire with the
18464        // listener: that is the whole bug this ordering exists to prevent.
18465        assert!(
18466            server_source
18467                .contains("let replication_shutdown = tokio_util::sync::CancellationToken::new();"),
18468            "the replication token must be independent of `server_shutdown`, not a child of it"
18469        );
18470        assert!(
18471            !server_source[..wait].contains("let replication_shutdown = server_shutdown"),
18472            "the replication token must not be derived from `server_shutdown`"
18473        );
18474        // A blocking join cannot be abandoned when the budget runs out.
18475        assert!(
18476            !server_source[wait..].contains("spawn_blocking"),
18477            "the bounded wait must not block on a join it cannot cancel"
18478        );
18479    }
18480
18481    /// The cluster must outlive the in-flight request drain.
18482    ///
18483    /// `server_shutdown` fires at phase 5, when the listener stops accepting —
18484    /// requests keep draining after it for up to `shutdown_timeout_secs`, and a
18485    /// request served in that window can still increment a cluster counter. If
18486    /// the cluster's loops were children of that token they would already have
18487    /// departed, so the increment would land in a document with nothing left to
18488    /// replicate it and die with the process: a write accepted and silently
18489    /// thrown away. Source-order test in the house style, because the ordering
18490    /// is a property of this function and nothing smaller.
18491    #[test]
18492    fn cluster_departs_after_the_request_drain() {
18493        let source = include_str!("app.rs").replace("\r\n", "\n");
18494        let server_start = source
18495            .find("pub async fn run(self)")
18496            .expect("normal server path should exist");
18497        // Bounded at the next path so the search cannot match this test's own
18498        // source, which necessarily quotes the strings it is looking for.
18499        let build_mode_start = source
18500            .find("async fn run_build_mode(self)")
18501            .expect("static build path should follow server path");
18502        let server_source = &source[server_start..build_mode_start];
18503
18504        let install = server_source
18505            .find("crate::cluster::install_from_config(&state, &config.cluster, &cluster_shutdown)")
18506            .expect("the cluster must be installed on its own token, not on server_shutdown");
18507        let drain = server_source
18508            .find("let server_result = server_task.await")
18509            .expect("the normal server path should await the drain");
18510        let depart = server_source
18511            .find("cluster_shutdown.cancel();")
18512            .expect("the cluster token should be cancelled explicitly");
18513
18514        assert!(
18515            install < drain && drain < depart,
18516            "the cluster must be installed before the drain and cancelled only \
18517             after it completes, so an increment accepted while requests drain \
18518             still has a push loop to replicate it"
18519        );
18520        assert!(
18521            !server_source.contains("&config.cluster, &server_shutdown"),
18522            "the cluster must not ride server_shutdown: that token fires when \
18523             the listener closes, with the request drain still ahead of it"
18524        );
18525    }
18526
18527    /// The fleet's final flush bounds blocking I/O, so it spends the larger of
18528    /// the injected-clock and the real elapsed time. A fixed or unadvanced
18529    /// clock reads zero while a real drain runs, and must not hand the flush
18530    /// a fresh full budget.
18531    #[cfg(feature = "sqlite")]
18532    #[test]
18533    fn the_fleet_flush_budget_counts_real_time_a_pinned_clock_misses() {
18534        use std::time::Duration;
18535
18536        let budget = Duration::from_secs(30);
18537        // A pinned injected clock: only the real reading shows the drain.
18538        assert_eq!(
18539            fleet_flush_budget(budget, Duration::ZERO, Duration::from_secs(25)),
18540            Duration::from_secs(5)
18541        );
18542        // A simulated clock that ran ahead of real time wins instead.
18543        assert_eq!(
18544            fleet_flush_budget(budget, Duration::from_secs(20), Duration::from_secs(1)),
18545            Duration::from_secs(10)
18546        );
18547        // Nothing is left once either reading spent the budget.
18548        assert_eq!(
18549            fleet_flush_budget(budget, Duration::ZERO, budget),
18550            Duration::ZERO
18551        );
18552    }
18553
18554    /// …and it must depart *inside* the shutdown budget, not after it.
18555    ///
18556    /// Drain, departure and `on_shutdown` hooks share one
18557    /// `shutdown_timeout_secs`: a supervisor that times the process out at that
18558    /// deadline `SIGKILL`s whatever is still running. An unconditional
18559    /// `LEAVE_BUDGET` sleep between the drain and the hooks would make the
18560    /// worst case `shutdown_timeout_secs + LEAVE_BUDGET`, so the wait is
18561    /// clamped to the budget the drain left over.
18562    #[test]
18563    fn the_cluster_departure_wait_fits_inside_the_shutdown_budget() {
18564        use std::time::Duration;
18565
18566        let budget = Duration::from_secs(30);
18567
18568        // A fast drain: the departure gets its whole budget…
18569        let quick = Duration::from_secs(1);
18570        assert_eq!(
18571            cluster_departure_wait(budget, quick),
18572            crate::cluster::LEAVE_BUDGET,
18573            "a drain that finished early must leave room for the full departure"
18574        );
18575
18576        // …but never more than the node itself will spend on it.
18577        assert!(
18578            cluster_departure_wait(budget, Duration::ZERO) <= crate::cluster::LEAVE_BUDGET,
18579            "waiting longer than LEAVE_BUDGET would idle past the node's own bound"
18580        );
18581
18582        // A drain that ran to the deadline gets no departure at all: the peer
18583        // converges on the suspicion timeout instead.
18584        assert_eq!(
18585            cluster_departure_wait(budget, budget),
18586            Duration::ZERO,
18587            "a drain that consumed the budget must not buy extra shutdown time"
18588        );
18589        assert_eq!(
18590            cluster_departure_wait(budget, budget.saturating_add(Duration::from_secs(5))),
18591            Duration::ZERO,
18592            "an overrun drain must saturate, not wrap into a fresh wait"
18593        );
18594
18595        // The property the finding is about: drain, departure and hooks are one
18596        // budget. `hook_budget` is `budget - elapsed` measured *after* the
18597        // departure wait, so whatever the drain leaves over has to cover both
18598        // of the phases that follow it — never budget + LEAVE_BUDGET.
18599        for elapsed_ms in [0_u64, 250, 29_800, 29_999, 30_000, 45_000] {
18600            let drained = Duration::from_millis(elapsed_ms);
18601            let departure = cluster_departure_wait(budget, drained);
18602            let hooks = budget.saturating_sub(drained.saturating_add(departure));
18603            let left_by_the_drain = budget.saturating_sub(drained);
18604            assert!(
18605                departure.saturating_add(hooks) <= left_by_the_drain,
18606                "after a {drained:?} drain only {left_by_the_drain:?} of the \
18607                 {budget:?} budget is left, but the departure ({departure:?}) \
18608                 and hooks ({hooks:?}) would spend more"
18609            );
18610        }
18611    }
18612
18613    /// The accept loop must go live only after the startup hooks succeed —
18614    /// but only for an in-place-upgrade successor.
18615    ///
18616    /// During an in-place upgrade (#1674) the successor adopts the
18617    /// predecessor's listening socket up front, so both processes are
18618    /// accepting on the same socket and the kernel hands new connections to
18619    /// either — and every connection the successor wins before it can serve is
18620    /// answered by the startup barrier with a 503 while the predecessor is
18621    /// right there, healthy. Spawning the accept loop only once
18622    /// `run_startup_hooks` has returned `Ok` keeps the predecessor serving for
18623    /// the whole window, so a slow or failing successor can no longer 503 real
18624    /// users (#2368).
18625    ///
18626    /// The deferral must NOT apply to a cold start: nothing accepts until the
18627    /// loop is polled, so holding it back would leave `/live` and `/startup`
18628    /// unreachable behind the startup barrier for the whole hook window, and a
18629    /// hook that outlasts a probe threshold would read as a dead pod. Source-
18630    /// order test in the house style: the ordering is a property of this
18631    /// function, and a two-process upgrade test with fd handoff cannot run in
18632    /// CI.
18633    #[test]
18634    fn accept_loop_defers_only_for_upgrade_successor() {
18635        let source = include_str!("app.rs").replace("\r\n", "\n");
18636        let server_start = source
18637            .find("pub async fn run(self)")
18638            .expect("normal server path should exist");
18639        // Bounded at the next path so the search cannot match this test's own
18640        // source, which necessarily quotes the strings it is looking for.
18641        let build_mode_start = source
18642            .find("async fn run_build_mode(self)")
18643            .expect("static build path should follow server path");
18644        let server_source = &source[server_start..build_mode_start];
18645
18646        let bound = server_source
18647            .find("let server_future:")
18648            .expect("the accept-loop future must be built from the bound listener");
18649        // The deferral is gated on this process being an upgrade successor —
18650        // the one shape where the socket is shared with a live predecessor.
18651        let gate = server_source
18652            .find("let defer_accept_loop = crate::upgrade::handoff_requested();")
18653            .expect("the deferral must be gated on the upgrade handoff");
18654        let cold_spawn = server_source
18655            .find("server_task = Some(tokio::spawn(server_future));")
18656            .expect("a cold start must spawn the accept loop up front, as before");
18657        let hooks = server_source
18658            .find("run_startup_hooks(&startup_hooks, state.clone())")
18659            .expect("startup hooks must run on the normal server path");
18660        let deferred_spawn = server_source
18661            .find("tokio::spawn(future)")
18662            .expect("an upgrade successor must spawn its deferred accept loop after the hooks");
18663
18664        assert!(
18665            bound < gate && gate < cold_spawn && cold_spawn < hooks && hooks < deferred_spawn,
18666            "ordering must be: bound listener -> deferral gate -> cold-start spawn \
18667             -> startup hooks -> successor's deferred spawn \
18668             (bound={bound}, gate={gate}, cold_spawn={cold_spawn}, hooks={hooks}, \
18669             deferred_spawn={deferred_spawn})"
18670        );
18671        // A cold start that already spawned must still abort its loop when a
18672        // hook fails; a successor has nothing to abort.
18673        assert!(
18674            server_source.contains("if let Some(task) = server_task.take()"),
18675            "the hook-failure path must abort the cold-start accept loop"
18676        );
18677        // The old shapes must not come back.
18678        assert!(
18679            !server_source.contains("let server_task = match bound_listener"),
18680            "the accept loop must not be spawned from the bound-listener match"
18681        );
18682        assert!(
18683            !server_source.contains("let server_task = tokio::spawn(server_future);"),
18684            "the accept loop must not be unconditionally deferred past the hooks"
18685        );
18686    }
18687
18688    #[test]
18689    fn state_initializers_run_before_job_runtime_initialization() {
18690        let source = include_str!("app.rs").replace("\r\n", "\n");
18691        let server_start = source
18692            .find("pub async fn run(self)")
18693            .expect("normal server path should exist");
18694        let build_mode_start = source
18695            .find("async fn run_build_mode(self)")
18696            .expect("static build path should follow server path");
18697        let task_start = source
18698            .find("async fn run_one_off_task_mode(self, requested_name: String)")
18699            .expect("task runner path should exist");
18700        let server_source = &source[server_start..build_mode_start];
18701        let task_source = &source[task_start..];
18702        let server_init = "initialize_job_runtime(
18703            jobs,
18704            &state,
18705            &server_shutdown,";
18706        let task_init = "initialize_job_runtime(jobs, &state, &task_shutdown, &config.jobs, true)";
18707        let server_initializer = server_source
18708            .find("run_state_initializers(state_initializers, &state);")
18709            .expect("normal server path should run state initializers");
18710        let task_initializer = task_source
18711            .find("run_state_initializers(state_initializers, &state);")
18712            .expect("task runner path should run state initializers");
18713        let server_job = server_source
18714            .find(server_init)
18715            .expect("normal server path should initialize jobs");
18716        let task_job = task_source
18717            .find(task_init)
18718            .expect("task runner path should initialize jobs");
18719
18720        assert!(
18721            server_initializer < server_job,
18722            "normal server startup must install state-initialized resources before job workers start"
18723        );
18724        assert!(
18725            task_initializer < task_job,
18726            "task runner startup must install state-initialized resources before job workers start"
18727        );
18728    }
18729
18730    #[test]
18731    fn optional_interceptor_wiring_matches_across_all_three_run_modes() {
18732        // See the clone-class comment above the `mail_interceptor` arm in
18733        // each of the three functions: this exact block (mail/job/db/
18734        // channels+presence+collab/http) is meant to stay byte-for-byte
18735        // identical across `run()`, `run_build_mode()` and
18736        // `run_one_off_task_mode()`. #862's own history shows what drift
18737        // here costs (a silently-inert one-off-task interceptor), so pin
18738        // the invariant rather than rely on reviewers to notice a divergent
18739        // copy-paste.
18740        let source = include_str!("app.rs").replace("\r\n", "\n");
18741        let start_marker =
18742            "#[cfg(feature = \"mail\")]\n        if let Some(interceptor) = mail_interceptor {";
18743        let end_marker = "if let Some(interceptor) = http_interceptor {\n            \
18744                           state.insert_extension(interceptor);\n        }\n";
18745
18746        let run_start = source.find("pub async fn run(self)").expect("run() exists");
18747        let build_start = source
18748            .find("async fn run_build_mode(self)")
18749            .expect("run_build_mode() follows run()");
18750        let task_start = source
18751            .find("async fn run_one_off_task_mode(self, requested_name: String)")
18752            .expect("run_one_off_task_mode() follows run_build_mode()");
18753
18754        let extract = |region: &str| -> String {
18755            let s = region
18756                .find(start_marker)
18757                .expect("the interceptor-wiring block starts with the mail arm");
18758            let e = region[s..]
18759                .find(end_marker)
18760                .map(|offset| s + offset + end_marker.len())
18761                .expect("the interceptor-wiring block ends with the http arm");
18762            region[s..e].to_owned()
18763        };
18764
18765        let run_block = extract(&source[run_start..build_start]);
18766        let build_block = extract(&source[build_start..task_start]);
18767        // `run_one_off_task_mode` is the last fn in this `impl AppBuilder`;
18768        // a generous bound stands in for hunting down the next `fn`.
18769        let task_block = extract(&source[task_start..(task_start + 20_000).min(source.len())]);
18770
18771        assert_eq!(
18772            run_block, build_block,
18773            "run() and run_build_mode() must wire the five optional interceptors identically"
18774        );
18775        assert_eq!(
18776            run_block, task_block,
18777            "run() and run_one_off_task_mode() must wire the five optional interceptors \
18778             identically (this exact drift shipped once, fixed by #862's \"wire interceptors \
18779             into one-off task mode\")"
18780        );
18781    }
18782
18783    #[test]
18784    fn flag_service_preloads_before_later_state_initializers() {
18785        // Every mode runs state initializers in order, before job workers,
18786        // startup hooks and routing. So a preload here covers them all.
18787        let store = Arc::new(PreloadProbe::default());
18788        let seen = Arc::new(std::sync::atomic::AtomicUsize::new(usize::MAX));
18789        let (seen_in, probe) = (Arc::clone(&seen), Arc::clone(&store));
18790        let builder = app()
18791            .with_flag_store(Arc::clone(&store))
18792            .state_initializer(move |_| {
18793                let preloads = probe.preloads.load(std::sync::atomic::Ordering::SeqCst);
18794                seen_in.store(preloads, std::sync::atomic::Ordering::SeqCst);
18795            });
18796        run_state_initializers(builder.state_initializers, &AppState::for_test());
18797        assert_eq!(seen.load(std::sync::atomic::Ordering::SeqCst), 1);
18798    }
18799
18800    #[test]
18801    fn experiment_store_preloads_before_later_state_initializers() {
18802        let store = Arc::new(crate::experiments::ProbeStore::default());
18803        let seen = Arc::new(std::sync::atomic::AtomicUsize::new(usize::MAX));
18804        let (seen_in, probe) = (Arc::clone(&seen), Arc::clone(&store));
18805        let builder = app()
18806            .with_experiment_store(Arc::clone(&store))
18807            .state_initializer(move |_| {
18808                let preloads = probe.preloads.load(std::sync::atomic::Ordering::SeqCst);
18809                seen_in.store(preloads, std::sync::atomic::Ordering::SeqCst);
18810            });
18811        run_state_initializers(builder.state_initializers, &AppState::for_test());
18812        assert_eq!(seen.load(std::sync::atomic::Ordering::SeqCst), 1);
18813    }
18814
18815    #[test]
18816    fn experiment_store_with_sink_preloads_too() {
18817        let store = Arc::new(crate::experiments::ProbeStore::default());
18818        let builder = app().with_experiment_store_and_sink(
18819            Arc::clone(&store),
18820            Arc::new(crate::experiments::NoOpExposureSink),
18821        );
18822        run_state_initializers(builder.state_initializers, &AppState::for_test());
18823        assert_eq!(store.preloads.load(std::sync::atomic::Ordering::SeqCst), 1);
18824    }
18825
18826    #[test]
18827    fn static_builds_run_state_initializers_before_router_build() {
18828        let source = include_str!("app.rs").replace("\r\n", "\n");
18829        let build_mode_start = source
18830            .find("async fn run_build_mode(self)")
18831            .expect("static build path should exist");
18832        let dump_mode_start = source
18833            .find("async fn run_dump_routes_mode(self)")
18834            .expect("route dump path should follow static build path");
18835        let build_mode_source = &source[build_mode_start..dump_mode_start];
18836        let state_initializer = build_mode_source
18837            .find("run_state_initializers(state_initializers, &state);")
18838            .expect("static build path should run state initializers");
18839        let router_build = build_mode_source
18840            .find("let router = crate::router::try_build_router_inner(")
18841            .expect("static build path should build a router");
18842
18843        assert!(
18844            state_initializer < router_build,
18845            "static builds must install state-initialized resources before rendering routes"
18846        );
18847    }
18848
18849    #[test]
18850    fn graph_installed_before_every_router_build() {
18851        // The architecture graph (#1747) is published by whichever path is
18852        // about to build a router, and `/actuator/graph` answers from what was
18853        // published. Codex round 1 found the install wired into the
18854        // static-build and capsule-replay paths but NOT into `run()` — so every
18855        // unit test passed while a normally running app answered 503 forever.
18856        // A structural assertion, because the serving path ends in `serve()`
18857        // and cannot be driven from a unit test.
18858        let whole = include_str!("app.rs").replace("\r\n", "\n");
18859        // Only the non-test source: this test names both strings itself.
18860        let source = whole
18861            .split_once("\n#[cfg(test)]\nmod tests {")
18862            .map_or(whole.as_str(), |(before, _)| before)
18863            .to_owned();
18864        let builds: Vec<usize> = source
18865            .match_indices("crate::router::try_build_router")
18866            .map(|(i, _)| i)
18867            // Skip doc-comment and test references; only real call sites are
18868            // followed by an open paren on the same expression.
18869            .filter(|i| source[*i..].starts_with("crate::router::try_build_router"))
18870            .filter(|i| {
18871                let tail = &source[*i..*i + 80];
18872                tail.contains("_inner(") || tail.contains("_with_static_inner(")
18873            })
18874            .collect();
18875        assert!(
18876            builds.len() >= 3,
18877            "expected the serving, static-build and replay router builds: {}",
18878            builds.len()
18879        );
18880        let installs: Vec<usize> = source
18881            .match_indices("crate::graph::install(")
18882            .map(|(i, _)| i)
18883            .collect();
18884        assert_eq!(
18885            installs.len(),
18886            3,
18887            "every path that builds an application router must publish the graph \
18888             it is about to serve"
18889        );
18890        // Each install precedes a router build with no other install between.
18891        for install in &installs {
18892            assert!(
18893                builds.iter().any(|build| build > install),
18894                "an install with no router build after it is dead code"
18895            );
18896        }
18897        for build in &builds {
18898            // A probe-only router (worker role) serves the actuator too, but it
18899            // is built inside the same `run()` block the serving install covers.
18900            assert!(
18901                installs.iter().any(|install| install < build),
18902                "a router built with no graph published before it answers 503 at \
18903                 /actuator/graph"
18904            );
18905        }
18906    }
18907
18908    // ── Architecture-graph census (#2520) ──────────────────────────────────
18909
18910    fn census_paths(mounted: &[crate::graph::MountedRoute]) -> Vec<&str> {
18911        mounted.iter().map(|m| m.path.as_str()).collect()
18912    }
18913
18914    /// The router merges the `OpenAPI` routes when the feature is on and the
18915    /// runtime gate is open, so the census must name them (#2520 item 1).
18916    #[cfg(feature = "openapi")]
18917    #[test]
18918    fn graph_census_names_openapi_routes_when_enabled() {
18919        let config = AutumnConfig::default();
18920        let openapi =
18921            crate::openapi::OpenApiConfig::new("Demo", "1.0.0").openapi_json_path("/spec.json");
18922        let mounted = graph_mounted_routes(&[], &[], &[], &config, Some(&openapi));
18923        assert!(
18924            census_paths(&mounted).contains(&"/spec.json"),
18925            "{:?}",
18926            census_paths(&mounted)
18927        );
18928    }
18929
18930    #[cfg(feature = "openapi")]
18931    #[test]
18932    fn graph_census_omits_openapi_routes_when_runtime_disabled() {
18933        let mut config = AutumnConfig::default();
18934        config.openapi_runtime.enabled = false;
18935        let openapi =
18936            crate::openapi::OpenApiConfig::new("Demo", "1.0.0").openapi_json_path("/spec.json");
18937        let mounted = graph_mounted_routes(&[], &[], &[], &config, Some(&openapi));
18938        assert!(
18939            !census_paths(&mounted).contains(&"/spec.json"),
18940            "{:?}",
18941            census_paths(&mounted)
18942        );
18943    }
18944
18945    /// `health.enabled = false` mounts no probe, so the census lists none
18946    /// (#2520 item 12).
18947    #[test]
18948    fn graph_census_lists_no_probe_when_health_is_off() {
18949        let mut config = AutumnConfig::default();
18950        config.health.enabled = false;
18951        let mounted = graph_mounted_routes(
18952            &[],
18953            &[],
18954            &[],
18955            &config,
18956            #[cfg(feature = "openapi")]
18957            None,
18958        );
18959        let paths = census_paths(&mounted);
18960        for probe in [
18961            &config.health.path,
18962            &config.health.live_path,
18963            &config.health.ready_path,
18964            &config.health.startup_path,
18965        ] {
18966            assert!(!paths.contains(&probe.as_str()), "{probe} in {paths:?}");
18967        }
18968    }
18969
18970    /// A worker mounts the probes and the actuator, nothing else (#2520 item 2).
18971    #[test]
18972    fn graph_worker_census_is_probes_and_actuator_only() {
18973        let mut config = AutumnConfig::default();
18974        config.role = crate::config::ProcessRole::Worker;
18975        config.actuator.sensitive = true;
18976        let mounted = graph_worker_mounted_routes(&config);
18977        let paths = census_paths(&mounted);
18978        assert!(
18979            paths.contains(&config.health.live_path.as_str()),
18980            "{paths:?}"
18981        );
18982        assert!(
18983            paths.iter().any(|p| p.ends_with("/graph")),
18984            "the actuator graph endpoint is served: {paths:?}"
18985        );
18986        assert!(!paths.contains(&"/static/{*path}"), "{paths:?}");
18987        assert!(
18988            !paths.iter().any(|p| p.starts_with("/static/")),
18989            "no asset routes on a worker: {paths:?}"
18990        );
18991        assert!(
18992            mounted.iter().all(|route| !route.auth_declared),
18993            "framework endpoints are not declared plugin mounts"
18994        );
18995    }
18996
18997    /// Only a proven posture is a declaration. An unclassified plugin route
18998    /// stays name-only, so the graph never claims `auth none` for it.
18999    #[test]
19000    fn graph_declared_summary_declares_auth_only_when_classified() {
19001        use crate::route_listing::{RouteClassification, RouteInfo};
19002        let summary = |classification| {
19003            let mut info = RouteInfo::framework_get("/plugin/ping".to_owned(), "ping");
19004            info.classification = classification;
19005            graph_declared_route_summary(&info).auth_declared
19006        };
19007        assert!(summary(RouteClassification::Gated));
19008        assert!(summary(RouteClassification::Public));
19009        assert!(!summary(RouteClassification::Unclassified));
19010    }
19011
19012    #[derive(Clone)]
19013    struct CensusLoader(Result<AutumnConfig, String>);
19014
19015    impl crate::config::ConfigLoader for CensusLoader {
19016        async fn load(&self) -> Result<AutumnConfig, crate::config::ConfigError> {
19017            self.0
19018                .clone()
19019                .map_err(crate::config::ConfigError::Validation)
19020        }
19021    }
19022
19023    /// The dump reads config through the builder's loader and picks the worker
19024    /// census for a worker role (#2520 items 4 and 6).
19025    #[tokio::test]
19026    #[allow(
19027        clippy::await_holding_lock,
19028        reason = "the guard spans the config load, which resets the global metric limits"
19029    )]
19030    async fn dump_graph_surface_uses_builder_loader_and_worker_role() {
19031        // `try_load_config` resets the process-global metric limits. See
19032        // `metrics::LIMITS_TEST_LOCK`.
19033        let _limits_lock = crate::metrics::LIMITS_TEST_LOCK
19034            .lock()
19035            .unwrap_or_else(std::sync::PoisonError::into_inner);
19036        let mut config = AutumnConfig::default();
19037        config.role = crate::config::ProcessRole::Worker;
19038        config.health.live_path = "/custom-live".to_owned();
19039        let builder = app().with_config_loader(CensusLoader(Ok(config)));
19040        let (mounted, opaque) = builder.dump_graph_surface().await;
19041        let paths = census_paths(&mounted);
19042        assert!(paths.contains(&"/custom-live"), "{paths:?}");
19043        assert!(!paths.contains(&"/static/{*path}"), "{paths:?}");
19044        assert_eq!(opaque, 0);
19045    }
19046
19047    #[tokio::test]
19048    #[allow(
19049        clippy::await_holding_lock,
19050        reason = "the guard spans the config load, which resets the global metric limits"
19051    )]
19052    async fn dump_graph_surface_falls_back_to_defaults_when_loader_fails() {
19053        // `try_load_config` resets the process-global metric limits. See
19054        // `metrics::LIMITS_TEST_LOCK`.
19055        let _limits_lock = crate::metrics::LIMITS_TEST_LOCK
19056            .lock()
19057            .unwrap_or_else(std::sync::PoisonError::into_inner);
19058        let builder = app().with_config_loader(CensusLoader(Err("boom".to_owned())));
19059        let (mounted, _) = builder.dump_graph_surface().await;
19060        assert!(
19061            census_paths(&mounted).contains(&"/static/{*path}"),
19062            "{:?}",
19063            census_paths(&mounted)
19064        );
19065    }
19066
19067    #[test]
19068    fn migrate_only_one_shot_applies_and_exits_without_serving() {
19069        // The runtime effect (applying against Postgres, exiting without binding a
19070        // port) needs a DB + a subprocess harness because `run()` ends in
19071        // `process::exit`; that live apply is exercised by the shared
19072        // `run_pending_locked` engine's own DB-backed tests. Here we lock the
19073        // *dispatch decision* and the *reuse/exit contract* structurally: with
19074        // AUTUMN_MIGRATE=1 the migrate-and-exit path is chosen and the
19075        // server-start path is NOT taken.
19076        let source = include_str!("app.rs").replace("\r\n", "\n");
19077        let run_start = source.find("pub async fn run(self)").expect("run() exists");
19078        let run_end = source
19079            .find("async fn run_build_mode(self)")
19080            .expect("build mode follows run()");
19081        let run_body = &source[run_start..run_end];
19082
19083        // The AUTUMN_MIGRATE=1 dispatch is an early one-shot: it sits BEFORE the
19084        // server-start machinery (the `let Self {` destructure that begins the
19085        // serving path) and returns, so a migrate run never binds a port.
19086        let dispatch = run_body
19087            .find("if is_migrate_only_mode() {")
19088            .expect("run() dispatches the migrate one-shot");
19089        let server_start = run_body
19090            .find("let Self {")
19091            .expect("run() destructures self to start the server");
19092        assert!(
19093            dispatch < server_start,
19094            "AUTUMN_MIGRATE must be handled before the server-start path"
19095        );
19096        let migrate_branch = &run_body[dispatch..server_start];
19097        assert!(
19098            migrate_branch.contains("self.run_migrate_only_mode().await;")
19099                && migrate_branch.contains("return;"),
19100            "the migrate one-shot must run then return before server start"
19101        );
19102
19103        // The handler applies per target and exits — never starting the server.
19104        let handler_start = source
19105            .find("async fn run_migrate_only_mode(self)")
19106            .expect("migrate handler exists");
19107        let handler_end = source
19108            .find("async fn run_one_off_task_mode(self, requested_name: String)")
19109            .expect("one-off task handler follows the migrate handler");
19110        let handler = &source[handler_start..handler_end];
19111        assert!(
19112            handler.contains("apply_pending_or_exit"),
19113            "the migrate handler applies pending migrations per target"
19114        );
19115        assert!(
19116            handler.contains("std::process::exit(0)"),
19117            "the migrate handler exits after applying"
19118        );
19119
19120        // Issue #1614, PR3: the migrate one-shot must apply the SAME SQLite
19121        // sharding guard as normal boot BEFORE its migration loop, so a sharded
19122        // `sqlite:` target exits with the actionable sharding error instead of a
19123        // generic `PgConnection` failure — and the two paths cannot drift because
19124        // both call `sqlite_sharding_unsupported_guard`.
19125        let guard_call = handler
19126            .find("sqlite_sharding_unsupported_guard(")
19127            .expect("migrate handler applies the SQLite sharding guard");
19128        let first_apply = handler
19129            .find("apply_pending_or_exit")
19130            .expect("migrate handler applies per target");
19131        assert!(
19132            guard_call < first_apply,
19133            "the SQLite guard must run BEFORE the migration loop / apply_pending_or_exit"
19134        );
19135        assert!(
19136            !handler.contains("initialize_job_runtime")
19137                && !handler.contains("try_build_router_inner"),
19138            "the migrate one-shot must not start the server"
19139        );
19140
19141        // The per-target applier reuses `run_pending_locked_with_policy` (the
19142        // exact engine `auto_migrate` drives — no duplicated migration logic) and exits
19143        // non-zero on failure so a bad migration aborts before cutover (AC-3).
19144        let helper_start = source
19145            .find("fn apply_pending_or_exit(")
19146            .expect("apply_pending_or_exit exists");
19147        let helper_end = source[helper_start..]
19148            .find("\n}\n")
19149            .map_or(source.len(), |end| helper_start + end);
19150        let helper = &source[helper_start..helper_end];
19151        assert!(
19152            helper.contains("crate::migrate::run_pending_locked_with_policy("),
19153            "must reuse the shared locked applier, not duplicate migration logic"
19154        );
19155        assert!(
19156            helper.contains("std::process::exit(1)"),
19157            "a failed migration must exit non-zero (abort before cutover)"
19158        );
19159    }
19160
19161    /// The replay handler's source, between its signature and the next
19162    /// top-level item after the `AppBuilder` impl block.
19163    #[cfg(feature = "reporting")]
19164    fn replay_mode_source(source: &str) -> &str {
19165        let start = source
19166            .find("async fn run_replay_mode(self, capsule_path: String)")
19167            .expect("replay handler exists");
19168        let end = source
19169            .find("pub(crate) fn is_static_build_mode()")
19170            .expect("the mode predicates follow the AppBuilder impl block");
19171        source
19172            .get(start..end)
19173            .expect("the replay handler precedes the mode predicates")
19174    }
19175
19176    #[cfg(feature = "reporting")]
19177    #[test]
19178    fn replay_one_shot_runs_before_server_start() {
19179        // `AUTUMN_REPLAY_CAPSULE=<path>` (set by `autumn replay`) must select an
19180        // early one-shot: rebuild the app, replay the recorded request, print a
19181        // verdict, exit. The runtime effect ends in `process::exit`, so — like
19182        // the migrate one-shot above — the *dispatch decision* and the
19183        // *offline contract* are locked structurally here, and the behaviour is
19184        // exercised end-to-end by `failure_capsule_end_to_end`.
19185        let source = include_str!("app.rs").replace("\r\n", "\n");
19186        let run_start = source.find("pub async fn run(self)").expect("run() exists");
19187        let run_end = source
19188            .find("async fn run_build_mode(self)")
19189            .expect("build mode follows run()");
19190        let run_body = source
19191            .get(run_start..run_end)
19192            .expect("run() precedes build mode");
19193
19194        let dispatch = run_body
19195            .find("if is_replay_mode()")
19196            .expect("run() dispatches the replay one-shot");
19197        let server_start = run_body
19198            .find("let Self {")
19199            .expect("run() destructures self to start the server");
19200        assert!(
19201            dispatch < server_start,
19202            "AUTUMN_REPLAY_CAPSULE must be handled before the server-start path"
19203        );
19204        let replay_branch = run_body
19205            .get(dispatch..server_start)
19206            .expect("the dispatch precedes server start");
19207        assert!(
19208            replay_branch.contains("self.run_replay_mode(capsule_path).await;")
19209                && replay_branch.contains("return;"),
19210            "the replay one-shot must run then return before server start"
19211        );
19212
19213        let handler = replay_mode_source(&source);
19214        assert!(
19215            handler.contains("std::process::exit("),
19216            "the replay handler exits with the verdict's code"
19217        );
19218        assert!(
19219            !handler.contains("axum::serve") && !handler.contains("http_server::serve"),
19220            "the replay one-shot must never start the server"
19221        );
19222
19223        // It rebuilds the real thing: same router builder, same state
19224        // initializers, same policy registrations as the serving path — a
19225        // replay against a stripped-down router would not be a replay (R6).
19226        assert!(
19227            handler.contains("crate::capsule::load_capsule(")
19228                && handler.contains("crate::capsule::execute(")
19229                && handler.contains("crate::capsule::print_verdict("),
19230            "the replay handler loads the capsule, executes it and prints a verdict"
19231        );
19232        assert!(
19233            handler.contains("crate::router::try_build_router_inner(")
19234                && handler.contains("run_state_initializers(state_initializers, &state);")
19235                && handler.contains("register(state.policy_registry());"),
19236            "the replay handler must rebuild the app's real router and state"
19237        );
19238
19239        // F15: replay is offline. No migrations, no job runtime, no scheduler,
19240        // no external session/cache backend — and it must not arm capture
19241        // against the stub pool.
19242        assert!(
19243            !handler.contains("setup_database(")
19244                && !handler.contains("initialize_job_runtime")
19245                && !handler.contains("start_task_scheduler")
19246                && !handler.contains("preflight_storage("),
19247            "replay must not run migrations, job workers or storage preflight"
19248        );
19249        assert!(
19250            handler.contains("force_offline_replay_config(&mut config);"),
19251            "the replay handler must force the offline configuration knobs"
19252        );
19253
19254        let forcer_start = source
19255            .find("fn force_offline_replay_config(")
19256            .expect("the offline-knob helper exists");
19257        let forcer = source
19258            .get(forcer_start..forcer_start.saturating_add(2_000))
19259            .unwrap_or_default();
19260        assert!(
19261            forcer.contains("SessionBackend::Memory"),
19262            "replay must force the in-memory session store (no Redis)"
19263        );
19264        assert!(
19265            forcer.contains("failure_capture.enabled = false"),
19266            "replay must not capture a capsule of the replay itself"
19267        );
19268    }
19269
19270    /// A replay must not reach anything the capsule does not contain. Three
19271    /// live-service escapes are closed here, in the same structural style as
19272    /// the session store below: the outbound HTTP client (a handler that calls
19273    /// a third party would otherwise call the *real* one), the channels backend
19274    /// (whose Redis form spawns a publisher and a listener against the
19275    /// application's live fan-out as soon as the state is built), and the
19276    /// request timeout (real tokio timers, so a debugger breakpoint would
19277    /// cancel the handler and print a mismatch that never happened).
19278    #[cfg(feature = "reporting")]
19279    #[test]
19280    fn replay_reaches_nothing_outside_the_capsule() {
19281        let source = include_str!("app.rs").replace("\r\n", "\n");
19282        let handler = replay_mode_source(&source);
19283
19284        assert!(
19285            handler.contains("crate::http_client::block_outbound_for_replay();"),
19286            "the replay handler must block outbound HTTP before rebuilding the app"
19287        );
19288        assert!(
19289            handler.contains("crate::capsule::boundary::block_egress_for_replay();"),
19290            "the replay handler must block egress outside the HTTP client in every build"
19291        );
19292        assert!(
19293            handler.contains("crate::cache::block_backends_for_replay();"),
19294            "the replay handler must keep every cache backend offline"
19295        );
19296        assert!(
19297            handler.contains("channels_backend: _replay_ignores_custom_channels_backend,"),
19298            "the replay handler must drop a custom channels backend, which outranks the \
19299             forced in-process one"
19300        );
19301        assert!(
19302            !handler.contains("            channels_backend,\n"),
19303            "the replay handler must not forward the builder's channels backend"
19304        );
19305
19306        let forcer_start = source
19307            .find("fn force_offline_replay_config(")
19308            .expect("the offline-knob helper exists");
19309        // Wide enough for the whole helper: it forces every config-driven
19310        // store the request path can reach, so the window has to outgrow the
19311        // list rather than the list being trimmed to the window.
19312        let forcer = source
19313            .get(forcer_start..forcer_start.saturating_add(6_000))
19314            .unwrap_or_default();
19315        assert!(
19316            forcer.contains("config.channels.backend = crate::config::ChannelBackend::InProcess;"),
19317            "replay must force the in-process channels backend (no Redis fan-out)"
19318        );
19319        assert!(
19320            forcer.contains("config.server.timeouts.request_timeout_ms = None;"),
19321            "replay must clear the wall-clock request deadline"
19322        );
19323        // #2351 item 1: a shadow mirror has its own client and is not a
19324        // handler effect.
19325        assert!(
19326            forcer.contains("config.shadow.enabled = false;"),
19327            "replay must not mirror traffic to a shadow target"
19328        );
19329    }
19330
19331    /// #2351 item 7: a replayed mail-sending route reaches the mail seam. The
19332    /// replay installs a mailer whose sends are served from the tape.
19333    #[cfg(all(feature = "reporting", feature = "mail"))]
19334    #[test]
19335    fn replay_installs_a_tape_backed_mailer() {
19336        let source = include_str!("app.rs").replace("\r\n", "\n");
19337        let handler = replay_mode_source(&source);
19338        let install =
19339            "crate::mail::install_replay_mailer(&state, &config.mail, builder_mail_queue);";
19340        assert!(
19341            handler.contains(install),
19342            "the replay handler must install the tape-backed mailer"
19343        );
19344        // Codex review on #3222: production installs the mailer before the
19345        // state initializers, so an initializer can read it. Replay does too,
19346        // and again after them, to replace a live one an initializer installed.
19347        let initializers = handler
19348            .find("run_state_initializers(state_initializers, &state);")
19349            .expect("the replay handler runs the state initializers");
19350        let first = handler.find(install).expect("installed");
19351        let last = handler.rfind(install).expect("installed");
19352        assert!(first < initializers, "installed before the initializers");
19353        assert!(last > initializers, "and again after them");
19354        // Codex review on #3222: with `outbox.enabled`, production's mailer
19355        // has the outbox's queue. Replay counts it before the first install.
19356        #[cfg(feature = "db")]
19357        {
19358            let outbox = handler
19359                .find("crate::outbox::installs_mail_queue(&state, &config.outbox)")
19360                .expect("the replay handler counts the outbox's mail queue");
19361            assert!(outbox < first, "counted before the mailer is installed");
19362        }
19363    }
19364
19365    /// Codex review on #3222: replay installs a builder's cache before the
19366    /// state initializers, and a startup hook's cache after them, as
19367    /// production did.
19368    #[cfg(feature = "reporting")]
19369    #[test]
19370    fn replay_installs_each_cache_where_production_did() {
19371        let source = include_str!("app.rs").replace("\r\n", "\n");
19372        let handler = replay_mode_source(&source);
19373        let initializers = handler
19374            .find("run_state_initializers(state_initializers, &state);")
19375            .expect("the replay handler runs the state initializers");
19376        let builder = handler
19377            .find("crate::cache::install_replay_cache(cache_backend.is_some())")
19378            .expect("installs a builder's cache");
19379        let late = handler
19380            .find("crate::cache::install_late_replay_cache(&state, &capsule.effects)")
19381            .expect("installs a startup hook's cache");
19382        assert!(builder < initializers && late > initializers);
19383    }
19384
19385    /// The knobs the helper forces, checked on a real configuration rather than
19386    /// on its source: everything a replay must not honour, in one place.
19387    #[cfg(feature = "reporting")]
19388    #[test]
19389    fn the_offline_replay_config_reaches_no_live_service() {
19390        let mut config = AutumnConfig::default();
19391        config.failure_capture.enabled = true;
19392        config.session.backend = crate::session::SessionBackend::Redis;
19393        config.channels.backend = crate::config::ChannelBackend::Redis;
19394        config.server.timeouts.request_timeout_ms = Some(30_000);
19395        // Every remaining config-driven store the request path can reach, set
19396        // the way a production recording would have had them.
19397        config.security.rate_limit.backend = crate::security::config::RateLimitBackend::Redis;
19398        config.idempotency.backend = crate::config::IdempotencyBackend::Redis;
19399        config.security.submit_token.backend = Some(crate::config::IdempotencyBackend::Redis);
19400        config.security.webhooks.replay.backend = crate::webhook::WebhookReplayBackend::Redis;
19401        config.cache.backend = crate::config::CacheBackend::Redis;
19402        config.jobs.backend = "redis".to_owned();
19403
19404        force_offline_replay_config(&mut config);
19405
19406        assert!(
19407            !config.failure_capture.enabled,
19408            "a replay must not capture a capsule of itself"
19409        );
19410        assert_eq!(
19411            config.session.backend,
19412            crate::session::SessionBackend::Memory
19413        );
19414        assert!(config.session.allow_memory_in_production);
19415        assert_eq!(
19416            config.channels.backend,
19417            crate::config::ChannelBackend::InProcess,
19418            "a replayed app must not dial the application's Redis fan-out"
19419        );
19420        assert_eq!(
19421            config.server.timeouts.request_timeout_ms, None,
19422            "a deterministic offline replay has no wall-clock deadline — and a \
19423             breakpoint held in a debugger must not cancel the handler"
19424        );
19425        // The middleware stores. Each of these is *written* by a replayed
19426        // request, so leaving any of them pointed at the recording
19427        // deployment's Redis would make diagnosing a failure mutate live state.
19428        assert_eq!(
19429            config.security.rate_limit.backend,
19430            crate::security::config::RateLimitBackend::Memory,
19431            "a replay must not consume the deployment's shared rate-limit budget"
19432        );
19433        assert_eq!(
19434            config.idempotency.backend,
19435            crate::config::IdempotencyBackend::Memory,
19436            "a replay must not take an idempotency key or its in-flight lock"
19437        );
19438        assert!(config.idempotency.allow_memory_in_production);
19439        assert_eq!(
19440            config.security.submit_token.backend,
19441            Some(crate::config::IdempotencyBackend::Memory),
19442            "submit tokens inherit the idempotency backend when unset, so a replay \
19443             must pin them explicitly rather than rely on that inheritance"
19444        );
19445        assert_eq!(
19446            config.security.webhooks.replay.backend,
19447            crate::webhook::WebhookReplayBackend::Memory,
19448            "a replay must not insert or delete webhook replay-protection keys"
19449        );
19450        assert!(config.security.webhooks.replay.allow_memory_in_production);
19451        assert_eq!(
19452            config.cache.backend,
19453            crate::config::CacheBackend::Memory,
19454            "a `#[cached]` handler must not read or populate the shared cache"
19455        );
19456        assert_eq!(
19457            config.jobs.backend, "local",
19458            "a job enqueued during a replay must not reach a queue a live worker drains"
19459        );
19460    }
19461
19462    /// Forcing `session.backend = memory` is not enough on its own: a store
19463    /// installed with `with_session_store(...)` *outranks* the config in
19464    /// `apply_session_layer`, so passing it through would let a replay dial —
19465    /// and mutate — the application's live Redis or database session backend,
19466    /// or 503 when that backend is unreachable. The replay router must be built
19467    /// with no custom store so the forced memory backend is what applies.
19468    #[cfg(feature = "reporting")]
19469    #[test]
19470    fn replay_never_uses_the_applications_custom_session_store() {
19471        let source = include_str!("app.rs").replace("\r\n", "\n");
19472        let handler = replay_mode_source(&source);
19473
19474        assert!(
19475            handler.contains("session_store: None,"),
19476            "the replay router context must be built with no custom session store"
19477        );
19478        assert!(
19479            !handler.contains("            session_store,"),
19480            "the replay handler must not forward the builder's session store; \
19481             a custom store outranks the forced memory backend"
19482        );
19483    }
19484
19485    #[cfg(feature = "reporting")]
19486    #[test]
19487    fn replay_never_invokes_a_custom_config_loader() {
19488        // Normalized like the other source assertions: a Windows checkout
19489        // (`core.autocrlf`) hands `include_str!` CRLF line endings, and the
19490        // `\n`-embedding pattern below would never match.
19491        let source = include_str!("app.rs").replace("\r\n", "\n");
19492        let handler = replay_mode_source(&source);
19493
19494        assert!(
19495            handler.contains("_replay_ignores_custom_config_loader = config_loader_factory"),
19496            "the replay handler must drop the builder's config loader — the documented \
19497             implementations call live services (Secrets Manager, Vault, Consul, HTTP)"
19498        );
19499        assert!(
19500            handler.contains("load_config_and_telemetry(\n            None,"),
19501            "replay must load configuration through the default local path, never a \
19502             custom loader"
19503        );
19504    }
19505
19506    #[cfg(feature = "reporting")]
19507    #[test]
19508    fn replay_mode_reads_capsule_env_var() {
19509        temp_env::with_var(
19510            "AUTUMN_REPLAY_CAPSULE",
19511            Some("tmp/capsules/abc.json"),
19512            || {
19513                assert!(is_replay_mode(), "a capsule path selects the replay path");
19514                assert_eq!(
19515                    replay_capsule_from_env().as_deref(),
19516                    Some("tmp/capsules/abc.json")
19517                );
19518            },
19519        );
19520        temp_env::with_var("AUTUMN_REPLAY_CAPSULE", Some("  spaced.json  "), || {
19521            assert_eq!(
19522                replay_capsule_from_env().as_deref(),
19523                Some("spaced.json"),
19524                "a shell-quoted path must be trimmed"
19525            );
19526        });
19527        temp_env::with_var("AUTUMN_REPLAY_CAPSULE", Some("   "), || {
19528            assert!(
19529                !is_replay_mode(),
19530                "a blank value must fall through to the normal boot path"
19531            );
19532        });
19533        temp_env::with_var("AUTUMN_REPLAY_CAPSULE", None::<&str>, || {
19534            assert!(!is_replay_mode(), "unset must not select the replay path");
19535        });
19536    }
19537
19538    #[cfg(feature = "db")]
19539    #[test]
19540    fn hooked_repository_apps_include_hook_queue_framework_migration() {
19541        let migrations = migrations_with_repository_framework_migrations(
19542            vec![("app", APP_TEST_MIGRATIONS)],
19543            true,
19544            false,
19545            false,
19546            RepositoryCommitHookQueueMigrationMode::Runtime,
19547        );
19548        let names = migration_names(&migrations);
19549
19550        assert!(
19551            names
19552                .iter()
19553                .any(|name| name == REPOSITORY_COMMIT_HOOK_QUEUE_MIGRATION),
19554            "hooked repository apps must auto-register the durable hook queue migration"
19555        );
19556        assert!(
19557            names.iter().all(|name| !name.contains("api_tokens")),
19558            "hooked repository apps must not auto-register unrelated framework migrations: {names:?}"
19559        );
19560    }
19561
19562    #[cfg(feature = "db")]
19563    #[test]
19564    fn runtime_hooked_apps_include_hook_queue_framework_migration_without_app_migrations() {
19565        let migrations = migrations_with_repository_framework_migrations(
19566            Vec::new(),
19567            true,
19568            false,
19569            false,
19570            RepositoryCommitHookQueueMigrationMode::Runtime,
19571        );
19572        let names = migration_names(&migrations);
19573
19574        assert!(
19575            names
19576                .iter()
19577                .any(|name| name == REPOSITORY_COMMIT_HOOK_QUEUE_MIGRATION),
19578            "runtime hooked repository apps must install the durable hook queue even when app migrations are managed elsewhere"
19579        );
19580    }
19581
19582    #[cfg(feature = "db")]
19583    #[test]
19584    fn versioned_repository_apps_include_version_history_framework_migration() {
19585        let migrations = migrations_with_repository_framework_migrations(
19586            vec![("app", APP_TEST_MIGRATIONS)],
19587            false,
19588            true,
19589            false,
19590            RepositoryCommitHookQueueMigrationMode::Runtime,
19591        );
19592        let names = migration_names(&migrations);
19593
19594        assert!(
19595            names.iter().any(|name| name == VERSION_HISTORY_MIGRATION),
19596            "versioned repository apps must auto-register the version-history migration"
19597        );
19598        assert!(
19599            names
19600                .iter()
19601                .all(|name| !name.contains("repository_commit_hook_queue")),
19602            "versioned-only repository apps must not auto-register the durable hook queue: {names:?}"
19603        );
19604    }
19605
19606    #[cfg(feature = "db")]
19607    #[test]
19608    fn runtime_versioned_apps_include_version_history_framework_migration_without_app_migrations() {
19609        let migrations = migrations_with_repository_framework_migrations(
19610            Vec::new(),
19611            false,
19612            true,
19613            false,
19614            RepositoryCommitHookQueueMigrationMode::Runtime,
19615        );
19616        let names = migration_names(&migrations);
19617
19618        assert!(
19619            names.iter().any(|name| name == VERSION_HISTORY_MIGRATION),
19620            "runtime versioned repository apps must install version history even when app migrations are managed elsewhere"
19621        );
19622    }
19623
19624    #[cfg(feature = "db")]
19625    #[test]
19626    fn static_builds_do_not_auto_add_hook_queue_when_no_migrations_registered() {
19627        let migrations = migrations_with_repository_framework_migrations(
19628            Vec::new(),
19629            true,
19630            true,
19631            true,
19632            RepositoryCommitHookQueueMigrationMode::StaticBuild,
19633        );
19634
19635        assert!(
19636            migrations.is_empty(),
19637            "static/export builds that pass no migrations must not mutate the database"
19638        );
19639    }
19640
19641    #[cfg(feature = "db")]
19642    #[test]
19643    fn directory_migration_required_only_at_runtime_with_shards_and_routing() {
19644        use RepositoryCommitHookQueueMigrationMode::{Runtime, StaticBuild};
19645
19646        // The happy path: routing on, shards present, real runtime boot.
19647        assert!(directory_migration_is_required(true, true, Runtime));
19648
19649        // A static build must never create the directory table, even with
19650        // routing enabled and shards configured.
19651        assert!(!directory_migration_is_required(true, true, StaticBuild));
19652
19653        // Routing disabled, or no shards, means no directory table at all.
19654        assert!(!directory_migration_is_required(false, true, Runtime));
19655        assert!(!directory_migration_is_required(true, false, Runtime));
19656    }
19657
19658    #[test]
19659    fn shard_map_migration_required_only_at_runtime_with_shards() {
19660        use RepositoryCommitHookQueueMigrationMode::{Runtime, StaticBuild};
19661
19662        // The happy path: shards present, real runtime boot.
19663        assert!(shard_map_migration_is_required(true, Runtime));
19664
19665        // A static build must never create the shard-map table.
19666        assert!(!shard_map_migration_is_required(true, StaticBuild));
19667
19668        // No shards means no shard-map table.
19669        assert!(!shard_map_migration_is_required(false, Runtime));
19670    }
19671
19672    #[cfg(feature = "db")]
19673    #[test]
19674    fn unhooked_apps_do_not_auto_add_hook_queue_framework_migration() {
19675        let migrations = migrations_with_repository_framework_migrations(
19676            Vec::new(),
19677            false,
19678            false,
19679            false,
19680            RepositoryCommitHookQueueMigrationMode::Runtime,
19681        );
19682
19683        assert!(
19684            migrations.is_empty(),
19685            "unhooked apps should not get durable hook queue migrations for free"
19686        );
19687    }
19688
19689    #[cfg(feature = "db")]
19690    #[test]
19691    fn apps_with_a_derivation_include_the_derivation_state_migration() {
19692        let migrations = migrations_with_repository_framework_migrations(
19693            vec![("app", APP_TEST_MIGRATIONS)],
19694            false,
19695            false,
19696            true,
19697            RepositoryCommitHookQueueMigrationMode::Runtime,
19698        );
19699        let names = migration_names(&migrations);
19700
19701        assert!(
19702            names.iter().any(|name| name == DERIVATION_MIGRATION),
19703            "an app that declares a `#[derivation]` must auto-register its \
19704             backfill state table: {names:?}"
19705        );
19706        assert!(
19707            names.iter().all(|name| !name.contains("version_history")),
19708            "a derivation alone must not drag in unrelated framework tables: {names:?}"
19709        );
19710    }
19711
19712    #[cfg(feature = "db")]
19713    #[test]
19714    fn apps_without_a_derivation_do_not_get_the_state_table() {
19715        // The whole feature is gated on a linked descriptor, so an app that
19716        // declares none pays for none of it, not even an empty table.
19717        let migrations = migrations_with_repository_framework_migrations(
19718            vec![("app", APP_TEST_MIGRATIONS)],
19719            false,
19720            false,
19721            false,
19722            RepositoryCommitHookQueueMigrationMode::Runtime,
19723        );
19724        assert!(
19725            !migration_names(&migrations)
19726                .iter()
19727                .any(|name| name == DERIVATION_MIGRATION)
19728        );
19729    }
19730
19731    #[cfg(feature = "db")]
19732    fn migration_names(migrations: &[(&str, crate::migrate::EmbeddedMigrations)]) -> Vec<String> {
19733        use diesel::migration::{Migration, MigrationSource as _};
19734        use diesel::pg::Pg;
19735
19736        migrations
19737            .iter()
19738            .flat_map(|(_, source)| {
19739                let migrations: Vec<Box<dyn Migration<Pg>>> = source.migrations().unwrap();
19740                migrations
19741            })
19742            .map(|migration| migration.name().to_string())
19743            .collect()
19744    }
19745
19746    #[cfg(feature = "db")]
19747    #[test]
19748    fn control_framework_filter_skips_control_but_keeps_shard_required_sets() {
19749        // The full control set is skipped on shards...
19750        assert!(migration_set_is_control_framework(
19751            &crate::migrate::FRAMEWORK_MIGRATIONS
19752        ));
19753        // ...but the standalone shard-required sets are kept (not flagged),
19754        // even though their migrations are duplicated into the control
19755        // `migrations/` directory.
19756        assert!(!migration_set_is_control_framework(
19757            &crate::version_history::VERSION_HISTORY_MIGRATIONS
19758        ));
19759        assert!(!migration_set_is_control_framework(
19760            &crate::repository_commit_hooks::REPOSITORY_COMMIT_HOOK_MIGRATIONS
19761        ));
19762        assert!(!migration_set_is_control_framework(
19763            &crate::derivation::DERIVATION_MIGRATIONS
19764        ));
19765    }
19766
19767    #[cfg(feature = "db")]
19768    #[test]
19769    fn sharded_app_with_full_framework_still_gets_shard_required_sets() {
19770        use diesel::migration::{Migration, MigrationSource as _};
19771        use diesel::pg::Pg;
19772
19773        // A sharded app that registers the full control FRAMEWORK_MIGRATIONS and
19774        // also uses commit hooks + versioning. The hook-queue / version-history
19775        // migrations are present *inside* the control set, but that set is
19776        // stripped from shard targets by `migration_set_is_control_framework`, so
19777        // the standalone shard-required sets must still be appended — otherwise
19778        // shards never get those tables.
19779        let migrations = migrations_with_repository_framework_migrations(
19780            vec![("app", crate::migrate::FRAMEWORK_MIGRATIONS)],
19781            true,
19782            true,
19783            true,
19784            RepositoryCommitHookQueueMigrationMode::Runtime,
19785        );
19786
19787        // The migration names the shard apply loop will actually run: every set
19788        // that is not the control framework set (which gets stripped on shards).
19789        let shard_names: Vec<String> = migrations
19790            .iter()
19791            .filter(|(_, set)| !migration_set_is_control_framework(set))
19792            .flat_map(|(_, set)| {
19793                let ms: Vec<Box<dyn Migration<Pg>>> = set.migrations().unwrap_or_default();
19794                ms.into_iter()
19795                    .map(|m| m.name().to_string())
19796                    .collect::<Vec<_>>()
19797            })
19798            .collect();
19799
19800        assert!(
19801            shard_names
19802                .iter()
19803                .any(|name| name == REPOSITORY_COMMIT_HOOK_QUEUE_MIGRATION),
19804            "shards must receive the commit-hook queue migration even when the full \
19805             control framework set is also registered: {shard_names:?}"
19806        );
19807        assert!(
19808            shard_names
19809                .iter()
19810                .any(|name| name == VERSION_HISTORY_MIGRATION),
19811            "shards must receive the version-history migration even when the full \
19812             control framework set is also registered: {shard_names:?}"
19813        );
19814        assert!(
19815            shard_names.iter().any(|name| name == DERIVATION_MIGRATION),
19816            "shards maintain derivations too, so they need the state table: \
19817             {shard_names:?}"
19818        );
19819    }
19820
19821    #[cfg(feature = "db")]
19822    #[test]
19823    fn plugin_migrations_registers_alongside_app_migrations() {
19824        const APP_MIGRATIONS: crate::migrate::EmbeddedMigrations =
19825            diesel_migrations::embed_migrations!("../examples/todo-app/migrations");
19826        const PLUGIN_MIGRATIONS: crate::migrate::EmbeddedMigrations =
19827            diesel_migrations::embed_migrations!("tests/fixtures/plugin_migrations_ok");
19828
19829        let builder = app()
19830            .migrations(APP_MIGRATIONS)
19831            .plugin_migrations("test-plugin", PLUGIN_MIGRATIONS);
19832
19833        let names = migration_names(&builder.migrations);
19834        assert!(
19835            names.iter().any(|n| n == "00000000000000_create_todos"),
19836            "app-registered migrations must still be applied: {names:?}"
19837        );
19838        assert!(
19839            names.iter().any(|n| n == "20260101000000_create_gizmos"),
19840            "plugin-registered migrations must be applied too: {names:?}"
19841        );
19842    }
19843
19844    #[cfg(feature = "db")]
19845    #[test]
19846    fn plugin_migrations_registration_never_panics_on_version_collision() {
19847        // The shape this guards against: an app's own first migration and a
19848        // plugin's migration both using the placeholder version
19849        // (`00000000000000`) with different content — exactly what
19850        // `examples/todo-app` hits against the framework's legacy
19851        // `create_api_tokens` migration. Registration must always succeed; the
19852        // collision is resolved at apply time instead (see
19853        // `compute_migration_disambiguation`'s own tests). Rejecting it here would
19854        // leave an app unable to use a plugin until someone renamed a migration in
19855        // a dependency they may not control.
19856        const APP_MIGRATIONS: crate::migrate::EmbeddedMigrations =
19857            diesel_migrations::embed_migrations!("../examples/todo-app/migrations");
19858        const COLLIDING_PLUGIN_MIGRATIONS: crate::migrate::EmbeddedMigrations =
19859            diesel_migrations::embed_migrations!("tests/fixtures/plugin_migrations_collision");
19860
19861        let builder = app()
19862            .migrations(APP_MIGRATIONS)
19863            .plugin_migrations("test-plugin", COLLIDING_PLUGIN_MIGRATIONS);
19864
19865        let names = migration_names(&builder.migrations);
19866        assert!(
19867            names.iter().any(|n| n == "00000000000000_create_todos"),
19868            "the app's own colliding migration must still be registered: {names:?}"
19869        );
19870        assert!(
19871            names.iter().any(|n| n == "00000000000000_create_gadgets"),
19872            "the plugin's colliding migration must still be registered: {names:?}"
19873        );
19874    }
19875
19876    #[cfg(feature = "db")]
19877    #[test]
19878    fn plugin_migrations_does_not_panic_on_identical_resubmission() {
19879        // Registering the exact same set twice (e.g. two plugins that both
19880        // depend on a shared migrations bundle) reuses the same versions
19881        // AND full names — the intentional, harmless duplication case, not a
19882        // collision.
19883        const PLUGIN_MIGRATIONS: crate::migrate::EmbeddedMigrations =
19884            diesel_migrations::embed_migrations!("tests/fixtures/plugin_migrations_ok");
19885
19886        let _ = app()
19887            .plugin_migrations("plugin-a", PLUGIN_MIGRATIONS)
19888            .plugin_migrations("plugin-b", PLUGIN_MIGRATIONS);
19889    }
19890
19891    // Postgres-only fixture: it configures a distinct `replica_url`, which
19892    // SQLite refuses at both layers — `database_backend_consistency` at config
19893    // time, and `reject_unusable_sqlite_replica` at topology build, which takes
19894    // only a replica naming the same file as its primary. That replica is the
19895    // blocker; the shards some of these fixtures also declare are refused at
19896    // BOOT (`sqlite_sharding_unsupported_guard`) but not by the topology
19897    // builder, so they would not fail here on their own. Gated rather than
19898    // swapped, so the test never asserts a configuration production refuses;
19899    // same treatment `db::`'s replica tests got.
19900    #[cfg(not(feature = "sqlite"))]
19901    #[cfg(feature = "db")]
19902    #[test]
19903    fn configure_replica_migration_check_stores_recheck_urls() {
19904        let mut config = AutumnConfig::default();
19905        config.database.primary_url = Some("postgres://localhost/primary".to_owned());
19906        config.database.replica_url = Some("postgres://localhost/replica".to_owned());
19907        let topology = crate::db::create_topology(&config.database)
19908            .expect("topology should build")
19909            .expect("database should be configured");
19910
19911        let state = build_state(
19912            &config,
19913            Some(&topology),
19914            None,
19915            #[cfg(feature = "ws")]
19916            None,
19917        );
19918
19919        assert!(
19920            state.probes().replica_migration_check().is_none(),
19921            "build_state should not enable migration checks without registered migrations"
19922        );
19923
19924        configure_replica_migration_check(
19925            &state,
19926            Some((
19927                "postgres://localhost/primary".to_owned(),
19928                "postgres://localhost/replica".to_owned(),
19929            )),
19930        );
19931
19932        let check = state
19933            .probes()
19934            .replica_migration_check()
19935            .expect("replica migration check should be configured");
19936
19937        assert_eq!(check.primary_url, "postgres://localhost/primary");
19938        assert_eq!(check.replica_url, "postgres://localhost/replica");
19939    }
19940
19941    // Postgres-only fixture: it configures a distinct `replica_url`, which
19942    // SQLite refuses at both layers — `database_backend_consistency` at config
19943    // time, and `reject_unusable_sqlite_replica` at topology build, which takes
19944    // only a replica naming the same file as its primary. That replica is the
19945    // blocker; the shards some of these fixtures also declare are refused at
19946    // BOOT (`sqlite_sharding_unsupported_guard`) but not by the topology
19947    // builder, so they would not fail here on their own. Gated rather than
19948    // swapped, so the test never asserts a configuration production refuses;
19949    // same treatment `db::`'s replica tests got.
19950    #[cfg(not(feature = "sqlite"))]
19951    #[cfg(feature = "db")]
19952    #[tokio::test]
19953    async fn replica_migration_readiness_marks_ready_endpoint_degraded() {
19954        let mut config = AutumnConfig::default();
19955        config.database.primary_url = Some("postgres://localhost/primary".to_owned());
19956        config.database.primary_pool_size = Some(5);
19957        config.database.replica_url = Some("postgres://localhost/replica".to_owned());
19958        config.database.replica_pool_size = Some(2);
19959        config.database.replica_fallback = crate::config::ReplicaFallback::FailReadiness;
19960        let topology = crate::db::create_topology(&config.database)
19961            .expect("topology should build")
19962            .expect("database should be configured");
19963        let state = build_state(
19964            &config,
19965            Some(&topology),
19966            None,
19967            #[cfg(feature = "ws")]
19968            None,
19969        );
19970
19971        apply_replica_migration_readiness(
19972            &state,
19973            Some(crate::migrate::ReplicaMigrationReadiness::Stale {
19974                primary_latest: Some("00000000000002".to_owned()),
19975                replica_latest: Some("00000000000001".to_owned()),
19976            }),
19977        );
19978
19979        let (status, _) = crate::probe::readiness_response(&state).await;
19980
19981        assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE);
19982    }
19983
19984    #[cfg(feature = "db")]
19985    #[tokio::test]
19986    async fn blocking_replica_migration_readiness_reports_unknown_connection_errors() {
19987        let readiness = crate::migrate::check_replica_migration_readiness_blocking(
19988            "not-a-primary-url".to_owned(),
19989            "not-a-replica-url".to_owned(),
19990        )
19991        .await;
19992
19993        assert!(matches!(
19994            readiness,
19995            crate::migrate::ReplicaMigrationReadiness::Unknown(_)
19996        ));
19997    }
19998
19999    #[cfg(feature = "ws")]
20000    #[test]
20001    fn with_channels_backend_overrides_config_driven_backend_selection() {
20002        let builder = app().with_channels_backend(crate::channels::LocalChannelsBackend::new(4));
20003        let AppBuilder {
20004            channels_backend, ..
20005        } = builder;
20006        assert!(channels_backend.is_some());
20007
20008        let mut config = AutumnConfig::default();
20009        config.channels.backend = crate::config::ChannelBackend::Redis;
20010        config.channels.redis.url = None;
20011
20012        let state = build_state(
20013            &config,
20014            #[cfg(feature = "db")]
20015            None,
20016            #[cfg(feature = "db")]
20017            None,
20018            #[cfg(feature = "ws")]
20019            channels_backend,
20020        );
20021        let mut rx = state.channels().subscribe("override");
20022
20023        state
20024            .broadcast()
20025            .publish("override", "ok")
20026            .expect("custom local backend should publish");
20027
20028        assert_eq!(rx.try_recv().expect("message should arrive").as_str(), "ok");
20029    }
20030
20031    /// Helper to create a simple GET route for testing.
20032    pub fn test_get_route(path: &'static str, name: &'static str) -> Route {
20033        Route {
20034            method: http::Method::GET,
20035            path,
20036            handler: axum::routing::get(|| async { "ok" }),
20037            name,
20038            api_doc: crate::openapi::ApiDoc {
20039                method: "GET",
20040                path,
20041                operation_id: name,
20042                success_status: 200,
20043                ..Default::default()
20044            },
20045            repository: None,
20046            idempotency: crate::route::RouteIdempotency::Direct,
20047            timeout: crate::route::RouteTimeout::Inherit,
20048            criticality: crate::admission::Criticality::Default,
20049            seo: crate::seo::SeoRouteDefaults::EMPTY,
20050            api_version: None,
20051            sunset_opt_out: false,
20052        }
20053    }
20054
20055    #[cfg(feature = "i18n")]
20056    fn test_i18n_bundle(key: &str, value: &str) -> Arc<crate::i18n::Bundle> {
20057        let mut messages = std::collections::HashMap::new();
20058        let mut en = std::collections::HashMap::new();
20059        en.insert(key.to_owned(), value.to_owned());
20060        messages.insert("en".to_owned(), en);
20061        Arc::new(crate::i18n::Bundle::from_messages(
20062            messages,
20063            &crate::i18n::I18nConfig::default(),
20064        ))
20065    }
20066
20067    #[cfg(feature = "i18n")]
20068    #[test]
20069    fn i18n_auto_defers_loading_until_runtime_config_is_available() {
20070        let builder = app().i18n_auto();
20071
20072        assert!(builder.i18n_bundle.is_none());
20073        assert!(builder.i18n_auto_load);
20074    }
20075
20076    // ── exclude_static_routes_from_locale_prefix (issue #1251, Codex review) ──
20077    //
20078    // `#[static_get]` pre-rendering requests each route's single, unprefixed
20079    // path and rejects any non-2xx response; without this exclusion,
20080    // enabling `locale_prefix_enabled` would replace that path with a 308
20081    // redirect and break `autumn build` for every app with a static route.
20082
20083    #[cfg(feature = "i18n")]
20084    fn static_meta(path: &'static str) -> crate::static_gen::StaticRouteMeta {
20085        crate::static_gen::StaticRouteMeta {
20086            path,
20087            name: "test_static_route",
20088            revalidate: None,
20089            params_fn: None,
20090            seo: crate::seo::SeoRouteDefaults::EMPTY,
20091            content_type: None,
20092        }
20093    }
20094
20095    #[cfg(feature = "i18n")]
20096    #[test]
20097    fn static_routes_are_excluded_when_locale_prefix_is_enabled() {
20098        let mut config = AutumnConfig::default();
20099        config.i18n.locale_prefix_enabled = true;
20100        let metas = vec![static_meta("/about"), static_meta("/pricing")];
20101
20102        exclude_static_routes_from_locale_prefix(&mut config, &metas);
20103
20104        assert_eq!(
20105            config.i18n.locale_prefix_exclude_exact,
20106            vec!["/about".to_owned(), "/pricing".to_owned()],
20107            "static routes must be tracked as EXACT exclusions, not prefix \
20108             exclusions — see the sibling `static_route_exclusion_does_not_leak_into_a_dynamic_sibling` test"
20109        );
20110        assert!(
20111            config.i18n.locale_prefix_exclude.is_empty(),
20112            "must not write static route paths into the prefix-matched exclude list"
20113        );
20114    }
20115
20116    #[cfg(feature = "i18n")]
20117    #[test]
20118    fn static_route_exclusion_is_a_noop_when_locale_prefix_is_disabled() {
20119        let mut config = AutumnConfig::default();
20120        assert!(!config.i18n.locale_prefix_enabled);
20121        let metas = vec![static_meta("/about")];
20122
20123        exclude_static_routes_from_locale_prefix(&mut config, &metas);
20124
20125        assert!(
20126            config.i18n.locale_prefix_exclude_exact.is_empty(),
20127            "must not touch the exact-exclude list when the feature is off"
20128        );
20129    }
20130
20131    #[cfg(feature = "i18n")]
20132    #[test]
20133    fn static_route_exclusion_preserves_existing_exclude_entries() {
20134        let mut config = AutumnConfig::default();
20135        config.i18n.locale_prefix_enabled = true;
20136        config.i18n.locale_prefix_exclude = vec!["/api".to_owned()];
20137        config.i18n.locale_prefix_exclude_exact = vec!["/contact".to_owned()];
20138        let metas = vec![static_meta("/about")];
20139
20140        exclude_static_routes_from_locale_prefix(&mut config, &metas);
20141
20142        assert_eq!(
20143            config.i18n.locale_prefix_exclude,
20144            vec!["/api".to_owned()],
20145            "must not touch the user-configured prefix-exclude list"
20146        );
20147        assert_eq!(
20148            config.i18n.locale_prefix_exclude_exact,
20149            vec!["/contact".to_owned(), "/about".to_owned()]
20150        );
20151    }
20152
20153    #[cfg(feature = "i18n")]
20154    #[test]
20155    fn static_route_exclusion_preserves_a_root_static_route_path_verbatim() {
20156        let mut config = AutumnConfig::default();
20157        config.i18n.locale_prefix_enabled = true;
20158        let metas = vec![static_meta("/")];
20159
20160        exclude_static_routes_from_locale_prefix(&mut config, &metas);
20161
20162        assert_eq!(
20163            config.i18n.locale_prefix_exclude_exact,
20164            vec!["/".to_owned()]
20165        );
20166    }
20167
20168    /// Codex review (P1): `AppState` caches an `AutumnConfig` snapshot
20169    /// (`build_state` clones it in before `exclude_static_routes_from_locale_prefix`
20170    /// runs) — `run()`/`run_build_mode()` must re-`insert_extension` the
20171    /// mutated config afterward, or `tenancy_middleware` (which reads config
20172    /// via `state.extension::<AutumnConfig>()`) would keep seeing the stale,
20173    /// pre-exclusion copy and could misjudge whether a `/{locale}`-look-alike
20174    /// path was ever actually locale-prefixed.
20175    #[cfg(feature = "i18n")]
20176    #[test]
20177    fn appstate_config_extension_reflects_static_route_exclusions_after_refresh() {
20178        let mut config = AutumnConfig::default();
20179        config.i18n.locale_prefix_enabled = true;
20180        let state = crate::state::AppState::for_test();
20181        // Simulates `build_state`'s clone, captured BEFORE the exclusion.
20182        state.insert_extension(config.clone());
20183
20184        let metas = vec![static_meta("/about")];
20185        exclude_static_routes_from_locale_prefix(&mut config, &metas);
20186        // The fix: re-insert the mutated config so the stored snapshot
20187        // matches what the router was actually built with.
20188        state.insert_extension(config.clone());
20189
20190        let stored = state
20191            .extension::<AutumnConfig>()
20192            .expect("config extension must be installed");
20193        assert_eq!(
20194            stored.i18n.locale_prefix_exclude_exact,
20195            vec!["/about".to_owned()],
20196            "AppState's config snapshot must reflect the static-route exclusion, \
20197             not the stale pre-mutation copy"
20198        );
20199    }
20200
20201    #[cfg(feature = "i18n")]
20202    #[derive(Clone)]
20203    struct StaticConfigLoader {
20204        config: AutumnConfig,
20205    }
20206
20207    #[cfg(feature = "i18n")]
20208    impl crate::config::ConfigLoader for StaticConfigLoader {
20209        async fn load(&self) -> Result<AutumnConfig, crate::config::ConfigError> {
20210            Ok(self.config.clone())
20211        }
20212    }
20213
20214    #[cfg(feature = "i18n")]
20215    struct NoopTelemetryProvider;
20216
20217    #[cfg(feature = "i18n")]
20218    impl crate::telemetry::TelemetryProvider for NoopTelemetryProvider {
20219        fn init(
20220            &self,
20221            _log: &crate::config::LogConfig,
20222            _telemetry: &crate::config::TelemetryConfig,
20223            _profile: Option<&str>,
20224        ) -> Result<crate::telemetry::TelemetryGuard, crate::telemetry::TelemetryInitError>
20225        {
20226            Ok(crate::telemetry::TelemetryGuard::disabled())
20227        }
20228    }
20229
20230    #[cfg(feature = "i18n")]
20231    #[tokio::test]
20232    #[allow(
20233        clippy::await_holding_lock,
20234        reason = "the guard must span the `load_config_and_telemetry` await — that \
20235                  await is what mutates the limits, so dropping the lock before it \
20236                  would serialize nothing. Same shape, and the same reason, as \
20237                  `config_runtime_drift_actuator_prefix_is_mounted`'s circuit-breaker \
20238                  guard: the contending holders are sibling libtest threads, not \
20239                  tasks on this runtime, so blocking here cannot starve the future \
20240                  that would release it."
20241    )]
20242    async fn i18n_auto_uses_config_loader_output_for_bundle_dir() {
20243        // `load_config_and_telemetry` installs the `[metrics]` section, so
20244        // calling it here resets the process-global metric limits as a side
20245        // effect. Take the same lock the metrics tests use, or this can land
20246        // between a limits test raising a cap and the loop that depends on it
20247        // — dropping samples at the default while that test expects the
20248        // raised value. See `metrics::LIMITS_TEST_LOCK`.
20249        let _limits_lock = crate::metrics::LIMITS_TEST_LOCK
20250            .lock()
20251            .unwrap_or_else(std::sync::PoisonError::into_inner);
20252
20253        let project = tempfile::tempdir().expect("project dir");
20254        let i18n_dir = project.path().join("custom-i18n");
20255        std::fs::create_dir_all(&i18n_dir).expect("i18n dir");
20256        std::fs::write(i18n_dir.join("en.ftl"), "nav.home = Loader Home\n").expect("bundle");
20257
20258        let mut config = AutumnConfig::default();
20259        config.i18n.dir = "custom-i18n".to_owned();
20260        let builder = app()
20261            .with_config_loader(StaticConfigLoader { config })
20262            .with_telemetry_provider(NoopTelemetryProvider)
20263            .i18n_auto();
20264        let AppBuilder {
20265            config_loader_factory,
20266            telemetry_provider,
20267            i18n_bundle,
20268            i18n_auto_load,
20269            plugin_config_roots,
20270            ..
20271        } = builder;
20272
20273        let (loaded_config, _guard) = load_config_and_telemetry(
20274            config_loader_factory,
20275            telemetry_provider,
20276            plugin_config_roots,
20277        )
20278        .await;
20279        let env = crate::config::MockEnv::new().with(
20280            "AUTUMN_MANIFEST_DIR",
20281            project.path().to_str().expect("utf-8 path"),
20282        );
20283        let bundle = resolve_i18n_bundle(i18n_bundle, i18n_auto_load, &loaded_config, &env)
20284            .expect("bundle loaded from configured dir");
20285
20286        assert_eq!(bundle.translate("en", "nav.home", &[]), "Loader Home");
20287    }
20288
20289    /// #1384: `locale_prefix_enabled` is supported WITHOUT `.i18n()` /
20290    /// `.i18n_auto()` — the router builds its `/{locale}` nests straight from
20291    /// `I18nConfig` — so no `Bundle` exists in that shape. Column decoding
20292    /// still needs the app's default locale: without it a `default_locale =
20293    /// "fr"` app attributes every legacy plain-text value to the last-resort
20294    /// `"en"`, so a `/fr/...` request renders upgraded content as empty and a
20295    /// later write can persist it under the wrong locale.
20296    #[cfg(feature = "i18n")]
20297    #[test]
20298    fn locale_defaults_are_installed_even_with_no_bundle() {
20299        let mut config = AutumnConfig::default();
20300        config.i18n.default_locale = "fr".to_owned();
20301        config.i18n.supported_locales = vec!["fr".to_owned(), "en".to_owned()];
20302        let state = AppState::for_test();
20303
20304        let layers = install_i18n_bundle_layer(Vec::new(), &state, None, &config.i18n);
20305
20306        assert!(
20307            layers.is_empty(),
20308            "no bundle means no Extension and no ambient layer registration"
20309        );
20310        assert_eq!(
20311            &*crate::i18n::default_locale_snapshot(),
20312            "fr",
20313            "the configured default must reach column decoding without a bundle"
20314        );
20315        // A legacy plain-text column is now attributed to `fr`, so a `/fr/...`
20316        // request resolves it instead of rendering empty.
20317        let legacy = crate::i18n::Translated::decode_column(
20318            "Bonjour le monde",
20319            &crate::i18n::default_locale_snapshot(),
20320        );
20321        assert_eq!(legacy.get("fr"), Some("Bonjour le monde"));
20322
20323        // Restore the framework default for the rest of this binary.
20324        crate::i18n::install_locale_defaults("en", vec!["en".to_owned()]);
20325    }
20326
20327    /// #1384: drive the REAL wiring — `install_i18n_bundle_layer` +
20328    /// `try_build_router_inner` — and assert both the ambient locale and a
20329    /// `Translated` field resolve for a handler that takes NO locale argument.
20330    ///
20331    /// This pins the layer-ORDERING invariant the feature rests on: the bundle
20332    /// `Extension` must be registered first (outermost) so the ambient layer,
20333    /// registered second, can read it while negotiating. Swap the two pushes
20334    /// and the layer negotiates against an empty supported-list, pins every
20335    /// request to the default locale, and this test goes red — where a
20336    /// hand-built router stack would not notice.
20337    #[cfg(feature = "i18n")]
20338    #[tokio::test]
20339    async fn ambient_locale_layer_resolves_translatable_content_through_the_real_stack() {
20340        use tower::ServiceExt as _;
20341
20342        async fn show() -> String {
20343            let title =
20344                crate::i18n::Translated::from_pairs([("en", "Hello world"), ("es", "Hola mundo")]);
20345            // No `Locale` parameter anywhere in this signature.
20346            format!(
20347                "{}|{}",
20348                title,
20349                crate::i18n::ambient_locale().unwrap_or_default()
20350            )
20351        }
20352
20353        let mut config = AutumnConfig::default();
20354        config.i18n.supported_locales = vec!["en".to_owned(), "es".to_owned()];
20355        let bundle = Arc::new(crate::i18n::Bundle::from_messages(
20356            std::collections::HashMap::new(),
20357            &config.i18n,
20358        ));
20359        let state = AppState::for_test();
20360        let custom_layers =
20361            install_i18n_bundle_layer(Vec::new(), &state, Some(bundle), &config.i18n);
20362
20363        let router = crate::router::try_build_router_inner(
20364            vec![Route {
20365                method: http::Method::GET,
20366                path: "/show",
20367                handler: axum::routing::get(show),
20368                name: "show",
20369                api_doc: crate::openapi::ApiDoc {
20370                    method: "GET",
20371                    path: "/show",
20372                    operation_id: "show",
20373                    success_status: 200,
20374                    ..Default::default()
20375                },
20376                repository: None,
20377                idempotency: crate::route::RouteIdempotency::Direct,
20378                timeout: crate::route::RouteTimeout::Inherit,
20379                criticality: crate::admission::Criticality::Default,
20380                seo: crate::seo::SeoRouteDefaults::EMPTY,
20381                api_version: None,
20382                sunset_opt_out: false,
20383            }],
20384            &config,
20385            state,
20386            crate::router::RouterContext {
20387                exception_filters: Vec::new(),
20388                scoped_groups: Vec::new(),
20389                merge_routers: Vec::new(),
20390                nest_routers: Vec::new(),
20391                declared_routes: Vec::new(),
20392                custom_layers,
20393                static_gate_layers: Vec::new(),
20394                #[cfg(feature = "maud")]
20395                error_page_renderer: None,
20396                session_store: None,
20397                #[cfg(feature = "openapi")]
20398                openapi: None,
20399                #[cfg(feature = "mcp")]
20400                mcp: None,
20401                #[cfg(feature = "webmcp")]
20402                webmcp: None,
20403            },
20404        )
20405        .expect("router builds");
20406
20407        for (accept_language, expected) in [
20408            ("es", "Hola mundo|es"),
20409            // Untranslated: falls back through the configured chain to `en`.
20410            ("fr", "Hello world|en"),
20411        ] {
20412            let request = axum::http::Request::builder()
20413                .uri("/show")
20414                .header(http::header::ACCEPT_LANGUAGE, accept_language)
20415                .body(axum::body::Body::empty())
20416                .expect("request");
20417            let response = router.clone().oneshot(request).await.expect("response");
20418            let bytes = axum::body::to_bytes(response.into_body(), 4096)
20419                .await
20420                .expect("body");
20421            assert_eq!(
20422                String::from_utf8(bytes.to_vec()).expect("utf-8"),
20423                expected,
20424                "Accept-Language: {accept_language}"
20425            );
20426        }
20427    }
20428
20429    // ── The origin never leaks the edge lane's internal sentinel (issue
20430    //    #2244, item 4) ────────────────────────────────────────────────────
20431    //
20432    // `EdgeCacheUnavailable` (autumn-edge's `extract.rs`) answers with the
20433    // fallthrough sentinel header so the EDGE CAPSULE runtime knows to fall
20434    // through to the origin. The same handler code also runs at the origin —
20435    // `extract.rs` cannot special-case which substrate it is on — so an app
20436    // that forgets to call `with_edge_kv` (a wiring bug) hits this same 500
20437    // at the origin, and a real HTTP client must never see the internal
20438    // header.
20439    #[cfg(feature = "edge")]
20440    #[tokio::test]
20441    async fn an_uninjected_edge_seam_never_leaks_the_fallthrough_sentinel_to_a_real_client() {
20442        async fn note(_cache: autumn_edge::EdgeCache) -> &'static str {
20443            "never reached: extraction fails first"
20444        }
20445
20446        // The real `app()` entry point, `with_edge_kv` NEVER called — the
20447        // wiring bug this test is about.
20448        let custom_layers = app().custom_layers;
20449
20450        let router = crate::router::try_build_router_inner(
20451            vec![Route {
20452                method: http::Method::GET,
20453                path: "/edge/note",
20454                handler: axum::routing::get(note),
20455                name: "note",
20456                api_doc: crate::openapi::ApiDoc {
20457                    method: "GET",
20458                    path: "/edge/note",
20459                    operation_id: "note",
20460                    success_status: 200,
20461                    ..Default::default()
20462                },
20463                repository: None,
20464                idempotency: crate::route::RouteIdempotency::Direct,
20465                timeout: crate::route::RouteTimeout::Inherit,
20466                criticality: crate::admission::Criticality::Default,
20467                seo: crate::seo::SeoRouteDefaults::EMPTY,
20468                api_version: None,
20469                sunset_opt_out: false,
20470            }],
20471            &AutumnConfig::default(),
20472            AppState::for_test(),
20473            crate::router::RouterContext {
20474                exception_filters: Vec::new(),
20475                scoped_groups: Vec::new(),
20476                merge_routers: Vec::new(),
20477                nest_routers: Vec::new(),
20478                declared_routes: Vec::new(),
20479                custom_layers,
20480                static_gate_layers: Vec::new(),
20481                #[cfg(feature = "maud")]
20482                error_page_renderer: None,
20483                session_store: None,
20484                #[cfg(feature = "openapi")]
20485                openapi: None,
20486                #[cfg(feature = "mcp")]
20487                mcp: None,
20488                #[cfg(feature = "webmcp")]
20489                webmcp: None,
20490            },
20491        )
20492        .expect("router builds");
20493
20494        let request = axum::http::Request::builder()
20495            .uri("/edge/note")
20496            .body(axum::body::Body::empty())
20497            .expect("request");
20498        let response = router.oneshot(request).await.expect("response");
20499
20500        // Existing behavior, unchanged: still a 500 with an actionable body.
20501        assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
20502        // The fix: the internal sentinel never reaches a real client.
20503        assert!(
20504            !response
20505                .headers()
20506                .contains_key(autumn_edge::FALLTHROUGH_SENTINEL),
20507            "the origin must strip the internal fallthrough sentinel: {:?}",
20508            response.headers()
20509        );
20510
20511        let body = axum::body::to_bytes(response.into_body(), usize::MAX)
20512            .await
20513            .expect("body");
20514        let body = String::from_utf8_lossy(&body);
20515        assert!(
20516            body.contains("with_edge_kv"),
20517            "the actionable message must survive: {body}"
20518        );
20519    }
20520
20521    /// `app()` registers the sentinel-strip layer through the ordinary
20522    /// `AppBuilder::layer` path, which the idempotency machinery otherwise
20523    /// treats as "opaque" (forcing fail-closed replay) for any custom layer
20524    /// it does not specifically recognize — see
20525    /// `router::is_idempotency_transparent_app_layer`. Without that
20526    /// recognition, every app built with the `edge` feature on would force
20527    /// fail-closed idempotency, whether or not it ever calls `with_edge_kv`.
20528    /// This pins the real registration's `type_name` against the substring
20529    /// that recognizer matches on, so the two sides cannot drift apart.
20530    #[cfg(feature = "edge")]
20531    #[test]
20532    fn the_sentinel_strip_layer_is_recognized_as_idempotency_transparent() {
20533        let registration = &app().custom_layers[0];
20534        assert_eq!(
20535            registration.type_id,
20536            std::any::TypeId::of::<StripEdgeFallthroughSentinelLayer>(),
20537            "the real registration's type_id no longer matches what \
20538             router::is_idempotency_transparent_app_layer looks for"
20539        );
20540    }
20541
20542    /// `get_layer_types()` documents "only user-installed layers", but the
20543    /// sentinel-strip layer above shares its underlying storage
20544    /// (`custom_layers`) with real `AppBuilder::layer` calls so the
20545    /// router-build step applies both the same way. Without filtering it
20546    /// back out, a plugin (or a test like
20547    /// `middleware_introspection::get_layer_types_returns_registration_order`)
20548    /// asserting an exact layer list sees this internal registration leak in
20549    /// as an unexpected leading entry (Codex review on #2739, round 6, P1).
20550    #[cfg(feature = "edge")]
20551    #[test]
20552    fn get_layer_types_excludes_the_framework_owned_sentinel_strip_layer() {
20553        #[derive(Clone, Copy)]
20554        struct UserLayer;
20555        impl<S> tower::Layer<S> for UserLayer {
20556            type Service = S;
20557            fn layer(&self, inner: S) -> S {
20558                inner
20559            }
20560        }
20561
20562        let builder = app().layer(UserLayer);
20563        assert_eq!(
20564            builder.get_layer_types(),
20565            vec![std::any::TypeId::of::<UserLayer>()],
20566            "the framework's own sentinel-strip registration must not appear \
20567             in the user-facing layer list"
20568        );
20569    }
20570
20571    /// The header-stripping behavior itself, independent of the router-level
20572    /// idempotency classification test above.
20573    #[cfg(feature = "edge")]
20574    #[tokio::test]
20575    async fn the_sentinel_strip_service_removes_the_header_and_keeps_the_body() {
20576        use axum::response::IntoResponse as _;
20577        use tower::{Layer as _, Service as _, ServiceExt as _};
20578
20579        let inner = tower::service_fn(|_req: axum::extract::Request| async move {
20580            Ok::<_, std::convert::Infallible>(
20581                (
20582                    [(autumn_edge::FALLTHROUGH_SENTINEL, "missing_capability")],
20583                    "actionable message",
20584                )
20585                    .into_response(),
20586            )
20587        });
20588        let mut service = StripEdgeFallthroughSentinelLayer.layer(inner);
20589        let request = axum::extract::Request::builder()
20590            .uri("/")
20591            .body(axum::body::Body::empty())
20592            .expect("request");
20593        let response: axum::response::Response = service
20594            .ready()
20595            .await
20596            .expect("ready")
20597            .call(request)
20598            .await
20599            .expect("infallible");
20600
20601        assert!(
20602            !response
20603                .headers()
20604                .contains_key(autumn_edge::FALLTHROUGH_SENTINEL)
20605        );
20606        let body = axum::body::to_bytes(response.into_body(), usize::MAX)
20607            .await
20608            .expect("body");
20609        assert_eq!(body, b"actionable message".as_slice());
20610    }
20611
20612    #[cfg(feature = "i18n")]
20613    #[tokio::test]
20614    async fn i18n_bundle_layer_is_applied_to_static_route_rendering() {
20615        async fn localized(locale: crate::i18n::Locale) -> String {
20616            locale.t("nav.home")
20617        }
20618
20619        let config = AutumnConfig::default();
20620        let state = AppState::for_test();
20621        let custom_layers = install_i18n_bundle_layer(
20622            Vec::new(),
20623            &state,
20624            Some(test_i18n_bundle("nav.home", "Home")),
20625            &config.i18n,
20626        );
20627        let router = crate::router::try_build_router_inner(
20628            vec![Route {
20629                method: http::Method::GET,
20630                path: "/about",
20631                handler: axum::routing::get(localized),
20632                name: "localized",
20633                api_doc: crate::openapi::ApiDoc {
20634                    method: "GET",
20635                    path: "/about",
20636                    operation_id: "localized",
20637                    success_status: 200,
20638                    ..Default::default()
20639                },
20640                repository: None,
20641                idempotency: crate::route::RouteIdempotency::Direct,
20642                timeout: crate::route::RouteTimeout::Inherit,
20643                criticality: crate::admission::Criticality::Default,
20644                seo: crate::seo::SeoRouteDefaults::EMPTY,
20645                api_version: None,
20646                sunset_opt_out: false,
20647            }],
20648            &config,
20649            state,
20650            crate::router::RouterContext {
20651                exception_filters: Vec::new(),
20652                scoped_groups: Vec::new(),
20653                merge_routers: Vec::new(),
20654                nest_routers: Vec::new(),
20655                declared_routes: Vec::new(),
20656                custom_layers,
20657                static_gate_layers: Vec::new(),
20658                #[cfg(feature = "maud")]
20659                error_page_renderer: None,
20660                session_store: None,
20661                #[cfg(feature = "openapi")]
20662                openapi: None,
20663                #[cfg(feature = "mcp")]
20664                mcp: None,
20665                #[cfg(feature = "webmcp")]
20666                webmcp: None,
20667            },
20668        )
20669        .expect("router builds");
20670        let tmp = tempfile::tempdir().expect("dist parent");
20671        let dist = tmp.path().join("dist");
20672
20673        crate::static_gen::render_static_routes(
20674            router,
20675            &[crate::static_gen::StaticRouteMeta {
20676                path: "/about",
20677                name: "localized",
20678                revalidate: None,
20679                params_fn: None,
20680                seo: crate::seo::SeoRouteDefaults::EMPTY,
20681                content_type: None,
20682            }],
20683            &dist,
20684        )
20685        .await
20686        .expect("static render succeeds");
20687
20688        let html = std::fs::read_to_string(dist.join("about/index.html")).expect("rendered html");
20689        assert_eq!(html, "Home");
20690    }
20691
20692    // ── Warden 2026-09-04: `#[static_get]` × multi-tenancy fails closed ──────
20693    //
20694    // Hypothesis: an app with documented multi-tenancy (`[tenancy] enabled =
20695    // true`, any `source`) pre-renders a `#[static_get]` route that reads
20696    // tenant-scoped state — a per-tenant storefront via the `Tenant` extractor, or
20697    // a `#[repository]` query scoped by `CURRENT_TENANT`. `autumn build` or ISR
20698    // regeneration might silently resolve a missing or default tenant and bake
20699    // that tenant's response into the single `dist/` file every tenant then
20700    // shares: a cross-tenant read through a first-class feature composition, with
20701    // no app-level analogue.
20702    //
20703    // It does not happen. `render_static_routes` (`autumn build`) and ISR's
20704    // `regenerate_page` (`static_gen/middleware.rs`) both send a bare synthetic
20705    // request through the full router — path only, no `Host`, no tenant header, no
20706    // session, no `Authorization` — and every tenancy `source` in `tenancy.rs`
20707    // (`extract_tenant_from_parts_inner`) rejects with a non-2xx `AutumnError`
20708    // when its required signal is absent, rather than resolving a default tenant.
20709    // A non-2xx is `BuildError::NonSuccessStatus` to the build/ISR caller, so the
20710    // handler's output never reaches disk. Asserted per tenancy source, so a
20711    // future change that makes any one of them fail open is caught rather than
20712    // silently shipping a cross-tenant static-file leak.
20713    #[tokio::test]
20714    async fn static_get_route_reading_tenant_fails_closed_for_every_tenancy_source() {
20715        async fn tenant_page(tenant: crate::tenancy::Tenant) -> String {
20716            tenant.0
20717        }
20718
20719        for source in ["header", "subdomain", "session", "jwt"] {
20720            let mut config = AutumnConfig::default();
20721            config.tenancy.enabled = true;
20722            config.tenancy.source = source.to_owned();
20723            // Exempt the route from `tenancy_middleware` itself (Codex review,
20724            // PR #2505) so the request reaches `tenant_page` and the assertion
20725            // below exercises `Tenant::from_request_parts`'s own fallback call to
20726            // `extract_tenant_from_parts`, not just the middleware's earlier call
20727            // to the same function. Without this, every synthetic build/ISR
20728            // request is non-public and `tenancy_middleware` rejects it before the
20729            // handler runs, so a regression making the extractor's fallback fail
20730            // open would go undetected on a route listed in
20731            // `[tenancy].public_paths` whose handler still reads `Tenant`
20732            // directly. The `#[public]` macro attribute is a compile-time
20733            // route-audit marker and has no effect on `tenancy_middleware`.
20734            config.tenancy.public_paths = vec!["/storefront".to_owned()];
20735
20736            let state = AppState::for_test();
20737            state.insert_extension(config.clone());
20738
20739            let router = crate::router::try_build_router_inner(
20740                vec![Route {
20741                    method: http::Method::GET,
20742                    path: "/storefront",
20743                    handler: axum::routing::get(tenant_page),
20744                    name: "tenant_page",
20745                    api_doc: crate::openapi::ApiDoc {
20746                        method: "GET",
20747                        path: "/storefront",
20748                        operation_id: "tenant_page",
20749                        success_status: 200,
20750                        ..Default::default()
20751                    },
20752                    repository: None,
20753                    idempotency: crate::route::RouteIdempotency::Direct,
20754                    timeout: crate::route::RouteTimeout::Inherit,
20755                    criticality: crate::admission::Criticality::Default,
20756                    seo: crate::seo::SeoRouteDefaults::EMPTY,
20757                    api_version: None,
20758                    sunset_opt_out: false,
20759                }],
20760                &config,
20761                state,
20762                crate::router::RouterContext {
20763                    exception_filters: Vec::new(),
20764                    scoped_groups: Vec::new(),
20765                    merge_routers: Vec::new(),
20766                    nest_routers: Vec::new(),
20767                    declared_routes: Vec::new(),
20768                    custom_layers: Vec::new(),
20769                    static_gate_layers: Vec::new(),
20770                    #[cfg(feature = "maud")]
20771                    error_page_renderer: None,
20772                    session_store: None,
20773                    #[cfg(feature = "openapi")]
20774                    openapi: None,
20775                    #[cfg(feature = "mcp")]
20776                    mcp: None,
20777                    #[cfg(feature = "webmcp")]
20778                    webmcp: None,
20779                },
20780            )
20781            .unwrap_or_else(|e| panic!("tenancy source {source:?}: router builds: {e}"));
20782
20783            let tmp = tempfile::tempdir().expect("dist parent");
20784            let dist = tmp.path().join("dist");
20785
20786            let result = crate::static_gen::render_static_routes(
20787                router,
20788                &[crate::static_gen::StaticRouteMeta {
20789                    path: "/storefront",
20790                    name: "tenant_page",
20791                    revalidate: None,
20792                    params_fn: None,
20793                    seo: crate::seo::SeoRouteDefaults::EMPTY,
20794                    content_type: None,
20795                }],
20796                &dist,
20797            )
20798            .await;
20799
20800            assert!(
20801                result.is_err(),
20802                "tenancy source {source:?}: a #[static_get] route reading `Tenant` must fail \
20803                 the build rather than silently bake a default/missing tenant's response into \
20804                 a dist/ file every tenant's requests would then share"
20805            );
20806            assert!(
20807                !dist.join("storefront/index.html").exists(),
20808                "tenancy source {source:?}: no file must be written when tenant resolution fails"
20809            );
20810        }
20811    }
20812
20813    // ── Warden 2026-09-04 (Codex review, PR #2505): a failed rebuild does NOT
20814    // invalidate a pre-existing static file ──────────────────────────────────
20815    //
20816    // The test above starts from an empty `dist/`, so "no file is written on
20817    // failure" proves only that a fresh build never bakes in a cross-tenant
20818    // response. It says nothing about a `dist/<route>/index.html` left from an
20819    // earlier successful render. `render_static_routes` stages into a sibling
20820    // `dist.staging` directory and swaps it in only once every route rendered
20821    // successfully: the loop over `results` in `build.rs` returns on the first
20822    // `Err`, before the atomic remove-and-rename, so a failure leaves the existing
20823    // file untouched. ISR's `regenerate_page` has the same shape — it returns
20824    // `Err` before any `std::fs::write`/`rename` on a non-2xx response — so a
20825    // repeatedly-failing revalidation serves the same stale file forever, which is
20826    // the documented stale-while-revalidate contract.
20827    //
20828    // This does not reopen the cross-tenant hypothesis: nothing in Autumn writes a
20829    // tenant-scoped response into `dist/` without a resolved tenant (see the sweep
20830    // above). The realistic way a tenant-mismatched file lands there is
20831    // operational — building with a different `[tenancy]` config than the one
20832    // serving requests. But once such a file exists, this test shows Autumn has no
20833    // mechanism to detect, invalidate, or expire it: a later tenant-resolution
20834    // failure preserves it indefinitely, with no operator-visible signal beyond a
20835    // log line. Documented as a known limitation rather than left implicit.
20836    #[tokio::test]
20837    async fn failed_rebuild_leaves_preexisting_static_file_untouched() {
20838        async fn tenant_page(tenant: crate::tenancy::Tenant) -> String {
20839            tenant.0
20840        }
20841
20842        let mut config = AutumnConfig::default();
20843        config.tenancy.enabled = true;
20844        config.tenancy.source = "header".to_owned();
20845        config.tenancy.public_paths = vec!["/storefront".to_owned()];
20846
20847        let state = AppState::for_test();
20848        state.insert_extension(config.clone());
20849
20850        let router = crate::router::try_build_router_inner(
20851            vec![Route {
20852                method: http::Method::GET,
20853                path: "/storefront",
20854                handler: axum::routing::get(tenant_page),
20855                name: "tenant_page",
20856                api_doc: crate::openapi::ApiDoc {
20857                    method: "GET",
20858                    path: "/storefront",
20859                    operation_id: "tenant_page",
20860                    success_status: 200,
20861                    ..Default::default()
20862                },
20863                repository: None,
20864                idempotency: crate::route::RouteIdempotency::Direct,
20865                timeout: crate::route::RouteTimeout::Inherit,
20866                criticality: crate::admission::Criticality::Default,
20867                seo: crate::seo::SeoRouteDefaults::EMPTY,
20868                api_version: None,
20869                sunset_opt_out: false,
20870            }],
20871            &config,
20872            state,
20873            crate::router::RouterContext {
20874                exception_filters: Vec::new(),
20875                scoped_groups: Vec::new(),
20876                merge_routers: Vec::new(),
20877                nest_routers: Vec::new(),
20878                declared_routes: Vec::new(),
20879                custom_layers: Vec::new(),
20880                static_gate_layers: Vec::new(),
20881                #[cfg(feature = "maud")]
20882                error_page_renderer: None,
20883                session_store: None,
20884                #[cfg(feature = "openapi")]
20885                openapi: None,
20886                #[cfg(feature = "mcp")]
20887                mcp: None,
20888                #[cfg(feature = "webmcp")]
20889                webmcp: None,
20890            },
20891        )
20892        .expect("router builds");
20893
20894        let tmp = tempfile::tempdir().expect("dist parent");
20895        let dist = tmp.path().join("dist");
20896
20897        // Seed `dist/` as if an earlier, successful build/regeneration had
20898        // captured tenant A's response (the operationally-realistic route:
20899        // a build/serve `[tenancy]` config mismatch, not anything this
20900        // framework's own request handling can produce — see the sweep
20901        // above).
20902        std::fs::create_dir_all(dist.join("storefront")).expect("mkdir storefront");
20903        let sentinel = "tenant-a-sentinel-warden-2026-09-04";
20904        std::fs::write(dist.join("storefront/index.html"), sentinel).expect("seed stale file");
20905        let mut seed_routes = std::collections::HashMap::new();
20906        seed_routes.insert(
20907            "/storefront".to_owned(),
20908            crate::static_gen::ManifestEntry::new("storefront/index.html".to_owned()),
20909        );
20910        let manifest = crate::static_gen::StaticManifest::new(seed_routes);
20911        std::fs::write(
20912            dist.join("manifest.json"),
20913            serde_json::to_string(&manifest).expect("serialize manifest"),
20914        )
20915        .expect("write manifest");
20916
20917        let result = crate::static_gen::render_static_routes(
20918            router,
20919            &[crate::static_gen::StaticRouteMeta {
20920                path: "/storefront",
20921                name: "tenant_page",
20922                revalidate: None,
20923                params_fn: None,
20924                seo: crate::seo::SeoRouteDefaults::EMPTY,
20925                content_type: None,
20926            }],
20927            &dist,
20928        )
20929        .await;
20930
20931        assert!(
20932            result.is_err(),
20933            "the rebuild must still fail closed (tenant resolution rejects the headerless \
20934             synthetic request), same as the fresh-dist case above"
20935        );
20936
20937        let surviving = std::fs::read_to_string(dist.join("storefront/index.html"))
20938            .expect("the pre-existing file must still be present after a failed rebuild");
20939        assert_eq!(
20940            surviving, sentinel,
20941            "a failed rebuild must not silently alter or remove a pre-existing static file. \
20942             This is Autumn's documented stale-while-revalidate/atomic-swap design working as \
20943             intended, but it also means a tenant-mismatched file that reached dist/ by some \
20944             other means (an operational build/serve config mismatch, not a code path in this \
20945             framework) is served indefinitely with no automatic invalidation — see \
20946             docs/security/2026-09-04-static-gen-tenancy-fails-closed/README.md"
20947        );
20948    }
20949
20950    #[test]
20951    fn app_builder_routes_adds_routes() {
20952        let builder = app();
20953        assert_eq!(builder.routes.len(), 0);
20954
20955        let builder = builder.routes(vec![test_get_route("/1", "route1")]);
20956        assert_eq!(builder.routes.len(), 1);
20957
20958        let builder = builder.routes(vec![
20959            test_get_route("/2", "route2"),
20960            test_get_route("/3", "route3"),
20961        ]);
20962        assert_eq!(builder.routes.len(), 3);
20963
20964        assert_eq!(builder.routes[0].path, "/1");
20965        assert_eq!(builder.routes[1].path, "/2");
20966        assert_eq!(builder.routes[2].path, "/3");
20967    }
20968
20969    #[test]
20970    fn app_builder_extensions_store_and_update_typed_values() {
20971        let builder = app()
20972            .with_extension::<String>("haunted".into())
20973            .update_extension::<String, _, _>(String::new, |value| value.push_str(" harvest"));
20974
20975        let value = builder
20976            .extension::<String>()
20977            .expect("string extension should be present");
20978        assert_eq!(value, "haunted harvest");
20979    }
20980
20981    #[cfg(feature = "mail")]
20982    #[tokio::test]
20983    async fn app_builder_with_mail_delivery_queue_stores_queue_for_install() {
20984        let builder = app().with_mail_delivery_queue(MailTestNoopQueue);
20985        let factory = builder
20986            .mail_delivery_queue_factory
20987            .expect("with_mail_delivery_queue should store a factory on the builder");
20988
20989        // Invoke the trivial wrapper closure built by with_mail_delivery_queue
20990        // and verify it returns the wrapped queue successfully.
20991        let state = AppState::for_test();
20992        let queue = factory(&state).expect("trivial factory should produce the queue");
20993        assert!(Arc::strong_count(&queue) >= 1);
20994        // Cover the enqueue method body by invoking it once.
20995        queue
20996            .enqueue(test_mail())
20997            .await
20998            .expect("noop queue should always succeed");
20999    }
21000
21001    #[cfg(feature = "mail")]
21002    #[test]
21003    fn app_builder_with_mail_delivery_queue_factory_runs_with_app_state() {
21004        let observed_profile: Arc<std::sync::Mutex<Option<String>>> =
21005            Arc::new(std::sync::Mutex::new(None));
21006        let captured = Arc::clone(&observed_profile);
21007        let builder = app().with_mail_delivery_queue_factory(move |state| {
21008            *captured.lock().expect("lock") = Some(state.profile().to_owned());
21009            Ok::<_, crate::AutumnError>(MailTestNoopQueue)
21010        });
21011
21012        let factory = builder
21013            .mail_delivery_queue_factory
21014            .expect("factory should be stored on the builder");
21015        let state = AppState::for_test().with_profile("dev");
21016        let _queue = factory(&state).expect("factory should succeed");
21017
21018        assert_eq!(
21019            observed_profile.lock().expect("lock").as_deref(),
21020            Some("dev"),
21021            "factory must run with the live AppState"
21022        );
21023    }
21024
21025    #[cfg(feature = "mail")]
21026    #[test]
21027    fn app_builder_with_mail_delivery_queue_factory_propagates_errors() {
21028        let builder = app().with_mail_delivery_queue_factory(|_state| {
21029            Err::<MailTestNoopQueue, _>(crate::AutumnError::service_unavailable_msg("factory boom"))
21030        });
21031
21032        let factory = builder
21033            .mail_delivery_queue_factory
21034            .expect("factory present");
21035        let state = AppState::for_test();
21036        match factory(&state) {
21037            Ok(_) => panic!("factory should have errored"),
21038            Err(err) => assert!(err.to_string().contains("factory boom")),
21039        }
21040    }
21041
21042    #[tokio::test]
21043    async fn startup_and_shutdown_hooks_run_in_expected_order() {
21044        let events = Arc::new(std::sync::Mutex::new(Vec::<&'static str>::new()));
21045        let startup_events = Arc::clone(&events);
21046        let shutdown_a = Arc::clone(&events);
21047        let shutdown_b = Arc::clone(&events);
21048        let builder = app()
21049            .on_startup(move |_state| {
21050                let startup_events = Arc::clone(&startup_events);
21051                async move {
21052                    startup_events
21053                        .lock()
21054                        .expect("events lock poisoned")
21055                        .push("start");
21056                    Ok(())
21057                }
21058            })
21059            .on_shutdown(move || {
21060                let shutdown_a = Arc::clone(&shutdown_a);
21061                async move {
21062                    shutdown_a
21063                        .lock()
21064                        .expect("events lock poisoned")
21065                        .push("stop-a");
21066                }
21067            })
21068            .on_shutdown(move || {
21069                let shutdown_b = Arc::clone(&shutdown_b);
21070                async move {
21071                    shutdown_b
21072                        .lock()
21073                        .expect("events lock poisoned")
21074                        .push("stop-b");
21075                }
21076            });
21077
21078        run_startup_hooks(&builder.startup_hooks, AppState::for_test())
21079            .await
21080            .expect("startup hooks should succeed");
21081        run_shutdown_hooks(&builder.shutdown_hooks).await;
21082
21083        let recorded_events = events.lock().expect("events lock poisoned").clone();
21084        assert_eq!(recorded_events, vec!["start", "stop-b", "stop-a"]);
21085    }
21086
21087    /// Counts `preload` calls. `fail` makes `preload` return an error.
21088    #[derive(Default)]
21089    struct PreloadProbe {
21090        inner: crate::feature_flags::InMemoryFlagStore,
21091        preloads: std::sync::atomic::AtomicUsize,
21092        fail: bool,
21093    }
21094
21095    impl crate::feature_flags::FlagStore for PreloadProbe {
21096        fn get(
21097            &self,
21098            key: &str,
21099        ) -> Result<Option<crate::feature_flags::FlagConfig>, crate::feature_flags::FlagStoreError>
21100        {
21101            self.inner.get(key)
21102        }
21103        fn list(
21104            &self,
21105        ) -> Result<Vec<crate::feature_flags::FlagConfig>, crate::feature_flags::FlagStoreError>
21106        {
21107            self.inner.list()
21108        }
21109        fn enable(
21110            &self,
21111            key: &str,
21112            actor: Option<&str>,
21113        ) -> Result<(), crate::feature_flags::FlagStoreError> {
21114            self.inner.enable(key, actor)
21115        }
21116        fn disable(
21117            &self,
21118            key: &str,
21119            actor: Option<&str>,
21120        ) -> Result<(), crate::feature_flags::FlagStoreError> {
21121            self.inner.disable(key, actor)
21122        }
21123        fn set_rollout(
21124            &self,
21125            key: &str,
21126            pct: u8,
21127            actor: Option<&str>,
21128        ) -> Result<(), crate::feature_flags::FlagStoreError> {
21129            self.inner.set_rollout(key, pct, actor)
21130        }
21131        fn allow_actor(
21132            &self,
21133            key: &str,
21134            actor_id: &str,
21135            actor: Option<&str>,
21136        ) -> Result<(), crate::feature_flags::FlagStoreError> {
21137            self.inner.allow_actor(key, actor_id, actor)
21138        }
21139        fn add_group(
21140            &self,
21141            key: &str,
21142            group: &str,
21143            actor: Option<&str>,
21144        ) -> Result<(), crate::feature_flags::FlagStoreError> {
21145            self.inner.add_group(key, group, actor)
21146        }
21147        fn history(
21148            &self,
21149            key: &str,
21150            limit: usize,
21151        ) -> Result<Vec<crate::feature_flags::FlagChangeRecord>, crate::feature_flags::FlagStoreError>
21152        {
21153            self.inner.history(key, limit)
21154        }
21155        fn preload(&self) -> Result<(), crate::feature_flags::FlagStoreError> {
21156            self.preloads
21157                .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
21158            if self.fail {
21159                return Err(crate::feature_flags::FlagStoreError::Backend(
21160                    "brownout".to_owned(),
21161                ));
21162            }
21163            Ok(())
21164        }
21165    }
21166
21167    #[tokio::test]
21168    async fn with_flag_store_preloads_the_store_at_startup() {
21169        for fail in [false, true] {
21170            let store = Arc::new(PreloadProbe {
21171                fail,
21172                ..PreloadProbe::default()
21173            });
21174            let builder = app().with_flag_store(Arc::clone(&store));
21175            let state = AppState::for_test();
21176            run_state_initializers(builder.state_initializers, &state);
21177
21178            // The state initializer preloaded; a failure only warns.
21179            assert_eq!(
21180                store.preloads.load(std::sync::atomic::Ordering::SeqCst),
21181                1,
21182                "fail = {fail}"
21183            );
21184        }
21185    }
21186
21187    #[test]
21188    fn with_flag_service_keeps_declared_defaults() {
21189        let service = crate::feature_flags::FeatureFlagService::new(Arc::new(
21190            crate::feature_flags::InMemoryFlagStore::new(),
21191        ))
21192        .with_default("checkout_v2", true);
21193        let builder = app().with_flag_service(service);
21194        let state = AppState::for_test();
21195        run_state_initializers(builder.state_initializers, &state);
21196
21197        let installed = state
21198            .extension::<crate::feature_flags::FeatureFlagService>()
21199            .expect("service registered");
21200        assert!(installed.is_enabled("checkout_v2", None));
21201        assert!(
21202            builder.startup_hooks.is_empty(),
21203            "the state initializer preloads flags; no startup hook does"
21204        );
21205    }
21206
21207    fn startup_noop_job_handler(
21208        _state: AppState,
21209        _payload: serde_json::Value,
21210    ) -> Pin<Box<dyn Future<Output = crate::AutumnResult<()>> + Send + 'static>> {
21211        Box::pin(async move { Ok(()) })
21212    }
21213
21214    #[tokio::test]
21215    async fn startup_hooks_can_enqueue_jobs_after_runtime_init() {
21216        let _guard = crate::job::global_job_runtime_test_lock().lock().await;
21217        crate::job::clear_global_job_client();
21218
21219        let builder = app()
21220            .jobs(vec![crate::job::JobInfo {
21221                version: 1,
21222                name: "startup-seed".to_string(),
21223                max_attempts: 1,
21224                initial_backoff_ms: 1,
21225                queue: "default".to_string(),
21226                uniqueness: None,
21227                concurrency: None,
21228                handler: startup_noop_job_handler,
21229                timeout: None,
21230            }])
21231            .on_startup(|_state| async {
21232                crate::job::enqueue("startup-seed", serde_json::json!({ "kind": "warmup" })).await
21233            });
21234
21235        let state = AppState::for_test().with_profile("dev");
21236        let shutdown = tokio_util::sync::CancellationToken::new();
21237
21238        initialize_job_runtime(
21239            builder.jobs.clone(),
21240            &state,
21241            &shutdown,
21242            &crate::config::JobConfig::default(),
21243            true,
21244        )
21245        .expect("job runtime should initialize before startup hooks");
21246
21247        run_startup_hooks(&builder.startup_hooks, state.clone())
21248            .await
21249            .expect("startup hook should be able to enqueue jobs");
21250
21251        tokio::time::timeout(std::time::Duration::from_secs(1), async {
21252            loop {
21253                let snapshot = state.job_registry().snapshot();
21254                let status = snapshot
21255                    .get("startup-seed")
21256                    .expect("job should be registered before startup hooks run");
21257                if status.total_successes == 1 {
21258                    break;
21259                }
21260                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
21261            }
21262        })
21263        .await
21264        .expect("startup-enqueued job should complete");
21265
21266        shutdown.cancel();
21267        crate::job::clear_global_job_client();
21268    }
21269
21270    #[tokio::test]
21271    async fn initialize_job_runtime_propagates_redis_init_errors() {
21272        let _guard = crate::job::global_job_runtime_test_lock().lock().await;
21273        crate::job::clear_global_job_client();
21274
21275        let state = AppState::for_test().with_profile("dev");
21276        let shutdown = tokio_util::sync::CancellationToken::new();
21277        let config = crate::config::JobConfig {
21278            backend: "redis".to_string(),
21279            ..Default::default()
21280        };
21281
21282        let error = initialize_job_runtime(
21283            vec![crate::job::JobInfo {
21284                version: 1,
21285                name: "startup-seed".to_string(),
21286                max_attempts: 1,
21287                initial_backoff_ms: 1,
21288                queue: "default".to_string(),
21289                uniqueness: None,
21290                concurrency: None,
21291                handler: startup_noop_job_handler,
21292                timeout: None,
21293            }],
21294            &state,
21295            &shutdown,
21296            &config,
21297            true,
21298        )
21299        .expect_err("redis init errors should abort startup");
21300
21301        #[cfg(feature = "redis")]
21302        assert!(
21303            error
21304                .to_string()
21305                .contains("jobs.backend=redis requires jobs.redis.url"),
21306            "unexpected error: {error}"
21307        );
21308
21309        #[cfg(not(feature = "redis"))]
21310        assert!(
21311            error
21312                .to_string()
21313                .contains("jobs.backend=redis requested but redis feature is disabled"),
21314            "unexpected error: {error}"
21315        );
21316    }
21317
21318    #[tokio::test]
21319    async fn startup_hook_errors_propagate() {
21320        let builder = app().on_startup(|_state| async {
21321            Err(crate::AutumnError::service_unavailable_msg(
21322                "startup ritual failed",
21323            ))
21324        });
21325
21326        let error = run_startup_hooks(&builder.startup_hooks, AppState::for_test())
21327            .await
21328            .expect_err("startup hook should fail");
21329        assert!(error.to_string().contains("startup ritual failed"));
21330    }
21331
21332    #[tokio::test]
21333    async fn build_router_mounts_user_routes() {
21334        let router = test_router(vec![test_get_route("/test", "test_handler")]);
21335
21336        let response = router
21337            .oneshot(Request::builder().uri("/test").body(Body::empty()).unwrap())
21338            .await
21339            .unwrap();
21340
21341        assert_eq!(response.status(), StatusCode::OK);
21342        let body = axum::body::to_bytes(response.into_body(), usize::MAX)
21343            .await
21344            .unwrap();
21345        assert_eq!(&body[..], b"ok");
21346    }
21347
21348    #[tokio::test]
21349    async fn build_router_mounts_health_check_at_default_path() {
21350        let router = test_router(vec![test_get_route("/dummy", "dummy")]);
21351
21352        let response = router
21353            .oneshot(
21354                Request::builder()
21355                    .uri("/health")
21356                    .body(Body::empty())
21357                    .unwrap(),
21358            )
21359            .await
21360            .unwrap();
21361
21362        assert_eq!(response.status(), StatusCode::OK);
21363        let body = axum::body::to_bytes(response.into_body(), usize::MAX)
21364            .await
21365            .unwrap();
21366        let json: serde_json::Value = serde_json::from_slice(&body).unwrap();
21367        assert_eq!(json["status"], "ok");
21368    }
21369
21370    #[tokio::test]
21371    async fn build_router_mounts_health_check_at_custom_path() {
21372        let mut config = AutumnConfig::default();
21373        config.health.path = "/healthz".to_owned();
21374        let router = test_router_with_config(vec![test_get_route("/dummy", "dummy")], &config);
21375
21376        let response = router
21377            .oneshot(
21378                Request::builder()
21379                    .uri("/healthz")
21380                    .body(Body::empty())
21381                    .unwrap(),
21382            )
21383            .await
21384            .unwrap();
21385
21386        assert_eq!(response.status(), StatusCode::OK);
21387    }
21388
21389    #[tokio::test]
21390    async fn build_router_adds_request_id_header() {
21391        let router = test_router(vec![test_get_route("/test", "test")]);
21392
21393        let response = router
21394            .oneshot(Request::builder().uri("/test").body(Body::empty()).unwrap())
21395            .await
21396            .unwrap();
21397
21398        assert!(response.headers().contains_key("x-request-id"));
21399    }
21400
21401    #[tokio::test]
21402    async fn build_router_unknown_route_returns_404() {
21403        let router = test_router(vec![test_get_route("/exists", "exists")]);
21404
21405        let response = router
21406            .oneshot(Request::builder().uri("/nope").body(Body::empty()).unwrap())
21407            .await
21408            .unwrap();
21409
21410        assert_eq!(response.status(), StatusCode::NOT_FOUND);
21411    }
21412
21413    #[tokio::test]
21414    async fn build_router_multiple_routes() {
21415        let router = test_router(vec![test_get_route("/a", "a"), test_get_route("/b", "b")]);
21416
21417        let resp_a = router
21418            .clone()
21419            .oneshot(Request::builder().uri("/a").body(Body::empty()).unwrap())
21420            .await
21421            .unwrap();
21422        assert_eq!(resp_a.status(), StatusCode::OK);
21423
21424        let resp_b = router
21425            .oneshot(Request::builder().uri("/b").body(Body::empty()).unwrap())
21426            .await
21427            .unwrap();
21428        assert_eq!(resp_b.status(), StatusCode::OK);
21429    }
21430
21431    #[tokio::test]
21432    async fn build_router_post_route() {
21433        let post_routes = vec![Route {
21434            method: http::Method::POST,
21435            path: "/submit",
21436            handler: axum::routing::post(|| async { "posted" }),
21437            name: "submit",
21438            api_doc: crate::openapi::ApiDoc {
21439                method: "POST",
21440                path: "/submit",
21441                operation_id: "submit",
21442                success_status: 200,
21443                ..Default::default()
21444            },
21445            repository: None,
21446            idempotency: crate::route::RouteIdempotency::Direct,
21447            timeout: crate::route::RouteTimeout::Inherit,
21448            criticality: crate::admission::Criticality::Default,
21449            seo: crate::seo::SeoRouteDefaults::EMPTY,
21450            api_version: None,
21451            sunset_opt_out: false,
21452        }];
21453        let router = test_router(post_routes);
21454
21455        let response = router
21456            .oneshot(
21457                Request::builder()
21458                    .method("POST")
21459                    .uri("/submit")
21460                    .body(Body::empty())
21461                    .unwrap(),
21462            )
21463            .await
21464            .unwrap();
21465
21466        assert_eq!(response.status(), StatusCode::OK);
21467    }
21468
21469    #[tokio::test]
21470    async fn build_router_merges_methods_on_same_path() {
21471        let route_list = vec![
21472            Route {
21473                method: http::Method::GET,
21474                path: "/admin",
21475                handler: axum::routing::get(|| async { "list" }),
21476                name: "admin_list",
21477                api_doc: crate::openapi::ApiDoc {
21478                    method: "GET",
21479                    path: "/admin",
21480                    operation_id: "admin_list",
21481                    success_status: 200,
21482                    ..Default::default()
21483                },
21484                repository: None,
21485                idempotency: crate::route::RouteIdempotency::Direct,
21486                timeout: crate::route::RouteTimeout::Inherit,
21487                criticality: crate::admission::Criticality::Default,
21488                seo: crate::seo::SeoRouteDefaults::EMPTY,
21489                api_version: None,
21490                sunset_opt_out: false,
21491            },
21492            Route {
21493                method: http::Method::POST,
21494                path: "/admin",
21495                handler: axum::routing::post(|| async { "created" }),
21496                name: "create",
21497                api_doc: crate::openapi::ApiDoc {
21498                    method: "POST",
21499                    path: "/admin",
21500                    operation_id: "create",
21501                    success_status: 200,
21502                    ..Default::default()
21503                },
21504                repository: None,
21505                idempotency: crate::route::RouteIdempotency::Direct,
21506                timeout: crate::route::RouteTimeout::Inherit,
21507                criticality: crate::admission::Criticality::Default,
21508                seo: crate::seo::SeoRouteDefaults::EMPTY,
21509                api_version: None,
21510                sunset_opt_out: false,
21511            },
21512        ];
21513        let config = AutumnConfig::default();
21514        let router = crate::router::build_router(route_list, &config, AppState::for_test());
21515
21516        // GET /admin should return "list"
21517        let resp = router
21518            .clone()
21519            .oneshot(
21520                Request::builder()
21521                    .uri("/admin")
21522                    .body(Body::empty())
21523                    .unwrap(),
21524            )
21525            .await
21526            .unwrap();
21527        assert_eq!(resp.status(), StatusCode::OK);
21528        let body = axum::body::to_bytes(resp.into_body(), usize::MAX)
21529            .await
21530            .unwrap();
21531        assert_eq!(&body[..], b"list");
21532
21533        // POST /admin should return "created" (not 405!)
21534        let resp = router
21535            .oneshot(
21536                Request::builder()
21537                    .method("POST")
21538                    .uri("/admin")
21539                    .body(Body::empty())
21540                    .unwrap(),
21541            )
21542            .await
21543            .unwrap();
21544        assert_eq!(resp.status(), StatusCode::OK);
21545        let body = axum::body::to_bytes(resp.into_body(), usize::MAX)
21546            .await
21547            .unwrap();
21548        assert_eq!(&body[..], b"created");
21549    }
21550
21551    /// Through the full router, so the global `AssetCacheControlLayer` (which
21552    /// re-stamps `Cache-Control` on every successful `/static/` response) is
21553    /// in play: the plain htmx path revalidates and the hashed one, which
21554    /// that layer must recognise as fingerprinted, stays `immutable`.
21555    #[cfg(feature = "htmx")]
21556    #[tokio::test]
21557    async fn htmx_is_served_at_plain_and_fingerprinted_paths_with_matching_cache_policy() {
21558        let router = test_router(vec![test_get_route("/dummy", "dummy")]);
21559        let get = |uri: String| {
21560            let router = router.clone();
21561            async move {
21562                router
21563                    .oneshot(Request::builder().uri(uri).body(Body::empty()).unwrap())
21564                    .await
21565                    .unwrap()
21566            }
21567        };
21568
21569        let plain = get(crate::htmx::HTMX_JS_PATH.to_owned()).await;
21570        assert_eq!(plain.status(), StatusCode::OK);
21571        let content_type = plain.headers()["content-type"].to_str().unwrap();
21572        assert!(
21573            content_type.contains("javascript"),
21574            "Expected JavaScript, got {content_type}"
21575        );
21576        let cache_control = plain.headers()["cache-control"].to_str().unwrap();
21577        assert_eq!(cache_control, "public, max-age=0, must-revalidate");
21578        assert!(plain.headers().contains_key("etag"));
21579        let body = axum::body::to_bytes(plain.into_body(), usize::MAX)
21580            .await
21581            .unwrap();
21582        assert_eq!(&body[..], crate::htmx::HTMX_JS);
21583
21584        let hashed_url = crate::assets::asset_url("js/htmx.min.js");
21585        assert_ne!(hashed_url, crate::htmx::HTMX_JS_PATH);
21586        let hashed = get(hashed_url).await;
21587        assert_eq!(hashed.status(), StatusCode::OK);
21588        assert_eq!(
21589            hashed.headers()["cache-control"].to_str().unwrap(),
21590            "public, max-age=31536000, immutable"
21591        );
21592        let body = axum::body::to_bytes(hashed.into_body(), usize::MAX)
21593            .await
21594            .unwrap();
21595        assert_eq!(&body[..], crate::htmx::HTMX_JS);
21596    }
21597
21598    #[cfg(feature = "htmx")]
21599    #[tokio::test]
21600    async fn htmx_csrf_helper_is_csp_compatible_javascript() {
21601        let router = test_router(vec![test_get_route("/dummy", "dummy")]);
21602
21603        let response = router
21604            .oneshot(
21605                Request::builder()
21606                    .uri(crate::htmx::HTMX_CSRF_JS_PATH)
21607                    .body(Body::empty())
21608                    .unwrap(),
21609            )
21610            .await
21611            .unwrap();
21612
21613        assert_eq!(response.status(), StatusCode::OK);
21614        assert_eq!(
21615            response
21616                .headers()
21617                .get("content-type")
21618                .and_then(|value| value.to_str().ok()),
21619            Some("text/javascript; charset=utf-8")
21620        );
21621
21622        let body = axum::body::to_bytes(response.into_body(), usize::MAX)
21623            .await
21624            .unwrap();
21625        let js = std::str::from_utf8(&body).expect("csrf helper should be valid utf-8");
21626
21627        assert!(js.contains("htmx:configRequest"));
21628        assert!(js.contains("X-CSRF-Token"));
21629        assert!(!js.contains("<script"));
21630    }
21631
21632    #[cfg(feature = "htmx")]
21633    #[tokio::test]
21634    async fn build_router_serves_htmx_js() {
21635        let router = test_router(vec![test_get_route("/dummy", "dummy")]);
21636
21637        let response = router
21638            .oneshot(
21639                Request::builder()
21640                    .uri(crate::htmx::HTMX_JS_PATH)
21641                    .body(Body::empty())
21642                    .unwrap(),
21643            )
21644            .await
21645            .unwrap();
21646
21647        assert_eq!(response.status(), StatusCode::OK);
21648        let ct = response
21649            .headers()
21650            .get("content-type")
21651            .unwrap()
21652            .to_str()
21653            .unwrap();
21654        assert!(ct.contains("javascript"));
21655    }
21656
21657    #[cfg(feature = "htmx")]
21658    #[tokio::test]
21659    async fn build_router_serves_htmx_csrf_js() {
21660        let router = test_router(vec![test_get_route("/dummy", "dummy")]);
21661
21662        let response = router
21663            .oneshot(
21664                Request::builder()
21665                    .uri(crate::htmx::HTMX_CSRF_JS_PATH)
21666                    .body(Body::empty())
21667                    .unwrap(),
21668            )
21669            .await
21670            .unwrap();
21671
21672        assert_eq!(response.status(), StatusCode::OK);
21673        let csp = response
21674            .headers()
21675            .get("content-security-policy")
21676            .expect("framework JS should still receive security headers")
21677            .to_str()
21678            .unwrap();
21679        assert!(csp.contains("script-src 'self'"), "csp = {csp}");
21680        let body = axum::body::to_bytes(response.into_body(), usize::MAX)
21681            .await
21682            .unwrap();
21683        let js = std::str::from_utf8(&body).expect("csrf helper should be valid utf-8");
21684        assert!(js.contains("htmx:configRequest"));
21685        assert!(js.contains("X-CSRF-Token"));
21686    }
21687
21688    #[tokio::test]
21689    async fn build_router_serves_default_favicon_without_404() {
21690        let router = test_router(vec![test_get_route("/dummy", "dummy")]);
21691
21692        let response = router
21693            .oneshot(
21694                Request::builder()
21695                    .uri(crate::router::DEFAULT_FAVICON_PATH)
21696                    .body(Body::empty())
21697                    .unwrap(),
21698            )
21699            .await
21700            .unwrap();
21701
21702        assert_eq!(response.status(), StatusCode::NO_CONTENT);
21703        assert!(
21704            response.headers().contains_key("content-security-policy"),
21705            "framework fallback responses should still receive security headers"
21706        );
21707        let body = axum::body::to_bytes(response.into_body(), usize::MAX)
21708            .await
21709            .unwrap();
21710        assert!(body.is_empty());
21711    }
21712
21713    #[tokio::test]
21714    async fn build_router_does_not_override_user_favicon_route() {
21715        let router = test_router(vec![test_get_route(
21716            crate::router::DEFAULT_FAVICON_PATH,
21717            "favicon",
21718        )]);
21719
21720        let response = router
21721            .oneshot(
21722                Request::builder()
21723                    .uri(crate::router::DEFAULT_FAVICON_PATH)
21724                    .body(Body::empty())
21725                    .unwrap(),
21726            )
21727            .await
21728            .unwrap();
21729
21730        assert_eq!(response.status(), StatusCode::OK);
21731        let body = axum::body::to_bytes(response.into_body(), usize::MAX)
21732            .await
21733            .unwrap();
21734        assert_eq!(&body[..], b"ok");
21735    }
21736
21737    #[tokio::test]
21738    async fn build_router_serves_static_files_for_unmatched_paths() {
21739        use std::collections::HashMap;
21740
21741        // Create a temp dist/ with a static page
21742        let tmp = tempfile::tempdir().expect("tempdir");
21743        let dist = tmp.path().join("dist");
21744        std::fs::create_dir_all(dist.join("docs")).expect("mkdir");
21745        std::fs::write(dist.join("docs/index.html"), "<h1>Static Docs</h1>").expect("write");
21746
21747        let manifest = crate::static_gen::StaticManifest::new(HashMap::from([(
21748            "/docs".to_owned(),
21749            crate::static_gen::ManifestEntry::new("docs/index.html".to_owned()),
21750        )]))
21751        .with_generated_at("2026-03-27T00:00:00Z");
21752        let json = serde_json::to_string(&manifest).expect("serialize");
21753        std::fs::write(dist.join("manifest.json"), json).expect("write manifest");
21754
21755        // No dynamic route for /docs — only a static file.
21756        let config = AutumnConfig::default();
21757        let state = AppState {
21758            health_detailed: true,
21759            ..AppState::test_default()
21760        };
21761        let router = crate::router::build_router_with_static(
21762            vec![test_get_route("/other", "other_page")],
21763            &config,
21764            state,
21765            Some(dist.as_path()),
21766        );
21767
21768        // GET /docs/ should serve the pre-built HTML via static-first
21769        // middleware (manifest lookup with trailing-slash normalization).
21770        let response = router
21771            .oneshot(
21772                Request::builder()
21773                    .uri("/docs/")
21774                    .body(Body::empty())
21775                    .unwrap(),
21776            )
21777            .await
21778            .unwrap();
21779
21780        assert_eq!(response.status(), StatusCode::OK);
21781        let csp = response
21782            .headers()
21783            .get("content-security-policy")
21784            .expect("static-first HTML should still receive security headers")
21785            .to_str()
21786            .unwrap();
21787        assert!(csp.contains("script-src 'self'"), "csp = {csp}");
21788        let body = axum::body::to_bytes(response.into_body(), usize::MAX)
21789            .await
21790            .unwrap();
21791        assert_eq!(std::str::from_utf8(&body).unwrap(), "<h1>Static Docs</h1>");
21792    }
21793
21794    /// #1832 through the composed `build_router_with_static` stack rather than
21795    /// the router tests' narrower helper: the manifest's recorded type must
21796    /// survive the security-headers layer and arrive alongside
21797    /// `X-Content-Type-Options: nosniff` — which is precisely why it has to be
21798    /// right. `/feed` is extensionless and stored as `feed/index.html`, so both
21799    /// legacy clues say `text/html`; only the recorded value makes it RSS.
21800    #[tokio::test]
21801    async fn build_router_serves_recorded_content_type_with_nosniff() {
21802        use std::collections::HashMap;
21803
21804        let tmp = tempfile::tempdir().expect("tempdir");
21805        let dist = tmp.path().join("dist");
21806        std::fs::create_dir_all(dist.join("feed")).expect("mkdir");
21807        std::fs::write(dist.join("feed/index.html"), "<rss/>").expect("write");
21808
21809        let manifest = crate::static_gen::StaticManifest::new(HashMap::from([(
21810            "/feed".to_owned(),
21811            crate::static_gen::ManifestEntry::new("feed/index.html".to_owned())
21812                .with_content_type(Some("application/rss+xml".to_owned())),
21813        )]));
21814        std::fs::write(
21815            dist.join("manifest.json"),
21816            serde_json::to_string(&manifest).expect("serialize"),
21817        )
21818        .expect("write manifest");
21819
21820        let config = AutumnConfig::default();
21821        let router = crate::router::build_router_with_static(
21822            Vec::new(),
21823            &config,
21824            AppState::for_test(),
21825            Some(dist.as_path()),
21826        );
21827
21828        let response = router
21829            .oneshot(Request::builder().uri("/feed").body(Body::empty()).unwrap())
21830            .await
21831            .unwrap();
21832
21833        assert_eq!(response.status(), StatusCode::OK);
21834        assert_eq!(
21835            response
21836                .headers()
21837                .get(http::header::CONTENT_TYPE)
21838                .and_then(|v| v.to_str().ok()),
21839            Some("application/rss+xml"),
21840            "the recorded type must survive the full composed stack"
21841        );
21842        assert_eq!(
21843            response
21844                .headers()
21845                .get("x-content-type-options")
21846                .and_then(|v| v.to_str().ok()),
21847            Some("nosniff"),
21848            "nosniff is why the recorded type has to be correct: the browser \
21849             will not second-guess it"
21850        );
21851    }
21852
21853    /// An edge-only build has no static pages. Old ones in `dist/` must not
21854    /// ship with it.
21855    #[test]
21856    fn an_empty_static_build_clears_stale_output() {
21857        let tmp = tempfile::tempdir().expect("temp dir");
21858        let dist = tmp.path().join("dist");
21859        std::fs::create_dir_all(dist.join("about")).expect("mkdir");
21860        std::fs::write(dist.join("about/index.html"), "stale").expect("write");
21861
21862        clear_stale_static_output(&dist).expect("clears");
21863        assert!(!dist.exists(), "stale dist/ must be gone");
21864
21865        clear_stale_static_output(&dist).expect("a missing dist/ is fine");
21866    }
21867
21868    #[test]
21869    fn no_static_routes_fails_the_static_build_unless_empty_is_allowed() {
21870        assert_eq!(no_static_routes_exit_code(false), 1);
21871        assert_eq!(no_static_routes_exit_code(true), 0);
21872        assert_eq!(
21873            BUILD_STATIC_ALLOW_EMPTY_ENV,
21874            "AUTUMN_BUILD_STATIC_ALLOW_EMPTY"
21875        );
21876    }
21877
21878    #[tokio::test]
21879    async fn build_mode_static_rendering_bypasses_startup_barrier() {
21880        temp_env::async_with_vars([("AUTUMN_BUILD_STATIC", Some("1"))], async {
21881            let config = AutumnConfig::default();
21882            let state = AppState::for_test().with_startup_complete(false);
21883            let router = crate::router::build_router(
21884                vec![Route {
21885                    method: http::Method::GET,
21886                    path: "/about",
21887                    handler: axum::routing::get(|| async { "About Page Content" }),
21888                    name: "about",
21889                    api_doc: crate::openapi::ApiDoc {
21890                        method: "GET",
21891                        path: "/about",
21892                        operation_id: "about",
21893                        success_status: 200,
21894                        ..Default::default()
21895                    },
21896                    repository: None,
21897                    idempotency: crate::route::RouteIdempotency::Direct,
21898                    timeout: crate::route::RouteTimeout::Inherit,
21899                    criticality: crate::admission::Criticality::Default,
21900                    seo: crate::seo::SeoRouteDefaults::EMPTY,
21901                    api_version: None,
21902                    sunset_opt_out: false,
21903                }],
21904                &config,
21905                state,
21906            );
21907            let tmp = tempfile::tempdir().unwrap();
21908            let dist = tmp.path().join("dist");
21909
21910            let result = crate::static_gen::render_static_routes(
21911                router,
21912                &[crate::static_gen::StaticRouteMeta {
21913                    path: "/about",
21914                    name: "about",
21915                    revalidate: None,
21916                    params_fn: None,
21917                    seo: crate::seo::SeoRouteDefaults::EMPTY,
21918                    content_type: None,
21919                }],
21920                &dist,
21921            )
21922            .await;
21923
21924            assert!(result.is_ok(), "build failed: {:?}", result.err());
21925            let html = std::fs::read_to_string(dist.join("about/index.html")).unwrap();
21926            assert_eq!(html, "About Page Content");
21927        })
21928        .await;
21929    }
21930
21931    #[tokio::test]
21932    async fn build_router_injects_live_reload_script_when_enabled() {
21933        let reload_file = tempfile::NamedTempFile::new().expect("reload state file");
21934        std::fs::write(reload_file.path(), r#"{"version":0,"kind":"full"}"#).expect("write");
21935        temp_env::async_with_vars(
21936            [
21937                ("AUTUMN_DEV_RELOAD", Some("1")),
21938                (
21939                    "AUTUMN_DEV_RELOAD_STATE",
21940                    Some(reload_file.path().to_str().expect("utf-8 path")),
21941                ),
21942            ],
21943            async {
21944                let router = test_router(vec![Route {
21945                    method: http::Method::GET,
21946                    path: "/page",
21947                    handler: axum::routing::get(|| async {
21948                        axum::response::Html("<html><body><main>ok</main></body></html>")
21949                    }),
21950                    name: "page",
21951                    api_doc: crate::openapi::ApiDoc {
21952                        method: "GET",
21953                        path: "/page",
21954                        operation_id: "page",
21955                        success_status: 200,
21956                        ..Default::default()
21957                    },
21958                    repository: None,
21959                    idempotency: crate::route::RouteIdempotency::Direct,
21960                    timeout: crate::route::RouteTimeout::Inherit,
21961                    criticality: crate::admission::Criticality::Default,
21962                    seo: crate::seo::SeoRouteDefaults::EMPTY,
21963                    api_version: None,
21964                    sunset_opt_out: false,
21965                }]);
21966
21967                let response = router
21968                    .oneshot(Request::builder().uri("/page").body(Body::empty()).unwrap())
21969                    .await
21970                    .unwrap();
21971
21972                let body = axum::body::to_bytes(response.into_body(), usize::MAX)
21973                    .await
21974                    .unwrap();
21975                let html = std::str::from_utf8(&body).expect("utf-8");
21976                assert!(html.contains("/__autumn/live-reload"));
21977            },
21978        )
21979        .await;
21980    }
21981
21982    #[tokio::test]
21983    async fn build_router_mounts_dev_reload_script_endpoint_when_enabled() {
21984        // The injected <script src="/__autumn/live-reload.js"> tag only works
21985        // under the default CSP (`script-src 'self'`) if the framework
21986        // actually serves the JS at that path. This guards against the
21987        // regression where the script endpoint is forgotten.
21988        let reload_file = tempfile::NamedTempFile::new().expect("reload state file");
21989        std::fs::write(reload_file.path(), r#"{"version":0,"kind":"full"}"#).expect("write");
21990        temp_env::async_with_vars(
21991            [
21992                ("AUTUMN_DEV_RELOAD", Some("1")),
21993                (
21994                    "AUTUMN_DEV_RELOAD_STATE",
21995                    Some(reload_file.path().to_str().expect("utf-8 path")),
21996                ),
21997            ],
21998            async {
21999                let router = test_router(vec![test_get_route("/dummy", "dummy")]);
22000
22001                let response = router
22002                    .oneshot(
22003                        Request::builder()
22004                            .uri("/__autumn/live-reload.js")
22005                            .body(Body::empty())
22006                            .unwrap(),
22007                    )
22008                    .await
22009                    .unwrap();
22010
22011                assert_eq!(response.status(), StatusCode::OK);
22012                assert_eq!(
22013                    response
22014                        .headers()
22015                        .get("content-type")
22016                        .and_then(|v| v.to_str().ok()),
22017                    Some("application/javascript; charset=utf-8")
22018                );
22019                let body = axum::body::to_bytes(response.into_body(), usize::MAX)
22020                    .await
22021                    .unwrap();
22022                let js = std::str::from_utf8(&body).expect("utf-8");
22023                assert!(js.contains("fetch("), "js body: {js}");
22024            },
22025        )
22026        .await;
22027    }
22028
22029    #[tokio::test]
22030    async fn build_router_mounts_dev_reload_endpoint_when_enabled() {
22031        let reload_file = tempfile::NamedTempFile::new().expect("reload state file");
22032        std::fs::write(reload_file.path(), r#"{"version":7,"kind":"css"}"#).expect("write");
22033        temp_env::async_with_vars(
22034            [
22035                ("AUTUMN_DEV_RELOAD", Some("1")),
22036                (
22037                    "AUTUMN_DEV_RELOAD_STATE",
22038                    Some(reload_file.path().to_str().expect("utf-8 path")),
22039                ),
22040            ],
22041            async {
22042                let router = test_router(vec![test_get_route("/dummy", "dummy")]);
22043
22044                let response = router
22045                    .oneshot(
22046                        Request::builder()
22047                            .uri("/__autumn/live-reload")
22048                            .body(Body::empty())
22049                            .unwrap(),
22050                    )
22051                    .await
22052                    .unwrap();
22053
22054                assert_eq!(response.status(), StatusCode::OK);
22055                assert_eq!(
22056                    response.headers().get("cache-control").unwrap(),
22057                    "no-store, no-cache, must-revalidate"
22058                );
22059                let body = axum::body::to_bytes(response.into_body(), usize::MAX)
22060                    .await
22061                    .unwrap();
22062                assert_eq!(&body[..], br#"{"version":7,"kind":"css"}"#);
22063            },
22064        )
22065        .await;
22066    }
22067
22068    #[tokio::test]
22069    async fn build_router_disables_cache_for_static_assets_in_dev_reload_mode() {
22070        let project = tempfile::tempdir().expect("project dir");
22071        let static_dir = project.path().join("static");
22072        std::fs::create_dir_all(&static_dir).expect("mkdir");
22073        std::fs::write(static_dir.join("demo.txt"), "hello").expect("write static file");
22074        let reload_file = tempfile::NamedTempFile::new().expect("reload state file");
22075        std::fs::write(reload_file.path(), r#"{"version":0,"kind":"full"}"#).expect("write");
22076        temp_env::async_with_vars(
22077            [
22078                (
22079                    "AUTUMN_MANIFEST_DIR",
22080                    Some(project.path().to_str().expect("utf-8 path")),
22081                ),
22082                ("AUTUMN_DEV_RELOAD", Some("1")),
22083                (
22084                    "AUTUMN_DEV_RELOAD_STATE",
22085                    Some(reload_file.path().to_str().expect("utf-8 path")),
22086                ),
22087            ],
22088            async {
22089                let router = test_router(vec![test_get_route("/dummy", "dummy")]);
22090
22091                let response = router
22092                    .oneshot(
22093                        Request::builder()
22094                            .uri("/static/demo.txt")
22095                            .body(Body::empty())
22096                            .unwrap(),
22097                    )
22098                    .await
22099                    .unwrap();
22100
22101                assert_eq!(response.status(), StatusCode::OK);
22102                assert_eq!(
22103                    response.headers().get("cache-control").unwrap(),
22104                    "no-store, no-cache, must-revalidate"
22105                );
22106            },
22107        )
22108        .await;
22109    }
22110
22111    #[test]
22112    fn app_builder_accepts_static_routes() {
22113        use crate::static_gen::StaticRouteMeta;
22114        let metas = vec![StaticRouteMeta {
22115            path: "/about",
22116            name: "about",
22117            revalidate: None,
22118            params_fn: None,
22119            seo: crate::seo::SeoRouteDefaults::EMPTY,
22120            content_type: None,
22121        }];
22122        let builder = app().static_routes(metas);
22123        assert_eq!(builder.static_metas.len(), 1);
22124    }
22125
22126    #[test]
22127    fn project_dir_defaults_to_subdir() {
22128        // When AUTUMN_MANIFEST_DIR is not set, project_dir returns the
22129        // subdir name as-is (relative to cwd).
22130        let env = crate::config::MockEnv::new();
22131        let dir = super::project_dir("dist", &env);
22132        assert_eq!(dir, std::path::PathBuf::from("dist"));
22133    }
22134
22135    /// Helper to build a test router with custom config.
22136    pub fn test_router_with_config(routes: Vec<Route>, config: &AutumnConfig) -> axum::Router {
22137        let state = AppState {
22138            health_detailed: true,
22139            ..AppState::test_default()
22140        };
22141        crate::router::build_router(routes, config, state)
22142    }
22143
22144    #[tokio::test]
22145    async fn cors_wildcard_allows_any_origin() {
22146        let mut config = AutumnConfig::default();
22147        config.cors.allowed_origins = vec!["*".to_owned()];
22148        let router = test_router_with_config(vec![test_get_route("/test", "test")], &config);
22149
22150        let response = router
22151            .oneshot(
22152                Request::builder()
22153                    .uri("/test")
22154                    .header("Origin", "https://example.com")
22155                    .body(Body::empty())
22156                    .unwrap(),
22157            )
22158            .await
22159            .unwrap();
22160
22161        assert_eq!(response.status(), StatusCode::OK);
22162        assert_eq!(
22163            response
22164                .headers()
22165                .get("access-control-allow-origin")
22166                .unwrap(),
22167            "*"
22168        );
22169    }
22170
22171    #[tokio::test]
22172    async fn cors_specific_origin_reflected() {
22173        let mut config = AutumnConfig::default();
22174        config.cors.allowed_origins = vec!["https://example.com".to_owned()];
22175        let router = test_router_with_config(vec![test_get_route("/test", "test")], &config);
22176
22177        let response = router
22178            .oneshot(
22179                Request::builder()
22180                    .uri("/test")
22181                    .header("Origin", "https://example.com")
22182                    .body(Body::empty())
22183                    .unwrap(),
22184            )
22185            .await
22186            .unwrap();
22187
22188        assert_eq!(response.status(), StatusCode::OK);
22189        assert_eq!(
22190            response
22191                .headers()
22192                .get("access-control-allow-origin")
22193                .unwrap(),
22194            "https://example.com"
22195        );
22196    }
22197
22198    #[tokio::test]
22199    async fn cors_disabled_when_no_origins() {
22200        let config = AutumnConfig::default();
22201        assert!(config.cors.allowed_origins.is_empty());
22202        let router = test_router_with_config(vec![test_get_route("/test", "test")], &config);
22203
22204        let response = router
22205            .oneshot(
22206                Request::builder()
22207                    .uri("/test")
22208                    .header("Origin", "https://example.com")
22209                    .body(Body::empty())
22210                    .unwrap(),
22211            )
22212            .await
22213            .unwrap();
22214
22215        assert_eq!(response.status(), StatusCode::OK);
22216        assert!(
22217            response
22218                .headers()
22219                .get("access-control-allow-origin")
22220                .is_none()
22221        );
22222    }
22223
22224    #[tokio::test]
22225    async fn cors_preflight_returns_204() {
22226        let mut config = AutumnConfig::default();
22227        config.cors.allowed_origins = vec!["https://example.com".to_owned()];
22228        let router = test_router_with_config(vec![test_get_route("/test", "test")], &config);
22229
22230        let response = router
22231            .oneshot(
22232                Request::builder()
22233                    .method("OPTIONS")
22234                    .uri("/test")
22235                    .header("Origin", "https://example.com")
22236                    .header("Access-Control-Request-Method", "GET")
22237                    .body(Body::empty())
22238                    .unwrap(),
22239            )
22240            .await
22241            .unwrap();
22242
22243        assert_eq!(response.status(), StatusCode::OK);
22244        assert!(
22245            response
22246                .headers()
22247                .contains_key("access-control-allow-methods")
22248        );
22249    }
22250
22251    #[tokio::test]
22252    async fn build_router_with_static_skips_without_manifest() {
22253        // When dist/ exists but has no manifest.json, fall back to
22254        // the app router without the static layer.
22255        let tmp = tempfile::tempdir().expect("tempdir");
22256        let dist = tmp.path().join("dist");
22257        std::fs::create_dir_all(&dist).expect("mkdir");
22258        // No manifest.json — just an empty dist/
22259
22260        let config = AutumnConfig::default();
22261        let state = AppState {
22262            health_detailed: true,
22263            ..AppState::test_default()
22264        };
22265        let router = crate::router::build_router_with_static(
22266            vec![test_get_route("/test", "test")],
22267            &config,
22268            state,
22269            Some(dist.as_path()),
22270        );
22271
22272        let response = router
22273            .oneshot(Request::builder().uri("/test").body(Body::empty()).unwrap())
22274            .await
22275            .unwrap();
22276        assert_eq!(response.status(), StatusCode::OK);
22277    }
22278
22279    #[tokio::test]
22280    async fn build_router_with_static_none_dist() {
22281        // When dist_dir is None, return the app router directly.
22282        let config = AutumnConfig::default();
22283        let state = AppState {
22284            health_detailed: true,
22285            ..AppState::test_default()
22286        };
22287        let router = crate::router::build_router_with_static(
22288            vec![test_get_route("/test", "test")],
22289            &config,
22290            state,
22291            None,
22292        );
22293
22294        let response = router
22295            .oneshot(Request::builder().uri("/test").body(Body::empty()).unwrap())
22296            .await
22297            .unwrap();
22298        assert_eq!(response.status(), StatusCode::OK);
22299    }
22300
22301    // ── Startup transparency helper tests ─────────────────────────
22302
22303    #[test]
22304    fn format_route_lines_lists_user_routes() {
22305        let routes = vec![
22306            test_get_route("/", "index"),
22307            test_get_route("/users/{id}", "get_user"),
22308        ];
22309        let config = AutumnConfig::default();
22310        let output = format_route_lines(&routes, &[], &config);
22311        assert!(output.contains("-> index"));
22312        assert!(output.contains("/ GET"));
22313        assert!(output.contains("/users/{id}"));
22314        assert!(output.contains("-> get_user"));
22315    }
22316
22317    #[test]
22318    fn config_runtime_drift_format_route_lines_uses_actuator_prefix() {
22319        let mut config = AutumnConfig::default();
22320        config.actuator.prefix = "/ops".to_owned();
22321        let output = format_route_lines(&[], &[], &config);
22322        assert!(output.contains("-> health"));
22323        assert!(output.contains("/ops/*"));
22324    }
22325
22326    #[test]
22327    fn format_task_lines_none_when_empty() {
22328        assert!(format_task_lines(&[]).is_none());
22329    }
22330
22331    #[test]
22332    fn format_task_lines_fixed_delay() {
22333        let tasks = vec![crate::task::TaskInfo {
22334            name: "cleanup".into(),
22335            schedule: crate::task::Schedule::FixedDelay(std::time::Duration::from_secs(300)),
22336            coordination: crate::task::TaskCoordination::Fleet,
22337            handler: |_| Box::pin(async { Ok(()) }),
22338        }];
22339        let output = format_task_lines(&tasks).unwrap();
22340        assert!(output.contains("cleanup (every 300s)"));
22341    }
22342
22343    #[test]
22344    fn format_task_lines_cron() {
22345        let tasks = vec![crate::task::TaskInfo {
22346            name: "nightly".into(),
22347            schedule: crate::task::Schedule::Cron {
22348                expression: "0 0 * * *".into(),
22349                timezone: None,
22350            },
22351            coordination: crate::task::TaskCoordination::Fleet,
22352            handler: |_| Box::pin(async { Ok(()) }),
22353        }];
22354        let output = format_task_lines(&tasks).unwrap();
22355        assert!(output.contains("nightly (cron 0 0 * * *)"));
22356    }
22357
22358    #[test]
22359    fn format_middleware_list_default() {
22360        let config = AutumnConfig::default();
22361        let output = format_middleware_list(&config);
22362        assert!(output.contains("RequestId"));
22363        assert!(output.contains("SecurityHeaders"));
22364        assert!(output.contains("Session (in-memory)"));
22365        assert!(output.contains("Metrics"));
22366        // CORS and CSRF should not be present with defaults
22367        assert!(!output.contains("CORS"));
22368        assert!(!output.contains("CSRF"));
22369    }
22370
22371    #[test]
22372    fn format_middleware_list_with_cors_and_csrf() {
22373        let config = AutumnConfig {
22374            cors: crate::config::CorsConfig {
22375                allowed_origins: vec!["https://example.com".into()],
22376                ..crate::config::CorsConfig::default()
22377            },
22378            security: crate::security::config::SecurityConfig {
22379                csrf: crate::security::config::CsrfConfig {
22380                    enabled: true,
22381                    ..crate::security::config::CsrfConfig::default()
22382                },
22383                ..crate::security::config::SecurityConfig::default()
22384            },
22385            ..AutumnConfig::default()
22386        };
22387        let output = format_middleware_list(&config);
22388        assert!(output.contains("CORS"));
22389        assert!(output.contains("CSRF"));
22390    }
22391
22392    #[test]
22393    fn mask_database_url_with_password() {
22394        let masked = mask_database_url("postgres://user:secret@localhost:5432/mydb", 10);
22395        assert!(masked.contains("****"));
22396        assert!(!masked.contains("secret"));
22397        assert!(masked.contains("postgres://user:****@localhost:5432/mydb"));
22398        assert!(masked.contains("pool_size=10"));
22399    }
22400
22401    #[test]
22402    fn mask_database_url_without_password() {
22403        let masked = mask_database_url("postgres://localhost/mydb", 5);
22404        assert!(!masked.contains("****"));
22405        assert!(masked.contains("postgres://localhost/mydb"));
22406        assert!(masked.contains("pool_size=5"));
22407    }
22408
22409    #[test]
22410    fn mask_database_url_edge_cases() {
22411        // Special chars in password
22412        // The url crate parses `p@ssw:rd!` where `@` creates problems if unencoded,
22413        // but url crate seems to treat `user:p` as auth and `@ssw:rd!` as host if it's poorly formed,
22414        // let's stick to valid URL formats for testing.
22415
22416        // URL encoded characters
22417        let masked2 = mask_database_url("postgres://user:p%40ssw%3Ard%21@localhost:5432/mydb", 10);
22418        assert!(masked2.contains("****"));
22419        assert!(!masked2.contains("p%40ssw%3Ard%21"));
22420        assert!(masked2.contains("postgres://user:****@localhost:5432/mydb"));
22421
22422        // No user, just password
22423        let masked3 = mask_database_url("postgres://:secret@localhost:5432/mydb", 10);
22424        assert!(masked3.contains("****"));
22425        assert!(!masked3.contains("secret"));
22426        assert!(masked3.contains("postgres://:****@localhost:5432/mydb"));
22427    }
22428    // The point of routing this through the shared redactor: a SQLite operator
22429    // reads their own path back out of the boot summary, and a Postgres
22430    // operator keeps the connection policy they debug TLS with — while the
22431    // shapes that carry a secret still go.
22432    #[test]
22433    fn mask_database_url_keeps_the_diagnostic_parts_of_a_target() {
22434        let sqlite = mask_database_url("sqlite:///var/lib/app.db", 1);
22435        assert!(sqlite.contains("sqlite:///var/lib/app.db"), "{sqlite}");
22436
22437        let read_only = mask_database_url("sqlite://file:app.db?mode=ro", 1);
22438        assert!(read_only.contains("mode=ro"), "{read_only}");
22439
22440        let pg = mask_database_url("postgres://app@db/app?sslmode=verify-full", 10);
22441        assert!(pg.contains("sslmode=verify-full"), "{pg}");
22442
22443        let secret = mask_database_url("postgres://app@db/app?sslpassword=hunter2", 10);
22444        assert!(!secret.contains("hunter2"), "{secret}");
22445    }
22446
22447    #[test]
22448    fn mask_database_url_invalid_url_fallback() {
22449        let masked = mask_database_url("this is completely invalid as a URL with supersecret", 10);
22450        assert!(masked.contains("****"));
22451        assert!(!masked.contains("supersecret"));
22452        assert!(masked.contains("pool_size=10"));
22453    }
22454
22455    #[test]
22456    fn format_config_summary_defaults() {
22457        let config = AutumnConfig::default();
22458        let output = format_config_summary(&config);
22459        assert!(output.contains("profile:    none"));
22460        assert!(output.contains("server:     127.0.0.1:3000"));
22461        assert!(output.contains("database:   not configured"));
22462        assert!(output.contains("log_level:"));
22463        assert!(output.contains("telemetry:  disabled"));
22464        assert!(output.contains("health:     /health"));
22465    }
22466
22467    #[test]
22468    fn format_config_summary_with_db() {
22469        let config = AutumnConfig {
22470            database: crate::config::DatabaseConfig {
22471                url: Some("postgres://user:pass@host/db".into()),
22472                pool_size: 20,
22473                ..crate::config::DatabaseConfig::default()
22474            },
22475            ..AutumnConfig::default()
22476        };
22477        let output = format_config_summary(&config);
22478        assert!(output.contains("user:****@host/db"));
22479        assert!(output.contains("pool_size=20"));
22480        assert!(!output.contains("pass"));
22481    }
22482
22483    #[test]
22484    fn format_config_summary_with_profile() {
22485        let config = AutumnConfig {
22486            profile: Some("prod".into()),
22487            ..AutumnConfig::default()
22488        };
22489        let output = format_config_summary(&config);
22490        assert!(output.contains("profile:    prod"));
22491    }
22492
22493    #[test]
22494    fn format_config_summary_with_telemetry() {
22495        let config = AutumnConfig {
22496            telemetry: crate::config::TelemetryConfig {
22497                enabled: true,
22498                service_name: "orders-api".into(),
22499                otlp_endpoint: Some("http://otel-collector:4317".into()),
22500                ..crate::config::TelemetryConfig::default()
22501            },
22502            ..AutumnConfig::default()
22503        };
22504        let output = format_config_summary(&config);
22505        assert!(output.contains("telemetry:  Grpc -> http://otel-collector:4317"));
22506    }
22507
22508    #[test]
22509    fn log_startup_transparency_runs_without_panic() {
22510        // Exercises the tracing::info! calls inside log_startup_transparency.
22511        // No subscriber installed, so output is discarded -- we just verify
22512        // the function doesn't panic.
22513        let routes = vec![test_get_route("/", "index")];
22514        let tasks = vec![crate::task::TaskInfo {
22515            name: "cleanup".into(),
22516            schedule: crate::task::Schedule::FixedDelay(std::time::Duration::from_secs(60)),
22517            coordination: crate::task::TaskCoordination::Fleet,
22518            handler: |_| Box::pin(async { Ok(()) }),
22519        }];
22520        let config = AutumnConfig::default();
22521        log_startup_transparency(&routes, &tasks, &[], &config);
22522    }
22523
22524    #[test]
22525    fn log_startup_transparency_no_tasks() {
22526        let routes = vec![test_get_route("/health", "check")];
22527        let config = AutumnConfig::default();
22528        log_startup_transparency(&routes, &[], &[], &config);
22529    }
22530
22531    #[cfg(feature = "ws")]
22532    #[tokio::test]
22533    async fn start_task_scheduler_broadcasts_events() {
22534        let state = AppState {
22535            health_detailed: true,
22536            ..AppState::test_default()
22537        };
22538
22539        let mut rx = state.channels().subscribe("sys:tasks");
22540
22541        let task = crate::task::TaskInfo {
22542            name: "test_broadcaster".into(),
22543            // 1ms delay so it fires immediately
22544            schedule: crate::task::Schedule::FixedDelay(std::time::Duration::from_millis(1)),
22545            coordination: crate::task::TaskCoordination::Fleet,
22546            handler: |_| Box::pin(async { Ok(()) }),
22547        };
22548
22549        // Start scheduler in background so we don't block
22550        let state_clone = state.clone();
22551        tokio::spawn(async move {
22552            super::start_task_scheduler(
22553                vec![task],
22554                &state_clone,
22555                &tokio_util::sync::CancellationToken::new(),
22556            );
22557        });
22558
22559        // First message should be "started"
22560        let msg1 = tokio::time::timeout(std::time::Duration::from_secs(1), rx.recv())
22561            .await
22562            .expect("timeout waiting for start event")
22563            .expect("channel closed");
22564        let json1: serde_json::Value = serde_json::from_str(msg1.as_str()).unwrap();
22565        assert_eq!(json1["event"], "started");
22566        assert_eq!(json1["task"], "test_broadcaster");
22567
22568        // Second message should be "success"
22569        let msg2 = tokio::time::timeout(std::time::Duration::from_secs(1), rx.recv())
22570            .await
22571            .expect("timeout waiting for success event")
22572            .expect("channel closed");
22573        let json2: serde_json::Value = serde_json::from_str(msg2.as_str()).unwrap();
22574        assert_eq!(json2["event"], "success");
22575        assert_eq!(json2["task"], "test_broadcaster");
22576        assert!(json2.get("duration_ms").is_some());
22577    }
22578
22579    #[cfg(feature = "ws")]
22580    #[tokio::test]
22581    async fn start_task_scheduler_broadcasts_failure_events() {
22582        let state = AppState {
22583            health_detailed: true,
22584            ..AppState::test_default()
22585        };
22586
22587        let mut rx = state.channels().subscribe("sys:tasks");
22588
22589        let task = crate::task::TaskInfo {
22590            name: "test_failing_task".into(),
22591            schedule: crate::task::Schedule::FixedDelay(std::time::Duration::from_millis(1)),
22592            coordination: crate::task::TaskCoordination::Fleet,
22593            handler: |_| {
22594                Box::pin(async { Err(crate::AutumnError::bad_request_msg("forced error")) })
22595            },
22596        };
22597
22598        let state_clone = state.clone();
22599        tokio::spawn(async move {
22600            super::start_task_scheduler(
22601                vec![task],
22602                &state_clone,
22603                &tokio_util::sync::CancellationToken::new(),
22604            );
22605        });
22606
22607        // First message: started
22608        let _ = rx.recv().await.unwrap();
22609
22610        // Second message: failure
22611        let msg2 = tokio::time::timeout(std::time::Duration::from_secs(1), rx.recv())
22612            .await
22613            .expect("timeout waiting for failure event")
22614            .expect("channel closed");
22615        let json2: serde_json::Value = serde_json::from_str(msg2.as_str()).unwrap();
22616        assert_eq!(json2["event"], "failure");
22617        assert_eq!(json2["task"], "test_failing_task");
22618        assert_eq!(json2["error"], "forced error");
22619    }
22620
22621    #[tokio::test]
22622    async fn execute_task_result_ok_returns_duration() {
22623        let state = AppState::for_test();
22624        let handler: crate::task::TaskHandler = |_| Box::pin(async { Ok(()) });
22625        let start = state.monotonic();
22626        let result =
22627            super::execute_task_result(&state, handler, start, "test_task", "fixed_delay", false)
22628                .await;
22629        assert!(result.is_ok(), "expected Ok from successful handler");
22630        // duration_ms should be a reasonable value (not MAX)
22631        assert!(result.unwrap() < u64::MAX);
22632    }
22633
22634    #[tokio::test]
22635    async fn execute_task_result_err_returns_duration_and_message() {
22636        let state = AppState::for_test();
22637        let handler: crate::task::TaskHandler =
22638            |_| Box::pin(async { Err(crate::AutumnError::bad_request_msg("test error")) });
22639        let start = state.monotonic();
22640        let result =
22641            super::execute_task_result(&state, handler, start, "test_task", "fixed_delay", false)
22642                .await;
22643        assert!(result.is_err(), "expected Err from failing handler");
22644        let (duration_ms, msg) = result.unwrap_err();
22645        assert!(duration_ms < u64::MAX);
22646        assert!(msg.contains("test error"));
22647    }
22648
22649    fn instantly_panicking_scheduled_handler(
22650        _state: AppState,
22651    ) -> std::pin::Pin<Box<dyn std::future::Future<Output = crate::AutumnResult<()>> + Send>> {
22652        panic!("panic before scheduled future")
22653    }
22654
22655    #[tokio::test]
22656    async fn execute_task_result_reports_immediate_handler_panics() {
22657        let state = AppState::for_test();
22658        let start = state.monotonic();
22659        let result = super::execute_task_result(
22660            &state,
22661            instantly_panicking_scheduled_handler,
22662            start,
22663            "test_task",
22664            "fixed_delay",
22665            false,
22666        )
22667        .await;
22668
22669        let (duration_ms, msg) = result.expect_err("expected Err from panicking handler");
22670        assert!(duration_ms < u64::MAX);
22671        assert!(msg.contains("scheduled task handler panicked: panic before scheduled future"));
22672    }
22673
22674    /// A tick whose handler panics before it returns its future is metered
22675    /// too (issue #1720).
22676    #[tokio::test]
22677    async fn execute_task_result_meters_an_immediate_handler_panic() {
22678        let state = AppState::for_test();
22679        let accountant = crate::cost::CostAccountant::new(10);
22680        state.insert_extension(accountant.clone());
22681        let start = state.monotonic();
22682        let result = super::execute_task_result(
22683            &state,
22684            instantly_panicking_scheduled_handler,
22685            start,
22686            "test_task",
22687            "fixed_delay",
22688            false,
22689        )
22690        .await;
22691
22692        assert!(result.is_err());
22693        assert_eq!(accountant.snapshot().tasks.total.runs, 1);
22694    }
22695
22696    #[tokio::test]
22697    async fn execute_fixed_delay_task_does_not_timeout_in_process_runs() {
22698        let state = AppState::for_test();
22699        state.task_registry.register_scheduled(
22700            "slow_task",
22701            "every 1s",
22702            crate::task::TaskCoordination::Fleet,
22703            "in_process",
22704            "replica-a",
22705        );
22706        let handler: crate::task::TaskHandler = |_| {
22707            Box::pin(async {
22708                tokio::time::sleep(std::time::Duration::from_millis(30)).await;
22709                Ok(())
22710            })
22711        };
22712        let coordinator = std::sync::Arc::new(
22713            crate::scheduler::InProcessSchedulerCoordinator::new("replica-a"),
22714        );
22715
22716        super::execute_fixed_delay_task(
22717            "slow_task".to_owned(),
22718            state.clone(),
22719            handler,
22720            std::time::Duration::from_secs(1),
22721            crate::task::TaskCoordination::Fleet,
22722            coordinator,
22723            std::time::Duration::from_millis(10),
22724            tokio_util::sync::CancellationToken::new(),
22725        )
22726        .await;
22727
22728        let snapshot = state.task_registry.snapshot();
22729        let status = &snapshot["slow_task"];
22730        assert_eq!(status.status, "idle");
22731        assert_eq!(status.last_result.as_deref(), Some("ok"));
22732        assert_eq!(status.total_runs, 1);
22733        assert_eq!(status.total_failures, 0);
22734        assert!(status.last_error.is_none());
22735    }
22736
22737    static SKIPPED_LEASE_HANDLER_CALLS: AtomicUsize = AtomicUsize::new(0);
22738
22739    struct DenyingSchedulerCoordinator;
22740
22741    impl crate::scheduler::SchedulerCoordinator for DenyingSchedulerCoordinator {
22742        fn backend(&self) -> &'static str {
22743            "postgres"
22744        }
22745
22746        fn replica_id(&self) -> &'static str {
22747            "replica-a"
22748        }
22749
22750        fn try_acquire<'a>(
22751            &'a self,
22752            _task_name: &'a str,
22753            _tick_key: &'a str,
22754            _coordination: crate::task::TaskCoordination,
22755        ) -> crate::scheduler::SchedulerFuture<
22756            'a,
22757            crate::AutumnResult<Option<crate::scheduler::SchedulerLease>>,
22758        > {
22759            Box::pin(async { Ok(None) })
22760        }
22761    }
22762
22763    struct GrantingSchedulerCoordinator {
22764        backend: &'static str,
22765        tick_keys: std::sync::Arc<std::sync::Mutex<Vec<String>>>,
22766        release_count: Option<std::sync::Arc<AtomicUsize>>,
22767    }
22768
22769    impl crate::scheduler::SchedulerCoordinator for GrantingSchedulerCoordinator {
22770        fn backend(&self) -> &'static str {
22771            self.backend
22772        }
22773
22774        fn replica_id(&self) -> &'static str {
22775            "replica-a"
22776        }
22777
22778        fn try_acquire<'a>(
22779            &'a self,
22780            _task_name: &'a str,
22781            tick_key: &'a str,
22782            _coordination: crate::task::TaskCoordination,
22783        ) -> crate::scheduler::SchedulerFuture<
22784            'a,
22785            crate::AutumnResult<Option<crate::scheduler::SchedulerLease>>,
22786        > {
22787            Box::pin(async move {
22788                self.tick_keys.lock().unwrap().push(tick_key.to_owned());
22789                let lease = self.release_count.as_ref().map_or_else(
22790                    || crate::scheduler::SchedulerLease::local(self.backend, "replica-a"),
22791                    |release_count| {
22792                        crate::scheduler::SchedulerLease::tracked(
22793                            self.backend,
22794                            "replica-a",
22795                            std::sync::Arc::clone(release_count),
22796                        )
22797                    },
22798                );
22799                Ok(Some(lease))
22800            })
22801        }
22802    }
22803
22804    fn counted_scheduled_handler(
22805        _state: AppState,
22806    ) -> std::pin::Pin<Box<dyn std::future::Future<Output = crate::AutumnResult<()>> + Send>> {
22807        Box::pin(async {
22808            SKIPPED_LEASE_HANDLER_CALLS.fetch_add(1, Ordering::SeqCst);
22809            Ok(())
22810        })
22811    }
22812
22813    #[tokio::test]
22814    async fn execute_fixed_delay_task_skips_handler_when_lease_is_not_acquired() {
22815        SKIPPED_LEASE_HANDLER_CALLS.store(0, Ordering::SeqCst);
22816        let state = AppState::for_test();
22817        state.task_registry.register_scheduled(
22818            "claimed_elsewhere",
22819            "every 1s",
22820            crate::task::TaskCoordination::Fleet,
22821            "postgres",
22822            "replica-a",
22823        );
22824        let coordinator = std::sync::Arc::new(DenyingSchedulerCoordinator);
22825
22826        super::execute_fixed_delay_task(
22827            "claimed_elsewhere".to_owned(),
22828            state.clone(),
22829            counted_scheduled_handler,
22830            std::time::Duration::from_secs(1),
22831            crate::task::TaskCoordination::Fleet,
22832            coordinator,
22833            std::time::Duration::from_secs(1),
22834            tokio_util::sync::CancellationToken::new(),
22835        )
22836        .await;
22837
22838        let snapshot = state.task_registry.snapshot();
22839        let status = &snapshot["claimed_elsewhere"];
22840        assert_eq!(SKIPPED_LEASE_HANDLER_CALLS.load(Ordering::SeqCst), 0);
22841        assert_eq!(status.total_runs, 0);
22842        assert!(status.current_leader.is_none());
22843        assert!(status.last_tick.is_none());
22844    }
22845
22846    /// A `SQLite` lease and a Postgres tick row expire, so a deferred tick takes
22847    /// them only after the wait. An in-process lease does not expire, so the
22848    /// tick holds it while it waits (#1720).
22849    #[tokio::test(start_paused = true)]
22850    async fn deferred_tick_takes_an_expiring_lease_only_after_the_wait() {
22851        for (backend, acquired_while_high) in [("sqlite", 0), ("postgres", 0), ("in_process", 1)] {
22852            let name = format!("cost_lease_order_{backend}");
22853            crate::cost::mark_deferrable(crate::cost::WorkKind::Task, &name);
22854            let state = cron_test_state();
22855            let signal = crate::cost::CostSignal::new(Some(1.0));
22856            signal.set(5.0);
22857            state.insert_extension(signal.clone());
22858            let tick_keys = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
22859            let coordinator = std::sync::Arc::new(GrantingSchedulerCoordinator {
22860                backend,
22861                tick_keys: std::sync::Arc::clone(&tick_keys),
22862                release_count: None,
22863            });
22864            let handler: crate::task::TaskHandler = |_| Box::pin(async { Ok(()) });
22865            let run = tokio::spawn(super::execute_cron_task(
22866                name,
22867                state,
22868                handler,
22869                crate::task::TaskCoordination::Fleet,
22870                coordinator,
22871                std::time::Duration::from_secs(30),
22872                super::CronTick {
22873                    unix_secs: 1_700_000_000,
22874                    window: std::time::Duration::from_secs(60),
22875                },
22876                super::CostGate {
22877                    shutdown: tokio_util::sync::CancellationToken::new(),
22878                    waiting: None,
22879                    waited: std::sync::atomic::AtomicBool::new(false),
22880                },
22881            ));
22882
22883            tokio::time::sleep(std::time::Duration::from_secs(90)).await;
22884            assert_eq!(
22885                tick_keys.lock().unwrap().len(),
22886                acquired_while_high,
22887                "{backend}"
22888            );
22889
22890            signal.set(0.5);
22891            run.await.expect("the tick ends");
22892            assert_eq!(tick_keys.lock().unwrap().len(), 1, "{backend}");
22893        }
22894    }
22895
22896    /// A `SQLite` coordinator that raises the cost signal during its first
22897    /// lease attempt, and records the key and the fold flag of each attempt.
22898    struct RisingSignal {
22899        signal: crate::cost::CostSignal,
22900        acquired: std::sync::Arc<AtomicUsize>,
22901        /// The key and the fold flag at each lease attempt.
22902        seen: std::sync::Arc<std::sync::Mutex<Vec<(String, bool)>>>,
22903        flag: std::sync::Arc<std::sync::atomic::AtomicBool>,
22904    }
22905
22906    impl crate::scheduler::SchedulerCoordinator for RisingSignal {
22907        fn backend(&self) -> &'static str {
22908            "sqlite"
22909        }
22910
22911        fn replica_id(&self) -> &'static str {
22912            "replica-a"
22913        }
22914
22915        fn try_acquire<'a>(
22916            &'a self,
22917            _task_name: &'a str,
22918            tick_key: &'a str,
22919            _coordination: crate::task::TaskCoordination,
22920        ) -> crate::scheduler::SchedulerFuture<
22921            'a,
22922            crate::AutumnResult<Option<crate::scheduler::SchedulerLease>>,
22923        > {
22924            Box::pin(async move {
22925                self.seen
22926                    .lock()
22927                    .unwrap()
22928                    .push((tick_key.to_owned(), self.flag.load(Ordering::SeqCst)));
22929                if self.acquired.fetch_add(1, Ordering::SeqCst) == 0 {
22930                    self.signal.set(5.0);
22931                }
22932                Ok(Some(crate::scheduler::SchedulerLease::local(
22933                    "sqlite",
22934                    "replica-a",
22935                )))
22936            })
22937        }
22938    }
22939
22940    /// A `SQLite` lease attempt that the signal outruns: the signal rises while
22941    /// the tick takes its lease. The tick gives the lease back, waits, and runs
22942    /// only after the signal falls (#1720).
22943    #[tokio::test(start_paused = true)]
22944    async fn deferred_tick_gives_back_a_sqlite_lease_when_the_signal_rises() {
22945        static RAN: AtomicUsize = AtomicUsize::new(0);
22946        static SEEN_RETRY_TICK: std::sync::Mutex<Option<String>> = std::sync::Mutex::new(None);
22947        let name = "cost_sqlite_rising_signal".to_owned();
22948        crate::cost::mark_deferrable(crate::cost::WorkKind::Task, &name);
22949        let state = cron_test_state();
22950        let signal = crate::cost::CostSignal::new(Some(1.0));
22951        state.insert_extension(signal.clone());
22952        let acquired = std::sync::Arc::new(AtomicUsize::new(0));
22953        let seen = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
22954        // The cron loop reserves the fold flag before it spawns the tick.
22955        let flag = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(true));
22956        let coordinator = std::sync::Arc::new(RisingSignal {
22957            signal: signal.clone(),
22958            acquired: std::sync::Arc::clone(&acquired),
22959            seen: std::sync::Arc::clone(&seen),
22960            flag: std::sync::Arc::clone(&flag),
22961        });
22962        let handler: crate::task::TaskHandler = |_| {
22963            Box::pin(async {
22964                RAN.fetch_add(1, Ordering::SeqCst);
22965                *SEEN_RETRY_TICK.lock().unwrap() =
22966                    crate::scheduler::current_tick().map(|tick| tick.tick_key().to_owned());
22967                Ok(())
22968            })
22969        };
22970        let run = tokio::spawn(super::execute_cron_task(
22971            name,
22972            state,
22973            handler,
22974            crate::task::TaskCoordination::Fleet,
22975            coordinator,
22976            std::time::Duration::from_secs(30),
22977            super::CronTick {
22978                unix_secs: 1_700_000_000,
22979                window: std::time::Duration::from_secs(60),
22980            },
22981            super::CostGate {
22982                shutdown: tokio_util::sync::CancellationToken::new(),
22983                waiting: Some(std::sync::Arc::clone(&flag)),
22984                waited: std::sync::atomic::AtomicBool::new(false),
22985            },
22986        ));
22987
22988        tokio::time::sleep(std::time::Duration::from_secs(90)).await;
22989        assert_eq!(
22990            RAN.load(Ordering::SeqCst),
22991            0,
22992            "no run while the signal is high"
22993        );
22994
22995        signal.set(0.5);
22996        run.await.expect("the tick ends");
22997        assert_eq!(RAN.load(Ordering::SeqCst), 1);
22998        assert_eq!(
22999            acquired.load(Ordering::SeqCst),
23000            2,
23001            "the lease is taken again"
23002        );
23003        let seen = seen.lock().unwrap().clone();
23004        assert_eq!(
23005            seen[0].0, "cost_sqlite_rising_signal:1700000000",
23006            "{seen:?}"
23007        );
23008        assert_eq!(
23009            seen[1].0, seen[0].0,
23010            "every replica resumes against the one tick key: {seen:?}"
23011        );
23012        assert!(
23013            seen.iter().all(|(_, reserved)| *reserved),
23014            "later ticks stay folded while a lease attempt is in flight: {seen:?}"
23015        );
23016        assert!(
23017            !flag.load(Ordering::SeqCst),
23018            "the tick clears the reservation"
23019        );
23020        assert_eq!(
23021            SEEN_RETRY_TICK.lock().unwrap().as_deref(),
23022            Some("cost_sqlite_rising_signal:1700000000"),
23023            "the task sees the scheduled tick, not the retry key"
23024        );
23025    }
23026
23027    /// A cron tick that does not get its lease clears the cost-gate flag, or
23028    /// every later tick would fold into a wait that never happens (#1720).
23029    #[tokio::test]
23030    async fn execute_cron_task_clears_the_cost_flag_when_the_lease_is_taken() {
23031        let state = cron_test_state();
23032        let waiting = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(true));
23033        let handler: crate::task::TaskHandler = |_| Box::pin(async { Ok(()) });
23034
23035        super::execute_cron_task(
23036            "cron_lease_taken".to_owned(),
23037            state,
23038            handler,
23039            crate::task::TaskCoordination::Fleet,
23040            std::sync::Arc::new(DenyingSchedulerCoordinator),
23041            std::time::Duration::from_secs(30),
23042            super::CronTick {
23043                unix_secs: 1_700_000_000,
23044                window: std::time::Duration::from_secs(60),
23045            },
23046            super::CostGate {
23047                shutdown: tokio_util::sync::CancellationToken::new(),
23048                waiting: Some(std::sync::Arc::clone(&waiting)),
23049                waited: std::sync::atomic::AtomicBool::new(false),
23050            },
23051        )
23052        .await;
23053
23054        assert!(!waiting.load(Ordering::SeqCst), "later ticks run again");
23055    }
23056
23057    #[tokio::test]
23058    async fn execute_fixed_delay_task_records_distributed_lease_ttl_timeout() {
23059        let state = AppState::for_test();
23060        state.task_registry.register_scheduled(
23061            "slow_distributed_task",
23062            "every 1s",
23063            crate::task::TaskCoordination::Fleet,
23064            "postgres",
23065            "replica-a",
23066        );
23067        let handler: crate::task::TaskHandler = |_| {
23068            Box::pin(async {
23069                tokio::time::sleep(std::time::Duration::from_secs(5)).await;
23070                Ok(())
23071            })
23072        };
23073        let coordinator = std::sync::Arc::new(GrantingSchedulerCoordinator {
23074            backend: "postgres",
23075            tick_keys: std::sync::Arc::new(std::sync::Mutex::new(Vec::new())),
23076            release_count: None,
23077        });
23078
23079        super::execute_fixed_delay_task(
23080            "slow_distributed_task".to_owned(),
23081            state.clone(),
23082            handler,
23083            std::time::Duration::from_secs(1),
23084            crate::task::TaskCoordination::Fleet,
23085            coordinator,
23086            std::time::Duration::from_millis(10),
23087            tokio_util::sync::CancellationToken::new(),
23088        )
23089        .await;
23090
23091        let snapshot = state.task_registry.snapshot();
23092        let status = &snapshot["slow_distributed_task"];
23093        assert_eq!(status.status, "idle");
23094        assert_eq!(status.last_result.as_deref(), Some("failed"));
23095        assert_eq!(status.total_runs, 1);
23096        assert_eq!(status.total_failures, 1);
23097        assert!(
23098            status
23099                .last_error
23100                .as_deref()
23101                .is_some_and(|error| error.contains("lease TTL"))
23102        );
23103    }
23104
23105    static SEEN_TICK: std::sync::Mutex<Option<crate::scheduler::ScheduledTick>> =
23106        std::sync::Mutex::new(None);
23107
23108    /// Grants every tick with fencing token 99.
23109    struct FencingSchedulerCoordinator;
23110
23111    impl crate::scheduler::SchedulerCoordinator for FencingSchedulerCoordinator {
23112        fn backend(&self) -> &'static str {
23113            "postgres"
23114        }
23115
23116        fn replica_id(&self) -> &'static str {
23117            "replica-a"
23118        }
23119
23120        fn try_acquire<'a>(
23121            &'a self,
23122            _task_name: &'a str,
23123            _tick_key: &'a str,
23124            _coordination: crate::task::TaskCoordination,
23125        ) -> crate::scheduler::SchedulerFuture<
23126            'a,
23127            crate::AutumnResult<Option<crate::scheduler::SchedulerLease>>,
23128        > {
23129            Box::pin(async {
23130                Ok(Some(
23131                    crate::scheduler::SchedulerLease::local("postgres", "replica-a")
23132                        .with_fencing_token(99),
23133                ))
23134            })
23135        }
23136    }
23137
23138    /// Records the period that `try_acquire_for_period` gets.
23139    struct PeriodRecordingCoordinator {
23140        period: std::sync::Mutex<Option<std::time::Duration>>,
23141    }
23142
23143    impl crate::scheduler::SchedulerCoordinator for PeriodRecordingCoordinator {
23144        fn backend(&self) -> &'static str {
23145            "postgres"
23146        }
23147
23148        fn replica_id(&self) -> &'static str {
23149            "replica-a"
23150        }
23151
23152        fn try_acquire<'a>(
23153            &'a self,
23154            _task_name: &'a str,
23155            _tick_key: &'a str,
23156            _coordination: crate::task::TaskCoordination,
23157        ) -> crate::scheduler::SchedulerFuture<
23158            'a,
23159            crate::AutumnResult<Option<crate::scheduler::SchedulerLease>>,
23160        > {
23161            Box::pin(async { Ok(None) })
23162        }
23163
23164        fn try_acquire_for_period<'a>(
23165            &'a self,
23166            _task_name: &'a str,
23167            _tick_key: &'a str,
23168            _coordination: crate::task::TaskCoordination,
23169            period: std::time::Duration,
23170        ) -> crate::scheduler::SchedulerFuture<
23171            'a,
23172            crate::AutumnResult<Option<crate::scheduler::SchedulerLease>>,
23173        > {
23174            *self.period.lock().unwrap() = Some(period);
23175            Box::pin(async { Ok(None) })
23176        }
23177    }
23178
23179    // Issue #3052: a fixed-delay tick stays claimed for its whole delay.
23180    #[tokio::test]
23181    async fn fixed_delay_task_claims_its_tick_for_the_delay() {
23182        let coordinator = std::sync::Arc::new(PeriodRecordingCoordinator {
23183            period: std::sync::Mutex::new(None),
23184        });
23185        let handler: crate::task::TaskHandler = |_| Box::pin(async { Ok(()) });
23186
23187        super::execute_fixed_delay_task(
23188            "hourly_task".to_owned(),
23189            AppState::for_test(),
23190            handler,
23191            std::time::Duration::from_secs(3_600),
23192            crate::task::TaskCoordination::Fleet,
23193            std::sync::Arc::clone(&coordinator) as _,
23194            std::time::Duration::from_secs(300),
23195            tokio_util::sync::CancellationToken::new(),
23196        )
23197        .await;
23198
23199        assert_eq!(
23200            *coordinator.period.lock().unwrap(),
23201            Some(std::time::Duration::from_secs(3_600))
23202        );
23203    }
23204
23205    fn fleet_task_info(coordination: crate::task::TaskCoordination) -> crate::task::TaskInfo {
23206        crate::task::TaskInfo {
23207            name: "nightly".to_owned(),
23208            schedule: crate::task::Schedule::FixedDelay(std::time::Duration::from_secs(60)),
23209            coordination,
23210            handler: |_| Box::pin(async { Ok(()) }),
23211        }
23212    }
23213
23214    // Issue #3052: the boot warning reads its hints through the env seam.
23215    #[test]
23216    fn in_process_fleet_warning_names_the_hint_and_the_fix() {
23217        let config = AutumnConfig::default();
23218        let tasks = [fleet_task_info(crate::task::TaskCoordination::Fleet)];
23219        let env = crate::config::MockEnv::new().with("AUTUMN_REPLICAS", "3");
23220
23221        let warning = super::in_process_fleet_warning(&config, &tasks, &env)
23222            .expect("three replicas on in_process must warn");
23223        assert!(
23224            warning.contains("AUTUMN_REPLICAS is more than 1"),
23225            "{warning}"
23226        );
23227        assert!(warning.contains("scheduler.backend = \""), "{warning}");
23228
23229        let k8s = crate::config::MockEnv::new().with("KUBERNETES_SERVICE_HOST", "10.0.0.1");
23230        assert!(super::in_process_fleet_warning(&config, &tasks, &k8s).is_some());
23231    }
23232
23233    #[test]
23234    fn in_process_fleet_warning_is_silent_without_a_hint_or_a_fleet_task() {
23235        let config = AutumnConfig::default();
23236        let fleet = [fleet_task_info(crate::task::TaskCoordination::Fleet)];
23237        let local = [fleet_task_info(crate::task::TaskCoordination::PerReplica)];
23238        let crowd = crate::config::MockEnv::new().with("AUTUMN_REPLICAS", "5");
23239
23240        assert!(
23241            super::in_process_fleet_warning(&config, &fleet, &crate::config::MockEnv::new())
23242                .is_none(),
23243            "no hint"
23244        );
23245        assert!(
23246            super::in_process_fleet_warning(&config, &local, &crowd).is_none(),
23247            "per-replica tasks run on every replica by design"
23248        );
23249
23250        let mut coordinated = AutumnConfig::default();
23251        coordinated.scheduler.backend = crate::config::SchedulerBackend::Postgres;
23252        assert!(super::in_process_fleet_warning(&coordinated, &fleet, &crowd).is_none());
23253    }
23254
23255    // Issue #3052: the handler reads its tick and fencing token.
23256    #[tokio::test]
23257    async fn scheduled_handler_sees_its_tick_and_fencing_token() {
23258        let state = cron_test_state();
23259        state.task_registry.register_scheduled(
23260            "cron_fence_task",
23261            "cron 0 * * * * *",
23262            crate::task::TaskCoordination::Fleet,
23263            "postgres",
23264            "replica-a",
23265        );
23266        let handler: crate::task::TaskHandler = |_| {
23267            Box::pin(async {
23268                *SEEN_TICK.lock().unwrap() = crate::scheduler::current_tick();
23269                Ok(())
23270            })
23271        };
23272
23273        super::execute_cron_task(
23274            "cron_fence_task".to_owned(),
23275            state,
23276            handler,
23277            crate::task::TaskCoordination::Fleet,
23278            std::sync::Arc::new(FencingSchedulerCoordinator),
23279            std::time::Duration::from_secs(5),
23280            super::CronTick {
23281                unix_secs: 1_700_000_000,
23282                window: std::time::Duration::from_secs(60),
23283            },
23284            super::CostGate {
23285                shutdown: tokio_util::sync::CancellationToken::new(),
23286                waiting: None,
23287                waited: std::sync::atomic::AtomicBool::new(false),
23288            },
23289        )
23290        .await;
23291
23292        let seen = SEEN_TICK
23293            .lock()
23294            .unwrap()
23295            .take()
23296            .expect("handler saw a tick");
23297        assert_eq!(seen.tick_key(), "cron_fence_task:1700000000");
23298        assert_eq!(seen.fencing_token(), Some(99));
23299    }
23300
23301    #[tokio::test]
23302    async fn execute_cron_task_uses_scheduled_occurrence_for_tick_key() {
23303        let state = AppState::for_test();
23304        state.task_registry.register_scheduled(
23305            "cron_review_task",
23306            "cron */10 * * * * *",
23307            crate::task::TaskCoordination::Fleet,
23308            "postgres",
23309            "replica-a",
23310        );
23311        let tick_keys = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
23312        let coordinator = std::sync::Arc::new(GrantingSchedulerCoordinator {
23313            backend: "postgres",
23314            tick_keys: std::sync::Arc::clone(&tick_keys),
23315            release_count: None,
23316        });
23317        let handler: crate::task::TaskHandler = |_| Box::pin(async { Ok(()) });
23318        let scheduled_unix_secs = 1_700_000_000;
23319
23320        super::execute_cron_task(
23321            "cron_review_task".to_owned(),
23322            state.clone(),
23323            handler,
23324            crate::task::TaskCoordination::Fleet,
23325            coordinator,
23326            std::time::Duration::from_secs(30),
23327            super::CronTick {
23328                unix_secs: scheduled_unix_secs,
23329                window: std::time::Duration::from_secs(10),
23330            },
23331            super::CostGate {
23332                shutdown: tokio_util::sync::CancellationToken::new(),
23333                waiting: None,
23334                waited: std::sync::atomic::AtomicBool::new(false),
23335            },
23336        )
23337        .await;
23338
23339        assert_eq!(
23340            tick_keys.lock().unwrap().as_slice(),
23341            ["cron_review_task:1700000000"]
23342        );
23343    }
23344
23345    /// A test state whose clock is one second into the cron occurrence at
23346    /// `1_700_000_000`, so the window check lets the occurrence run.
23347    fn cron_test_state() -> AppState {
23348        AppState::for_test().with_clock(std::sync::Arc::new(crate::time::FixedClock::at(
23349            chrono::DateTime::from_timestamp(1_700_000_001, 0).expect("a valid timestamp"),
23350        )))
23351    }
23352
23353    /// #3071: a cost wait, or a slow claim, can outlast the tick row.
23354    /// Another replica can have run the occurrence, and the prune can have
23355    /// deleted its row. An occurrence past its window does not run.
23356    #[tokio::test]
23357    async fn execute_cron_task_skips_an_occurrence_past_its_window() {
23358        static RUNS: AtomicUsize = AtomicUsize::new(0);
23359
23360        async fn claims(scheduled_unix_secs: u64, waited: bool) -> Vec<String> {
23361            let state = AppState::for_test();
23362            state.task_registry.register_scheduled(
23363                "late_cron_task",
23364                "cron */10 * * * * *",
23365                crate::task::TaskCoordination::Fleet,
23366                "postgres",
23367                "replica-a",
23368            );
23369            let tick_keys = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
23370            let coordinator = std::sync::Arc::new(GrantingSchedulerCoordinator {
23371                backend: "postgres",
23372                tick_keys: std::sync::Arc::clone(&tick_keys),
23373                release_count: None,
23374            });
23375            let handler: crate::task::TaskHandler = |_| {
23376                RUNS.fetch_add(1, Ordering::SeqCst);
23377                Box::pin(async { Ok(()) })
23378            };
23379            super::execute_cron_task(
23380                "late_cron_task".to_owned(),
23381                state.clone(),
23382                handler,
23383                crate::task::TaskCoordination::Fleet,
23384                coordinator,
23385                std::time::Duration::from_secs(30),
23386                super::CronTick {
23387                    unix_secs: scheduled_unix_secs,
23388                    window: std::time::Duration::from_secs(10),
23389                },
23390                super::CostGate {
23391                    shutdown: tokio_util::sync::CancellationToken::new(),
23392                    waiting: None,
23393                    waited: std::sync::atomic::AtomicBool::new(waited),
23394                },
23395            )
23396            .await;
23397            tick_keys.lock().unwrap().clone()
23398        }
23399
23400        let now = crate::time::clock_unix_secs(AppState::for_test().clock());
23401        // After a cost wait: not claimed.
23402        assert!(claims(1_700_000_000, true).await.is_empty());
23403        // A slow claim: claimed, but not run.
23404        assert_eq!(claims(1_700_000_000, false).await.len(), 1);
23405        assert_eq!(
23406            RUNS.load(Ordering::SeqCst),
23407            0,
23408            "a late occurrence does not run"
23409        );
23410        // In the window: claimed and run.
23411        assert_eq!(claims(now, true).await, [format!("late_cron_task:{now}")]);
23412        assert_eq!(RUNS.load(Ordering::SeqCst), 1);
23413    }
23414
23415    #[tokio::test]
23416    async fn execute_fixed_delay_task_releases_lease_when_handler_panics() {
23417        let state = AppState::for_test();
23418        state.task_registry.register_scheduled(
23419            "panic_task",
23420            "every 1s",
23421            crate::task::TaskCoordination::Fleet,
23422            "postgres",
23423            "replica-a",
23424        );
23425        let release_count = std::sync::Arc::new(AtomicUsize::new(0));
23426        let coordinator = std::sync::Arc::new(GrantingSchedulerCoordinator {
23427            backend: "postgres",
23428            tick_keys: std::sync::Arc::new(std::sync::Mutex::new(Vec::new())),
23429            release_count: Some(std::sync::Arc::clone(&release_count)),
23430        });
23431        let handler: crate::task::TaskHandler = |_| {
23432            Box::pin(async {
23433                panic!("forced scheduled panic");
23434                #[allow(unreachable_code)]
23435                Ok(())
23436            })
23437        };
23438
23439        super::execute_fixed_delay_task(
23440            "panic_task".to_owned(),
23441            state.clone(),
23442            handler,
23443            std::time::Duration::from_secs(1),
23444            crate::task::TaskCoordination::Fleet,
23445            coordinator,
23446            std::time::Duration::from_secs(30),
23447            tokio_util::sync::CancellationToken::new(),
23448        )
23449        .await;
23450
23451        let snapshot = state.task_registry.snapshot();
23452        let status = &snapshot["panic_task"];
23453        assert_eq!(release_count.load(Ordering::SeqCst), 1);
23454        assert_eq!(status.status, "idle");
23455        assert_eq!(status.last_result.as_deref(), Some("failed"));
23456        assert_eq!(status.total_runs, 1);
23457        assert_eq!(status.total_failures, 1);
23458        assert!(
23459            status
23460                .last_error
23461                .as_deref()
23462                .is_some_and(|error| error.contains("scheduled task handler panicked"))
23463        );
23464    }
23465
23466    // croner 4 made shorthand steps (`5/5`) opt-in; the scheduler keeps
23467    // accepting what croner 3 accepted, with the same meaning.
23468    #[test]
23469    fn cron_expressions_keep_accepting_shorthand_steps() {
23470        use chrono::TimeZone as _;
23471
23472        let shorthand = super::parse_cron_expression("5/5 * * * *").expect("shorthand step");
23473        let explicit = super::parse_cron_expression("5-59/5 * * * *").expect("explicit range");
23474        let now = chrono_tz::UTC
23475            .with_ymd_and_hms(2026, 5, 5, 12, 6, 0)
23476            .unwrap();
23477        let next = shorthand
23478            .find_next_occurrence(&now, false)
23479            .expect("next occurrence");
23480        assert_eq!(
23481            next,
23482            explicit
23483                .find_next_occurrence(&now, false)
23484                .expect("next occurrence")
23485        );
23486        assert_eq!(
23487            next,
23488            chrono_tz::UTC
23489                .with_ymd_and_hms(2026, 5, 5, 12, 10, 0)
23490                .unwrap()
23491        );
23492        assert!(super::parse_cron_expression("not a cron").is_err());
23493    }
23494
23495    #[test]
23496    fn next_cron_occurrence_skips_overdue_slots() {
23497        use chrono::TimeZone as _;
23498
23499        let cron = "0 * * * * *"
23500            .parse::<croner::Cron>()
23501            .expect("cron expression should parse");
23502        let stale_cursor = chrono_tz::UTC
23503            .with_ymd_and_hms(2026, 5, 5, 12, 0, 0)
23504            .unwrap();
23505        let now = chrono_tz::UTC
23506            .with_ymd_and_hms(2026, 5, 5, 12, 30, 5)
23507            .unwrap();
23508        let next = super::next_cron_occurrence_after(&cron, &stale_cursor, &now)
23509            .expect("next cron occurrence should resolve");
23510
23511        assert_eq!(
23512            next,
23513            chrono_tz::UTC
23514                .with_ymd_and_hms(2026, 5, 5, 12, 31, 0)
23515                .unwrap()
23516        );
23517    }
23518
23519    // Issue #3052: a late cron occurrence still runs until the next one, so
23520    // the claim must hold for that whole window.
23521    #[test]
23522    fn cron_occurrence_window_spans_to_the_next_occurrence() {
23523        use chrono::TimeZone as _;
23524
23525        let daily = "0 0 3 * * *".parse::<croner::Cron>().expect("parse");
23526        let at = chrono_tz::UTC
23527            .with_ymd_and_hms(2026, 5, 5, 3, 0, 0)
23528            .unwrap();
23529        assert_eq!(
23530            super::cron_occurrence_window(&daily, &at),
23531            std::time::Duration::from_secs(24 * 60 * 60)
23532        );
23533        let every_ten = "*/10 * * * * *".parse::<croner::Cron>().expect("parse");
23534        let at = chrono_tz::UTC
23535            .with_ymd_and_hms(2026, 5, 5, 3, 0, 10)
23536            .unwrap();
23537        assert_eq!(
23538            super::cron_occurrence_window(&every_ten, &at),
23539            std::time::Duration::from_secs(10)
23540        );
23541    }
23542
23543    #[tokio::test]
23544    async fn cron_task_claims_its_tick_for_the_occurrence_window() {
23545        let coordinator = std::sync::Arc::new(PeriodRecordingCoordinator {
23546            period: std::sync::Mutex::new(None),
23547        });
23548        let handler: crate::task::TaskHandler = |_| Box::pin(async { Ok(()) });
23549
23550        super::execute_cron_task(
23551            "daily_task".to_owned(),
23552            cron_test_state(),
23553            handler,
23554            crate::task::TaskCoordination::Fleet,
23555            std::sync::Arc::clone(&coordinator) as _,
23556            std::time::Duration::from_secs(300),
23557            super::CronTick {
23558                unix_secs: 1_700_000_000,
23559                window: std::time::Duration::from_secs(86_400),
23560            },
23561            super::CostGate {
23562                shutdown: tokio_util::sync::CancellationToken::new(),
23563                waiting: None,
23564                waited: std::sync::atomic::AtomicBool::new(false),
23565            },
23566        )
23567        .await;
23568
23569        assert_eq!(
23570            *coordinator.period.lock().unwrap(),
23571            Some(std::time::Duration::from_secs(86_400))
23572        );
23573    }
23574
23575    #[test]
23576    fn cron_occurrence_is_overdue_after_later_slot_passed() {
23577        use chrono::TimeZone as _;
23578
23579        let cron = "0 * * * * *"
23580            .parse::<croner::Cron>()
23581            .expect("cron expression should parse");
23582        let scheduled_at = chrono_tz::UTC
23583            .with_ymd_and_hms(2026, 5, 5, 12, 1, 0)
23584            .unwrap();
23585        let slightly_late = chrono_tz::UTC
23586            .with_ymd_and_hms(2026, 5, 5, 12, 1, 5)
23587            .unwrap();
23588        let after_later_slot = chrono_tz::UTC
23589            .with_ymd_and_hms(2026, 5, 5, 12, 30, 5)
23590            .unwrap();
23591
23592        assert!(
23593            !super::cron_occurrence_is_overdue(&cron, &scheduled_at, &slightly_late)
23594                .expect("overdue check should resolve")
23595        );
23596        assert!(
23597            super::cron_occurrence_is_overdue(&cron, &scheduled_at, &after_later_slot)
23598                .expect("overdue check should resolve")
23599        );
23600    }
23601
23602    #[cfg(feature = "storage")]
23603    mod storage_preflight {
23604        use super::super::{StorageBootstrap, preflight_storage};
23605        use crate::AppState;
23606        use crate::config::AutumnConfig;
23607        use crate::storage::{BlobStoreState, StorageBackend, StorageConfig, StorageLocalConfig};
23608
23609        fn config_with_storage(storage: StorageConfig) -> AutumnConfig {
23610            AutumnConfig {
23611                profile: Some("dev".into()),
23612                storage,
23613                ..AutumnConfig::default()
23614            }
23615        }
23616
23617        #[test]
23618        fn preflight_returns_none_when_disabled() {
23619            let cfg = config_with_storage(StorageConfig {
23620                backend: StorageBackend::Disabled,
23621                ..StorageConfig::default()
23622            });
23623            assert!(preflight_storage(&cfg).is_none());
23624        }
23625
23626        #[test]
23627        fn preflight_provisions_local_backend_against_tempdir() {
23628            let dir = tempfile::tempdir().unwrap();
23629            let cfg = config_with_storage(StorageConfig {
23630                backend: StorageBackend::Local,
23631                local: StorageLocalConfig {
23632                    root: dir.path().to_path_buf(),
23633                    ..StorageLocalConfig::default()
23634                },
23635                ..StorageConfig::default()
23636            });
23637            let bootstrap = preflight_storage(&cfg).expect("local backend should provision");
23638            assert_eq!(bootstrap.store.provider_id(), "default");
23639            assert!(bootstrap.serving.is_some(), "local backend mounts a route");
23640        }
23641
23642        #[tokio::test]
23643        async fn install_registers_blob_store_on_state() {
23644            let dir = tempfile::tempdir().unwrap();
23645            let cfg = config_with_storage(StorageConfig {
23646                backend: StorageBackend::Local,
23647                local: StorageLocalConfig {
23648                    root: dir.path().to_path_buf(),
23649                    ..StorageLocalConfig::default()
23650                },
23651                ..StorageConfig::default()
23652            });
23653            let bootstrap: StorageBootstrap = preflight_storage(&cfg).unwrap();
23654
23655            let state = AppState::for_test();
23656            assert!(state.extension::<BlobStoreState>().is_none());
23657            let serving = bootstrap.install(&state);
23658            assert!(serving.is_some());
23659            assert!(state.extension::<BlobStoreState>().is_some());
23660        }
23661
23662        #[test]
23663        fn with_blob_store_stores_custom_store() {
23664            use crate::storage::{
23665                Blob, BlobFuture, BlobMeta, BlobStore, BlobStoreError, ByteStream,
23666            };
23667            use bytes::Bytes;
23668            use std::time::Duration;
23669
23670            struct FakeStore;
23671            impl BlobStore for FakeStore {
23672                fn provider_id(&self) -> &'static str {
23673                    "fake"
23674                }
23675                fn put<'a>(&'a self, _k: &'a str, _ct: &'a str, _b: Bytes) -> BlobFuture<'a, Blob> {
23676                    Box::pin(async { Err(BlobStoreError::Unsupported("fake".into())) })
23677                }
23678                fn put_stream<'a>(
23679                    &'a self,
23680                    _k: &'a str,
23681                    _ct: &'a str,
23682                    _d: ByteStream<'a>,
23683                ) -> BlobFuture<'a, Blob> {
23684                    Box::pin(async { Err(BlobStoreError::Unsupported("fake".into())) })
23685                }
23686                fn get<'a>(&'a self, _k: &'a str) -> BlobFuture<'a, Bytes> {
23687                    Box::pin(async { Err(BlobStoreError::Unsupported("fake".into())) })
23688                }
23689                fn delete<'a>(&'a self, _k: &'a str) -> BlobFuture<'a, ()> {
23690                    Box::pin(async { Err(BlobStoreError::Unsupported("fake".into())) })
23691                }
23692                fn head<'a>(&'a self, _k: &'a str) -> BlobFuture<'a, Option<BlobMeta>> {
23693                    Box::pin(async { Err(BlobStoreError::Unsupported("fake".into())) })
23694                }
23695                fn presigned_url<'a>(
23696                    &'a self,
23697                    _k: &'a str,
23698                    _e: Duration,
23699                ) -> BlobFuture<'a, String> {
23700                    Box::pin(async { Err(BlobStoreError::Unsupported("fake".into())) })
23701                }
23702            }
23703
23704            let builder = crate::app().with_blob_store(FakeStore);
23705            assert!(builder.blob_store.is_some());
23706        }
23707
23708        #[tokio::test]
23709        async fn with_blob_store_is_installed_on_state() {
23710            use crate::storage::{
23711                Blob, BlobFuture, BlobMeta, BlobStore, BlobStoreError, ByteStream,
23712            };
23713            use bytes::Bytes;
23714            use std::time::Duration;
23715
23716            struct FakeStore;
23717            impl BlobStore for FakeStore {
23718                fn provider_id(&self) -> &'static str {
23719                    "fake-installed"
23720                }
23721                fn put<'a>(&'a self, _k: &'a str, _ct: &'a str, _b: Bytes) -> BlobFuture<'a, Blob> {
23722                    Box::pin(async { Err(BlobStoreError::Unsupported("fake".into())) })
23723                }
23724                fn put_stream<'a>(
23725                    &'a self,
23726                    _k: &'a str,
23727                    _ct: &'a str,
23728                    _d: ByteStream<'a>,
23729                ) -> BlobFuture<'a, Blob> {
23730                    Box::pin(async { Err(BlobStoreError::Unsupported("fake".into())) })
23731                }
23732                fn get<'a>(&'a self, _k: &'a str) -> BlobFuture<'a, Bytes> {
23733                    Box::pin(async { Err(BlobStoreError::Unsupported("fake".into())) })
23734                }
23735                fn delete<'a>(&'a self, _k: &'a str) -> BlobFuture<'a, ()> {
23736                    Box::pin(async { Err(BlobStoreError::Unsupported("fake".into())) })
23737                }
23738                fn head<'a>(&'a self, _k: &'a str) -> BlobFuture<'a, Option<BlobMeta>> {
23739                    Box::pin(async { Err(BlobStoreError::Unsupported("fake".into())) })
23740                }
23741                fn presigned_url<'a>(
23742                    &'a self,
23743                    _k: &'a str,
23744                    _e: Duration,
23745                ) -> BlobFuture<'a, String> {
23746                    Box::pin(async { Err(BlobStoreError::Unsupported("fake".into())) })
23747                }
23748            }
23749
23750            let builder = crate::app().with_blob_store(FakeStore);
23751            let bootstrap = builder.blob_store.map(|store| StorageBootstrap {
23752                store,
23753                serving: None,
23754            });
23755            let state = AppState::for_test();
23756            assert!(state.extension::<BlobStoreState>().is_none());
23757            if let Some(b) = bootstrap {
23758                b.install(&state);
23759            }
23760            let installed = state
23761                .extension::<BlobStoreState>()
23762                .expect("store should be installed");
23763            assert_eq!(installed.store().provider_id(), "fake-installed");
23764        }
23765    }
23766
23767    // ── Route source attribution ───────────────────────────────────────────
23768
23769    /// A minimal plugin that registers one route with a known name.
23770    struct TestPlugin {
23771        name: &'static str,
23772        route: Route,
23773    }
23774
23775    impl crate::plugin::Plugin for TestPlugin {
23776        fn name(&self) -> std::borrow::Cow<'static, str> {
23777            std::borrow::Cow::Borrowed(self.name)
23778        }
23779
23780        fn build(self, app: AppBuilder) -> AppBuilder {
23781            app.routes(vec![self.route])
23782        }
23783    }
23784
23785    #[test]
23786    fn routes_registered_before_plugin_are_user_sourced() {
23787        let user_route = test_get_route("/home", "home");
23788        let builder = app().routes(vec![user_route]);
23789        assert_eq!(builder.route_sources.len(), 1);
23790        assert_eq!(
23791            builder.route_sources[0],
23792            crate::route_listing::RouteSource::User
23793        );
23794    }
23795
23796    #[test]
23797    fn routes_registered_inside_plugin_are_plugin_sourced() {
23798        let plugin_route = test_get_route("/plugin-page", "plugin_page");
23799        let plugin = TestPlugin {
23800            name: "my-plugin",
23801            route: plugin_route,
23802        };
23803        let builder = app().plugin(plugin);
23804        assert_eq!(builder.route_sources.len(), 1);
23805        assert_eq!(
23806            builder.route_sources[0],
23807            crate::route_listing::RouteSource::Plugin("my-plugin".to_owned())
23808        );
23809    }
23810
23811    #[test]
23812    fn routes_registered_after_plugin_revert_to_user_sourced() {
23813        let plugin_route = test_get_route("/plugin-page", "plugin_page");
23814        let user_route = test_get_route("/home", "home");
23815        let plugin = TestPlugin {
23816            name: "my-plugin",
23817            route: plugin_route,
23818        };
23819        let builder = app().plugin(plugin).routes(vec![user_route]);
23820        assert_eq!(builder.route_sources.len(), 2);
23821        assert_eq!(
23822            builder.route_sources[0],
23823            crate::route_listing::RouteSource::Plugin("my-plugin".to_owned())
23824        );
23825        assert_eq!(
23826            builder.route_sources[1],
23827            crate::route_listing::RouteSource::User
23828        );
23829    }
23830
23831    /// A plugin that registers a route and then registers a nested plugin.
23832    struct OuterPlugin;
23833
23834    impl crate::plugin::Plugin for OuterPlugin {
23835        fn name(&self) -> std::borrow::Cow<'static, str> {
23836            "outer".into()
23837        }
23838
23839        fn build(self, app: AppBuilder) -> AppBuilder {
23840            let inner = TestPlugin {
23841                name: "inner",
23842                route: test_get_route("/inner", "inner"),
23843            };
23844            app.plugin(inner)
23845                .routes(vec![test_get_route("/outer-after", "outer_after")])
23846        }
23847    }
23848
23849    #[test]
23850    fn outer_plugin_source_restored_after_nested_plugin() {
23851        let builder = app().plugin(OuterPlugin);
23852        // Routes: [/inner from "inner", /outer-after from "outer"]
23853        assert_eq!(builder.route_sources.len(), 2);
23854        assert_eq!(
23855            builder.route_sources[0],
23856            crate::route_listing::RouteSource::Plugin("inner".to_owned()),
23857            "first route should be attributed to inner plugin"
23858        );
23859        assert_eq!(
23860            builder.route_sources[1],
23861            crate::route_listing::RouteSource::Plugin("outer".to_owned()),
23862            "second route should be re-attributed to outer plugin after nested build"
23863        );
23864    }
23865
23866    // ── shutdown hook timeout tests ───────────────────────────────────────────
23867
23868    #[tokio::test]
23869    async fn shutdown_hooks_with_timeout_runs_all_fast_hooks() {
23870        use std::sync::atomic::{AtomicUsize, Ordering};
23871        let counter = Arc::new(AtomicUsize::new(0));
23872        let c1 = Arc::clone(&counter);
23873        let c2 = Arc::clone(&counter);
23874
23875        let hooks: Vec<ShutdownHook> = vec![
23876            Box::new(move || {
23877                let c = Arc::clone(&c1);
23878                Box::pin(async move {
23879                    c.fetch_add(1, Ordering::SeqCst);
23880                })
23881            }),
23882            Box::new(move || {
23883                let c = Arc::clone(&c2);
23884                Box::pin(async move {
23885                    c.fetch_add(1, Ordering::SeqCst);
23886                })
23887            }),
23888        ];
23889
23890        run_shutdown_hooks_with_timeout(
23891            &hooks,
23892            std::time::Duration::from_secs(2),
23893            std::time::Duration::from_secs(10),
23894        )
23895        .await;
23896
23897        assert_eq!(counter.load(Ordering::SeqCst), 2, "both hooks must run");
23898    }
23899
23900    #[tokio::test]
23901    async fn shutdown_hooks_with_timeout_tolerates_slow_hook_overrun() {
23902        use std::sync::atomic::{AtomicBool, Ordering};
23903        let fast_ran = Arc::new(AtomicBool::new(false));
23904        let fr = Arc::clone(&fast_ran);
23905
23906        let hooks: Vec<ShutdownHook> = vec![
23907            // hook 0 (first registered → runs LAST in LIFO): fast
23908            Box::new(move || {
23909                let fr = Arc::clone(&fr);
23910                Box::pin(async move {
23911                    fr.store(true, Ordering::SeqCst);
23912                })
23913            }),
23914            // hook 1 (last registered → runs FIRST in LIFO): slow, exceeds per-hook budget
23915            Box::new(|| {
23916                Box::pin(async move {
23917                    tokio::time::sleep(std::time::Duration::from_millis(200)).await;
23918                })
23919            }),
23920        ];
23921
23922        // Per-hook budget = 50 ms (hook 0 will overrun).
23923        // Total budget = 1 s (ample for hook 1 after the overrun is cut short).
23924        run_shutdown_hooks_with_timeout(
23925            &hooks,
23926            std::time::Duration::from_millis(50),
23927            std::time::Duration::from_secs(1),
23928        )
23929        .await;
23930
23931        assert!(
23932            fast_ran.load(Ordering::SeqCst),
23933            "fast hook must still run even after slow hook overruns its per-hook budget"
23934        );
23935    }
23936
23937    // Verify that build_state registers a SharedReqwestClient so that
23938    // Client::from_state can reuse the shared connection pool on every request.
23939    #[cfg(feature = "http-client")]
23940    #[test]
23941    fn build_state_registers_shared_reqwest_client() {
23942        let config = AutumnConfig::default();
23943        let state = build_state(
23944            &config,
23945            #[cfg(feature = "db")]
23946            None,
23947            #[cfg(feature = "db")]
23948            None,
23949            #[cfg(feature = "ws")]
23950            None,
23951        );
23952        assert!(
23953            state
23954                .extension::<crate::http_client::SharedReqwestClient>()
23955                .is_some(),
23956            "build_state must register a SharedReqwestClient for connection-pool sharing"
23957        );
23958    }
23959
23960    // AC5 plumbing (#1526): `with_story_gallery` stores the gallery on the
23961    // builder, and `install_story_registry` — the single install step shared
23962    // by both the run and build/SSG state-construction paths — publishes it
23963    // as the StoryRegistry extension the `/_stories` handlers read.
23964    #[cfg(feature = "maud")]
23965    #[test]
23966    fn with_story_gallery_installs_story_registry_extension() {
23967        let builder = crate::app().with_story_gallery(crate::stories::StoryGallery::builtin());
23968        let gallery = builder
23969            .story_gallery
23970            .expect("with_story_gallery must store the gallery on the builder");
23971        let expected_count = gallery.stories().len();
23972        assert!(expected_count > 0, "builtin gallery must not be empty");
23973
23974        let config = AutumnConfig::default();
23975        let state = build_state(
23976            &config,
23977            #[cfg(feature = "db")]
23978            None,
23979            #[cfg(feature = "db")]
23980            None,
23981            #[cfg(feature = "ws")]
23982            None,
23983        );
23984        install_story_registry(&state, Some(gallery));
23985        let registry = state
23986            .extension::<crate::stories::StoryRegistry>()
23987            .expect("install_story_registry must publish the StoryRegistry extension");
23988        assert_eq!(
23989            registry.stories().len(),
23990            expected_count,
23991            "every registered story must reach the state extension"
23992        );
23993
23994        // Without a registered gallery no extension is installed: the
23995        // handlers fall back to the empty default and serve the empty state.
23996        let bare_state = build_state(
23997            &config,
23998            #[cfg(feature = "db")]
23999            None,
24000            #[cfg(feature = "db")]
24001            None,
24002            #[cfg(feature = "ws")]
24003            None,
24004        );
24005        install_story_registry(&bare_state, None);
24006        assert!(
24007            bare_state
24008                .extension::<crate::stories::StoryRegistry>()
24009                .is_none(),
24010            "no gallery registered must mean no StoryRegistry extension"
24011        );
24012    }
24013
24014    /// `autumn openapi export --check` must refuse what startup refuses. A
24015    /// plugin route on the configured `OpenAPI` or MCP path fails the router
24016    /// build, so the no-boot preflight must fail it too.
24017    #[cfg(feature = "openapi")]
24018    #[test]
24019    fn export_preflight_refuses_a_declared_route_on_a_context_mount() {
24020        let config = crate::config::AutumnConfig::default();
24021        let openapi =
24022            crate::openapi::OpenApiConfig::new("Demo", "1.0.0").openapi_json_path("/spec.json");
24023        let declared = vec![crate::route_listing::RouteInfo {
24024            method: "GET".to_owned(),
24025            path: "/spec.json".to_owned(),
24026            handler: "sandbox:evil-plugin".to_owned(),
24027            source: crate::route_listing::RouteSource::Plugin("evil-plugin".to_owned()),
24028            ..crate::route_listing::RouteInfo::default()
24029        }];
24030        let error = export_preflight(&ExportPreflight {
24031            routes: &[test_get_route("/hello", "hello")],
24032            scoped_groups: &[],
24033            api_versions: &[],
24034            openapi_config: &openapi,
24035            merge_routers: &[],
24036            nest_routers: &[],
24037            declared_routes: &declared,
24038            config: &config,
24039            mcp_mount_path: None,
24040        })
24041        .expect_err("the router build refuses this, so the export check must too");
24042        assert!(error.contains("/spec.json"), "{error}");
24043    }
24044}
24045
24046#[cfg(all(test, unix))]
24047mod unix_socket_tests {
24048    use super::prepare_unix_socket_path;
24049
24050    #[test]
24051    fn prepare_unix_socket_path_noop_when_absent() {
24052        let dir = tempfile::tempdir().expect("tempdir");
24053        let path = dir.path().join("missing.sock");
24054        prepare_unix_socket_path(&path).expect("absent path is fine");
24055        assert!(!path.exists());
24056    }
24057
24058    #[test]
24059    fn prepare_unix_socket_path_removes_stale_socket() {
24060        let dir = tempfile::tempdir().expect("tempdir");
24061        let path = dir.path().join("stale.sock");
24062        // Bind then drop a real socket to leave a stale socket file behind.
24063        let listener = std::os::unix::net::UnixListener::bind(&path).expect("bind socket");
24064        drop(listener);
24065        assert!(path.exists(), "socket file should exist before prepare");
24066        prepare_unix_socket_path(&path).expect("stale socket should be removed");
24067        assert!(!path.exists(), "stale socket should be unlinked");
24068    }
24069
24070    #[test]
24071    fn prepare_unix_socket_path_refuses_live_socket() {
24072        let dir = tempfile::tempdir().expect("tempdir");
24073        let path = dir.path().join("live.sock");
24074        // Keep the listener bound so a connect probe succeeds.
24075        let _listener = std::os::unix::net::UnixListener::bind(&path).expect("bind socket");
24076        let err = prepare_unix_socket_path(&path).expect_err("must refuse a live socket");
24077        assert_eq!(err.kind(), std::io::ErrorKind::AddrInUse);
24078        assert!(path.exists(), "live socket must not be removed");
24079    }
24080
24081    #[test]
24082    fn prepare_unix_socket_path_errors_on_regular_file() {
24083        let dir = tempfile::tempdir().expect("tempdir");
24084        let path = dir.path().join("not-a-socket");
24085        std::fs::write(&path, b"i am a regular file").expect("write file");
24086        let err = prepare_unix_socket_path(&path).expect_err("must refuse a non-socket file");
24087        assert_eq!(err.kind(), std::io::ErrorKind::AlreadyExists);
24088        assert!(path.exists(), "regular file must not be removed");
24089    }
24090
24091    /// Regression (#2652): the one-shot `autumn db retention` path must
24092    /// install the same `dyn CustomDomainPruner` the server installs — the
24093    /// TLS bind path that used to do it never runs in a one-shot, so the
24094    /// `custom_domains` retention dataset always reported "custom domains
24095    /// are not enabled".
24096    #[cfg(feature = "acme")]
24097    #[tokio::test]
24098    async fn retention_mode_installs_the_custom_domain_pruner() {
24099        fn acme_with_custom_domains(enabled: bool) -> crate::config::AcmeConfig {
24100            let toml = format!(
24101                "domains = [\"app.example.com\"]\ncontact_email = \"ops@example.com\"\n\
24102                 [custom_domains]\nenabled = {enabled}\n"
24103            );
24104            toml::from_str(&toml).expect("test ACME config parses")
24105        }
24106
24107        // Enabled custom domains: the pruner is installed. It carries no ACME
24108        // issuer — pruning only touches the registry and the certificate
24109        // store, so no order is placed and no CA is contacted.
24110        let state = crate::state::AppState::for_test();
24111        super::install_custom_domain_retention_pruner(
24112            Some(&acme_with_custom_domains(true)),
24113            None,
24114            &state,
24115            true,
24116        )
24117        .await;
24118        assert!(
24119            state
24120                .extension::<std::sync::Arc<dyn crate::custom_domain::CustomDomainPruner>>()
24121                .is_some(),
24122            "one-shot retention must install the custom-domain pruner"
24123        );
24124
24125        // Custom domains disabled: nothing is installed, and the dataset keeps
24126        // its honest "not enabled" report.
24127        let state = crate::state::AppState::for_test();
24128        super::install_custom_domain_retention_pruner(
24129            Some(&acme_with_custom_domains(false)),
24130            None,
24131            &state,
24132            true,
24133        )
24134        .await;
24135        assert!(
24136            state
24137                .extension::<std::sync::Arc<dyn crate::custom_domain::CustomDomainPruner>>()
24138                .is_none()
24139        );
24140
24141        // No ACME at all: nothing is installed.
24142        let state = crate::state::AppState::for_test();
24143        super::install_custom_domain_retention_pruner(None, None, &state, true).await;
24144        assert!(
24145            state
24146                .extension::<std::sync::Arc<dyn crate::custom_domain::CustomDomainPruner>>()
24147                .is_none()
24148        );
24149    }
24150}