1#![cfg_attr(not(test), deny(clippy::disallowed_methods))]
34
35use std::any::{Any, TypeId};
36use std::collections::{BTreeSet, HashMap, HashSet};
37use std::future::Future;
38use std::pin::Pin;
39use std::sync::Arc;
40
41use futures::FutureExt as _;
42use tracing::Instrument as _;
43
44use crate::config::{AutumnConfig, ConfigLoader};
45#[cfg(feature = "maud")]
46use crate::error_pages::{ErrorPageRenderer, SharedRenderer};
47use crate::middleware::exception_filter::ExceptionFilter;
48#[cfg(feature = "db")]
49use crate::migrate;
50use crate::route::Route;
51use crate::state::AppState;
52
53#[must_use]
76pub fn app() -> AppBuilder {
77 let builder = AppBuilder {
78 routes: Vec::new(),
79 api_versions: Vec::new(),
80 route_sources: Vec::new(),
81 current_plugin: None,
82 tasks: Vec::new(),
83 one_off_tasks: Vec::new(),
84 jobs: Vec::new(),
85 listeners: Vec::new(),
86 #[cfg(feature = "db")]
87 outbox_handlers: crate::outbox::OutboxHandlers::default(),
88 static_metas: Vec::new(),
89 exception_filters: Vec::new(),
90 scoped_groups: Vec::new(),
91 merge_routers: Vec::new(),
92 nest_routers: Vec::new(),
93 custom_layers: Vec::new(),
94 static_gate_layers: Vec::new(),
95 startup_hooks: Vec::new(),
96 state_initializers: Vec::new(),
97 shutdown_hooks: Vec::new(),
98 extensions: HashMap::new(),
99 registered_plugins: HashSet::new(),
100 plugin_contracts: Vec::new(),
101 plugin_config_roots: BTreeSet::new(),
102 #[cfg(feature = "maud")]
103 error_page_renderer: None,
104 #[cfg(feature = "db")]
105 migrations: Vec::new(),
106 config_loader_factory: None,
107 #[cfg(feature = "db")]
108 pool_provider_factory: None,
109 #[cfg(feature = "db")]
110 shard_provider_factory: None,
111 #[cfg(feature = "db")]
112 shard_router: None,
113 #[cfg(feature = "db")]
114 directory_shard_router: false,
115 telemetry_provider: None,
116 session_store: None,
117 #[cfg(feature = "ws")]
118 channels_backend: None,
119 #[cfg(feature = "storage")]
120 blob_store: None,
121 cache_backend: None,
122 #[cfg(feature = "reporting")]
123 error_reporters: Vec::new(),
124 alert_channels: Vec::new(),
125 #[cfg(feature = "openapi")]
126 openapi: None,
127 #[cfg(feature = "mcp")]
128 mcp: None,
129 #[cfg(feature = "webmcp")]
130 webmcp: None,
131 audit_logger: None,
132 #[cfg(feature = "i18n")]
133 i18n_bundle: None,
134 #[cfg(feature = "i18n")]
135 i18n_auto_load: false,
136 #[cfg(feature = "embed-assets")]
137 embedded_static: None,
138 #[cfg(all(feature = "embed-assets", feature = "i18n"))]
139 embedded_locales: None,
140 policy_registrations: Vec::new(),
141 #[cfg(feature = "mail")]
142 mail_delivery_queue_factory: None,
143 #[cfg(feature = "mail")]
144 suppression_store: None,
145 #[cfg(feature = "mail")]
146 mail_suppression_store: None,
147 #[cfg(feature = "mail")]
148 mount_unsubscribe_endpoint: false,
149 #[cfg(feature = "mail")]
150 mail_previews: Vec::new(),
151 #[cfg(feature = "maud")]
152 story_gallery: None,
153 declared_routes: Vec::new(),
154 plugin_asset_bundles: Vec::new(),
155 idempotency_enabled: false,
156 #[cfg(feature = "mail")]
157 mail_interceptor: None,
158 job_interceptor: None,
159 #[cfg(feature = "db")]
160 db_interceptor: None,
161 #[cfg(feature = "ws")]
162 channels_interceptor: None,
163 #[cfg(feature = "oauth2")]
164 http_interceptor: None,
165 seo_sources: Vec::new(),
166 agent_skills: Vec::new(),
167 commerce_docs: crate::aeo::RegisteredCommerceDocs::default(),
168 metrics_sources: Vec::new(),
169 health_indicators: Vec::new(),
170 #[cfg(feature = "inbound-mail")]
171 inbound_mail_router: None,
172 };
173 #[cfg(feature = "edge")]
183 let builder = builder.layer(StripEdgeFallthroughSentinelLayer);
184 builder
185}
186
187#[cfg(feature = "edge")]
204#[derive(Clone, Copy, Default)]
205pub(crate) struct StripEdgeFallthroughSentinelLayer;
206
207#[cfg(feature = "edge")]
208impl<S> tower::Layer<S> for StripEdgeFallthroughSentinelLayer {
209 type Service = StripEdgeFallthroughSentinelService<S>;
210
211 fn layer(&self, inner: S) -> Self::Service {
212 StripEdgeFallthroughSentinelService { inner }
213 }
214}
215
216#[cfg(feature = "edge")]
223fn is_framework_owned_layer(type_id: TypeId) -> bool {
224 type_id == TypeId::of::<StripEdgeFallthroughSentinelLayer>()
225}
226
227#[cfg(not(feature = "edge"))]
228const fn is_framework_owned_layer(_type_id: TypeId) -> bool {
229 false
230}
231
232#[cfg(feature = "edge")]
235#[derive(Clone, Debug)]
236pub(crate) struct StripEdgeFallthroughSentinelService<S> {
237 inner: S,
238}
239
240#[cfg(feature = "edge")]
241impl<S, ReqBody> tower::Service<axum::http::Request<ReqBody>>
242 for StripEdgeFallthroughSentinelService<S>
243where
244 S: tower::Service<axum::http::Request<ReqBody>, Response = axum::response::Response>
245 + Send
246 + 'static,
247 S::Future: Send + 'static,
248{
249 type Response = S::Response;
250 type Error = S::Error;
251 type Future = std::pin::Pin<
252 Box<dyn std::future::Future<Output = Result<Self::Response, Self::Error>> + Send>,
253 >;
254
255 fn poll_ready(
256 &mut self,
257 cx: &mut std::task::Context<'_>,
258 ) -> std::task::Poll<Result<(), Self::Error>> {
259 self.inner.poll_ready(cx)
260 }
261
262 fn call(&mut self, req: axum::http::Request<ReqBody>) -> Self::Future {
263 let response = self.inner.call(req);
264 Box::pin(async move {
265 let mut response = response.await?;
266 response
267 .headers_mut()
268 .remove(autumn_edge::FALLTHROUGH_SENTINEL);
269 Ok(response)
270 })
271 }
272}
273
274fn omitted_router_count<'a>(
291 merge_routers: usize,
292 nest_prefixes: impl IntoIterator<Item = &'a str>,
293 declared_routes: &[crate::route_listing::RouteInfo],
294) -> usize {
295 let uncovered_nests = nest_prefixes
296 .into_iter()
297 .filter(|prefix| !nest_prefix_is_covered(prefix, declared_routes))
298 .count();
299 merge_routers + uncovered_nests
300}
301
302fn nest_prefix_is_covered(
306 prefix: &str,
307 declared_routes: &[crate::route_listing::RouteInfo],
308) -> bool {
309 declared_routes
310 .iter()
311 .any(|route| path_is_under_prefix(&route.path, prefix))
312}
313
314fn path_is_under_prefix(path: &str, prefix: &str) -> bool {
319 let prefix = prefix.trim_end_matches('/');
320 if prefix.is_empty() {
321 return true;
322 }
323 path == prefix
324 || path
325 .strip_prefix(prefix)
326 .is_some_and(|rest| rest.starts_with('/'))
327}
328
329type StartupHookFuture = Pin<Box<dyn Future<Output = crate::AutumnResult<()>> + Send>>;
330type StartupHook = Box<dyn Fn(AppState) -> StartupHookFuture + Send + Sync>;
331type StateInitializer = Box<dyn FnOnce(&AppState) + Send>;
332type ShutdownHookFuture = Pin<Box<dyn Future<Output = ()> + Send>>;
333type ShutdownHook = Box<dyn Fn() -> ShutdownHookFuture + Send + Sync>;
334
335type ConfigLoaderFactory = Box<
343 dyn FnOnce() -> Pin<
344 Box<dyn Future<Output = Result<AutumnConfig, crate::config::ConfigError>> + Send>,
345 > + Send,
346>;
347#[cfg(feature = "db")]
348type PoolProviderFactory = Box<
349 dyn FnOnce(
350 crate::config::DatabaseConfig,
351 ) -> Pin<
352 Box<
353 dyn Future<
354 Output = Result<Option<crate::db::DatabaseTopology>, crate::db::PoolError>,
355 > + Send,
356 >,
357 > + Send,
358>;
359#[cfg(feature = "db")]
362type ShardProviderFactory = Box<
363 dyn FnOnce(
364 crate::config::DatabaseConfig,
365 ) -> Pin<
366 Box<
367 dyn Future<Output = Result<Vec<crate::db::DatabaseTopology>, crate::db::PoolError>>
368 + Send,
369 >,
370 > + Send,
371>;
372
373type PolicyRegistration = Box<dyn FnOnce(&crate::authorization::PolicyRegistry) + Send>;
376
377#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)]
379pub struct ApiVersion {
380 pub version: String,
382 pub deprecated_at: Option<chrono::DateTime<chrono::Utc>>,
384 pub sunset_at: Option<chrono::DateTime<chrono::Utc>>,
386}
387
388#[derive(Clone, Debug)]
390pub struct RegisteredApiVersions(pub Vec<ApiVersion>);
391
392#[allow(clippy::struct_excessive_bools)]
421pub struct AppBuilder {
422 pub(crate) routes: Vec<Route>,
423 pub api_versions: Vec<ApiVersion>,
425 route_sources: Vec<crate::route_listing::RouteSource>,
427 current_plugin: Option<String>,
430 pub(crate) tasks: Vec<crate::task::TaskInfo>,
431 one_off_tasks: Vec<crate::task::OneOffTaskInfo>,
432 pub(crate) jobs: Vec<crate::job::JobInfo>,
433 pub(crate) listeners: Vec<crate::events::ListenerInfo>,
436 #[cfg(feature = "db")]
438 pub(crate) outbox_handlers: crate::outbox::OutboxHandlers,
439 pub(crate) static_metas: Vec<crate::static_gen::StaticRouteMeta>,
440 pub(crate) exception_filters: Vec<Arc<dyn ExceptionFilter>>,
441 pub(crate) scoped_groups: Vec<ScopedGroup>,
442 pub(crate) merge_routers: Vec<axum::Router<AppState>>,
443 pub(crate) nest_routers: Vec<(String, axum::Router<AppState>)>,
444 pub(crate) custom_layers: Vec<CustomLayerRegistration>,
447 pub(crate) static_gate_layers: Vec<CustomLayerRegistration>,
452 pub(crate) startup_hooks: Vec<StartupHook>,
453 pub(crate) state_initializers: Vec<StateInitializer>,
454 pub(crate) shutdown_hooks: Vec<ShutdownHook>,
455 pub(crate) extensions: HashMap<TypeId, Box<dyn Any + Send>>,
456 pub(crate) registered_plugins: HashSet<String>,
458 pub(crate) plugin_contracts: Vec<crate::plugin_contract::PluginContract>,
464 pub(crate) plugin_config_roots: BTreeSet<String>,
469 #[cfg(feature = "maud")]
471 error_page_renderer: Option<SharedRenderer>,
472 #[cfg(feature = "db")]
476 migrations: Vec<(&'static str, migrate::EmbeddedMigrations)>,
477 config_loader_factory: Option<ConfigLoaderFactory>,
480 #[cfg(feature = "db")]
483 pool_provider_factory: Option<PoolProviderFactory>,
484 #[cfg(feature = "db")]
487 shard_provider_factory: Option<ShardProviderFactory>,
488 #[cfg(feature = "db")]
492 shard_router: Option<Arc<dyn crate::sharding::ShardRouter>>,
493 #[cfg(feature = "db")]
496 directory_shard_router: bool,
497 telemetry_provider: Option<Box<dyn crate::telemetry::TelemetryProvider>>,
500 session_store: Option<Arc<dyn crate::session::BoxedSessionStore>>,
504 #[cfg(feature = "ws")]
507 channels_backend: Option<Arc<dyn crate::channels::ChannelsBackend>>,
508 #[cfg(feature = "storage")]
512 blob_store: Option<crate::storage::SharedBlobStore>,
513 cache_backend: Option<Arc<dyn crate::cache::Cache>>,
517 #[cfg(feature = "reporting")]
523 pub(crate) error_reporters: Vec<Arc<dyn crate::reporting::ErrorReporter>>,
524 pub(crate) alert_channels: Vec<Arc<dyn crate::alerts::AlertChannel>>,
530 #[cfg(feature = "openapi")]
538 openapi: Option<crate::openapi::OpenApiConfig>,
539 #[cfg(feature = "mcp")]
544 mcp: Option<crate::mcp::McpRuntime>,
545 #[cfg(feature = "webmcp")]
548 webmcp: Option<crate::webmcp::WebMcpRuntime>,
549 audit_logger: Option<Arc<crate::audit::AuditLogger>>,
551 #[cfg(feature = "i18n")]
555 i18n_bundle: Option<Arc<crate::i18n::Bundle>>,
556 #[cfg(feature = "i18n")]
560 i18n_auto_load: bool,
561 #[cfg(feature = "embed-assets")]
566 embedded_static: Option<crate::assets::EmbeddedStaticDir>,
567 #[cfg(all(feature = "embed-assets", feature = "i18n"))]
572 embedded_locales: Option<&'static include_dir::Dir<'static>>,
573 policy_registrations: Vec<PolicyRegistration>,
579 #[cfg(feature = "mail")]
583 mail_delivery_queue_factory: Option<MailDeliveryQueueFactory>,
584 #[cfg(feature = "mail")]
585 pub(crate) suppression_store: Option<crate::mail::SuppressionStoreHandle>,
586 #[cfg(feature = "mail")]
587 pub(crate) mail_suppression_store: Option<crate::mail::suppression::SuppressionStoreHandle>,
588 #[cfg(feature = "mail")]
589 pub(crate) mount_unsubscribe_endpoint: bool,
590 #[cfg(feature = "mail")]
592 mail_previews: Vec<crate::mail::MailPreview>,
593 #[cfg(feature = "maud")]
595 story_gallery: Option<crate::stories::StoryGallery>,
596 pub(crate) declared_routes: Vec<crate::route_listing::RouteInfo>,
606 pub(crate) plugin_asset_bundles: Vec<&'static crate::assets::PluginAssets>,
610 idempotency_enabled: bool,
614 #[cfg(feature = "mail")]
615 mail_interceptor: Option<Arc<dyn crate::interceptor::MailInterceptor>>,
616 job_interceptor: Option<Arc<dyn crate::interceptor::JobInterceptor>>,
617 #[cfg(feature = "db")]
618 db_interceptor: Option<Arc<dyn crate::interceptor::DbConnectionInterceptor>>,
619 #[cfg(feature = "ws")]
620 channels_interceptor: Option<Arc<dyn crate::interceptor::ChannelsInterceptor>>,
621 #[cfg(feature = "oauth2")]
622 http_interceptor: Option<Arc<dyn crate::interceptor::HttpInterceptor>>,
623 seo_sources: Vec<Arc<dyn crate::seo::SitemapSource>>,
626 agent_skills: Vec<crate::aeo::AgentSkill>,
628 commerce_docs: crate::aeo::RegisteredCommerceDocs,
631
632 pub(crate) metrics_sources: Vec<(String, Arc<dyn crate::actuator::MetricsSource>)>,
634 pub(crate) health_indicators: Vec<(
636 String,
637 crate::actuator::IndicatorGroup,
638 Arc<dyn crate::actuator::HealthIndicator>,
639 )>,
640 #[cfg(feature = "inbound-mail")]
644 pub(crate) inbound_mail_router: Option<Arc<crate::inbound_mail::InboundMailRouter>>,
645}
646
647#[cfg(feature = "mail")]
651pub(crate) type MailDeliveryQueueFactory = Box<
652 dyn FnOnce(&AppState) -> crate::AutumnResult<Arc<dyn crate::mail::MailDeliveryQueue>> + Send,
653>;
654
655pub struct ScopedGroup {
660 pub prefix: String,
661 pub routes: Vec<Route>,
662 pub source: crate::route_listing::RouteSource,
664 pub layer_guarded: bool,
667 pub apply_layer: Box<dyn FnOnce(axum::Router<AppState>) -> axum::Router<AppState> + Send>,
669}
670
671pub type ErasedAppService = tower::util::BoxCloneSyncService<
693 axum::extract::Request,
694 axum::response::Response,
695 std::convert::Infallible,
696>;
697
698pub(crate) type ErasedAppLayer = tower::util::BoxCloneSyncServiceLayer<
701 ErasedAppService,
702 axum::extract::Request,
703 axum::response::Response,
704 std::convert::Infallible,
705>;
706
707#[derive(Clone)]
716pub(crate) struct CustomLayerRegistration {
717 pub(crate) type_id: TypeId,
719 pub(crate) type_name: &'static str,
722 pub(crate) layer: ErasedAppLayer,
725}
726
727mod sealed {
728 pub trait Sealed {}
729}
730
731#[diagnostic::on_unimplemented(
751 message = "`{Self}` is not a usable Autumn app-wide Tower layer",
752 label = "this type is not a `tower::Layer` over Autumn's ingress service with the required service bounds",
753 note = "`AppBuilder::layer(..)` requires a layer that is generic over the service it wraps (or written against `autumn_web::app::ErasedAppService`), producing:\n L::Service: Service<axum::extract::Request, Response = axum::response::Response, Error = Infallible> + Clone + Send + Sync + 'static,\n <L::Service as Service<axum::extract::Request>>::Future: Send + 'static\nand the layer itself must be Clone + Send + Sync + 'static.\nSee docs/guide/middleware.md for common patterns and how to wrap raw-error layers (e.g. TimeoutLayer) with HandleErrorLayer."
754)]
755pub trait IntoAppLayer: sealed::Sealed + Send + Sync + 'static {
756 #[doc(hidden)]
760 fn erase(self) -> ErasedAppLayer;
761}
762
763impl<L> sealed::Sealed for L
764where
765 L: tower::Layer<ErasedAppService> + Clone + Send + Sync + 'static,
766 L::Service: tower::Service<
767 axum::extract::Request,
768 Response = axum::response::Response,
769 Error = std::convert::Infallible,
770 > + Clone
771 + Send
772 + Sync
773 + 'static,
774 <L::Service as tower::Service<axum::extract::Request>>::Future: Send + 'static,
775{
776}
777
778impl<L> IntoAppLayer for L
779where
780 L: tower::Layer<ErasedAppService> + Clone + Send + Sync + 'static,
781 L::Service: tower::Service<
782 axum::extract::Request,
783 Response = axum::response::Response,
784 Error = std::convert::Infallible,
785 > + Clone
786 + Send
787 + Sync
788 + 'static,
789 <L::Service as tower::Service<axum::extract::Request>>::Future: Send + 'static,
790{
791 fn erase(self) -> ErasedAppLayer {
792 tower::util::BoxCloneSyncServiceLayer::new(self)
793 }
794}
795
796impl AppBuilder {
797 #[must_use]
819 pub fn routes(mut self, routes: Vec<Route>) -> Self {
820 let source = self
821 .current_plugin
822 .as_ref()
823 .map_or(crate::route_listing::RouteSource::User, |name| {
824 crate::route_listing::RouteSource::Plugin(name.clone())
825 });
826 for _ in &routes {
827 self.route_sources.push(source.clone());
828 }
829 self.routes.extend(routes);
830 self
831 }
832
833 #[must_use]
839 pub fn tasks(mut self, tasks: Vec<crate::task::TaskInfo>) -> Self {
840 self.tasks.extend(tasks);
841 self
842 }
843
844 #[must_use]
849 pub fn one_off_tasks(mut self, tasks: Vec<crate::task::OneOffTaskInfo>) -> Self {
850 self.one_off_tasks.extend(tasks);
851 self
852 }
853
854 #[must_use]
856 pub fn jobs(mut self, jobs: Vec<crate::job::JobInfo>) -> Self {
857 self.jobs.extend(jobs);
858 self
859 }
860
861 #[must_use]
868 pub fn listeners(mut self, listeners: Vec<crate::events::ListenerInfo>) -> Self {
869 self.listeners.extend(listeners);
870 self
871 }
872
873 #[cfg(feature = "db")]
884 #[must_use]
885 pub fn outbox_handler<F, Fut>(mut self, topic: impl Into<String>, handler: F) -> Self
886 where
887 F: Fn(AppState, crate::outbox::OutboxMessage) -> Fut + Send + Sync + 'static,
888 Fut: std::future::Future<Output = crate::AutumnResult<()>> + Send + 'static,
889 {
890 self.outbox_handlers.insert(topic, handler);
891 self
892 }
893
894 #[must_use]
899 pub fn static_routes(mut self, metas: Vec<crate::static_gen::StaticRouteMeta>) -> Self {
900 self.static_metas.extend(metas);
901 self
902 }
903
904 #[must_use]
942 pub fn seo_source<S: crate::seo::SitemapSource + 'static>(mut self, source: S) -> Self {
943 self.seo_sources.push(Arc::new(source));
944 self
945 }
946
947 #[must_use]
964 pub fn agent_skill(mut self, skill: crate::aeo::AgentSkill) -> Self {
965 self.agent_skills.push(skill);
966 self
967 }
968
969 #[must_use]
971 pub fn ucp_profile(mut self, profile: crate::aeo::commerce::UcpProfile) -> Self {
972 self.commerce_docs.ucp = Some(profile);
973 self
974 }
975
976 #[must_use]
978 pub fn acp_discovery(mut self, document: crate::aeo::commerce::AcpDiscovery) -> Self {
979 self.commerce_docs.acp = Some(document);
980 self
981 }
982
983 #[cfg(feature = "openapi")]
1033 #[must_use]
1034 pub fn openapi(mut self, config: crate::openapi::OpenApiConfig) -> Self {
1035 self.openapi = Some(config);
1036 self
1037 }
1038
1039 #[cfg(feature = "mcp")]
1079 #[must_use]
1080 pub fn mount_mcp(mut self, path: impl Into<String>) -> Self {
1081 let path = path.into();
1082 if let Some(rt) = self.mcp.as_mut() {
1083 rt.mount_path = path;
1084 } else {
1085 self.mcp = Some(crate::mcp::McpRuntime::new(path));
1086 }
1087 self
1088 }
1089
1090 #[cfg(feature = "mcp")]
1104 #[must_use]
1105 pub fn expose_all_as_mcp(mut self) -> Self {
1106 if let Some(rt) = self.mcp.as_mut() {
1107 rt.expose_all = true;
1108 } else {
1109 let mut rt = crate::mcp::McpRuntime::new("/mcp");
1110 rt.expose_all = true;
1111 self.mcp = Some(rt);
1112 }
1113 self
1114 }
1115
1116 #[cfg(feature = "mcp")]
1125 #[must_use]
1126 pub fn mcp_tasks(mut self, config: crate::mcp::McpTaskConfig) -> Self {
1127 self.mcp
1128 .get_or_insert_with(|| crate::mcp::McpRuntime::new("/mcp"))
1129 .tasks = config;
1130 self
1131 }
1132
1133 #[cfg(feature = "mcp")]
1144 #[must_use]
1145 pub fn mcp_instructions(mut self, text: impl Into<String>) -> Self {
1146 self.mcp
1147 .get_or_insert_with(|| crate::mcp::McpRuntime::new("/mcp"))
1148 .instructions = Some(text.into());
1149 self
1150 }
1151
1152 #[cfg(feature = "mcp")]
1166 #[must_use]
1167 pub fn mcp_catalog(mut self, catalog: crate::mcp::McpCatalog) -> Self {
1168 self.mcp
1169 .get_or_insert_with(|| crate::mcp::McpRuntime::new("/mcp"))
1170 .catalog = catalog;
1171 self
1172 }
1173
1174 #[cfg(feature = "mcp")]
1187 #[must_use]
1188 pub fn secure_mcp<L>(mut self, layer: L) -> Self
1189 where
1190 L: tower::Layer<axum::routing::Route> + Clone + Send + Sync + 'static,
1191 L::Service: tower::Service<
1192 axum::http::Request<axum::body::Body>,
1193 Response = axum::http::Response<axum::body::Body>,
1194 Error = std::convert::Infallible,
1195 > + Clone
1196 + Send
1197 + Sync
1198 + 'static,
1199 <L::Service as tower::Service<axum::http::Request<axum::body::Body>>>::Future:
1200 Send + 'static,
1201 {
1202 let applier: crate::mcp::McpEndpointLayer = Box::new(move |router| router.layer(layer));
1203 if let Some(rt) = self.mcp.as_mut() {
1204 rt.endpoint_layer = Some(applier);
1205 } else {
1206 let mut rt = crate::mcp::McpRuntime::new("/mcp");
1207 rt.endpoint_layer = Some(applier);
1208 self.mcp = Some(rt);
1209 }
1210 self
1211 }
1212
1213 #[cfg(feature = "webmcp")]
1234 #[must_use]
1235 pub fn mount_webmcp(mut self) -> Self {
1236 self.webmcp.get_or_insert_with(Default::default);
1237 self
1238 }
1239
1240 #[cfg(feature = "webmcp")]
1246 #[must_use]
1247 pub fn expose_all_as_webmcp(mut self) -> Self {
1248 self.webmcp.get_or_insert_with(Default::default).expose_all = true;
1249 self
1250 }
1251
1252 #[must_use]
1284 pub fn exception_filter(mut self, filter: impl ExceptionFilter) -> Self {
1285 self.exception_filters.push(Arc::new(filter));
1286 self
1287 }
1288
1289 #[must_use]
1328 #[cfg(feature = "maud")]
1329 pub fn error_pages(mut self, renderer: impl ErrorPageRenderer) -> Self {
1330 self.error_page_renderer = Some(Arc::new(renderer));
1331 self
1332 }
1333
1334 #[must_use]
1357 pub fn scoped<L>(self, prefix: &str, layer: L, routes: Vec<Route>) -> Self
1358 where
1359 L: tower::Layer<axum::routing::Route> + Clone + Send + Sync + 'static,
1360 L::Service: tower::Service<
1361 axum::http::Request<axum::body::Body>,
1362 Response = axum::http::Response<axum::body::Body>,
1363 Error = std::convert::Infallible,
1364 > + Clone
1365 + Send
1366 + Sync
1367 + 'static,
1368 <L::Service as tower::Service<axum::http::Request<axum::body::Body>>>::Future:
1369 Send + 'static,
1370 {
1371 self.push_scoped_group(prefix, layer, routes, false)
1372 }
1373
1374 #[must_use]
1387 pub fn scoped_guarded<L>(self, prefix: &str, layer: L, routes: Vec<Route>) -> Self
1388 where
1389 L: tower::Layer<axum::routing::Route> + Clone + Send + Sync + 'static,
1390 L::Service: tower::Service<
1391 axum::http::Request<axum::body::Body>,
1392 Response = axum::http::Response<axum::body::Body>,
1393 Error = std::convert::Infallible,
1394 > + Clone
1395 + Send
1396 + Sync
1397 + 'static,
1398 <L::Service as tower::Service<axum::http::Request<axum::body::Body>>>::Future:
1399 Send + 'static,
1400 {
1401 self.push_scoped_group(prefix, layer, routes, true)
1402 }
1403
1404 fn push_scoped_group<L>(
1405 mut self,
1406 prefix: &str,
1407 layer: L,
1408 routes: Vec<Route>,
1409 layer_guarded: bool,
1410 ) -> Self
1411 where
1412 L: tower::Layer<axum::routing::Route> + Clone + Send + Sync + 'static,
1413 L::Service: tower::Service<
1414 axum::http::Request<axum::body::Body>,
1415 Response = axum::http::Response<axum::body::Body>,
1416 Error = std::convert::Infallible,
1417 > + Clone
1418 + Send
1419 + Sync
1420 + 'static,
1421 <L::Service as tower::Service<axum::http::Request<axum::body::Body>>>::Future:
1422 Send + 'static,
1423 {
1424 let source = self
1425 .current_plugin
1426 .as_ref()
1427 .map_or(crate::route_listing::RouteSource::User, |name| {
1428 crate::route_listing::RouteSource::Plugin(name.clone())
1429 });
1430 self.scoped_groups.push(ScopedGroup {
1431 prefix: prefix.to_owned(),
1432 routes,
1433 source,
1434 layer_guarded,
1435 apply_layer: Box::new(move |router| router.layer(layer)),
1436 });
1437 self
1438 }
1439
1440 #[must_use]
1512 pub fn layer<L: IntoAppLayer>(mut self, layer: L) -> Self {
1513 self.custom_layers.push(CustomLayerRegistration {
1514 type_id: TypeId::of::<L>(),
1515 type_name: std::any::type_name::<L>(),
1516 layer: layer.erase(),
1517 });
1518 self
1519 }
1520
1521 #[must_use]
1526 pub fn has_layer<L: 'static>(&self) -> bool {
1527 let layer_type = TypeId::of::<L>();
1528 self.custom_layers
1529 .iter()
1530 .any(|registered| registered.type_id == layer_type)
1531 }
1532
1533 #[must_use]
1566 pub const fn idempotent(mut self) -> Self {
1567 self.idempotency_enabled = true;
1568 self
1569 }
1570
1571 #[must_use]
1579 pub fn get_layer_types(&self) -> Vec<TypeId> {
1580 self.custom_layers
1581 .iter()
1582 .map(|registered| registered.type_id)
1583 .filter(|type_id| !is_framework_owned_layer(*type_id))
1584 .collect()
1585 }
1586
1587 #[must_use]
1674 pub fn static_gate<L: IntoAppLayer>(mut self, layer: L) -> Self {
1675 self.static_gate_layers.push(CustomLayerRegistration {
1676 type_id: TypeId::of::<L>(),
1677 type_name: std::any::type_name::<L>(),
1678 layer: layer.erase(),
1679 });
1680 self
1681 }
1682
1683 #[must_use]
1688 pub fn has_static_gate<L: 'static>(&self) -> bool {
1689 let layer_type = TypeId::of::<L>();
1690 self.static_gate_layers
1691 .iter()
1692 .any(|registered| registered.type_id == layer_type)
1693 }
1694
1695 #[must_use]
1702 pub fn get_static_gate_types(&self) -> Vec<TypeId> {
1703 self.static_gate_layers
1704 .iter()
1705 .map(|registered| registered.type_id)
1706 .collect()
1707 }
1708
1709 #[must_use]
1752 pub fn merge(mut self, router: axum::Router<AppState>) -> Self {
1753 self.merge_routers.push(router);
1754 self
1755 }
1756
1757 #[must_use]
1794 pub fn nest(mut self, path: &str, router: axum::Router<AppState>) -> Self {
1795 self.nest_routers.push((path.to_owned(), router));
1796 self
1797 }
1798
1799 #[must_use]
1819 pub fn declare_plugin_routes(
1820 mut self,
1821 routes: impl IntoIterator<Item = crate::route_listing::RouteInfo>,
1822 ) -> Self {
1823 let source = self.current_route_source();
1824 for mut route in routes {
1825 route.source = source.clone();
1826 route
1830 .middleware
1831 .retain(|label| label != crate::assets::PLUGIN_ASSETS_ROUTE_MARKER);
1832 self.declared_routes.push(route);
1833 }
1834 self
1835 }
1836
1837 fn current_route_source(&self) -> crate::route_listing::RouteSource {
1840 self.current_plugin
1841 .as_deref()
1842 .map_or(crate::route_listing::RouteSource::User, |name| {
1843 crate::route_listing::RouteSource::Plugin(name.to_owned())
1844 })
1845 }
1846
1847 #[must_use]
1877 pub fn plugin_assets(mut self, assets: &'static crate::assets::PluginAssets) -> Self {
1878 if let Some(existing) = self
1879 .plugin_asset_bundles
1880 .iter()
1881 .find(|installed| installed.namespace() == assets.namespace())
1882 {
1883 assert!(
1884 std::ptr::eq(*existing, assets),
1885 "two different PluginAssets bundles use the namespace `{}`; each bundle needs \
1886 its own namespace because it becomes the URL path {}",
1887 assets.namespace(),
1888 assets.mount_path(),
1889 );
1890 return self;
1891 }
1892 self.plugin_asset_bundles.push(assets);
1893 crate::assets::plugin::register(assets);
1894 if assets.iter().next().is_none() {
1895 tracing::warn!(
1896 namespace = assets.namespace(),
1897 "PluginAssets bundle has no files; nothing to serve"
1898 );
1899 return self;
1900 }
1901 let source = self.current_route_source();
1904 for mut route in assets.route_infos() {
1905 route.source = source.clone();
1906 self.declared_routes.push(route);
1907 }
1908 let mount = assets.mount_path();
1909 self.nest(&mount, assets.nested_router())
1910 }
1911
1912 pub fn plugin_route_infos(
1931 &self,
1932 ) -> Result<Vec<crate::route_listing::RouteInfo>, crate::RouterBuildError> {
1933 let mut infos = crate::route_listing::collect_route_infos(
1934 &self.routes,
1935 &self.route_sources,
1936 &self.scoped_groups,
1937 &self.api_versions,
1938 )?;
1939 infos.extend(self.declared_routes.iter().cloned());
1940 Ok(infos)
1941 }
1942
1943 #[must_use]
1949 pub fn on_startup<F, Fut>(mut self, hook: F) -> Self
1950 where
1951 F: Fn(AppState) -> Fut + Send + Sync + 'static,
1952 Fut: Future<Output = crate::AutumnResult<()>> + Send + 'static,
1953 {
1954 self.startup_hooks
1955 .push(Box::new(move |state| Box::pin(hook(state))));
1956 self
1957 }
1958
1959 #[must_use]
1962 pub fn state_initializer<F>(mut self, initializer: F) -> Self
1963 where
1964 F: FnOnce(&AppState) + Send + 'static,
1965 {
1966 self.state_initializers.push(Box::new(initializer));
1967 self
1968 }
1969
1970 #[cfg(feature = "plugin-sandbox")]
1976 #[must_use]
1977 pub fn plugin_mounts(self, mounts: crate::plugin_sandbox::PluginMounts) -> Self {
1978 self.state_initializer(move |state| {
1979 mounts.reserve(crate::plugin_sandbox::framework_prefixes(&state.config()));
1980 state.insert_extension(mounts);
1981 })
1982 }
1983
1984 #[must_use]
2023 pub fn with_live_state<T>(self, initial: T) -> Self
2024 where
2025 T: crate::upgrade::LiveState,
2026 {
2027 self.state_initializer(move |state| {
2028 install_live_state(state, initial, crate::upgrade::decode::<T>);
2029 })
2030 }
2031
2032 #[must_use]
2073 pub fn with_live_state_from<Old, T>(self, initial: T) -> Self
2074 where
2075 Old: crate::upgrade::LiveState,
2076 T: crate::upgrade::MigrateFrom<Old>,
2077 {
2078 self.state_initializer(move |state| {
2079 install_live_state(state, initial, crate::upgrade::decode_migrating::<Old, T>);
2080 })
2081 }
2082
2083 #[must_use]
2088 pub fn on_shutdown<F, Fut>(mut self, hook: F) -> Self
2089 where
2090 F: Fn() -> Fut + Send + Sync + 'static,
2091 Fut: Future<Output = ()> + Send + 'static,
2092 {
2093 self.shutdown_hooks.push(Box::new(move || Box::pin(hook())));
2094 self
2095 }
2096
2097 #[must_use]
2099 pub fn api_version(mut self, version: ApiVersion) -> Self {
2100 if let Some(pos) = self
2101 .api_versions
2102 .iter()
2103 .position(|v| v.version == version.version)
2104 {
2105 self.api_versions[pos] = version;
2106 } else {
2107 self.api_versions.push(version);
2108 }
2109 self
2110 }
2111
2112 #[must_use]
2114 pub fn api_versions(mut self, versions: impl IntoIterator<Item = ApiVersion>) -> Self {
2115 for version in versions {
2116 if let Some(pos) = self
2117 .api_versions
2118 .iter()
2119 .position(|v| v.version == version.version)
2120 {
2121 self.api_versions[pos] = version;
2122 } else {
2123 self.api_versions.push(version);
2124 }
2125 }
2126 self
2127 }
2128
2129 #[must_use]
2145 pub fn impersonation_gate(self, gate: crate::auth::impersonation::ImpersonationGate) -> Self {
2146 self.state_initializer(move |state| {
2147 let auth_key = state.auth_session_key();
2150 if crate::auth::impersonation::is_reserved_session_key(auth_key) {
2151 tracing::error!(
2152 auth_session_key = %auth_key,
2153 "impersonation is enabled but `auth.session_key` collides with a key the \
2154 impersonation record reserves; every attempt will be refused"
2155 );
2156 }
2157 state.insert_extension(gate);
2158 })
2159 }
2160
2161 #[must_use]
2168 pub fn on_session_rotation<F, Fut>(self, hook: F) -> Self
2169 where
2170 F: Fn(crate::session::SessionRotation) -> Fut + Send + Sync + 'static,
2171 Fut: Future<Output = Result<(), crate::session::SessionRotationError>> + Send + 'static,
2172 {
2173 self.state_initializer(move |state| state.session_rotation_hooks().register(hook))
2174 }
2175
2176 #[must_use]
2181 pub fn with_extension<T>(mut self, value: T) -> Self
2182 where
2183 T: Any + Send + 'static,
2184 {
2185 self.extensions.insert(TypeId::of::<T>(), Box::new(value));
2186 self
2187 }
2188
2189 #[must_use]
2197 pub fn update_extension<T, Init, Update>(mut self, init: Init, update: Update) -> Self
2198 where
2199 T: Any + Send + 'static,
2200 Init: FnOnce() -> T,
2201 Update: FnOnce(&mut T),
2202 {
2203 let type_id = TypeId::of::<T>();
2204 let entry = self
2205 .extensions
2206 .entry(type_id)
2207 .or_insert_with(|| Box::new(init()));
2208 let typed = entry
2209 .downcast_mut::<T>()
2210 .expect("extension type map corrupted");
2211 update(typed);
2212 self
2213 }
2214
2215 #[must_use]
2217 pub fn extension<T>(&self) -> Option<&T>
2218 where
2219 T: Any + Send + 'static,
2220 {
2221 self.extensions.get(&TypeId::of::<T>())?.downcast_ref::<T>()
2222 }
2223
2224 #[cfg(feature = "mail")]
2225 #[must_use]
2226 pub fn with_mail_interceptor(
2227 mut self,
2228 interceptor: impl crate::interceptor::MailInterceptor,
2229 ) -> Self {
2230 self.mail_interceptor = Some(Arc::new(interceptor));
2231 self
2232 }
2233
2234 #[must_use]
2235 pub fn with_job_interceptor(
2236 mut self,
2237 interceptor: impl crate::interceptor::JobInterceptor,
2238 ) -> Self {
2239 self.job_interceptor = Some(Arc::new(interceptor));
2240 self
2241 }
2242
2243 #[cfg(feature = "db")]
2244 #[must_use]
2245 pub fn with_db_interceptor(
2246 mut self,
2247 interceptor: impl crate::interceptor::DbConnectionInterceptor,
2248 ) -> Self {
2249 self.db_interceptor = Some(Arc::new(interceptor));
2250 self
2251 }
2252
2253 #[cfg(feature = "ws")]
2254 #[must_use]
2255 pub fn with_channels_interceptor(
2256 mut self,
2257 interceptor: impl crate::interceptor::ChannelsInterceptor,
2258 ) -> Self {
2259 self.channels_interceptor = Some(Arc::new(interceptor));
2260 self
2261 }
2262
2263 #[cfg(feature = "oauth2")]
2264 #[must_use]
2265 pub fn with_http_interceptor(
2266 mut self,
2267 interceptor: impl crate::interceptor::HttpInterceptor,
2268 ) -> Self {
2269 self.http_interceptor = Some(Arc::new(interceptor));
2270 self
2271 }
2272
2273 #[cfg(feature = "i18n")]
2281 #[must_use]
2282 pub fn i18n(mut self, bundle: crate::i18n::Bundle) -> Self {
2283 self.i18n_bundle = Some(Arc::new(bundle));
2284 self.i18n_auto_load = false;
2285 self
2286 }
2287
2288 #[cfg(feature = "i18n")]
2324 #[must_use]
2325 pub fn i18n_auto(mut self) -> Self {
2326 self.i18n_bundle = None;
2327 self.i18n_auto_load = true;
2328 self
2329 }
2330
2331 #[must_use]
2346 pub fn with_config_loader<L>(mut self, loader: L) -> Self
2347 where
2348 L: crate::config::ConfigLoader,
2349 {
2350 if self.config_loader_factory.is_some() {
2351 tracing::warn!(
2352 "config loader replaced; the previously-installed loader was overwritten"
2353 );
2354 }
2355 self.config_loader_factory = Some(Box::new(move || {
2356 Box::pin(async move { loader.load().await })
2357 }));
2358 self
2359 }
2360
2361 #[cfg(feature = "db")]
2369 #[must_use]
2370 pub fn with_pool_provider<P>(mut self, provider: P) -> Self
2371 where
2372 P: crate::db::DatabasePoolProvider,
2373 {
2374 if self.pool_provider_factory.is_some() {
2375 tracing::warn!(
2376 "database pool provider replaced; the previously-installed provider was overwritten"
2377 );
2378 }
2379 let provider = Arc::new(provider);
2382 let shard_provider = Arc::clone(&provider);
2383 self.pool_provider_factory =
2384 Some(Box::new(move |config: crate::config::DatabaseConfig| {
2385 Box::pin(async move { provider.create_topology(&config).await })
2386 }));
2387 self.shard_provider_factory =
2388 Some(Box::new(move |config: crate::config::DatabaseConfig| {
2389 Box::pin(async move {
2390 let mut topologies = Vec::with_capacity(config.shards.len());
2391 for shard in &config.shards {
2392 topologies
2393 .push(shard_provider.create_shard_topology(shard, &config).await?);
2394 }
2395 Ok(topologies)
2396 })
2397 }));
2398 self
2399 }
2400
2401 #[cfg(feature = "db")]
2412 #[must_use]
2413 pub fn with_shard_router<R>(mut self, router: R) -> Self
2414 where
2415 R: crate::sharding::ShardRouter,
2416 {
2417 if self.shard_router.is_some() {
2418 tracing::warn!(
2419 "shard router replaced; the previously-installed router was overwritten"
2420 );
2421 }
2422 self.shard_router = Some(Arc::new(router));
2423 self
2424 }
2425
2426 #[cfg(feature = "db")]
2437 #[must_use]
2438 pub const fn with_directory_shard_router(mut self) -> Self {
2439 self.directory_shard_router = true;
2440 self
2441 }
2442
2443 #[must_use]
2450 pub fn with_telemetry_provider<T>(mut self, provider: T) -> Self
2451 where
2452 T: crate::telemetry::TelemetryProvider,
2453 {
2454 if self.telemetry_provider.is_some() {
2455 tracing::warn!(
2456 "telemetry provider replaced; the previously-installed provider was overwritten"
2457 );
2458 }
2459 self.telemetry_provider = Some(Box::new(provider));
2460 self
2461 }
2462
2463 #[must_use]
2470 pub fn with_session_store<S>(mut self, store: S) -> Self
2471 where
2472 S: crate::session::SessionStore,
2473 {
2474 if self.session_store.is_some() {
2475 tracing::warn!(
2476 "session store replaced; the previously-installed store was overwritten"
2477 );
2478 }
2479 self.session_store = Some(Arc::new(store));
2480 self
2481 }
2482
2483 #[cfg(feature = "ws")]
2490 #[must_use]
2491 pub fn with_channels_backend<B>(mut self, backend: B) -> Self
2492 where
2493 B: crate::channels::ChannelsBackend,
2494 {
2495 if self.channels_backend.is_some() {
2496 tracing::warn!(
2497 "channels backend replaced; the previously-installed backend was overwritten"
2498 );
2499 }
2500 self.channels_backend = Some(Arc::new(backend));
2501 self
2502 }
2503
2504 #[cfg(feature = "storage")]
2538 #[must_use]
2539 pub fn with_blob_store<B>(mut self, store: B) -> Self
2540 where
2541 B: crate::storage::BlobStore,
2542 {
2543 if self.blob_store.is_some() {
2544 tracing::warn!("blob store replaced; the previously-installed store was overwritten");
2545 }
2546 self.blob_store = Some(std::sync::Arc::new(store));
2547 self
2548 }
2549
2550 #[must_use]
2569 pub fn with_cache_backend<C: crate::cache::Cache>(mut self, cache: C) -> Self {
2570 if self.cache_backend.is_some() {
2571 tracing::warn!(
2572 "cache backend replaced; the previously-installed backend was overwritten"
2573 );
2574 }
2575 self.cache_backend = Some(Arc::new(cache) as Arc<dyn crate::cache::Cache>);
2576 self
2577 }
2578
2579 #[cfg(feature = "edge")]
2624 #[must_use]
2625 pub fn with_edge_kv(self, kv: Arc<dyn autumn_edge::EdgeKv>) -> Self {
2626 self.layer(autumn_edge::EdgeCache::layer(kv))
2631 }
2632
2633 #[cfg(feature = "edge")]
2638 #[must_use]
2639 pub fn with_edge_identity_provider<P>(self, provider: P) -> Self
2640 where
2641 P: crate::edge_support::EdgeIdentityProvider,
2642 {
2643 self.layer(crate::edge_support::EdgeIdentityLayer::new(provider))
2644 }
2645
2646 #[cfg(feature = "reporting")]
2681 #[must_use]
2682 pub fn with_error_reporter<R: crate::reporting::ErrorReporter>(mut self, reporter: R) -> Self {
2683 self.error_reporters
2684 .push(Arc::new(reporter) as Arc<dyn crate::reporting::ErrorReporter>);
2685 self
2686 }
2687
2688 #[must_use]
2724 pub fn with_alert_channel<C: crate::alerts::AlertChannel>(mut self, channel: C) -> Self {
2725 self.alert_channels
2726 .push(Arc::new(channel) as Arc<dyn crate::alerts::AlertChannel>);
2727 self
2728 }
2729
2730 #[must_use]
2782 pub fn with_flag_store<S>(self, store: S) -> Self
2783 where
2784 S: crate::feature_flags::FlagStore,
2785 {
2786 let service = crate::feature_flags::FeatureFlagService::new(Arc::new(store) as Arc<_>);
2787 self.with_flag_service(service)
2788 }
2789
2790 #[must_use]
2805 pub fn with_flag_service(self, service: crate::feature_flags::FeatureFlagService) -> Self {
2806 self.state_initializer(move |state| {
2807 crate::feature_flags::preload_blocking(&service, crate::feature_flags::PRELOAD_WAIT);
2809 state.insert_extension(service);
2810 })
2811 }
2812
2813 #[must_use]
2835 pub fn with_flag_store_and_resolver<S>(
2836 self,
2837 store: S,
2838 resolver: crate::feature_flags::GroupResolver,
2839 ) -> Self
2840 where
2841 S: crate::feature_flags::FlagStore,
2842 {
2843 let service = crate::feature_flags::FeatureFlagService::new(Arc::new(store) as Arc<_>)
2844 .with_group_resolver(resolver);
2845 self.with_flag_service(service)
2846 }
2847
2848 #[must_use]
2893 pub fn with_experiment_store<S>(self, store: S) -> Self
2894 where
2895 S: crate::experiments::ExperimentStore,
2896 {
2897 let service = crate::experiments::ExperimentService::new(Arc::new(store) as Arc<_>);
2898 self.state_initializer(move |state| {
2899 crate::experiments::preload_blocking(&service, crate::feature_flags::PRELOAD_WAIT);
2900 state.insert_extension(service);
2901 })
2902 }
2903
2904 #[must_use]
2928 pub fn with_notification_store<S>(self, store: S) -> Self
2929 where
2930 S: crate::notifications::NotificationStore,
2931 {
2932 let service = crate::notifications::Notifications::new(store);
2933 self.state_initializer(move |state| {
2934 state.insert_extension(service);
2935 })
2936 }
2937
2938 #[must_use]
2963 pub fn with_push_subscription_store<S>(self, store: S) -> Self
2964 where
2965 S: crate::push::PushSubscriptionStore,
2966 {
2967 self.state_initializer(move |state| {
2968 state.insert_extension(crate::push::WebPush::from_state_with_store(state, store));
2969 })
2970 }
2971
2972 #[must_use]
3003 pub fn with_web_push(self, push: crate::push::WebPush) -> Self {
3004 self.state_initializer(move |state| {
3005 state.insert_extension(push);
3006 })
3007 }
3008
3009 #[must_use]
3031 pub fn with_experiment_store_and_sink<S>(
3032 self,
3033 store: S,
3034 sink: Arc<dyn crate::experiments::ExposureSink>,
3035 ) -> Self
3036 where
3037 S: crate::experiments::ExperimentStore,
3038 {
3039 let service = crate::experiments::ExperimentService::new(Arc::new(store) as Arc<_>)
3040 .with_exposure_sink(sink);
3041 self.state_initializer(move |state| {
3042 crate::experiments::preload_blocking(&service, crate::feature_flags::PRELOAD_WAIT);
3043 state.insert_extension(service);
3044 })
3045 }
3046
3047 #[cfg(feature = "mail")]
3058 #[must_use]
3059 pub fn with_mail_delivery_queue(
3060 mut self,
3061 queue: impl crate::mail::MailDeliveryQueue + 'static,
3062 ) -> Self {
3063 let arc: Arc<dyn crate::mail::MailDeliveryQueue> = Arc::new(queue);
3064 self.mail_delivery_queue_factory = Some(Box::new(move |_state| Ok(arc)));
3065 self
3066 }
3067
3068 #[cfg(feature = "mail")]
3078 #[must_use]
3079 pub fn with_mail_delivery_queue_factory<F, Q>(mut self, factory: F) -> Self
3080 where
3081 F: FnOnce(&AppState) -> crate::AutumnResult<Q> + Send + 'static,
3082 Q: crate::mail::MailDeliveryQueue + 'static,
3083 {
3084 self.mail_delivery_queue_factory = Some(Box::new(move |state| {
3085 factory(state).map(|q| Arc::new(q) as Arc<dyn crate::mail::MailDeliveryQueue>)
3086 }));
3087 self
3088 }
3089
3090 #[cfg(feature = "mail")]
3099 #[must_use]
3100 pub fn with_suppression_store(
3101 mut self,
3102 store: impl crate::mail::SuppressionStore + 'static,
3103 ) -> Self {
3104 self.suppression_store = Some(crate::mail::SuppressionStoreHandle::new(store));
3105 self
3106 }
3107
3108 #[cfg(feature = "mail")]
3119 #[must_use]
3120 pub fn with_mail_suppression_store(
3121 mut self,
3122 store: impl crate::mail::suppression::SuppressionStore + 'static,
3123 ) -> Self {
3124 self.mail_suppression_store =
3125 Some(crate::mail::suppression::SuppressionStoreHandle::new(store));
3126 self
3127 }
3128
3129 #[cfg(feature = "mail")]
3138 #[must_use]
3139 pub const fn mount_unsubscribe_endpoint(mut self) -> Self {
3140 self.mount_unsubscribe_endpoint = true;
3141 self
3142 }
3143
3144 #[cfg(feature = "inbound-mail")]
3174 #[must_use]
3175 pub fn inbound_mail_router(mut self, router: crate::inbound_mail::InboundMailRouter) -> Self {
3176 self.inbound_mail_router = Some(Arc::new(router));
3177 self
3178 }
3179
3180 #[cfg(feature = "mail")]
3184 #[must_use]
3185 pub fn mail_previews(
3186 mut self,
3187 previews: impl IntoIterator<Item = crate::mail::MailPreview>,
3188 ) -> Self {
3189 self.mail_previews.extend(previews);
3190 self
3191 }
3192
3193 #[cfg(feature = "maud")]
3202 #[must_use]
3203 pub fn with_story_gallery(mut self, gallery: crate::stories::StoryGallery) -> Self {
3204 self.story_gallery = Some(gallery);
3205 self
3206 }
3207
3208 #[must_use]
3213 pub fn with_audit_sink<S>(mut self, sink: S) -> Self
3214 where
3215 S: crate::audit::AuditSink,
3216 {
3217 let logger = self
3218 .audit_logger
3219 .take()
3220 .map_or_else(crate::audit::AuditLogger::new, |logger| (*logger).clone())
3221 .with_sink(Arc::new(sink));
3222 self.audit_logger = Some(Arc::new(logger));
3223 self
3224 }
3225
3226 #[must_use]
3249 pub fn policy<R, P>(mut self, policy: P) -> Self
3250 where
3251 R: Send + Sync + 'static,
3252 P: crate::authorization::Policy<R>,
3253 {
3254 self.policy_registrations.push(Box::new(move |registry| {
3255 registry.register_policy::<R, _>(policy);
3256 }));
3257 self
3258 }
3259
3260 #[must_use]
3268 pub fn scope<R, S>(mut self, scope: S) -> Self
3269 where
3270 R: Send + Sync + 'static,
3271 S: crate::authorization::Scope<R>,
3272 {
3273 self.policy_registrations.push(Box::new(move |registry| {
3274 registry.register_scope::<R, _>(scope);
3275 }));
3276 self
3277 }
3278
3279 #[must_use]
3287 #[track_caller]
3288 pub fn plugin<P>(mut self, plugin: P) -> Self
3289 where
3290 P: crate::plugin::Plugin,
3291 {
3292 let name = plugin.name();
3293 if self.registered_plugins.contains(name.as_ref()) {
3294 tracing::warn!(
3295 plugin = name.as_ref(),
3296 "plugin already registered; skipping duplicate"
3297 );
3298 return self;
3299 }
3300 if let Some(mut contract) = plugin.contract() {
3301 Self::enforce_plugin_contract(&contract);
3302 contract.registered_as = Some(name.as_ref().to_owned());
3308 self.plugin_contracts.push(contract);
3309 }
3310 let name_str = name.into_owned();
3311 self.registered_plugins.insert(name_str.clone());
3312 let outer_plugin = self.current_plugin.replace(name_str);
3315 let mut result = plugin.build(self);
3316 result.current_plugin = outer_plugin;
3317 result
3318 }
3319
3320 #[must_use]
3323 pub fn plugins<P>(self, plugins: P) -> Self
3324 where
3325 P: crate::plugin::Plugins,
3326 {
3327 plugins.apply(self)
3328 }
3329
3330 #[must_use]
3333 pub fn has_plugin(&self, name: &str) -> bool {
3334 self.registered_plugins.contains(name)
3335 }
3336
3337 #[must_use]
3345 pub fn plugin_contracts(&self) -> &[crate::plugin_contract::PluginContract] {
3346 &self.plugin_contracts
3347 }
3348
3349 #[track_caller]
3378 fn enforce_plugin_contract(contract: &crate::plugin_contract::PluginContract) {
3379 use crate::plugin_contract::{AUTUMN_WEB_VERSION, ContractVerdict, evaluate};
3380
3381 match evaluate(contract, AUTUMN_WEB_VERSION) {
3382 ContractVerdict::Compatible | ContractVerdict::Undeclared => {}
3383 ContractVerdict::Incompatible(err) => {
3384 if std::env::var("AUTUMN_PLUGIN_CONTRACT").as_deref() == Ok("warn") {
3385 tracing::warn!(
3386 plugin = contract.plugin.as_str(),
3387 "{err}\n (demoted to a warning by AUTUMN_PLUGIN_CONTRACT=warn)"
3388 );
3389 } else {
3390 panic!(
3391 "{err}\n \u{2192} or, to boot anyway while you sort it out, set \
3392 AUTUMN_PLUGIN_CONTRACT=warn"
3393 );
3394 }
3395 }
3396 ContractVerdict::Unparseable {
3397 requirement,
3398 reason,
3399 } => {
3400 tracing::warn!(
3401 plugin = contract.plugin.as_str(),
3402 requirement = requirement.as_str(),
3403 reason = reason.as_str(),
3404 "plugin declares an autumn-web requirement that cannot be evaluated; \
3405 compatibility was NOT checked (run `autumn plugin-check` on the plugin)"
3406 );
3407 }
3408 }
3409 }
3410
3411 #[must_use]
3453 pub fn config_section(mut self, root: impl Into<String>) -> Self {
3454 self.plugin_config_roots.insert(root.into());
3455 self
3456 }
3457
3458 #[must_use]
3464 pub fn has_config_section(&self, root: &str) -> bool {
3465 self.plugin_config_roots.contains(root)
3466 }
3467
3468 #[must_use]
3504 pub fn metrics_source(
3505 mut self,
3506 name: impl Into<String>,
3507 source: Arc<dyn crate::actuator::MetricsSource>,
3508 ) -> Self {
3509 let name = name.into();
3510 if self.metrics_sources.iter().any(|(n, _)| n == &name) {
3511 tracing::warn!(
3512 source_name = %name,
3513 "MetricsSource '{}' is already registered; skipping duplicate",
3514 name
3515 );
3516 return self;
3517 }
3518 self.metrics_sources.push((name, source));
3519 self
3520 }
3521
3522 #[must_use]
3546 pub fn health_indicator(
3547 mut self,
3548 name: impl Into<String>,
3549 indicator: Arc<dyn crate::actuator::HealthIndicator>,
3550 ) -> Self {
3551 let name = name.into();
3552 #[cfg(feature = "db")]
3559 if name == "db" || name == "db:fleet" || name.starts_with("db:shard:") {
3560 tracing::warn!(
3561 indicator_name = %name,
3562 "\"db\", \"db:fleet\" and \"db:shard:*\" are reserved built-in health indicator \
3563 names; registration skipped. Use a different name for your custom indicator."
3564 );
3565 return self;
3566 }
3567 if self.health_indicators.iter().any(|(n, _, _)| n == &name) {
3568 tracing::warn!(
3569 indicator_name = %name,
3570 "HealthIndicator '{}' is already registered; skipping duplicate",
3571 name
3572 );
3573 return self;
3574 }
3575 let group = indicator.group();
3576 self.health_indicators.push((name, group, indicator));
3577 self
3578 }
3579
3580 #[cfg(feature = "db")]
3608 #[must_use]
3609 pub fn migrations(mut self, migrations: migrate::EmbeddedMigrations) -> Self {
3610 self.migrations.push(("app", migrations));
3611 self
3612 }
3613
3614 #[cfg(feature = "db")]
3712 #[must_use]
3713 pub fn plugin_migrations(
3714 mut self,
3715 name: &'static str,
3716 migrations: migrate::EmbeddedMigrations,
3717 ) -> Self {
3718 self.migrations.push((name, migrations));
3719 self
3720 }
3721
3722 #[cfg(feature = "embed-assets")]
3744 #[must_use]
3745 pub const fn embedded_static(mut self, dir: &'static include_dir::Dir<'static>) -> Self {
3746 self.embedded_static = Some(crate::assets::EmbeddedStaticDir(dir));
3747 self
3748 }
3749
3750 #[cfg(all(feature = "embed-assets", feature = "i18n"))]
3766 #[must_use]
3767 pub const fn embedded_locales(mut self, dir: &'static include_dir::Dir<'static>) -> Self {
3768 self.embedded_locales = Some(dir);
3769 self
3770 }
3771
3772 #[allow(clippy::too_many_lines)]
3792 #[allow(clippy::cognitive_complexity)]
3793 #[allow(clippy::large_stack_frames)]
3795 pub async fn run(self) {
3796 crate::upgrade::record_startup_exe();
3801
3802 if is_static_build_mode() {
3806 Box::pin(self.run_build_mode()).await;
3807 return;
3808 }
3809
3810 if is_dump_routes_mode() {
3815 self.run_dump_routes_mode().await;
3816 return;
3817 }
3818
3819 if is_dump_openapi_mode() {
3827 #[cfg(feature = "openapi")]
3828 {
3829 self.run_dump_openapi_mode().await;
3830 return;
3831 }
3832 #[cfg(not(feature = "openapi"))]
3833 {
3834 eprintln!(
3835 "{marker}{reason}",
3836 marker = crate::openapi::OPENAPI_UNAVAILABLE_MARKER,
3837 reason = crate::openapi::OPENAPI_UNAVAILABLE_FEATURE,
3838 );
3839 std::process::exit(2);
3840 }
3841 }
3842
3843 if crate::cache::coherence::is_dump_mode() {
3851 crate::cache::coherence::print_manifest_dump(&crate::cache::coherence::audit());
3852 return;
3853 }
3854
3855 if crate::classify::manifest::is_dump_mode() {
3863 crate::classify::manifest::print_manifest_dump(&crate::classify::manifest::audit());
3864 return;
3865 }
3866
3867 if crate::agent_authority::manifest::is_dump_mode() {
3876 self.run_dump_agent_authority_mode();
3877 return;
3878 }
3879
3880 if crate::graph::manifest::is_dump_mode() {
3890 self.run_dump_graph_mode().await;
3891 return;
3892 }
3893
3894 if is_dump_jobs_mode() {
3900 self.run_dump_jobs_mode().await;
3901 return;
3902 }
3903
3904 if is_list_one_off_tasks_mode() {
3905 self.run_list_one_off_tasks_mode();
3906 return;
3907 }
3908
3909 if let Some(task_name) = one_off_task_name_from_env() {
3910 self.run_one_off_task_mode(task_name).await;
3911 return;
3912 }
3913
3914 if is_migrate_only_mode() {
3922 self.run_migrate_only_mode().await;
3923 return;
3924 }
3925
3926 if is_retention_dry_run_mode() {
3932 self.run_retention_dry_run_mode().await;
3933 return;
3934 }
3935
3936 if let Some(mode) = framework_retention_mode_from_env() {
3944 self.run_framework_retention_mode(mode).await;
3945 return;
3946 }
3947
3948 if let Some(mode) = data_capsule_mode_from_env() {
3954 self.run_data_capsule_mode(mode).await;
3955 return;
3956 }
3957
3958 if is_replay_mode()
3964 && let Some(capsule_path) = replay_capsule_from_env()
3965 {
3966 #[cfg(feature = "reporting")]
3967 {
3968 self.run_replay_mode(capsule_path).await;
3969 return;
3970 }
3971 #[cfg(not(feature = "reporting"))]
3975 {
3976 eprintln!(
3977 "REFUSED {capsule_path}\n this binary was built without the `reporting` \
3978 feature, which failure capsules require"
3979 );
3980 std::process::exit(2);
3981 }
3982 }
3983
3984 #[cfg(unix)]
3990 let upgrade_signal =
3991 match tokio::signal::unix::signal(tokio::signal::unix::SignalKind::user_defined2()) {
3992 Ok(signal) => Some(signal),
3993 Err(e) => {
3994 tracing::warn!(
3995 error = %e,
3996 "could not install the SIGUSR2 handler; in-place upgrade is unavailable"
3997 );
3998 None
3999 }
4000 };
4001
4002 let Self {
4003 routes,
4004 api_versions,
4005 route_sources: _,
4006 current_plugin: _,
4007 mut tasks,
4008 one_off_tasks: _,
4009 mut jobs,
4010 listeners,
4011 #[cfg(feature = "db")]
4012 outbox_handlers,
4013 static_metas,
4014 exception_filters,
4015 scoped_groups,
4016 merge_routers,
4017 nest_routers,
4018 custom_layers,
4019 static_gate_layers,
4020 startup_hooks,
4021 state_initializers,
4022 shutdown_hooks,
4023 extensions: _,
4024 registered_plugins: _,
4025 plugin_contracts: _,
4026 plugin_config_roots,
4027 #[cfg(feature = "maud")]
4028 error_page_renderer,
4029 #[cfg(feature = "db")]
4030 migrations,
4031 config_loader_factory,
4032 #[cfg(feature = "db")]
4033 pool_provider_factory,
4034 #[cfg(feature = "db")]
4035 shard_provider_factory,
4036 #[cfg(feature = "db")]
4037 shard_router,
4038 #[cfg(feature = "db")]
4039 directory_shard_router,
4040 telemetry_provider,
4041 session_store,
4042 #[cfg(feature = "ws")]
4043 channels_backend,
4044 #[cfg(feature = "storage")]
4045 blob_store,
4046 cache_backend,
4047 #[cfg(feature = "reporting")]
4048 error_reporters,
4049 alert_channels,
4050 #[cfg(feature = "openapi")]
4051 openapi,
4052 #[cfg(feature = "mcp")]
4053 mcp,
4054 #[cfg(feature = "webmcp")]
4055 webmcp,
4056 audit_logger,
4057 #[cfg(feature = "i18n")]
4058 i18n_bundle,
4059 #[cfg(feature = "i18n")]
4060 i18n_auto_load,
4061 #[cfg(feature = "embed-assets")]
4062 embedded_static,
4063 #[cfg(all(feature = "embed-assets", feature = "i18n"))]
4064 embedded_locales,
4065 policy_registrations,
4066 #[cfg(feature = "mail")]
4067 mail_delivery_queue_factory,
4068 #[cfg(feature = "mail")]
4069 suppression_store,
4070 #[cfg(feature = "mail")]
4071 mail_suppression_store,
4072 #[cfg(feature = "mail")]
4073 mount_unsubscribe_endpoint,
4074 #[cfg(feature = "mail")]
4075 mail_previews,
4076 #[cfg(feature = "maud")]
4077 story_gallery,
4078 declared_routes,
4079 plugin_asset_bundles: _,
4080 idempotency_enabled,
4081 #[cfg(feature = "mail")]
4082 mail_interceptor,
4083 job_interceptor,
4084 #[cfg(feature = "db")]
4085 db_interceptor,
4086 #[cfg(feature = "ws")]
4087 channels_interceptor,
4088 #[cfg(feature = "oauth2")]
4089 http_interceptor,
4090 seo_sources,
4091 agent_skills,
4092 commerce_docs,
4093 metrics_sources,
4094 health_indicators,
4095 #[cfg(feature = "inbound-mail")]
4096 inbound_mail_router,
4097 } = self;
4098
4099 #[cfg(feature = "db")]
4104 tasks.extend(crate::retention::collect_retention_tasks());
4105
4106 if let Err(error) = crate::task::validate_unique_scheduled_task_names(&tasks) {
4114 panic!("{error}");
4115 }
4116
4117 let all_routes = routes;
4118
4119 let (mut config, telemetry_guard) = load_config_and_telemetry(
4121 config_loader_factory,
4122 telemetry_provider,
4123 plugin_config_roots,
4124 )
4125 .await;
4126
4127 let retention_tasks = crate::data_retention::framework_retention_tasks(&config.retention);
4134 if !retention_tasks.is_empty() {
4135 tasks.extend(retention_tasks);
4136 if let Err(error) = crate::task::validate_unique_scheduled_task_names(&tasks) {
4141 panic!("{error}");
4142 }
4143 }
4144
4145 let role = config.role;
4154 if let Err(message) = validate_config_preconditions(&config, &tasks) {
4160 tracing::error!("{message}");
4161 #[cfg(feature = "managed-pg")]
4162 crate::managed_pg::emergency_stop_async().await;
4163 std::process::exit(1);
4164 }
4165
4166 if crate::config::split_role_requires_file_backed_sqlite(
4171 role,
4172 &config.jobs.backend,
4173 config.database.effective_primary_url(),
4174 ) {
4175 tracing::error!(
4176 role = role.as_str(),
4177 "process role '{}' with jobs.backend = \"sqlite\" requires a FILE-backed \
4178 database: an in-memory SQLite target is private to each process, so the web \
4179 replica would enqueue into a queue no worker process can see. Point \
4180 database.url at a sqlite:// file, or run the combined role.",
4181 role.as_str(),
4182 );
4183 #[cfg(feature = "managed-pg")]
4184 crate::managed_pg::emergency_stop_async().await;
4185 std::process::exit(1);
4186 }
4187
4188 #[cfg(feature = "mail")]
4189 apply_mail_builder_overrides(&mut config, mount_unsubscribe_endpoint);
4190
4191 if idempotency_enabled {
4197 let env_disabled = std::env::var("AUTUMN_IDEMPOTENCY__ENABLED")
4198 .is_ok_and(|v| matches!(v.to_lowercase().as_str(), "false" | "0" | "no" | "off"));
4199 if !env_disabled && config.idempotency.enabled != Some(false) {
4202 config.idempotency.enabled = Some(true);
4203 }
4204 }
4205
4206 #[cfg(feature = "embed-assets")]
4211 register_embedded_static_dir(embedded_static);
4212
4213 #[cfg(all(feature = "embed-assets", feature = "i18n"))]
4214 let i18n_bundle = embedded_i18n_bundle(i18n_bundle, embedded_locales, &config);
4215
4216 #[cfg(feature = "i18n")]
4217 let i18n_bundle =
4218 resolve_i18n_bundle(i18n_bundle, i18n_auto_load, &config, &crate::config::OsEnv);
4219
4220 if let Err(message) =
4228 validate_pre_router_preconditions(&all_routes, &scoped_groups, &config)
4229 {
4230 panic!("{message}");
4231 }
4232
4233 let profile_display = config.profile.as_deref().unwrap_or("none");
4235 tracing::info!(
4236 version = env!("CARGO_PKG_VERSION"),
4237 profile = profile_display,
4238 "Autumn starting"
4239 );
4240
4241 let show_config = std::env::var("AUTUMN_SHOW_CONFIG").as_deref() == Ok("1");
4243 if show_config {
4244 log_startup_transparency(&all_routes, &tasks, &scoped_groups, &config);
4245 }
4246
4247 fail_fast_on_invalid_session_config(&config, session_store.is_some());
4251
4252 fail_fast_on_invalid_signing_secret(&config);
4255 fail_fast_on_missing_encryption_keys(&config);
4256 fail_fast_on_invalid_trusted_hosts(&config);
4257
4258 fail_fast_on_invalid_webhook_config(&config);
4262
4263 fail_fast_on_invalid_idempotency_config(&config);
4265
4266 #[cfg(feature = "storage")]
4274 let storage_bootstrap = blob_store.map_or_else(
4275 || preflight_storage(&config),
4276 |store| {
4277 Some(StorageBootstrap {
4278 store,
4279 serving: None,
4280 })
4281 },
4282 );
4283
4284 #[cfg(all(feature = "db", feature = "reporting", not(feature = "sqlite")))]
4298 let pool_provider_factory =
4299 crate::capsule::record_db::maybe_capture_pool_provider(pool_provider_factory, &config);
4300 #[cfg(feature = "db")]
4301 let database = setup_database(
4302 &config,
4303 migrations,
4304 pool_provider_factory,
4305 shard_provider_factory,
4306 shard_router,
4307 directory_shard_router,
4308 RepositoryCommitHookQueueMigrationMode::Runtime,
4309 )
4310 .await
4311 .unwrap_or_else(|e| {
4312 tracing::error!("{e}");
4313 std::process::exit(1);
4314 });
4315 #[cfg(feature = "db")]
4316 let pool = database.topology;
4317 #[cfg(feature = "db")]
4318 let shards = database.shards;
4319 #[cfg(feature = "db")]
4320 let replica_readiness = database.replica_readiness;
4321 #[cfg(feature = "db")]
4322 let replica_migration_check = database.replica_migration_check;
4323
4324 #[cfg(feature = "db")]
4325 if pool.is_some() || shards.is_some() {
4326 let shard_max_connections = shards
4329 .as_ref()
4330 .map_or(0, crate::sharding::ShardSet::total_max_connections);
4331 let control_max_connections = pool.as_ref().map_or(0, |topology| {
4332 topology.primary().status().max_size
4333 + topology.replica().map_or(0, |p| p.status().max_size)
4334 });
4335 let total_max_connections = control_max_connections + shard_max_connections;
4336 tracing::info!(
4337 primary_max_connections = config.database.effective_primary_pool_size(),
4338 replica_configured = config.database.replica_url.is_some(),
4339 replica_max_connections = config.database.effective_replica_pool_size(),
4340 shard_count = shards.as_ref().map_or(0, crate::sharding::ShardSet::len),
4341 total_max_connections,
4342 "Database topology configured"
4343 );
4344 let warn_threshold = config.database.max_connections_warn_threshold;
4347 if crate::config::should_warn_total_connections(total_max_connections, warn_threshold) {
4348 tracing::warn!(
4349 total_max_connections,
4350 warn_threshold,
4351 "Aggregate database connection count is high: the control \
4352 topology and all shard pools together may open \
4353 {total_max_connections} connections (warn threshold \
4354 {warn_threshold}). Ensure each Postgres server's \
4355 max_connections (plus headroom for migrations and \
4356 psql) exceeds the pools that target it, or lower \
4357 database.pool_size. Set \
4358 database.max_connections_warn_threshold = 0 to silence."
4359 );
4360 }
4361 for warning in crate::db::pooler::pooler_warnings(&config.database) {
4363 tracing::warn!("{warning}");
4364 }
4365 if config.database.replica_max_lag_ms.is_some_and(|ms| ms > 0)
4366 && config.database.replica_url.is_none()
4367 {
4368 tracing::warn!(
4369 "database.replica_max_lag_ms has no effect: database.replica_url is not set"
4370 );
4371 }
4372 } else {
4373 tracing::info!("Database not configured");
4374 }
4375
4376 #[cfg(feature = "redis")]
4390 let mut unused_redis_subsystems: Vec<&'static str> = Vec::new();
4391 #[cfg(feature = "redis")]
4392 if session_store.is_some() {
4393 unused_redis_subsystems.push("sessions");
4394 }
4395 #[cfg(all(feature = "redis", feature = "ws"))]
4396 if channels_backend.is_some() {
4397 unused_redis_subsystems.push("channels");
4398 }
4399
4400 let mut state = build_state(
4402 &config,
4403 #[cfg(feature = "db")]
4404 pool.as_ref(),
4405 #[cfg(feature = "db")]
4406 shards,
4407 #[cfg(feature = "ws")]
4408 channels_backend,
4409 );
4410
4411 #[cfg(feature = "reporting")]
4415 if config.failure_capture.enabled {
4416 let recording =
4417 std::sync::Arc::new(crate::capsule::RecordingClock::new(state.clock_arc()))
4418 as std::sync::Arc<dyn crate::time::ClockSource>;
4419 state = state.with_clock(recording);
4420 }
4421 #[cfg(feature = "reporting")]
4426 if config.failure_capture.enabled {
4427 let recording =
4428 std::sync::Arc::new(crate::capsule::RecordingEntropy::new(state.entropy_arc()))
4429 as std::sync::Arc<dyn crate::entropy::Entropy>;
4430 state = state.with_entropy(recording);
4431 }
4432
4433 if let Some(buf) = telemetry_guard.log_buffer.clone() {
4436 state.insert_extension(buf);
4437 }
4438 if let Some(handle) = telemetry_guard.filter_reload.clone() {
4442 state.log_levels().attach_reload_handle(handle);
4443 }
4444
4445 let maintenance_state = crate::maintenance::MaintenanceState::new();
4458 let flag_path = crate::maintenance::resolve_flag_file_path();
4459 if let Ok(Some(cfg)) = crate::maintenance::MaintenanceState::load_from_file(&flag_path) {
4460 maintenance_state.enable(cfg);
4461 }
4462 state.insert_extension(maintenance_state.clone());
4463
4464 let poller_state = maintenance_state.clone();
4465 tokio::spawn(async move {
4466 let path = crate::maintenance::resolve_flag_file_path();
4467 let interval = std::time::Duration::from_millis(500);
4468 loop {
4469 let poll_path = path.clone();
4470 let load_res = crate::time::spawn_blocking(move || {
4471 crate::maintenance::MaintenanceState::load_from_file(&poll_path)
4472 })
4473 .await;
4474
4475 match load_res {
4476 Ok(Ok(Some(cfg))) => {
4477 if poller_state.get() != Some(cfg.clone()) {
4478 poller_state.enable(cfg);
4479 }
4480 }
4481 Ok(Ok(None)) => {
4482 if poller_state.is_active() {
4483 poller_state.disable();
4484 }
4485 }
4486 Ok(Err(e)) => {
4487 tracing::error!(error = %e, "failed to load maintenance flag file");
4488 }
4489 Err(e) => {
4490 tracing::error!(error = %e, "maintenance poller task panicked");
4491 }
4492 }
4493 tokio::time::sleep(interval).await;
4494 }
4495 });
4496
4497 let canary_state = crate::canary::CanaryState::from_env();
4501 if canary_state.is_canary() {
4502 tracing::info!(
4503 version = canary_state.version(),
4504 "canary: replica labelled as canary cohort"
4505 );
4506 }
4507 state.insert_extension(canary_state);
4508
4509 if crate::canary::CanaryState::rollback_flag_present(std::path::Path::new(
4514 crate::canary::CANARY_ROLLBACK_FLAG_FILE,
4515 )) {
4516 tracing::warn!(
4517 "canary: rollback flag present at startup; /ready will report draining until \
4518 the flag is cleared (`autumn canary promote`)"
4519 );
4520 state.begin_shutdown();
4521 }
4522
4523 #[cfg(feature = "mail")]
4536 if let Some(interceptor) = mail_interceptor {
4537 state.insert_extension(interceptor);
4538 }
4539 if let Some(interceptor) = job_interceptor {
4540 state.insert_extension(interceptor);
4541 }
4542 #[cfg(feature = "db")]
4543 if let Some(interceptor) = db_interceptor {
4544 state.insert_extension(interceptor);
4545 }
4546 #[cfg(feature = "ws")]
4547 if let Some(interceptor) = channels_interceptor {
4548 state.insert_extension(interceptor.clone());
4549 state.channels = crate::channels::Channels::with_shared_backend(std::sync::Arc::new(
4550 crate::channels::InterceptedChannelsBackend::new(
4551 state.channels.backend().clone(),
4552 vec![interceptor],
4553 ),
4554 ));
4555 #[cfg(feature = "presence")]
4556 {
4557 state.presence = crate::presence::Presence::new(state.channels.clone());
4558 #[cfg(feature = "collab")]
4562 {
4563 state.collab = crate::collab::CollabHub::new(
4564 state.channels.clone(),
4565 state.presence.clone(),
4566 );
4567 }
4568 }
4569 }
4570 #[cfg(feature = "oauth2")]
4571 if let Some(interceptor) = http_interceptor {
4572 state.insert_extension(interceptor);
4573 }
4574
4575 for (name, source) in metrics_sources {
4579 if let Err(e) = state.metrics_source_registry.register(name, source) {
4580 tracing::warn!("{e}");
4581 }
4582 }
4583
4584 for (name, group, indicator) in health_indicators {
4586 if let Err(e) = state
4587 .health_indicator_registry
4588 .register(name, group, indicator)
4589 {
4590 tracing::warn!("{e}");
4591 }
4592 }
4593
4594 #[cfg(feature = "db")]
4602 let mut replication_worker: Option<crate::replication::Replicator> = None;
4603 #[cfg(feature = "db")]
4604 if let Some(replication_config) = config.replication.clone() {
4605 let database_url = config
4606 .database
4607 .primary_url
4608 .clone()
4609 .or_else(|| config.database.url.clone())
4610 .unwrap_or_default();
4611 let profile = config.profile.clone().unwrap_or_else(|| "dev".to_owned());
4612 #[cfg(feature = "storage")]
4617 let storage_destination = (config.storage.backend
4618 == crate::storage::StorageBackend::S3)
4619 .then(|| {
4620 config
4621 .storage
4622 .s3
4623 .bucket
4624 .clone()
4625 .map(|bucket| (bucket, config.storage.s3.endpoint.clone()))
4626 })
4627 .flatten();
4628 #[cfg(not(feature = "storage"))]
4629 let storage_destination: Option<(String, Option<String>)> = None;
4630 let clock = state.clock_arc();
4634 #[cfg(feature = "sqlite")]
4635 let fleet_lag_alert_after = replication_config
4636 .rpo()
4637 .saturating_mul(crate::replication::LAG_ALERT_MULTIPLIER);
4638
4639 let built = crate::time::spawn_blocking(move || {
4640 crate::replication::build(
4641 &replication_config,
4642 &database_url,
4643 &profile,
4644 storage_destination.as_ref().map(|(bucket, endpoint)| {
4645 crate::replication::StorageDestination {
4646 bucket,
4647 endpoint: endpoint.as_deref(),
4648 }
4649 }),
4650 clock,
4651 )
4652 })
4653 .await;
4654
4655 match built {
4656 Ok(Ok(runtime)) => {
4657 tracing::info!(
4658 database = %runtime.settings.database_path.display(),
4659 destination = %runtime.status.snapshot().destination,
4660 prefix = %runtime.settings.root,
4661 sync_interval_secs = runtime.settings.sync_interval.as_secs(),
4662 "continuous SQLite replication is enabled"
4663 );
4664 if let Err(e) = state.health_indicator_registry.register(
4665 crate::replication::INDICATOR_NAME,
4666 crate::actuator::IndicatorGroup::HealthOnly,
4667 std::sync::Arc::clone(&runtime.indicator)
4668 as std::sync::Arc<dyn crate::actuator::HealthIndicator>,
4669 ) {
4670 tracing::warn!("{e}");
4671 }
4672 #[cfg(feature = "sqlite")]
4675 if let Err(e) =
4676 start_fleet_replication(&state, &config, &runtime, fleet_lag_alert_after)
4677 {
4678 tracing::error!("SQLite fleet replication could not start: {e}");
4679 std::process::exit(1);
4680 }
4681 replication_worker = Some(runtime.replicator);
4682 }
4683 Ok(Err(crate::replication::SetupError::Disabled)) => {}
4684 Ok(Err(e)) => {
4689 tracing::error!("Continuous SQLite replication could not start: {e}");
4690 #[cfg(feature = "managed-pg")]
4691 crate::managed_pg::emergency_stop_async().await;
4692 std::process::exit(1);
4693 }
4694 Err(e) => {
4695 tracing::error!("Continuous SQLite replication setup panicked: {e}");
4696 #[cfg(feature = "managed-pg")]
4697 crate::managed_pg::emergency_stop_async().await;
4698 std::process::exit(1);
4699 }
4700 }
4701 }
4702
4703 #[cfg(feature = "acme")]
4708 let acme_status: Option<crate::acme::renewal::AcmeStatus> =
4709 if let Some(acme_cfg) = config.server.tls.as_ref().and_then(|t| t.acme.as_ref()) {
4710 let status = crate::acme::renewal::AcmeStatus::new();
4711 let indicator = std::sync::Arc::new(
4712 crate::acme::renewal::AcmeHealthIndicator::new(
4713 status.clone(),
4714 acme_cfg.renew_before_days,
4715 )
4716 .with_dns_provider(acme_cfg.dns.as_ref().map(|dns| dns.provider.as_str())),
4720 );
4721 if let Err(e) = state.health_indicator_registry.register(
4722 "acme",
4723 crate::actuator::IndicatorGroup::HealthOnly,
4724 indicator,
4725 ) {
4726 tracing::warn!("{e}");
4727 }
4728 Some(status)
4729 } else {
4730 None
4731 };
4732
4733 #[cfg(feature = "db")]
4734 configure_replica_migration_check(&state, replica_migration_check);
4735 #[cfg(feature = "db")]
4736 apply_replica_migration_readiness(&state, replica_readiness);
4737 if let Some(cache) = cache_backend {
4738 let cache = crate::cache::with_capsule_seam(cache);
4740 crate::cache::set_global_cache(cache.clone());
4741 state.shared_cache = Some(cache);
4742 } else {
4743 crate::cache::clear_global_cache();
4744 }
4745 state.insert_extension(RegisteredApiVersions(api_versions));
4746
4747 #[cfg(all(feature = "acme", feature = "reporting"))]
4752 let acme_reporters = error_reporters.clone();
4753
4754 #[cfg(feature = "reporting")]
4758 if !error_reporters.is_empty() {
4759 state.insert_extension(crate::reporting::RegisteredReporters(error_reporters));
4760 }
4761 for register in policy_registrations {
4766 register(state.policy_registry());
4767 }
4768 validate_repository_policies_registered(
4774 &all_routes,
4775 &scoped_groups,
4776 state.policy_registry(),
4777 &config,
4778 );
4779 #[cfg(feature = "mail")]
4780 if let Some(handle) = suppression_store {
4781 state.insert_extension(handle);
4782 }
4783 #[cfg(feature = "mail")]
4784 if let Some(handle) = mail_suppression_store {
4785 state.insert_extension(handle);
4786 }
4787 #[cfg(feature = "db")]
4790 {
4791 crate::outbox::install(&state, &config.outbox, outbox_handlers);
4792 if state.extension::<crate::outbox::OutboxRelay>().is_some()
4793 && let Err(error) = crate::outbox::ensure_relay_schema(&state).await
4794 {
4795 tracing::error!(error = %error, "Failed to create the outbox tables");
4796 exit_stop_managed_pg();
4797 std::process::exit(1);
4798 }
4799 }
4800 #[cfg(feature = "mail")]
4801 crate::mail::install_mailer_with_factory(
4802 &state,
4803 &config.mail,
4804 mail_delivery_queue_factory,
4805 true,
4806 )
4807 .unwrap_or_else(|error| {
4808 tracing::error!(error = %error, "Failed to configure mailer");
4809 exit_stop_managed_pg();
4810 std::process::exit(1);
4811 });
4812 #[cfg(feature = "mail")]
4813 state.insert_extension(crate::mail::MailPreviewRegistry::new(mail_previews));
4814 #[cfg(feature = "maud")]
4815 install_story_registry(&state, story_gallery);
4816 crate::alerts::install_from_config(&state, &config.alerts, alert_channels);
4822 #[cfg(feature = "mcp")]
4828 if mcp.is_some()
4829 && !audit_logger
4830 .as_ref()
4831 .is_some_and(|logger| logger.is_enabled())
4832 {
4833 tracing::warn!(
4834 target: "autumn.agent",
4835 "MCP is mounted with no audit sink installed: agent tool calls will be traced \
4836 but not recorded. Install one with `AppBuilder::with_audit_sink(..)`. \
4837 See docs/guide/agent-authority.md"
4838 );
4839 }
4840 if let Some(logger) = audit_logger {
4841 state.insert_extension::<crate::audit::AuditLogger>((*logger).clone());
4842 }
4843 #[cfg(feature = "i18n")]
4844 let custom_layers =
4845 install_i18n_bundle_layer(custom_layers, &state, i18n_bundle, &config.i18n);
4846
4847 #[cfg(feature = "storage")]
4851 let storage_router = storage_bootstrap.and_then(|b| b.install(&state));
4852 install_webhook_registry(&state, &config);
4853 run_state_initializers(state_initializers, &state);
4854 crate::cost::install(&state, &config);
4855 if let Some(failure) = state.extension::<crate::upgrade::LiveStateInstallFailure>() {
4860 tracing::error!("refusing to start: {}", failure.0);
4861 #[cfg(feature = "managed-pg")]
4862 crate::managed_pg::emergency_stop_async().await;
4863 std::process::exit(1);
4864 }
4865 finalize_event_bus(listeners, &mut jobs, &state);
4866
4867 #[cfg(feature = "redis")]
4872 {
4873 if jobs.is_empty() {
4874 unused_redis_subsystems.push("jobs");
4875 }
4876 unused_redis_subsystems.extend(crate::redis_health::unused_for_role(role));
4877 crate::redis_health::register_redis_health_indicators(
4878 &config,
4879 &state.health_indicator_registry,
4880 &crate::redis_health::app_pingers(&state),
4881 &unused_redis_subsystems,
4882 );
4883 }
4884
4885 let env = crate::config::OsEnv;
4886 let dist_dir = project_dir("dist", &env);
4887 let dist_ref = if dist_dir.exists() {
4888 Some(dist_dir.as_path())
4889 } else {
4890 None
4891 };
4892 #[cfg_attr(
4893 not(any(feature = "storage", feature = "inbound-mail")),
4894 allow(unused_mut)
4895 )]
4896 let mut merge_routers = merge_routers;
4897 #[cfg(feature = "storage")]
4898 if let Some(router) = storage_router {
4899 merge_routers.push(router);
4900 }
4901
4902 exclude_static_routes_from_locale_prefix(&mut config, &static_metas);
4908 state.insert_extension(config.clone());
4916 state.insert_extension(crate::aeo::RegisteredAgentSkills(agent_skills));
4917 state.insert_extension(commerce_docs);
4918
4919 if !seo_sources.is_empty() || crate::seo::has_seo_config(&config.seo) {
4924 let seo_cfg = &config.seo;
4925 let raw_profile = config.profile.as_deref().unwrap_or("dev");
4926 let profile = crate::seo::effective_seo_profile(raw_profile, seo_cfg.robots.allow_all);
4927 let static_paths: Vec<&str> = static_metas
4931 .iter()
4932 .filter(|m| !crate::seo::defaults_exclude_from_sitemap(m.seo))
4933 .map(|m| m.path)
4934 .collect();
4935 let (robots_body, sitemap_body) = crate::seo::assemble_seo_bodies_with_policy(
4936 profile,
4937 seo_cfg.base_url.as_deref(),
4938 seo_cfg.robots.sitemap_url.as_deref(),
4939 &seo_cfg.robots.additional_rules,
4940 &seo_sources,
4941 &static_paths,
4942 sitemap_locale_config(&config),
4943 &crate::aeo::robots_policy(&config),
4944 )
4945 .await;
4946 let seo_router = crate::seo::build_seo_router_from_bodies(robots_body, sitemap_body);
4947 let is_seo_path = |p: &str| p == "/robots.txt" || p == "/sitemap.xml";
4948 let seo_collision = all_routes.iter().any(|r| is_seo_path(r.path))
4949 || static_metas.iter().any(|m| is_seo_path(m.path))
4950 || scoped_groups.iter().any(|g| {
4951 let prefix = g.prefix.trim_end_matches('/');
4952 g.routes
4953 .iter()
4954 .any(|r| is_seo_path(&format!("{prefix}{}", r.path)))
4955 })
4956 || declared_routes.iter().any(|r| {
4967 (r.method.eq_ignore_ascii_case("GET")
4975 || r.method.eq_ignore_ascii_case("WS"))
4976 && is_seo_path(&r.path)
4977 });
4978 if seo_collision {
4979 tracing::warn!(
4980 "seo: /robots.txt or /sitemap.xml is already registered by the application; \
4981 skipping automatic SEO routes to prevent a startup panic"
4982 );
4983 } else {
4984 merge_routers.push(seo_router);
4985 }
4986 }
4987
4988 #[cfg(feature = "inbound-mail")]
4989 if let Some(ref im_router) = inbound_mail_router {
4990 let mut registered_inbound: std::collections::HashSet<String> =
4991 std::collections::HashSet::new();
4992 for (path, axum_router) in crate::inbound_mail::build_routes(im_router) {
4993 if all_routes
4998 .iter()
4999 .any(|r| r.method == http::Method::POST && r.path == path)
5000 || scoped_groups.iter().any(|g| {
5001 g.routes.iter().any(|r| {
5002 r.method == http::Method::POST
5003 && crate::router::join_nested_path(&g.prefix, r.path)
5004 == path.as_str()
5005 })
5006 })
5007 || nest_routers.iter().any(|(nest_path, _)| {
5008 let p = nest_path.as_str();
5009 path.as_str() == p
5010 || path.starts_with(p)
5011 && (p.ends_with('/') || path.as_bytes().get(p.len()) == Some(&b'/'))
5012 })
5013 {
5014 tracing::warn!(
5015 path = %path,
5016 "inbound_mail: skipping webhook route — a POST handler is \
5017 already registered at this path by the application"
5018 );
5019 continue;
5020 }
5021 if !registered_inbound.insert(path.clone()) {
5024 tracing::warn!(
5025 path = %path,
5026 "inbound_mail: skipping duplicate inbound webhook path"
5027 );
5028 continue;
5029 }
5030 config.security.csrf.exempt_paths.push(path.clone());
5034 config.security.captcha_exempt_paths.push(path);
5035 merge_routers.push(axum_router);
5036 }
5037 }
5038
5039 let (graph_mounted, graph_opaque) = if role.serves_http() {
5062 (
5063 graph_mounted_routes(
5064 &all_routes,
5065 &scoped_groups,
5066 &declared_routes,
5067 &config,
5068 #[cfg(feature = "openapi")]
5069 openapi.as_ref(),
5070 ),
5071 omitted_router_count(
5072 merge_routers.len(),
5073 nest_routers.iter().map(|(prefix, _)| prefix.as_str()),
5074 &declared_routes,
5075 ),
5076 )
5077 } else {
5078 (graph_worker_mounted_routes(&config), 0)
5079 };
5080 crate::graph::install(
5081 &state,
5082 crate::graph::manifest::audit(&graph_mounted, graph_opaque),
5083 );
5084
5085 let router_build = if role.serves_http() {
5086 crate::router::try_build_router_with_static_inner(
5087 all_routes,
5088 &config,
5089 state.clone(),
5090 dist_ref,
5091 crate::router::RouterContext {
5092 exception_filters,
5093 scoped_groups,
5094 merge_routers,
5095 nest_routers,
5096 declared_routes,
5101 custom_layers,
5102 static_gate_layers,
5103 #[cfg(feature = "maud")]
5104 error_page_renderer,
5105 session_store,
5106 #[cfg(feature = "openapi")]
5109 openapi: if config.openapi_runtime.enabled {
5110 openapi
5111 } else {
5112 None
5113 },
5114 #[cfg(feature = "mcp")]
5115 mcp,
5116 #[cfg(feature = "webmcp")]
5117 webmcp,
5118 },
5119 )
5120 } else {
5121 crate::router::try_build_probe_only_router(&config, state.clone())
5122 };
5123 let router = router_build.unwrap_or_else(|error| {
5124 tracing::error!(error = %error, "Failed to build router");
5125 exit_stop_managed_pg();
5126 std::process::exit(1);
5127 });
5128
5129 if let Err(e) = config.validate_push() {
5145 tracing::error!("Invalid [push] configuration: {e}");
5146 #[cfg(feature = "managed-pg")]
5147 crate::managed_pg::emergency_stop_async().await;
5148 std::process::exit(1);
5149 }
5150
5151 if let Some(tls_cfg) = config.server.tls.as_ref() {
5159 if let Err(msg) = tls_cfg.validate() {
5164 tracing::error!("Invalid [server.tls] configuration: {msg}");
5165 #[cfg(feature = "managed-pg")]
5166 crate::managed_pg::emergency_stop_async().await;
5167 std::process::exit(1);
5168 }
5169 #[cfg(not(feature = "tls"))]
5170 {
5171 tracing::error!(
5172 "[server.tls] is configured but this binary was built without the `tls` \
5173 feature; rebuild with `--features tls`, or remove [server.tls] to serve \
5174 plain HTTP"
5175 );
5176 #[cfg(feature = "managed-pg")]
5177 crate::managed_pg::emergency_stop_async().await;
5178 std::process::exit(1);
5179 }
5180 #[cfg(all(feature = "tls", not(feature = "acme")))]
5184 if tls_cfg.acme.is_some() {
5185 tracing::error!(
5186 "[server.tls.acme] is configured but this binary was built without the \
5187 `acme` feature; rebuild with `--features acme`, or configure a static \
5188 cert_path/key_path instead"
5189 );
5190 #[cfg(feature = "managed-pg")]
5191 crate::managed_pg::emergency_stop_async().await;
5192 std::process::exit(1);
5193 }
5194 #[cfg(feature = "tls")]
5195 if config.server.unix_socket.is_some() {
5196 tracing::error!(
5197 "[server.tls] cannot be combined with server.unix_socket; direct TLS \
5198 terminates on host:port. Unset one of them"
5199 );
5200 #[cfg(feature = "managed-pg")]
5201 crate::managed_pg::emergency_stop_async().await;
5202 std::process::exit(1);
5203 }
5204 }
5205
5206 let server_shutdown = tokio_util::sync::CancellationToken::new();
5210
5211 #[cfg(feature = "tls")]
5214 let mut tls_reload_state: Option<crate::tls::CertReloader> = None;
5215 #[cfg(feature = "tls")]
5219 let mut client_trust_reload: Option<crate::tls::client_auth::ClientTrustReloader> = None;
5220
5221 #[cfg(feature = "acme")]
5224 let mut acme_bind_state: Option<AcmeBindState> = None;
5225
5226 let bound_endpoint: String;
5232 let (bound_listener, bound_desc, unix_socket_cleanup): (
5233 BoundListener,
5234 String,
5235 Option<(std::path::PathBuf, u64, u64)>,
5236 ) = if let Some(socket_path) = config.server.unix_socket.as_deref() {
5237 let _ = socket_path;
5238 #[cfg(unix)]
5239 {
5240 use std::os::unix::fs::PermissionsExt;
5241
5242 let path = std::path::Path::new(socket_path);
5243 if let Err(e) = prepare_unix_socket_path(path) {
5244 tracing::error!(socket = %socket_path, "Failed to prepare unix socket: {e}");
5245 #[cfg(feature = "managed-pg")]
5248 crate::managed_pg::emergency_stop_async().await;
5249 std::process::exit(1);
5250 }
5251 let bind_result = {
5262 static UMASK_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
5263 let _umask_guard = UMASK_LOCK
5264 .lock()
5265 .unwrap_or_else(std::sync::PoisonError::into_inner);
5266 let prev_umask =
5267 nix::sys::stat::umask(nix::sys::stat::Mode::from_bits_truncate(0o177));
5268 let result = tokio::net::UnixListener::bind(path);
5269 nix::sys::stat::umask(prev_umask);
5270 result
5271 };
5272 let listener = match bind_result {
5273 Ok(listener) => listener,
5274 Err(e) => {
5275 tracing::error!(socket = %socket_path, "Failed to bind unix socket: {e}");
5276 #[cfg(feature = "managed-pg")]
5277 crate::managed_pg::emergency_stop_async().await;
5278 std::process::exit(1);
5279 }
5280 };
5281 if let Err(e) =
5287 std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600))
5288 {
5289 tracing::error!(socket = %socket_path, "Failed to enforce owner-only permissions on unix socket: {e}");
5290 let _ = std::fs::remove_file(path);
5291 #[cfg(feature = "managed-pg")]
5292 crate::managed_pg::emergency_stop_async().await;
5293 std::process::exit(1);
5294 }
5295 let (dev, ino) = {
5298 use std::os::unix::fs::MetadataExt;
5299 std::fs::metadata(path).map_or((0, 0), |m| (m.dev(), m.ino()))
5300 };
5301 bound_endpoint = format!("unix {socket_path}");
5302 (
5303 BoundListener::Unix(listener),
5304 format!("unix:{socket_path}"),
5305 Some((path.to_path_buf(), dev, ino)),
5306 )
5307 }
5308 #[cfg(not(unix))]
5309 {
5310 tracing::error!(
5311 "server.unix_socket is only supported on Unix platforms; \
5312 unset it or use server.host/server.port"
5313 );
5314 std::process::exit(1);
5315 }
5316 } else {
5317 #[cfg(feature = "tls")]
5324 if config.server.tls.is_some() && crate::upgrade::handoff_requested() {
5325 tracing::error!(
5326 "refusing to start: this build terminates TLS ([server.tls]) but was \
5327 handed the previous build's plaintext listening socket. An in-place \
5328 upgrade cannot change the transport — restart the process to apply it. \
5329 The previous build keeps serving"
5330 );
5331 #[cfg(feature = "managed-pg")]
5332 crate::managed_pg::emergency_stop_async().await;
5333 std::process::exit(1);
5334 }
5335 let configured_addr = format!("{}:{}", config.server.host, config.server.port);
5336 let listener = bind_or_adopt_tcp_listener(&configured_addr).await;
5337 let addr = listener
5342 .local_addr()
5343 .map_or(configured_addr, |bound| bound.to_string());
5344 bound_endpoint = format!("tcp {}", dialable_endpoint(&addr));
5345 #[cfg(feature = "tls")]
5352 {
5353 if let Some(tls_cfg) = config.server.tls.as_ref() {
5354 #[cfg(feature = "acme")]
5358 if let Some(acme_cfg) = tls_cfg.acme.as_ref() {
5359 let https_port = config.server.port;
5360 match build_acme_tls_listener(
5361 listener,
5362 tls_cfg,
5363 acme_cfg,
5364 config.tenancy.base_domain.as_deref(),
5365 &config.credentials,
5366 https_port,
5367 acme_status.clone(),
5368 server_shutdown.child_token(),
5369 )
5370 .await
5371 {
5372 Ok((tls_listener, bind_state)) => {
5373 acme_bind_state = Some(bind_state);
5374 (
5375 BoundListener::Tls(tls_listener),
5376 format!("https://{addr} (ACME)"),
5377 None,
5378 )
5379 }
5380 Err(e) => {
5381 tracing::error!(error = %e, "Failed to configure [server.tls.acme]");
5382 #[cfg(feature = "managed-pg")]
5383 crate::managed_pg::emergency_stop_async().await;
5384 std::process::exit(1);
5385 }
5386 }
5387 } else {
5388 match build_tls_listener(listener, tls_cfg, server_shutdown.child_token()) {
5389 Ok((tls_listener, reload, client_reload)) => {
5390 tls_reload_state = Some(reload);
5391 client_trust_reload = client_reload;
5392 (
5393 BoundListener::Tls(tls_listener),
5394 format!("https://{addr}"),
5395 None,
5396 )
5397 }
5398 Err(e) => {
5399 tracing::error!(error = %e, "Failed to configure [server.tls]");
5400 #[cfg(feature = "managed-pg")]
5401 crate::managed_pg::emergency_stop_async().await;
5402 std::process::exit(1);
5403 }
5404 }
5405 }
5406 #[cfg(not(feature = "acme"))]
5407 match build_tls_listener(listener, tls_cfg, server_shutdown.child_token()) {
5408 Ok((tls_listener, reload, client_reload)) => {
5409 tls_reload_state = Some(reload);
5410 client_trust_reload = client_reload;
5411 (
5412 BoundListener::Tls(tls_listener),
5413 format!("https://{addr}"),
5414 None,
5415 )
5416 }
5417 Err(e) => {
5418 tracing::error!(error = %e, "Failed to configure [server.tls]");
5419 #[cfg(feature = "managed-pg")]
5420 crate::managed_pg::emergency_stop_async().await;
5421 std::process::exit(1);
5422 }
5423 }
5424 } else {
5425 (BoundListener::Tcp(listener), addr, None)
5426 }
5427 }
5428 #[cfg(not(feature = "tls"))]
5429 {
5430 (BoundListener::Tcp(listener), addr, None)
5431 }
5432 };
5433
5434 let shutdown_timeout = config.server.shutdown_timeout_secs;
5435 if let Some(warning) = config.server.drain_window_warning() {
5436 tracing::warn!(
5437 shutdown_timeout_secs = shutdown_timeout,
5438 request_timeout_ms = config.server.timeouts.request_timeout_ms,
5439 "{warning}"
5440 );
5441 }
5442 let prestop_grace = config.server.prestop_grace_secs;
5443
5444 if let Err(error) = initialize_job_runtime(
5445 jobs,
5446 &state,
5447 &server_shutdown,
5448 &config.jobs,
5449 role.runs_workers(),
5450 ) {
5451 tracing::error!(error = %error, "job runtime initialization failed");
5452 #[cfg(feature = "managed-pg")]
5455 crate::managed_pg::emergency_stop_async().await;
5456 std::process::exit(1);
5457 }
5458
5459 let cluster_shutdown = tokio_util::sync::CancellationToken::new();
5475 if let Err(error) =
5476 crate::cluster::install_from_config(&state, &config.cluster, &cluster_shutdown)
5477 {
5478 tracing::error!(error = %error, "cluster installation failed");
5479 #[cfg(feature = "managed-pg")]
5480 crate::managed_pg::emergency_stop_async().await;
5481 std::process::exit(1);
5482 }
5483
5484 #[cfg(feature = "db")]
5485 {
5486 #[cfg(feature = "ws")]
5487 crate::repository_commit_hooks::set_global_channels(state.channels().clone());
5488 }
5489
5490 #[cfg(all(feature = "db", not(feature = "sqlite")))]
5498 if role.runs_workers()
5499 && let Some(pool) = state.pool().cloned()
5500 {
5501 #[cfg(feature = "ws")]
5502 {
5503 let channels = state.channels().clone();
5504 crate::repository_commit_hooks::start_repository_commit_hook_worker(
5505 pool,
5506 Some(channels),
5507 server_shutdown.child_token(),
5508 );
5509 }
5510 #[cfg(not(feature = "ws"))]
5511 crate::repository_commit_hooks::start_repository_commit_hook_worker(
5512 pool,
5513 server_shutdown.child_token(),
5514 );
5515 }
5516 #[cfg(feature = "db")]
5519 if role.runs_workers() {
5520 drop(crate::outbox::start_relay_worker(
5521 state.clone(),
5522 server_shutdown.child_token(),
5523 ));
5524 }
5525 #[cfg(all(feature = "db", not(feature = "sqlite")))]
5530 if role.runs_workers()
5531 && let Some(shards) = state.shards()
5532 {
5533 for shard in shards.iter() {
5534 #[cfg(feature = "ws")]
5535 crate::repository_commit_hooks::start_repository_commit_hook_worker(
5536 shard.primary_pool().clone(),
5537 Some(state.channels().clone()),
5538 server_shutdown.child_token(),
5539 );
5540 #[cfg(not(feature = "ws"))]
5541 crate::repository_commit_hooks::start_repository_commit_hook_worker(
5542 shard.primary_pool().clone(),
5543 server_shutdown.child_token(),
5544 );
5545 }
5546 }
5547 #[cfg(all(feature = "db", feature = "sqlite"))]
5553 if role.runs_workers()
5554 && let Some(pool) = state.pool().cloned()
5555 {
5556 #[cfg(feature = "ws")]
5557 {
5558 let channels = state.channels().clone();
5559 crate::repository_commit_hooks::start_repository_commit_hook_worker(
5560 pool,
5561 Some(channels),
5562 server_shutdown.child_token(),
5563 );
5564 }
5565 #[cfg(not(feature = "ws"))]
5566 crate::repository_commit_hooks::start_repository_commit_hook_worker(
5567 pool,
5568 server_shutdown.child_token(),
5569 );
5570 }
5571 #[cfg(feature = "sqlite")]
5578 let fleet_shutdown = tokio_util::sync::CancellationToken::new();
5579 #[cfg(feature = "sqlite")]
5580 let mut fleet_maintenance = start_fleet_runtime(
5581 &state,
5582 fleet_shutdown.clone(),
5583 Some(server_shutdown.child_token()),
5584 )
5585 .await;
5586
5587 #[cfg(feature = "db")]
5599 let replication_shutdown = tokio_util::sync::CancellationToken::new();
5600 #[cfg(feature = "db")]
5607 let mut replication_done: Option<tokio::sync::oneshot::Receiver<()>> = None;
5608 #[cfg(feature = "db")]
5609 if let Some(replicator) = replication_worker.take() {
5610 let replication_shutdown = replication_shutdown.clone();
5611 let (finished, waiter) = tokio::sync::oneshot::channel();
5612 match std::thread::Builder::new()
5613 .name("autumn-sqlite-replication".to_owned())
5614 .spawn(move || {
5615 replicator.run(&replication_shutdown);
5616 let _ = finished.send(());
5619 }) {
5620 Ok(_handle) => replication_done = Some(waiter),
5621 Err(e) => {
5627 tracing::error!("Could not start the SQLite replication thread: {e}");
5628 #[cfg(feature = "managed-pg")]
5629 crate::managed_pg::emergency_stop_async().await;
5630 std::process::exit(1);
5631 }
5632 }
5633 }
5634
5635 #[cfg(feature = "db")]
5638 let _replica_lag_monitor = spawn_replica_lag_monitor(&state, server_shutdown.child_token());
5639
5640 #[cfg(feature = "presence")]
5641 {
5642 let presence = state.presence().clone();
5643 let sweep_shutdown = server_shutdown.child_token();
5644 tokio::spawn(async move {
5645 let interval = std::time::Duration::from_secs(15);
5646 loop {
5647 tokio::select! {
5648 () = tokio::time::sleep(interval) => {
5649 presence.sweep_expired();
5650 }
5651 () = sweep_shutdown.cancelled() => break,
5652 }
5653 }
5654 });
5655 }
5656
5657 #[cfg(feature = "tls")]
5665 if let Some(reload) = tls_reload_state.take() {
5666 let reload_shutdown = server_shutdown.child_token();
5667 tokio::spawn(async move {
5668 reload.run(reload_shutdown).await;
5669 });
5670 }
5671
5672 #[cfg(feature = "tls")]
5677 if let Some(reload) = client_trust_reload.take() {
5678 let reload_shutdown = server_shutdown.child_token();
5679 tokio::spawn(async move {
5680 reload.run(reload_shutdown).await;
5681 });
5682 }
5683
5684 #[cfg(feature = "acme")]
5691 if let Some(bind_state) = acme_bind_state.take() {
5692 let AcmeBindState {
5693 mut renewal_task,
5694 tokens,
5695 http_challenge_port,
5696 https_port,
5697 dns01,
5698 custom_domains,
5699 client_trust_reload: acme_client_trust_reload,
5700 } = bind_state;
5701 let custom_domains_enabled = custom_domains.is_some();
5703 #[cfg(feature = "tls")]
5708 if let Some(reload) = acme_client_trust_reload {
5709 let reload_shutdown = server_shutdown.child_token();
5710 tokio::spawn(async move {
5711 reload.run(reload_shutdown).await;
5712 });
5713 }
5714
5715 let challenge_listeners =
5730 match crate::acme::challenge::bind_challenge_listeners(http_challenge_port).await {
5731 Ok(listeners) => listeners,
5732 Err(e) if dns01 && !custom_domains_enabled => {
5739 tracing::warn!(
5740 port = http_challenge_port,
5741 error = %e,
5742 "Could not bind the ACME challenge/redirect listener. DNS-01 issuance \
5743 does not need it, so startup continues — but visitors who type \
5744 http:// will not be redirected to HTTPS. Grant \
5745 CAP_NET_BIND_SERVICE, or set [server.tls.acme] http_challenge_port \
5746 to a port this process may bind"
5747 );
5748 Vec::new()
5749 }
5750 Err(e) => {
5751 if dns01 {
5752 tracing::error!(
5753 port = http_challenge_port,
5754 "Failed to bind the ACME HTTP-01 challenge listener: {e}. This \
5755 deployment issues its own certificate over DNS-01, which does \
5756 not need the listener — but [server.tls.acme.custom_domains] is \
5757 enabled, and a tenant's domain can only be validated over \
5758 HTTP-01. Grant CAP_NET_BIND_SERVICE, set [server.tls.acme] \
5759 http_challenge_port to a port a front-end forwards :80 to, or \
5760 disable custom domains"
5761 );
5762 #[cfg(feature = "managed-pg")]
5763 crate::managed_pg::emergency_stop_async().await;
5764 std::process::exit(1);
5765 }
5766 tracing::error!(
5767 port = http_challenge_port,
5768 "Failed to bind the ACME HTTP-01 challenge listener: {e}. Port \
5769 {http_challenge_port} typically needs privilege (grant \
5770 CAP_NET_BIND_SERVICE), or set [server.tls.acme] http_challenge_port \
5771 to a port a front-end forwards :80 to"
5772 );
5773 #[cfg(feature = "managed-pg")]
5774 crate::managed_pg::emergency_stop_async().await;
5775 std::process::exit(1);
5776 }
5777 };
5778 let challenge_router =
5779 crate::acme::challenge::challenge_router(tokens.clone(), https_port);
5780 for challenge_listener in challenge_listeners {
5784 let router = challenge_router.clone();
5785 let challenge_shutdown = server_shutdown.child_token();
5786 let limits = crate::http_server::HttpLimits::from(&config.server.http);
5788 tokio::spawn(async move {
5789 if let Err(e) = crate::http_server::serve(
5790 challenge_listener,
5791 router.into_make_service(),
5792 limits,
5793 async move {
5794 challenge_shutdown.cancelled().await;
5795 },
5796 )
5797 .await
5798 {
5799 tracing::error!(
5800 error = %e,
5801 "ACME challenge listener stopped with an error"
5802 );
5803 }
5804 });
5805 }
5806
5807 let mut leadership_degraded = false;
5819 let coordinator =
5820 match crate::scheduler::coordinator_from_config(&config.scheduler, &state) {
5821 Ok(c) => c,
5822 Err(e) => {
5823 tracing::warn!(
5824 error = %e,
5825 "ACME renewal: falling back to an in-process coordinator"
5826 );
5827 leadership_degraded = config.scheduler.backend.is_fleet_distributed();
5828 std::sync::Arc::new(crate::scheduler::InProcessSchedulerCoordinator::new(
5829 config.scheduler.resolved_replica_id(),
5830 ))
5831 }
5832 };
5833 renewal_task.leadership_degraded = leadership_degraded;
5834
5835 if let Some(message) = acme_fleet_warning(config.scheduler.backend, dns01) {
5842 tracing::warn!(scheduler_backend = coordinator.backend(), "{message}");
5843 }
5844
5845 #[cfg(feature = "reporting")]
5846 let reporter = make_acme_reporter(acme_reporters);
5847 #[cfg(not(feature = "reporting"))]
5848 let reporter = make_acme_reporter();
5849 let reporter = compose_acme_alert_reporter(reporter, &state);
5856 renewal_task.recovery = Some(make_acme_alert_recovery(&state));
5857 let renewal_shutdown = server_shutdown.child_token();
5858 let renewal_coordinator = std::sync::Arc::clone(&coordinator);
5859 tokio::spawn(async move {
5860 renewal_task
5861 .run(renewal_coordinator, reporter, renewal_shutdown)
5862 .await;
5863 });
5864
5865 if let Some(cd) = custom_domains {
5869 spawn_custom_domain_task(
5870 cd,
5871 tokens,
5872 std::sync::Arc::clone(&coordinator),
5873 leadership_degraded,
5874 &state,
5875 server_shutdown.child_token(),
5876 );
5877 }
5878 }
5879
5880 tracing::info!(bound = %bound_desc, "Listening");
5881
5882 let server_shutdown_wait = server_shutdown.clone();
5883 let after_method = tower::Layer::layer(
5894 &crate::middleware::MethodOverrideLayer::new()
5895 .with_max_scan_bytes(config.security.upload.max_request_size_bytes),
5896 router,
5897 );
5898 let service = tower::Layer::layer(
5899 &crate::security::TrustedProxiesLayer::from_config(&config.security.trusted_proxies),
5900 after_method,
5901 );
5902 #[cfg(unix)]
5915 let mut handoff_socket: Option<crate::upgrade::HandoffSocket> = None;
5916 #[cfg(unix)]
5917 if let BoundListener::Tcp(listener) = &bound_listener {
5918 match crate::upgrade::HandoffSocket::from_listener(listener) {
5919 Ok(socket) => handoff_socket = Some(socket),
5920 Err(e) => tracing::warn!(
5921 error = %e,
5922 "could not duplicate the listening socket; in-place upgrade \
5923 (SIGUSR2) will be refused for this process"
5924 ),
5925 }
5926 }
5927
5928 let http_limits = crate::http_server::HttpLimits::from(&config.server.http);
5953 let server_future: std::pin::Pin<
5954 Box<dyn std::future::Future<Output = std::io::Result<()>> + Send + 'static>,
5955 > = match bound_listener {
5956 BoundListener::Tcp(listener) => {
5957 let service = tower::Layer::layer(
5961 &axum::middleware::from_fn(crate::accept_drain::stamp_tcp_peer),
5962 service,
5963 );
5964 let make_service =
5965 axum::ServiceExt::<axum::extract::Request>::into_make_service_with_connect_info::<
5966 crate::accept_drain::TcpPeer,
5967 >(service);
5968 Box::pin(async move {
5969 crate::http_server::serve(
5970 crate::accept_drain::StopAcceptingOnShutdown::new(
5971 listener,
5972 server_shutdown_wait.clone(),
5973 ),
5974 make_service,
5975 http_limits,
5976 crate::accept_drain::drain_signal(server_shutdown_wait),
5977 )
5978 .await
5979 })
5980 }
5981 #[cfg(unix)]
5982 BoundListener::Unix(listener) => {
5983 let service = tower::Layer::layer(
5988 &axum::middleware::from_fn(stamp_loopback_connect_info),
5989 service,
5990 );
5991 let make_service =
5992 axum::ServiceExt::<axum::extract::Request>::into_make_service_with_connect_info::<
5993 UdsConnectInfo,
5994 >(service);
5995 Box::pin(async move {
5996 crate::http_server::serve(
5997 crate::accept_drain::StopAcceptingOnShutdown::new(
5998 listener,
5999 server_shutdown_wait.clone(),
6000 ),
6001 make_service,
6002 http_limits,
6003 crate::accept_drain::drain_signal(server_shutdown_wait),
6004 )
6005 .await
6006 })
6007 }
6008 #[cfg(feature = "tls")]
6017 BoundListener::Tls(mut listener) => {
6018 let handoff = http_limits
6021 .max_connections
6022 .and_then(|max| listener.limit_connections(max));
6023 let service =
6036 tower::Layer::layer(&crate::tls::client_auth::ClientIdentityLayer, service);
6037 let make_service =
6038 axum::ServiceExt::<axum::extract::Request>::into_make_service_with_connect_info::<
6039 crate::tls::TlsConnectInfo,
6040 >(service);
6041 Box::pin(async move {
6042 crate::http_server::serve_with_handoff(
6043 crate::accept_drain::StopAcceptingOnShutdown::new(
6044 listener,
6045 server_shutdown_wait.clone(),
6046 ),
6047 make_service,
6048 http_limits,
6049 handoff,
6050 crate::accept_drain::drain_signal(server_shutdown_wait),
6051 )
6052 .await
6053 })
6054 }
6055 };
6056
6057 let defer_accept_loop = crate::upgrade::handoff_requested();
6063 let mut pending_accept: Option<
6066 std::pin::Pin<
6067 Box<dyn std::future::Future<Output = std::io::Result<()>> + Send + 'static>,
6068 >,
6069 > = None;
6070 let mut server_task: Option<tokio::task::JoinHandle<std::io::Result<()>>> = None;
6071 if defer_accept_loop {
6072 pending_accept = Some(server_future);
6073 } else {
6074 server_task = Some(tokio::spawn(server_future));
6075 }
6076
6077 let upgrade_cutover = tokio_util::sync::CancellationToken::new();
6081 let upgrade_cutover_wait = upgrade_cutover.clone();
6082
6083 #[cfg(unix)]
6088 {
6089 let upgrade_config = config.server.upgrade.clone();
6090 let upgrade_state = state.clone();
6091 let cutover = upgrade_cutover.clone();
6092 let socket = handoff_socket.take();
6093 let watcher_shutdown = server_shutdown.child_token();
6100 tokio::spawn(async move {
6101 tokio::select! {
6102 () = watch_for_in_place_upgrade(
6103 &upgrade_config,
6104 upgrade_signal,
6105 socket,
6106 upgrade_state,
6107 cutover,
6108 ) => {}
6109 () = watcher_shutdown.cancelled() => {
6110 tracing::debug!(
6111 "shutting down: in-place upgrade is no longer available in this \
6112 process"
6113 );
6114 }
6115 }
6116 });
6117 }
6118
6119 let shutdown_state = state.clone();
6120 let shutdown_signal_token = server_shutdown.clone();
6121 let handler_shutdown = state.probes.shutdown_signal().clone();
6122 let shutdown_metrics = state.metrics.clone();
6124
6125 let drain_started_at: std::sync::Arc<std::sync::OnceLock<crate::time::MonotonicInstant>> =
6129 std::sync::Arc::new(std::sync::OnceLock::new());
6130 let drain_started_clone = std::sync::Arc::clone(&drain_started_at);
6131 let drain_started_real: std::sync::Arc<std::sync::OnceLock<std::time::Instant>> =
6135 std::sync::Arc::new(std::sync::OnceLock::new());
6136 let drain_started_real_clone = std::sync::Arc::clone(&drain_started_real);
6137 let drain_clock = state.clock_arc();
6141 let drain_clock_for_task = std::sync::Arc::clone(&drain_clock);
6142
6143 let drain_phase_notify = std::sync::Arc::new(tokio::sync::Notify::new());
6147 let drain_phase_notify_for_watchdog = std::sync::Arc::clone(&drain_phase_notify);
6148 let server_entered_drain = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
6151 let server_entered_drain_for_watchdog = std::sync::Arc::clone(&server_entered_drain);
6152
6153 let shutdown_task = tokio::spawn(async move {
6169 let cause = shutdown_signal(upgrade_cutover_wait).await;
6172 let upgrade_cutover = matches!(cause, DrainCause::UpgradeCutover);
6173
6174 if upgrade_cutover {
6175 tracing::info!(
6182 phase = "upgrade_cutover",
6183 shutdown_timeout_secs = shutdown_timeout,
6184 "shutdown: successor is serving; draining without a readiness flip"
6185 );
6186 } else {
6187 tracing::info!(
6188 phase = "signal_received",
6189 prestop_grace_secs = prestop_grace,
6190 shutdown_timeout_secs = shutdown_timeout,
6191 "shutdown: graceful shutdown initiated"
6192 );
6193
6194 shutdown_state.begin_shutdown();
6196 tracing::info!(phase = "ready_draining", "shutdown: /ready now 503");
6197
6198 if prestop_grace > 0 {
6200 tokio::time::sleep(std::time::Duration::from_secs(prestop_grace)).await;
6201 }
6202 }
6203 tracing::info!(phase = "listener_stopping", "shutdown: stopping listener");
6204
6205 handler_shutdown.cancel();
6208
6209 let _ = drain_started_clone.set(drain_clock_for_task.monotonic());
6213 #[allow(
6214 clippy::disallowed_methods,
6215 reason = "real time: the fleet flush bounds blocking I/O"
6216 )]
6217 let _ = drain_started_real_clone.set(std::time::Instant::now());
6218 shutdown_signal_token.cancel();
6219
6220 if !server_entered_drain_for_watchdog.load(std::sync::atomic::Ordering::Acquire) {
6231 tracing::warn!(
6232 phase = "signal_during_startup",
6233 "shutdown: SIGTERM during startup hooks; waiting for drain phase \
6234 to begin before enforcing the drain deadline"
6235 );
6236 drain_phase_notify_for_watchdog.notified().await;
6240 }
6241 tokio::time::sleep(std::time::Duration::from_secs(shutdown_timeout)).await;
6242 if shutdown_metrics.snapshot().http.requests_active == 0 {
6247 return;
6248 }
6249 let aborted = shutdown_metrics.snapshot().http.requests_active;
6250 shutdown_metrics.record_shutdown_aborted(aborted);
6251 tracing::error!(
6252 phase = "in_flight_drain",
6253 timeout_secs = shutdown_timeout,
6254 autumn_shutdown_aborted_requests_total = aborted,
6255 exit_code = 1,
6256 "shutdown: in_flight_drain phase exceeded deadline; terminating"
6257 );
6258 #[cfg(feature = "managed-pg")]
6262 crate::managed_pg::emergency_stop_async().await;
6263 std::process::exit(1);
6264 });
6265
6266 if let Err(error) = run_startup_hooks(&startup_hooks, state.clone()).await {
6267 tracing::error!(error = %error, "startup hook failed");
6268 if let Some(task) = server_task.take() {
6274 task.abort();
6275 }
6276 server_shutdown.cancel();
6277 #[cfg(feature = "managed-pg")]
6279 crate::managed_pg::emergency_stop_async().await;
6280 std::process::exit(1);
6281 }
6282
6283 let server_task: tokio::task::JoinHandle<std::io::Result<()>> =
6290 match (server_task.take(), pending_accept.take()) {
6291 (Some(task), _) => task,
6292 (None, Some(future)) => tokio::spawn(future),
6293 (None, None) => {
6296 unreachable!("accept-loop future consumed without spawning its task")
6297 }
6298 };
6299
6300 if !state.probes().is_shutting_down() {
6301 if role.runs_workers() && !tasks.is_empty() {
6305 if let Some(warning) =
6306 in_process_fleet_warning(&config, &tasks, &crate::config::OsEnv)
6307 {
6308 tracing::warn!("{warning}");
6309 }
6310 let res = start_task_scheduler_with_config(
6311 tasks,
6312 &state,
6313 &server_shutdown,
6314 &config.scheduler,
6315 );
6316 if let Err(err) = res {
6317 tracing::error!(error = %err, "scheduled task runtime initialization failed");
6318 server_shutdown.cancel();
6319 server_task.abort();
6320 #[cfg(feature = "managed-pg")]
6322 crate::managed_pg::emergency_stop_async().await;
6323 std::process::exit(1);
6324 }
6325 }
6326 state.probes().mark_startup_complete();
6327 #[cfg(unix)]
6332 {
6333 if let Err(reason) = crate::upgrade::verify_handover_complete() {
6334 tracing::error!("{reason}");
6335 #[cfg(feature = "managed-pg")]
6337 crate::managed_pg::emergency_stop_async().await;
6338 std::process::exit(1);
6339 }
6340 match crate::upgrade::publish_upgrade_readiness(|| {
6349 unfreeze_adopted_live_state(&state);
6350 }) {
6351 Ok(_) => {}
6352 Err(error) => {
6353 tracing::error!(
6354 error = %error,
6355 "refusing to start: this build took over but could not tell the \
6356 previous build it is serving, so the handover cannot complete. \
6357 The previous build keeps serving"
6358 );
6359 #[cfg(feature = "managed-pg")]
6361 crate::managed_pg::emergency_stop_async().await;
6362 std::process::exit(1);
6363 }
6364 }
6365 }
6366 signal_serve_ready(
6367 config
6368 .server
6369 .prestop_grace_secs
6370 .saturating_add(config.server.shutdown_timeout_secs),
6371 &bound_endpoint,
6372 );
6373 }
6374
6375 server_entered_drain.store(true, std::sync::atomic::Ordering::Release);
6380 drain_phase_notify.notify_one();
6381
6382 let server_result = server_task.await.unwrap_or_else(|e| {
6385 tracing::error!("Server task join error: {e}");
6386 exit_stop_managed_pg();
6390 std::process::exit(1);
6391 });
6392 shutdown_task.abort();
6394 server_result.unwrap_or_else(|e| {
6395 tracing::error!("Server error: {e}");
6396 exit_stop_managed_pg();
6397 std::process::exit(1);
6398 });
6399
6400 let elapsed_since_drain_start = || {
6405 drain_started_at
6406 .get()
6407 .map_or(std::time::Duration::ZERO, |started| {
6408 drain_clock.monotonic().saturating_duration_since(*started)
6409 })
6410 };
6411 let shutdown_budget = std::time::Duration::from_secs(shutdown_timeout);
6412
6413 #[cfg(feature = "sqlite")]
6430 if let Some(FleetRuntime {
6431 maintenance,
6432 sweeper,
6433 ownership,
6434 }) = fleet_maintenance.take()
6435 {
6436 let flush_budget = || {
6437 let real_elapsed = drain_started_real
6438 .get()
6439 .map_or(std::time::Duration::ZERO, std::time::Instant::elapsed);
6440 fleet_flush_budget(shutdown_budget, elapsed_since_drain_start(), real_elapsed)
6441 };
6442 if let Some(sweeper) = sweeper {
6445 let abort = sweeper.abort_handle();
6446 if tokio::time::timeout(flush_budget(), sweeper).await.is_err() {
6447 abort.abort();
6448 tracing::warn!("shutdown: the fleet sweeper did not stop within the budget");
6449 }
6450 }
6451 let wait = flush_budget();
6452 if let Some(replication) = state
6455 .shards()
6456 .and_then(|s| s.fleet())
6457 .and_then(|f| f.replication())
6458 {
6459 replication.set_shutdown_deadline_in(wait);
6460 }
6461 fleet_shutdown.cancel();
6462 if tokio::time::timeout(wait, maintenance).await.is_ok() {
6463 tracing::info!("shutdown: SQLite fleet databases closed");
6464 if let Some(ownership) = ownership {
6466 ownership.finish().await;
6467 }
6468 } else {
6469 tracing::warn!(
6470 timeout_secs = wait.as_secs(),
6471 "shutdown: SQLite fleet databases did not all close within the shutdown \
6472 budget; a replicated fleet's last frames may not be offsite"
6473 );
6474 if let Some(ownership) = ownership {
6477 drop(ownership.abandon());
6478 }
6479 }
6480 }
6481
6482 #[cfg(feature = "db")]
6483 if let Some(waiter) = replication_done.take() {
6484 replication_shutdown.cancel();
6487 let wait = shutdown_budget.saturating_sub(elapsed_since_drain_start());
6488 match tokio::time::timeout(wait, waiter).await {
6489 Ok(Ok(())) => {
6490 tracing::info!("shutdown: SQLite replication flushed and stopped");
6491 }
6492 Ok(Err(_)) => tracing::error!(
6493 "shutdown: the SQLite replication thread ended without finishing; the \
6494 frames committed since the last successful tick may not be offsite"
6495 ),
6496 Err(_) => tracing::warn!(
6500 timeout_secs = wait.as_secs(),
6501 "shutdown: SQLite replication did not finish its final flush within the \
6502 shutdown budget; the last frames may not be offsite"
6503 ),
6504 }
6505 }
6506
6507 if config.cluster.enabled {
6520 cluster_shutdown.cancel();
6521 let departure_wait =
6522 cluster_departure_wait(shutdown_budget, elapsed_since_drain_start());
6523 if !departure_wait.is_zero() {
6524 tokio::time::sleep(departure_wait).await;
6525 }
6526 }
6527
6528 let hook_budget = shutdown_budget.saturating_sub(elapsed_since_drain_start());
6534 run_shutdown_hooks_with_timeout(&shutdown_hooks, hook_budget, hook_budget).await;
6535 #[cfg(feature = "managed-pg")]
6541 crate::managed_pg::emergency_stop_async().await;
6542
6543 #[cfg(unix)]
6550 if let Some((path, dev, ino)) = &unix_socket_cleanup {
6551 use std::os::unix::fs::MetadataExt;
6552 let still_ours =
6553 std::fs::metadata(path).is_ok_and(|m| m.dev() == *dev && m.ino() == *ino);
6554 if still_ours {
6555 let _ = std::fs::remove_file(path);
6556 }
6557 }
6558 #[cfg(not(unix))]
6559 let _ = &unix_socket_cleanup;
6560
6561 tracing::info!(exit_code = 0, "shutdown: all phases completed cleanly");
6562 }
6563
6564 #[allow(clippy::too_many_lines)]
6570 async fn run_build_mode(self) {
6571 let Self {
6572 routes,
6573 api_versions,
6574 route_sources: _,
6575 current_plugin: _,
6576 tasks: _,
6577 one_off_tasks: _,
6578 jobs: _,
6579 listeners,
6580 #[cfg(feature = "db")]
6581 outbox_handlers: _,
6582 static_metas,
6583 exception_filters: _,
6584 scoped_groups,
6585 merge_routers: _,
6586 nest_routers: _,
6587 custom_layers,
6588 static_gate_layers: _,
6589 startup_hooks: _,
6590 state_initializers,
6591 shutdown_hooks: _,
6592 extensions: _,
6593 registered_plugins: _,
6594 plugin_contracts: _,
6595 plugin_config_roots,
6596 #[cfg(feature = "maud")]
6597 error_page_renderer: _,
6598 #[cfg(feature = "db")]
6599 migrations: _,
6600 config_loader_factory,
6601 #[cfg(feature = "db")]
6602 pool_provider_factory,
6603 #[cfg(feature = "db")]
6604 shard_provider_factory,
6605 #[cfg(feature = "db")]
6606 shard_router,
6607 #[cfg(feature = "db")]
6608 directory_shard_router,
6609 telemetry_provider,
6610 session_store,
6611 #[cfg(feature = "ws")]
6612 channels_backend,
6613 #[cfg(feature = "storage")]
6614 blob_store,
6615 cache_backend,
6616 #[cfg(feature = "reporting")]
6617 error_reporters,
6618 alert_channels: _,
6619 #[cfg(feature = "openapi")]
6620 openapi,
6621 #[cfg(feature = "mcp")]
6622 mcp: _,
6623 #[cfg(feature = "webmcp")]
6624 webmcp: _,
6625 audit_logger: _,
6626 #[cfg(feature = "i18n")]
6627 i18n_bundle,
6628 #[cfg(feature = "i18n")]
6629 i18n_auto_load,
6630 #[cfg(feature = "embed-assets")]
6631 embedded_static,
6632 #[cfg(all(feature = "embed-assets", feature = "i18n"))]
6633 embedded_locales,
6634 policy_registrations,
6635 #[cfg(feature = "mail")]
6636 mail_delivery_queue_factory,
6637 #[cfg(feature = "mail")]
6638 suppression_store,
6639 #[cfg(feature = "mail")]
6640 mail_suppression_store,
6641 #[cfg(feature = "mail")]
6642 mount_unsubscribe_endpoint,
6643 #[cfg(feature = "mail")]
6644 mail_previews,
6645 #[cfg(feature = "maud")]
6646 story_gallery,
6647 declared_routes: _,
6648 plugin_asset_bundles: _,
6649 idempotency_enabled,
6650 #[cfg(feature = "mail")]
6651 mail_interceptor,
6652 job_interceptor,
6653 #[cfg(feature = "db")]
6654 db_interceptor,
6655 #[cfg(feature = "ws")]
6656 channels_interceptor,
6657 #[cfg(feature = "oauth2")]
6658 http_interceptor,
6659 seo_sources,
6660 agent_skills,
6661 commerce_docs,
6662 metrics_sources,
6663 health_indicators,
6664 #[cfg(feature = "inbound-mail")]
6665 inbound_mail_router: _,
6666 } = self;
6667
6668 let _ = &api_versions;
6669 let _ = &metrics_sources;
6670 let _ = &health_indicators;
6671 let all_routes = routes;
6672
6673 let (mut config, telemetry_guard) = load_config_and_telemetry(
6675 config_loader_factory,
6676 telemetry_provider,
6677 plugin_config_roots,
6678 )
6679 .await;
6680
6681 #[cfg(feature = "mail")]
6682 apply_mail_builder_overrides(&mut config, mount_unsubscribe_endpoint);
6683 if idempotency_enabled {
6684 let env_disabled = std::env::var("AUTUMN_IDEMPOTENCY__ENABLED")
6685 .is_ok_and(|v| matches!(v.to_lowercase().as_str(), "false" | "0" | "no" | "off"));
6686 if !env_disabled && config.idempotency.enabled != Some(false) {
6689 config.idempotency.enabled = Some(true);
6690 }
6691 }
6692
6693 #[cfg(feature = "embed-assets")]
6698 register_embedded_static_dir(embedded_static);
6699
6700 #[cfg(all(feature = "embed-assets", feature = "i18n"))]
6701 let i18n_bundle = embedded_i18n_bundle(i18n_bundle, embedded_locales, &config);
6702
6703 #[cfg(feature = "i18n")]
6704 let i18n_bundle =
6705 resolve_i18n_bundle(i18n_bundle, i18n_auto_load, &config, &crate::config::OsEnv);
6706
6707 #[cfg(feature = "openapi")]
6711 let api_docs_snapshot: Vec<crate::openapi::ApiDoc> = {
6712 let mut docs: Vec<crate::openapi::ApiDoc> = all_routes
6713 .iter()
6714 .map(|r| {
6715 let mut doc = r.api_doc.clone();
6716 doc.api_version = r.api_version;
6717 doc.sunset_opt_out = r.sunset_opt_out;
6718 doc
6719 })
6720 .collect();
6721 for group in &scoped_groups {
6722 let prefix_params = crate::router::extract_path_params(&group.prefix);
6726 for route in &group.routes {
6727 let mut doc = route.api_doc.clone();
6728 doc.api_version = route.api_version;
6729 doc.sunset_opt_out = route.sunset_opt_out;
6730 let full = crate::router::join_nested_path(&group.prefix, route.api_doc.path);
6731 doc.path = Box::leak(full.into_boxed_str());
6732 if !prefix_params.is_empty() {
6733 let mut merged: Vec<&'static str> = prefix_params
6734 .iter()
6735 .map(|p| &*Box::leak(p.clone().into_boxed_str()))
6736 .collect();
6737 merged.extend_from_slice(doc.path_params);
6738 doc.path_params = Box::leak(merged.into_boxed_slice());
6739 }
6740 docs.push(doc);
6741 }
6742 }
6743 docs
6744 };
6745
6746 if static_metas.is_empty() {
6747 let allow_empty = std::env::var(BUILD_STATIC_ALLOW_EMPTY_ENV).as_deref() == Ok("1");
6748 if allow_empty {
6749 let dist_dir = project_dir("dist", &crate::config::OsEnv);
6750 if let Err(error) = clear_stale_static_output(&dist_dir) {
6751 eprintln!("Failed to remove stale {}: {error}", dist_dir.display());
6752 std::process::exit(1);
6753 }
6754 eprintln!("No static routes registered. Nothing to render.");
6755 } else {
6756 eprintln!("No static routes registered. Nothing to build.");
6757 eprintln!("Hint: use .static_routes(static_routes![...]) on your AppBuilder.");
6758 }
6759 std::process::exit(no_static_routes_exit_code(allow_empty));
6760 }
6761
6762 fail_fast_on_invalid_session_config(&config, session_store.is_some());
6766 fail_fast_on_invalid_signing_secret(&config);
6767 fail_fast_on_missing_encryption_keys(&config);
6768 fail_fast_on_invalid_trusted_hosts(&config);
6769
6770 #[cfg(feature = "storage")]
6777 let storage_bootstrap = blob_store.map_or_else(
6778 || preflight_storage(&config),
6779 |store| {
6780 Some(StorageBootstrap {
6781 store,
6782 serving: None,
6783 })
6784 },
6785 );
6786
6787 #[cfg(feature = "db")]
6789 let database = setup_database(
6790 &config,
6791 vec![],
6792 pool_provider_factory,
6793 shard_provider_factory,
6794 shard_router,
6795 directory_shard_router,
6796 RepositoryCommitHookQueueMigrationMode::StaticBuild,
6797 )
6798 .await
6799 .unwrap_or_else(|e| {
6800 eprintln!("{e}");
6801 std::process::exit(1);
6802 });
6803 #[cfg(feature = "db")]
6804 let pool = database.topology;
6805 #[cfg(feature = "db")]
6806 let shards = database.shards;
6807 #[cfg(feature = "db")]
6808 let replica_readiness = database.replica_readiness;
6809 #[cfg(feature = "db")]
6810 let replica_migration_check = database.replica_migration_check;
6811
6812 let mut state = build_state(
6813 &config,
6814 #[cfg(feature = "db")]
6815 pool.as_ref(),
6816 #[cfg(feature = "db")]
6817 shards,
6818 #[cfg(feature = "ws")]
6819 channels_backend,
6820 );
6821 if let Some(buf) = telemetry_guard.log_buffer.clone() {
6822 state.insert_extension(buf);
6823 }
6824 if let Some(handle) = telemetry_guard.filter_reload.clone() {
6828 state.log_levels().attach_reload_handle(handle);
6829 }
6830 state.insert_extension(RegisteredApiVersions(api_versions.clone()));
6831 #[cfg(feature = "mail")]
6844 if let Some(interceptor) = mail_interceptor {
6845 state.insert_extension(interceptor);
6846 }
6847 if let Some(interceptor) = job_interceptor {
6848 state.insert_extension(interceptor);
6849 }
6850 #[cfg(feature = "db")]
6851 if let Some(interceptor) = db_interceptor {
6852 state.insert_extension(interceptor);
6853 }
6854 #[cfg(feature = "ws")]
6855 if let Some(interceptor) = channels_interceptor {
6856 state.insert_extension(interceptor.clone());
6857 state.channels = crate::channels::Channels::with_shared_backend(std::sync::Arc::new(
6858 crate::channels::InterceptedChannelsBackend::new(
6859 state.channels.backend().clone(),
6860 vec![interceptor],
6861 ),
6862 ));
6863 #[cfg(feature = "presence")]
6864 {
6865 state.presence = crate::presence::Presence::new(state.channels.clone());
6866 #[cfg(feature = "collab")]
6870 {
6871 state.collab = crate::collab::CollabHub::new(
6872 state.channels.clone(),
6873 state.presence.clone(),
6874 );
6875 }
6876 }
6877 }
6878 #[cfg(feature = "oauth2")]
6879 if let Some(interceptor) = http_interceptor {
6880 state.insert_extension(interceptor);
6881 }
6882 #[cfg(feature = "db")]
6883 configure_replica_migration_check(&state, replica_migration_check);
6884 #[cfg(feature = "db")]
6885 apply_replica_migration_readiness(&state, replica_readiness);
6886 if let Some(cache) = cache_backend {
6887 let cache = crate::cache::with_capsule_seam(cache);
6889 crate::cache::set_global_cache(cache.clone());
6890 state.shared_cache = Some(cache);
6891 } else {
6892 crate::cache::clear_global_cache();
6893 }
6894 #[cfg(feature = "reporting")]
6895 if !error_reporters.is_empty() {
6896 state.insert_extension(crate::reporting::RegisteredReporters(error_reporters));
6897 }
6898 #[cfg(feature = "mail")]
6905 if let Some(handle) = suppression_store {
6906 state.insert_extension(handle);
6907 }
6908 #[cfg(feature = "mail")]
6909 if let Some(handle) = mail_suppression_store {
6910 state.insert_extension(handle);
6911 }
6912 #[cfg(feature = "mail")]
6913 crate::mail::install_mailer_with_factory(
6914 &state,
6915 &config.mail,
6916 mail_delivery_queue_factory,
6917 false,
6918 )
6919 .unwrap_or_else(|error| {
6920 eprintln!("Failed to configure mailer: {error}");
6921 exit_stop_managed_pg();
6922 std::process::exit(1);
6923 });
6924 #[cfg(feature = "mail")]
6925 state.insert_extension(crate::mail::MailPreviewRegistry::new(mail_previews));
6926 #[cfg(feature = "maud")]
6927 install_story_registry(&state, story_gallery);
6928 state.probes = crate::probe::ProbeState::default();
6930 state.apply_health_config(&config.health);
6931
6932 for register in policy_registrations {
6940 register(state.policy_registry());
6941 }
6942
6943 #[cfg(feature = "i18n")]
6944 let custom_layers =
6945 install_i18n_bundle_layer(custom_layers, &state, i18n_bundle, &config.i18n);
6946
6947 let (custom_layers, _drained) =
6961 crate::router::partition_custom_layers_for_static_render(custom_layers);
6962
6963 #[cfg(feature = "storage")]
6967 let storage_router = storage_bootstrap.and_then(|b| b.install(&state));
6968 install_webhook_registry(&state, &config);
6969 run_state_initializers(state_initializers, &state);
6970 crate::cost::install(&state, &config);
6971 let sync_listeners: Vec<_> = listeners
6976 .into_iter()
6977 .filter(|listener| listener.mode == crate::events::DispatchMode::Sync)
6978 .collect();
6979 finalize_event_bus(sync_listeners, &mut Vec::new(), &state);
6980
6981 #[cfg_attr(not(feature = "storage"), allow(unused_mut))]
6994 let mut merge_routers: Vec<axum::Router<AppState>> = Vec::new();
6995 #[cfg(feature = "storage")]
6996 if let Some(router) = storage_router {
6997 merge_routers.push(router);
6998 }
6999 exclude_static_routes_from_locale_prefix(&mut config, &static_metas);
7003 state.insert_extension(config.clone());
7006 state.insert_extension(crate::aeo::RegisteredAgentSkills(agent_skills));
7007 state.insert_extension(commerce_docs);
7008 #[cfg(feature = "openapi")]
7017 let static_openapi = openapi
7018 .clone()
7019 .map(|api| api.openapi_json_path("/openapi.json").swagger_ui_path(None));
7020 #[cfg(feature = "openapi")]
7021 let census_config = {
7022 let mut census = config.clone();
7023 census.openapi_runtime.enabled = true;
7024 census
7025 };
7026 #[cfg(not(feature = "openapi"))]
7027 let census_config = config.clone();
7028 crate::graph::install(
7029 &state,
7030 crate::graph::manifest::audit(
7031 &graph_mounted_routes(
7032 &all_routes,
7033 &scoped_groups,
7034 &[],
7035 &census_config,
7036 #[cfg(feature = "openapi")]
7037 static_openapi.as_ref(),
7038 ),
7039 omitted_router_count(merge_routers.len(), std::iter::empty::<&str>(), &[]),
7043 ),
7044 );
7045 let aeo_state = state.clone();
7048 let router = crate::router::try_build_router_inner(
7049 all_routes,
7050 &config,
7051 state,
7052 crate::router::RouterContext {
7053 exception_filters: Vec::new(),
7054 scoped_groups,
7055 merge_routers,
7056 nest_routers: Vec::new(),
7057 declared_routes: Vec::new(),
7058 custom_layers,
7059 static_gate_layers: Vec::new(),
7060 #[cfg(feature = "maud")]
7061 error_page_renderer: None,
7062 session_store,
7063 #[cfg(feature = "openapi")]
7064 openapi: static_openapi,
7065 #[cfg(feature = "mcp")]
7066 mcp: None,
7067 #[cfg(feature = "webmcp")]
7068 webmcp: None,
7069 },
7070 )
7071 .unwrap_or_else(|error| {
7072 eprintln!("Failed to build router: {error}");
7073 exit_stop_managed_pg();
7074 std::process::exit(1);
7075 });
7076
7077 let env = crate::config::OsEnv;
7078 let dist_dir = project_dir("dist", &env);
7079
7080 eprintln!("Building {} static route(s)...", static_metas.len());
7081
7082 match crate::static_gen::render_static_routes(router.clone(), &static_metas, &dist_dir)
7083 .await
7084 {
7085 Ok(()) => {
7086 eprintln!(
7087 "\n \u{2713} Static build complete \u{2192} {}",
7088 dist_dir.display()
7089 );
7090 }
7091 Err(e) => {
7092 eprintln!("\n \u{2717} Static build failed: {e}");
7093 exit_stop_managed_pg();
7094 std::process::exit(1);
7095 }
7096 }
7097
7098 if config.aeo.enabled {
7101 let home_links = aeo_state
7102 .extension::<crate::aeo::AeoSite>()
7103 .map(|site| site.negotiate.home_links.clone())
7104 .unwrap_or_default();
7105 match crate::aeo::write_static_documents(
7106 router,
7107 config.seo.base_url.as_deref(),
7108 &home_links,
7109 &dist_dir,
7110 )
7111 .await
7112 {
7113 Ok(paths) => {
7114 for path in paths {
7115 eprintln!(" \u{2713} AEO: {path} written");
7116 }
7117 if config.seo.base_url.is_none() {
7118 eprintln!(
7119 " \u{2139} AEO: set [seo] base_url to write the agent documents"
7120 );
7121 }
7122 }
7123 Err(e) => eprintln!(" \u{26A0} Failed to write the agent documents: {e}"),
7124 }
7125 }
7126
7127 #[cfg(feature = "openapi")]
7130 if let Some(mut openapi_config) = openapi {
7131 openapi_config.api_versions = api_versions;
7132 let openapi_config =
7133 openapi_config.session_cookie_name(config.session.cookie_name.clone());
7134 let docs: Vec<&crate::openapi::ApiDoc> = api_docs_snapshot.iter().collect();
7135 let spec = crate::openapi::generate_spec(&openapi_config, &docs);
7136 let written = crate::openapi::write_openapi_spec_to_dist(&spec, &dist_dir)
7137 .and_then(|()| crate::aeo::commerce::write_mpp_spec(&spec, &config, &dist_dir));
7138 match written {
7139 Ok(()) => {
7140 eprintln!(
7141 " \u{2713} OpenAPI spec written \u{2192} {}/openapi.json",
7142 dist_dir.display()
7143 );
7144 }
7145 Err(e) => {
7146 eprintln!(" \u{26A0} Failed to write OpenAPI spec: {e}");
7147 }
7148 }
7149 }
7150
7151 if !seo_sources.is_empty() || crate::seo::has_seo_config(&config.seo) || config.aeo.enabled
7155 {
7156 let seo_cfg = &config.seo;
7157 let raw_profile = config.profile.as_deref().unwrap_or("dev");
7158 let aeo_only = seo_sources.is_empty() && !crate::seo::has_seo_config(seo_cfg);
7162 let profile = if aeo_only {
7163 crate::aeo::robots_profile(&config)
7164 } else {
7165 crate::seo::effective_seo_profile(raw_profile, seo_cfg.robots.allow_all)
7166 };
7167 let static_paths: Vec<&str> = static_metas
7171 .iter()
7172 .filter(|m| !crate::seo::defaults_exclude_from_sitemap(m.seo))
7173 .map(|m| m.path)
7174 .collect();
7175 let (robots_body, sitemap_body) = crate::seo::assemble_seo_bodies_with_policy(
7176 profile,
7177 seo_cfg.base_url.as_deref(),
7178 seo_cfg.robots.sitemap_url.as_deref(),
7179 &seo_cfg.robots.additional_rules,
7180 &seo_sources,
7181 &static_paths,
7182 sitemap_locale_config(&config),
7183 &crate::aeo::robots_policy(&config),
7184 )
7185 .await;
7186 let robots_path = dist_dir.join("robots.txt");
7189 let sitemap_path = dist_dir.join("sitemap.xml");
7190 if robots_path.exists() {
7191 eprintln!(
7192 " \u{2713} SEO: robots.txt already present (custom static route), skipping"
7193 );
7194 } else {
7195 match tokio::fs::write(&robots_path, robots_body).await {
7196 Ok(()) => eprintln!(
7197 " \u{2713} SEO: robots.txt written \u{2192} {}",
7198 robots_path.display()
7199 ),
7200 Err(e) => eprintln!(" \u{26A0} Failed to write robots.txt: {e}"),
7201 }
7202 }
7203 if sitemap_path.exists() {
7204 eprintln!(
7205 " \u{2713} SEO: sitemap.xml already present (custom static route), skipping"
7206 );
7207 } else {
7208 match tokio::fs::write(&sitemap_path, sitemap_body).await {
7209 Ok(()) => eprintln!(
7210 " \u{2713} SEO: sitemap.xml written \u{2192} {}",
7211 sitemap_path.display()
7212 ),
7213 Err(e) => eprintln!(" \u{26A0} Failed to write sitemap.xml: {e}"),
7214 }
7215 }
7216 }
7217
7218 #[cfg(feature = "managed-pg")]
7222 crate::managed_pg::emergency_stop_async().await;
7223 }
7224
7225 async fn run_dump_graph_mode(self) {
7230 let (mounted, opaque) = self.dump_graph_surface().await;
7231 crate::graph::manifest::print_manifest_dump(&crate::graph::manifest::audit(
7232 &mounted, opaque,
7233 ));
7234 }
7235
7236 async fn dump_graph_surface(self) -> (Vec<crate::graph::MountedRoute>, usize) {
7244 let opaque = self.graph_opaque_router_count();
7245 let config =
7246 match try_load_config(self.config_loader_factory, self.plugin_config_roots).await {
7247 Ok(config) => config,
7248 Err(error) => {
7249 eprintln!("warning: config failed to load, graph uses defaults: {error}");
7251 crate::config::AutumnConfig::default()
7252 }
7253 };
7254 if !config.role.serves_http() {
7255 return (graph_worker_mounted_routes(&config), 0);
7256 }
7257 let mounted = graph_mounted_routes(
7258 &self.routes,
7259 &self.scoped_groups,
7260 &self.declared_routes,
7261 &config,
7262 #[cfg(feature = "openapi")]
7263 self.openapi.as_ref(),
7264 );
7265 (mounted, opaque)
7266 }
7267
7268 fn graph_opaque_router_count(&self) -> usize {
7274 omitted_router_count(
7275 self.merge_routers.len(),
7276 self.nest_routers.iter().map(|(prefix, _)| prefix.as_str()),
7277 &self.declared_routes,
7278 )
7279 }
7280
7281 fn run_dump_agent_authority_mode(&self) {
7289 let audit_sink_configured = self
7293 .audit_logger
7294 .as_ref()
7295 .is_some_and(|logger| logger.is_enabled());
7296 #[cfg(feature = "mcp")]
7300 let expose_all = self.mcp.as_ref().is_some_and(|rt| rt.expose_all);
7301 #[cfg(not(feature = "mcp"))]
7302 let expose_all = false;
7303 #[cfg(feature = "webmcp")]
7306 let webmcp_expose_all = self.webmcp.is_some_and(|rt| rt.expose_all);
7307 #[cfg(not(feature = "webmcp"))]
7308 let webmcp_expose_all = false;
7309 let exposure = AgentExposure {
7310 mcp_expose_all: expose_all,
7311 webmcp_expose_all,
7312 };
7313 let mut routes: Vec<crate::agent_authority::manifest::RouteSummary> = self
7319 .routes
7320 .iter()
7321 .map(|route| agent_authority_route_summary(route, None, exposure))
7322 .chain(self.scoped_groups.iter().flat_map(|group| {
7323 group.routes.iter().map(move |route| {
7324 agent_authority_route_summary(route, Some(&group.prefix), exposure)
7325 })
7326 }))
7327 .collect();
7328 crate::agent_authority::manifest::dedupe_mcp_tool_names(&mut routes);
7330 crate::agent_authority::manifest::print_manifest_dump(
7331 &crate::agent_authority::manifest::build(&routes, audit_sink_configured),
7332 );
7333 }
7334
7335 #[allow(clippy::too_many_lines)]
7341 async fn run_dump_routes_mode(self) {
7342 let Self {
7343 routes,
7344 api_versions,
7345 route_sources,
7346 scoped_groups,
7347 merge_routers,
7348 nest_routers,
7349 declared_routes,
7350 config_loader_factory,
7351 telemetry_provider,
7352 #[cfg(feature = "openapi")]
7353 openapi,
7354 plugin_config_roots,
7355 plugin_contracts,
7356 ..
7357 } = self;
7358
7359 let registered_versions: std::collections::HashSet<&str> =
7361 api_versions.iter().map(|av| av.version.as_str()).collect();
7362
7363 for route in &routes {
7364 if let Some(ver) = route
7365 .api_version
7366 .filter(|ver| !registered_versions.contains(*ver))
7367 {
7368 eprintln!(
7369 "Failed to build router: route '{}' uses unregistered API version '{}'",
7370 route.name, ver
7371 );
7372 std::process::exit(1);
7373 }
7374 }
7375
7376 for group in &scoped_groups {
7377 for route in &group.routes {
7378 if let Some(ver) = route
7379 .api_version
7380 .filter(|ver| !registered_versions.contains(*ver))
7381 {
7382 eprintln!(
7383 "Failed to build router: route '{}' uses unregistered API version '{}'",
7384 route.name, ver
7385 );
7386 std::process::exit(1);
7387 }
7388 }
7389 }
7390
7391 let hidden = omitted_router_count(
7400 merge_routers.len(),
7401 nest_routers.iter().map(|(prefix, _)| prefix.as_str()),
7402 &declared_routes,
7403 );
7404 if hidden > 0 {
7405 eprintln!(
7406 "[autumn routes] warning: {hidden} raw router(s) added via \
7407 .merge()/.nest() are not enumerable and are omitted from this listing"
7408 );
7409 eprintln!(
7412 "{marker}{hidden}",
7413 marker = crate::route_listing::OMITTED_ROUTES_MARKER
7414 );
7415 }
7416
7417 let (config, _telemetry_guard) = load_config_and_telemetry(
7418 config_loader_factory,
7419 telemetry_provider,
7420 plugin_config_roots,
7421 )
7422 .await;
7423
7424 if is_dump_security_mode() {
7429 let security = crate::route_listing::SecurityDump::from_config(&config);
7430 match serde_json::to_string(&security) {
7431 Ok(json) => eprintln!(
7432 "{marker}{json}",
7433 marker = crate::route_listing::SECURITY_CONFIG_MARKER
7434 ),
7435 Err(e) => eprintln!("Failed to serialize security config: {e}"),
7436 }
7437 }
7438
7439 if is_dump_plugin_contract_mode() {
7445 match serde_json::to_string(&plugin_contracts) {
7446 Ok(json) => eprintln!(
7447 "{marker}{json}",
7448 marker = crate::plugin_contract::PLUGIN_CONTRACT_MARKER
7449 ),
7450 Err(e) => eprintln!("Failed to serialize plugin contracts: {e}"),
7451 }
7452 }
7453
7454 let mut infos = match crate::route_listing::collect_route_infos(
7455 &routes,
7456 &route_sources,
7457 &scoped_groups,
7458 &api_versions,
7459 ) {
7460 Ok(infos) => infos,
7461 Err(e) => {
7462 eprintln!("Failed to build router: {e}");
7463 std::process::exit(1);
7464 }
7465 };
7466 infos.extend(declared_routes);
7467 crate::route_listing::append_framework_routes(&mut infos, &config);
7468 #[cfg(feature = "openapi")]
7469 if let Some(ref oa) = openapi {
7470 crate::route_listing::append_openapi_routes(&mut infos, oa);
7471 }
7472 crate::route_listing::sort_route_infos(&mut infos);
7473
7474 let json = serde_json::to_string_pretty(&infos).unwrap_or_else(|e| {
7475 eprintln!("Failed to serialize route listing: {e}");
7476 std::process::exit(1);
7477 });
7478 println!("{json}");
7479 std::process::exit(0);
7480 }
7481
7482 #[cfg(feature = "openapi")]
7505 async fn run_dump_openapi_mode(self) {
7506 let Self {
7507 routes,
7508 scoped_groups,
7509 api_versions,
7510 openapi,
7511 config_loader_factory,
7512 plugin_config_roots,
7513 merge_routers,
7514 nest_routers,
7515 declared_routes,
7516 #[cfg(feature = "mcp")]
7517 mcp,
7518 #[cfg(feature = "mail")]
7519 mount_unsubscribe_endpoint,
7520 policy_registrations,
7521 tasks,
7522 ..
7523 } = self;
7524
7525 let Some(openapi_config) = openapi else {
7526 eprintln!(
7527 "{marker}{reason}",
7528 marker = crate::openapi::OPENAPI_UNAVAILABLE_MARKER,
7529 reason = crate::openapi::OPENAPI_UNAVAILABLE_UNCONFIGURED,
7530 );
7531 std::process::exit(2);
7532 };
7533
7534 #[cfg_attr(
7538 not(feature = "mail"),
7539 expect(unused_mut, reason = "only `mail` mutates it")
7540 )]
7541 let mut config = load_config_only(config_loader_factory, plugin_config_roots).await;
7542
7543 #[cfg(feature = "mail")]
7548 apply_mail_builder_overrides(&mut config, mount_unsubscribe_endpoint);
7549
7550 let tasks = merge_framework_scheduled_tasks(tasks, &config);
7570 if let Err(message) = validate_config_preconditions(&config, &tasks) {
7571 eprintln!("\u{2717} Cannot export a spec for an app that cannot start: {message}");
7572 std::process::exit(1);
7573 }
7574
7575 let export_registry = crate::authorization::PolicyRegistry::default();
7587 for register in policy_registrations {
7588 register(&export_registry);
7589 }
7590 validate_repository_policies_registered(&routes, &scoped_groups, &export_registry, &config);
7591
7592 let mcp_mount_path: Option<&str> = {
7593 #[cfg(feature = "mcp")]
7594 {
7595 mcp.as_ref().map(|rt| rt.mount_path.as_str())
7596 }
7597 #[cfg(not(feature = "mcp"))]
7598 {
7599 None
7600 }
7601 };
7602 if let Err(error) = export_preflight(&ExportPreflight {
7603 routes: &routes,
7604 scoped_groups: &scoped_groups,
7605 api_versions: &api_versions,
7606 openapi_config: &openapi_config,
7607 merge_routers: &merge_routers,
7608 nest_routers: &nest_routers,
7609 declared_routes: &declared_routes,
7610 config: &config,
7611 mcp_mount_path,
7612 }) {
7613 eprintln!("\u{2717} Cannot export a spec for a router that cannot be built: {error}");
7614 std::process::exit(1);
7615 }
7616
7617 let mut openapi_config = openapi_config;
7618 openapi_config.api_versions = api_versions;
7619 let openapi_config = openapi_config.session_cookie_name(config.session.cookie_name.clone());
7620
7621 let docs = crate::router::collect_openapi_docs(&routes, &scoped_groups);
7622 let refs: Vec<&crate::openapi::ApiDoc> = docs.iter().collect();
7623 let spec = crate::openapi::generate_spec(&openapi_config, &refs);
7624
7625 let json = crate::aeo::commerce::spec_json_with_mpp(&spec, &config).unwrap_or_else(|e| {
7626 eprintln!("Failed to serialize OpenAPI spec: {e}");
7627 std::process::exit(1);
7628 });
7629 println!("{json}");
7630 std::process::exit(0);
7631 }
7632
7633 async fn run_dump_jobs_mode(self) {
7642 let Self {
7643 jobs,
7644 listeners,
7645 config_loader_factory,
7646 telemetry_provider,
7647 plugin_config_roots,
7648 ..
7649 } = self;
7650
7651 let (config, _telemetry_guard) = load_config_and_telemetry(
7652 config_loader_factory,
7653 telemetry_provider,
7654 plugin_config_roots,
7655 )
7656 .await;
7657
7658 let manifest = dump_jobs_manifest(&config.jobs.queues, jobs, listeners);
7662 print!("{manifest}");
7663 std::process::exit(0);
7664 }
7665
7666 fn run_list_one_off_tasks_mode(self) {
7670 let Self { one_off_tasks, .. } = self;
7671
7672 if let Err(error) = crate::task::validate_unique_one_off_task_names(&one_off_tasks) {
7673 eprintln!("Invalid task registration: {error}");
7674 std::process::exit(1);
7675 }
7676
7677 let listing = crate::task::list_one_off_tasks(&one_off_tasks);
7678 let json = serde_json::to_string_pretty(&listing).unwrap_or_else(|error| {
7679 eprintln!("Failed to serialize task listing: {error}");
7680 std::process::exit(1);
7681 });
7682 println!("{json}");
7683 std::process::exit(0);
7684 }
7685
7686 #[cfg(feature = "db")]
7705 #[allow(clippy::too_many_lines)]
7706 async fn run_migrate_only_mode(self) {
7707 let Self {
7708 migrations,
7709 config_loader_factory,
7710 telemetry_provider,
7711 plugin_config_roots,
7712 ..
7713 } = self;
7714
7715 let (config, _telemetry_guard) = load_config_and_telemetry(
7719 config_loader_factory,
7720 telemetry_provider,
7721 plugin_config_roots,
7722 )
7723 .await;
7724
7725 let migrations = Arc::new(migrations_with_repository_framework_migrations(
7728 migrations,
7729 crate::repository_commit_hooks::has_repository_commit_hook_descriptors(),
7730 crate::version_history::has_versioned_repository_descriptors(),
7731 crate::derivation::has_derivation_descriptors(),
7732 RepositoryCommitHookQueueMigrationMode::Runtime,
7733 ));
7734 #[cfg(feature = "sqlite")]
7735 let fleet_migrations = Arc::clone(&migrations);
7736
7737 let control_url = config.database.effective_primary_url().map(str::to_owned);
7739 let lock_policy = crate::migrate::MigrationLockPolicy::from_config(&config.database);
7740 let shard_targets: Vec<(String, String)> = config
7741 .database
7742 .shards
7743 .iter()
7744 .map(|shard| (format!("shard:{}", shard.name), shard.primary_url.clone()))
7745 .collect();
7746
7747 if migrations.is_empty() || (control_url.is_none() && shard_targets.is_empty()) {
7748 eprintln!(
7749 "autumn migrate: no database configured or no migrations registered — nothing to apply"
7750 );
7751 std::process::exit(0);
7752 }
7753
7754 #[cfg(feature = "sqlite")]
7762 {
7763 let sqlite_guard_shard_urls: Vec<&str> =
7764 shard_targets.iter().map(|(_, url)| url.as_str()).collect();
7765 if let Err(e) = sqlite_sharding_unsupported_guard(
7766 control_url.as_deref(),
7767 !shard_targets.is_empty(),
7768 &sqlite_guard_shard_urls,
7769 ) {
7770 eprintln!("autumn migrate: {e}");
7771 #[cfg(feature = "managed-pg")]
7774 crate::managed_pg::emergency_stop();
7775 std::process::exit(1);
7776 }
7777 }
7778
7779 let disambiguation_sets =
7787 migration_sets_for_disambiguation(&migrations, config.database.has_shards());
7788 let disambiguated = crate::migrate::compute_migration_disambiguation(&disambiguation_sets);
7789 #[cfg(feature = "sqlite")]
7790 let sqlite_history_sets = crate::migrate::sqlite_collision_pairs(&disambiguation_sets);
7791
7792 let applied_total = crate::time::spawn_blocking(move || {
7795 let mut total = 0_usize;
7796 if let Some(url) = &control_url {
7797 #[cfg(feature = "sqlite")]
7803 let is_sqlite_control = crate::config::DatabaseBackend::detect(url)
7804 == Some(crate::config::DatabaseBackend::Sqlite);
7805 #[cfg(not(feature = "sqlite"))]
7806 let is_sqlite_control = false;
7807 if is_sqlite_control {
7808 #[cfg(feature = "sqlite")]
7813 if let Err(error) = crate::migrate::adopt_sqlite_collision_history(
7814 url,
7815 &sqlite_history_sets,
7816 &disambiguated,
7817 ) {
7818 eprintln!(
7819 "autumn migrate: could not move an already-applied migration's \
7820 version record (target control): {error}"
7821 );
7822 std::process::exit(1);
7823 }
7824 #[cfg(feature = "sqlite")]
7825 for (_, mig) in migrations.iter() {
7826 total += apply_pending_sqlite_or_exit(
7827 url,
7828 crate::migrate::DisambiguatedMigrations::new(mig, &disambiguated),
7829 "control",
7830 );
7831 }
7832 } else {
7833 for (_, mig) in migrations.iter() {
7834 total += apply_pending_or_exit(
7835 url,
7836 crate::migrate::DisambiguatedMigrations::new(mig, &disambiguated),
7837 "control",
7838 lock_policy,
7839 );
7840 }
7841 }
7842 }
7843 for (label, url) in &shard_targets {
7848 for (_, mig) in migrations
7849 .iter()
7850 .filter(|(_, mig)| !migration_set_is_control_framework(mig))
7851 {
7852 total += apply_pending_or_exit(
7853 url,
7854 crate::migrate::DisambiguatedMigrations::new(mig, &disambiguated),
7855 label,
7856 lock_policy,
7857 );
7858 }
7859 }
7860 total
7861 })
7862 .await
7863 .unwrap_or_else(|error| {
7864 eprintln!("autumn migrate: migration task panicked: {error}");
7865 std::process::exit(1);
7866 });
7867
7868 #[cfg(feature = "sqlite")]
7872 let applied_total = applied_total + migrate_fleet_or_exit(&config, fleet_migrations).await;
7873
7874 eprintln!(
7875 "autumn migrate: applied {applied_total} pending migration(s); database is up to date"
7876 );
7877 std::process::exit(0);
7878 }
7879
7880 #[cfg(not(feature = "db"))]
7884 #[allow(clippy::unused_async)]
7885 async fn run_migrate_only_mode(self) {
7886 eprintln!("autumn migrate: this build has no database support — nothing to migrate");
7887 std::process::exit(0);
7888 }
7889
7890 #[cfg(feature = "db")]
7899 async fn run_retention_dry_run_mode(self) {
7900 let Self {
7901 tasks,
7902 migrations,
7903 config_loader_factory,
7904 telemetry_provider,
7905 plugin_config_roots,
7906 pool_provider_factory,
7907 shard_provider_factory,
7908 shard_router,
7909 directory_shard_router,
7910 #[cfg(feature = "ws")]
7911 channels_backend,
7912 ..
7913 } = self;
7914
7915 let model_filter = retention_dry_run_model_filter_from_env();
7916 let descriptors =
7924 match crate::retention::resolve_retention_descriptors(model_filter.as_deref()) {
7925 Ok(descriptors) => descriptors,
7926 Err(error) => {
7927 eprintln!("retention dry-run: {error}");
7928 std::process::exit(1);
7929 }
7930 };
7931
7932 if let Err(error) =
7946 merge_and_validate_task_names(&crate::retention::all_retention_descriptors(), tasks)
7947 {
7948 eprintln!("retention dry-run: {error}");
7949 std::process::exit(1);
7950 }
7951
7952 if descriptors.is_empty() {
7953 println!("{RETENTION_DRY_RUN_JSON_PREFIX}[]");
7954 std::process::exit(0);
7955 }
7956
7957 let (config, _telemetry_guard) = load_config_and_telemetry(
7958 config_loader_factory,
7959 telemetry_provider,
7960 plugin_config_roots,
7961 )
7962 .await;
7963
7964 let database = match setup_database(
7972 &config,
7973 migrations,
7974 pool_provider_factory,
7975 shard_provider_factory,
7976 shard_router,
7977 directory_shard_router,
7978 RepositoryCommitHookQueueMigrationMode::Runtime,
7979 )
7980 .await
7981 {
7982 Ok(database) => database,
7983 Err(error) => {
7984 eprintln!("{error}");
7985 #[cfg(feature = "managed-pg")]
7986 crate::managed_pg::emergency_stop_async().await;
7987 std::process::exit(1);
7988 }
7989 };
7990
7991 let state = build_state(
7992 &config,
7993 database.topology.as_ref(),
7994 database.shards,
7995 #[cfg(feature = "ws")]
7996 channels_backend,
7997 );
7998
7999 match crate::retention::run_retention_dry_run(&state, model_filter.as_deref()).await {
8005 Ok(reports) => match serde_json::to_string(&reports) {
8006 Ok(json) => {
8007 println!("{RETENTION_DRY_RUN_JSON_PREFIX}{json}");
8008 #[cfg(feature = "managed-pg")]
8009 crate::managed_pg::emergency_stop_async().await;
8010 std::process::exit(0);
8011 }
8012 Err(error) => {
8013 eprintln!("retention dry-run: failed to serialize report: {error}");
8014 #[cfg(feature = "managed-pg")]
8015 crate::managed_pg::emergency_stop_async().await;
8016 std::process::exit(1);
8017 }
8018 },
8019 Err(error) => {
8020 eprintln!("retention dry-run: {error}");
8021 #[cfg(feature = "managed-pg")]
8022 crate::managed_pg::emergency_stop_async().await;
8023 std::process::exit(1);
8024 }
8025 }
8026 }
8027
8028 #[allow(clippy::too_many_lines)]
8044 async fn run_framework_retention_mode(self, mode: FrameworkRetentionMode) {
8045 let Self {
8046 state_initializers,
8047 audit_logger,
8048 config_loader_factory,
8049 telemetry_provider,
8050 plugin_config_roots,
8051 #[cfg(feature = "db")]
8052 migrations,
8053 #[cfg(feature = "db")]
8054 pool_provider_factory,
8055 #[cfg(feature = "db")]
8056 shard_provider_factory,
8057 #[cfg(feature = "db")]
8058 shard_router,
8059 #[cfg(feature = "db")]
8060 directory_shard_router,
8061 #[cfg(feature = "ws")]
8062 channels_backend,
8063 ..
8064 } = self;
8065
8066 let dataset_filter = framework_retention_dataset_from_env();
8067 if let Some(filter) = dataset_filter.as_deref()
8070 && crate::data_retention::RetentionDataset::from_key(filter).is_none()
8071 {
8072 let known: Vec<&str> = crate::data_retention::RETENTION_DATASETS
8073 .iter()
8074 .map(|dataset| dataset.key())
8075 .collect();
8076 eprintln!(
8077 "autumn db retention: unknown dataset {filter:?}; known datasets: {}",
8078 known.join(", ")
8079 );
8080 std::process::exit(1);
8081 }
8082
8083 let (config, _telemetry_guard) = load_config_and_telemetry(
8084 config_loader_factory,
8085 telemetry_provider,
8086 plugin_config_roots,
8087 )
8088 .await;
8089
8090 if let Some(refusal) = framework_purge_refusal(
8093 mode,
8094 config.profile.as_deref(),
8095 std::env::var(FRAMEWORK_RETENTION_FORCE_ENV).ok().as_deref(),
8096 ) {
8097 eprintln!("autumn db retention: {refusal}");
8098 std::process::exit(1);
8099 }
8100
8101 #[cfg(feature = "db")]
8102 let database = match setup_database(
8103 &config,
8104 migrations,
8105 pool_provider_factory,
8106 shard_provider_factory,
8107 shard_router,
8108 directory_shard_router,
8109 RepositoryCommitHookQueueMigrationMode::Runtime,
8110 )
8111 .await
8112 {
8113 Ok(database) => database,
8114 Err(error) => {
8115 eprintln!("{error}");
8116 #[cfg(feature = "managed-pg")]
8117 crate::managed_pg::emergency_stop_async().await;
8118 std::process::exit(1);
8119 }
8120 };
8121
8122 let state = build_state(
8123 &config,
8124 #[cfg(feature = "db")]
8125 database.topology.as_ref(),
8126 #[cfg(feature = "db")]
8127 database.shards,
8128 #[cfg(feature = "ws")]
8129 channels_backend,
8130 );
8131 if let Some(logger) = audit_logger {
8137 state.insert_extension::<crate::audit::AuditLogger>((*logger).clone());
8138 }
8139 #[cfg(feature = "acme")]
8147 install_custom_domain_retention_pruner(
8148 config.server.tls.as_ref().and_then(|tls| tls.acme.as_ref()),
8149 config.tenancy.base_domain.as_deref(),
8150 &state,
8151 mode != FrameworkRetentionMode::Report,
8152 )
8153 .await;
8154 run_state_initializers(state_initializers, &state);
8158
8159 let options = crate::data_retention::RetentionRunOptions {
8160 dry_run: mode == FrameworkRetentionMode::Report,
8161 dataset: dataset_filter.as_deref(),
8162 };
8163 match crate::data_retention::run_retention(&state, &options).await {
8167 Ok(reports) => match serde_json::to_string(&reports) {
8168 Ok(json) => {
8169 println!("{FRAMEWORK_RETENTION_JSON_PREFIX}{json}");
8170 #[cfg(feature = "managed-pg")]
8171 crate::managed_pg::emergency_stop_async().await;
8172 let exit_code = i32::from(reports.iter().any(|r| r.error.is_some()));
8177 std::process::exit(exit_code);
8178 }
8179 Err(error) => {
8180 eprintln!("autumn db retention: failed to serialize report: {error}");
8181 #[cfg(feature = "managed-pg")]
8182 crate::managed_pg::emergency_stop_async().await;
8183 std::process::exit(1);
8184 }
8185 },
8186 Err(error) => {
8187 eprintln!("autumn db retention: {error}");
8188 #[cfg(feature = "managed-pg")]
8189 crate::managed_pg::emergency_stop_async().await;
8190 std::process::exit(1);
8191 }
8192 }
8193 }
8194
8195 #[allow(clippy::too_many_lines)]
8204 async fn run_data_capsule_mode(self, mode: DataCapsuleMode) {
8205 let Self {
8206 state_initializers,
8207 config_loader_factory,
8208 telemetry_provider,
8209 plugin_config_roots,
8210 #[cfg(feature = "db")]
8211 migrations,
8212 #[cfg(feature = "db")]
8213 pool_provider_factory,
8214 #[cfg(feature = "db")]
8215 shard_provider_factory,
8216 #[cfg(feature = "db")]
8217 shard_router,
8218 #[cfg(feature = "db")]
8219 directory_shard_router,
8220 #[cfg(feature = "ws")]
8221 channels_backend,
8222 #[cfg(feature = "storage")]
8223 blob_store,
8224 ..
8225 } = self;
8226
8227 let Some(path) = std::env::var_os(DATA_CAPSULE_PATH_ENV)
8228 .filter(|p| !p.is_empty())
8229 .map(std::path::PathBuf::from)
8230 else {
8231 eprintln!("autumn data capsule: {DATA_CAPSULE_PATH_ENV} is not set");
8232 std::process::exit(1);
8233 };
8234 let subject = std::env::var(DATA_CAPSULE_SUBJECT_ENV).unwrap_or_default();
8235 if mode == DataCapsuleMode::Export && subject.is_empty() {
8236 eprintln!("autumn data capsule export: {DATA_CAPSULE_SUBJECT_ENV} is not set");
8237 std::process::exit(1);
8238 }
8239
8240 let (config, _telemetry_guard) = load_config_and_telemetry(
8241 config_loader_factory,
8242 telemetry_provider,
8243 plugin_config_roots,
8244 )
8245 .await;
8246 fail_fast_on_invalid_signing_secret(&config);
8248
8249 #[cfg(feature = "db")]
8252 let verify = mode == DataCapsuleMode::Verify;
8253
8254 #[cfg(feature = "storage")]
8255 let storage_bootstrap = blob_store.map_or_else(
8256 || preflight_storage(&config),
8257 |store| {
8258 Some(StorageBootstrap {
8259 store,
8260 serving: None,
8261 })
8262 },
8263 );
8264
8265 #[cfg(feature = "db")]
8266 let (topology, shards) = if verify {
8267 verify_database_parts(
8268 &config,
8269 pool_provider_factory,
8270 shard_provider_factory,
8271 shard_router,
8272 directory_shard_router,
8273 )
8274 .await
8275 } else {
8276 match setup_database(
8277 &config,
8278 migrations,
8279 pool_provider_factory,
8280 shard_provider_factory,
8281 shard_router,
8282 directory_shard_router,
8283 RepositoryCommitHookQueueMigrationMode::Runtime,
8284 )
8285 .await
8286 {
8287 Ok(database) => (database.topology, database.shards),
8288 Err(error) => {
8289 eprintln!("{error}");
8290 #[cfg(feature = "managed-pg")]
8291 crate::managed_pg::emergency_stop_async().await;
8292 std::process::exit(1);
8293 }
8294 }
8295 };
8296
8297 let state = build_state(
8298 &config,
8299 #[cfg(feature = "db")]
8300 topology.as_ref(),
8301 #[cfg(feature = "db")]
8302 shards,
8303 #[cfg(feature = "ws")]
8304 channels_backend,
8305 );
8306 #[cfg(feature = "storage")]
8307 if let Some(bootstrap) = storage_bootstrap {
8308 let _ = bootstrap.install(&state);
8309 }
8310 run_state_initializers(state_initializers, &state);
8311
8312 let result = match mode {
8313 DataCapsuleMode::Verify => {
8316 crate::gdpr::portability::CapsuleService::signer_for_state(&state)
8317 .and_then(|signer| crate::gdpr::portability::verify_dir(&path, &signer))
8318 .map(|report| serde_json::json!(report))
8319 }
8320 DataCapsuleMode::Export | DataCapsuleMode::Import => {
8321 match crate::gdpr::portability::CapsuleService::from_state(&state) {
8322 Ok(service) if mode == DataCapsuleMode::Export => service
8323 .export_to(&subject, &path)
8324 .await
8325 .map(|report| serde_json::json!(report)),
8326 Ok(service) => service
8327 .import_from(&path)
8328 .await
8329 .map(|summary| serde_json::json!(summary)),
8330 Err(error) => Err(error),
8331 }
8332 }
8333 };
8334 #[cfg(feature = "managed-pg")]
8336 crate::managed_pg::emergency_stop_async().await;
8337 emit_data_capsule_report(mode, &result);
8338 }
8339
8340 #[cfg(not(feature = "db"))]
8350 #[allow(clippy::unused_async)]
8351 async fn run_retention_dry_run_mode(self) {
8352 eprintln!(
8353 "autumn retention --dry-run: this build has no database support — nothing to report"
8354 );
8355 println!("{RETENTION_DRY_RUN_JSON_PREFIX}[]");
8356 std::process::exit(0);
8357 }
8358
8359 #[allow(clippy::too_many_lines)]
8363 #[allow(clippy::cognitive_complexity)]
8364 async fn run_one_off_task_mode(self, requested_name: String) {
8365 let Self {
8366 one_off_tasks,
8367 mut jobs,
8368 listeners,
8369 #[cfg(feature = "db")]
8370 outbox_handlers,
8371 #[cfg(feature = "i18n")]
8372 custom_layers,
8373 #[cfg(not(feature = "i18n"))]
8374 custom_layers: _,
8375 startup_hooks,
8376 state_initializers,
8377 shutdown_hooks,
8378 config_loader_factory,
8379 #[cfg(feature = "db")]
8380 migrations,
8381 #[cfg(feature = "db")]
8382 pool_provider_factory,
8383 #[cfg(feature = "db")]
8384 shard_provider_factory,
8385 #[cfg(feature = "db")]
8386 shard_router,
8387 #[cfg(feature = "db")]
8388 directory_shard_router,
8389 telemetry_provider,
8390 session_store,
8391 #[cfg(feature = "ws")]
8392 channels_backend,
8393 #[cfg(feature = "storage")]
8394 blob_store,
8395 audit_logger,
8396 #[cfg(feature = "i18n")]
8397 i18n_bundle,
8398 #[cfg(feature = "i18n")]
8399 i18n_auto_load,
8400 #[cfg(feature = "embed-assets")]
8401 embedded_static,
8402 #[cfg(all(feature = "embed-assets", feature = "i18n"))]
8403 embedded_locales,
8404 policy_registrations,
8405 cache_backend,
8406 #[cfg(feature = "mail")]
8407 mail_delivery_queue_factory,
8408 #[cfg(feature = "mail")]
8409 suppression_store,
8410 #[cfg(feature = "mail")]
8411 mail_suppression_store,
8412 #[cfg(feature = "mail")]
8413 mount_unsubscribe_endpoint: _,
8414 #[cfg(feature = "mail")]
8415 mail_interceptor,
8416 job_interceptor,
8417 #[cfg(feature = "db")]
8418 db_interceptor,
8419 #[cfg(feature = "ws")]
8420 channels_interceptor,
8421 #[cfg(feature = "oauth2")]
8422 http_interceptor,
8423 plugin_config_roots,
8424 ..
8425 } = self;
8426
8427 if let Err(error) = crate::task::validate_unique_one_off_task_names(&one_off_tasks) {
8428 eprintln!("Invalid task registration: {error}");
8429 std::process::exit(1);
8430 }
8431
8432 let Some((task_name, task_handler)) = one_off_tasks
8433 .iter()
8434 .find(|task| task.name == requested_name)
8435 .map(|task| (task.name.clone(), task.handler))
8436 else {
8437 eprintln!("No one-off task named '{requested_name}' is registered.");
8438 print_available_one_off_tasks(&one_off_tasks);
8439 std::process::exit(1);
8440 };
8441
8442 let args = one_off_task_args_from_env().unwrap_or_else(|error| {
8443 eprintln!("Invalid task args: {error}");
8444 std::process::exit(1);
8445 });
8446
8447 let (config, telemetry_guard) = load_config_and_telemetry(
8448 config_loader_factory,
8449 telemetry_provider,
8450 plugin_config_roots,
8451 )
8452 .await;
8453
8454 #[cfg(feature = "embed-assets")]
8459 register_embedded_static_dir(embedded_static);
8460
8461 #[cfg(all(feature = "embed-assets", feature = "i18n"))]
8462 let i18n_bundle = embedded_i18n_bundle(i18n_bundle, embedded_locales, &config);
8463
8464 #[cfg(feature = "i18n")]
8465 let i18n_bundle =
8466 resolve_i18n_bundle(i18n_bundle, i18n_auto_load, &config, &crate::config::OsEnv);
8467
8468 fail_fast_on_invalid_session_config(&config, session_store.is_some());
8469 fail_fast_on_invalid_signing_secret(&config);
8470 fail_fast_on_missing_encryption_keys(&config);
8471 fail_fast_on_invalid_trusted_hosts(&config);
8472
8473 #[cfg(feature = "storage")]
8474 let storage_bootstrap = blob_store.map_or_else(
8475 || preflight_storage(&config),
8476 |store| {
8477 Some(StorageBootstrap {
8478 store,
8479 serving: None,
8480 })
8481 },
8482 );
8483
8484 #[cfg(feature = "db")]
8485 let database = setup_database(
8486 &config,
8487 migrations,
8488 pool_provider_factory,
8489 shard_provider_factory,
8490 shard_router,
8491 directory_shard_router,
8492 RepositoryCommitHookQueueMigrationMode::Runtime,
8493 )
8494 .await
8495 .unwrap_or_else(|error| {
8496 eprintln!("{error}");
8497 std::process::exit(1);
8498 });
8499 #[cfg(feature = "db")]
8500 let pool = database.topology;
8501 #[cfg(feature = "db")]
8502 let shards = database.shards;
8503 #[cfg(feature = "db")]
8504 let replica_readiness = database.replica_readiness;
8505 #[cfg(feature = "db")]
8506 let replica_migration_check = database.replica_migration_check;
8507
8508 let mut state = build_state(
8509 &config,
8510 #[cfg(feature = "db")]
8511 pool.as_ref(),
8512 #[cfg(feature = "db")]
8513 shards,
8514 #[cfg(feature = "ws")]
8515 channels_backend,
8516 );
8517 if let Some(buf) = telemetry_guard.log_buffer.clone() {
8518 state.insert_extension(buf);
8519 }
8520 if let Some(handle) = telemetry_guard.filter_reload.clone() {
8524 state.log_levels().attach_reload_handle(handle);
8525 }
8526 #[cfg(feature = "mail")]
8539 if let Some(interceptor) = mail_interceptor {
8540 state.insert_extension(interceptor);
8541 }
8542 if let Some(interceptor) = job_interceptor {
8543 state.insert_extension(interceptor);
8544 }
8545 #[cfg(feature = "db")]
8546 if let Some(interceptor) = db_interceptor {
8547 state.insert_extension(interceptor);
8548 }
8549 #[cfg(feature = "ws")]
8550 if let Some(interceptor) = channels_interceptor {
8551 state.insert_extension(interceptor.clone());
8552 state.channels = crate::channels::Channels::with_shared_backend(std::sync::Arc::new(
8553 crate::channels::InterceptedChannelsBackend::new(
8554 state.channels.backend().clone(),
8555 vec![interceptor],
8556 ),
8557 ));
8558 #[cfg(feature = "presence")]
8559 {
8560 state.presence = crate::presence::Presence::new(state.channels.clone());
8561 #[cfg(feature = "collab")]
8565 {
8566 state.collab = crate::collab::CollabHub::new(
8567 state.channels.clone(),
8568 state.presence.clone(),
8569 );
8570 }
8571 }
8572 }
8573 #[cfg(feature = "oauth2")]
8574 if let Some(interceptor) = http_interceptor {
8575 state.insert_extension(interceptor);
8576 }
8577 #[cfg(feature = "db")]
8578 configure_replica_migration_check(&state, replica_migration_check);
8579 #[cfg(feature = "db")]
8580 apply_replica_migration_readiness(&state, replica_readiness);
8581 if let Some(cache) = cache_backend {
8582 let cache = crate::cache::with_capsule_seam(cache);
8584 crate::cache::set_global_cache(cache.clone());
8585 state.shared_cache = Some(cache);
8586 } else {
8587 crate::cache::clear_global_cache();
8588 }
8589
8590 for register in policy_registrations {
8591 register(state.policy_registry());
8592 }
8593
8594 #[cfg(feature = "mail")]
8595 if let Some(handle) = suppression_store {
8596 state.insert_extension(handle);
8597 }
8598 #[cfg(feature = "mail")]
8599 if let Some(handle) = mail_suppression_store {
8600 state.insert_extension(handle);
8601 }
8602 #[cfg(feature = "db")]
8604 {
8605 crate::outbox::install(&state, &config.outbox, outbox_handlers);
8606 if state.extension::<crate::outbox::OutboxRelay>().is_some()
8607 && let Err(error) = crate::outbox::ensure_relay_schema(&state).await
8608 {
8609 eprintln!("Failed to create the outbox tables: {error}");
8610 exit_stop_managed_pg();
8611 std::process::exit(1);
8612 }
8613 }
8614 #[cfg(feature = "mail")]
8615 crate::mail::install_mailer_with_factory(
8616 &state,
8617 &config.mail,
8618 mail_delivery_queue_factory,
8619 true,
8620 )
8621 .unwrap_or_else(|error| {
8622 eprintln!("Failed to configure mailer: {error}");
8623 exit_stop_managed_pg();
8624 std::process::exit(1);
8625 });
8626
8627 if let Some(logger) = audit_logger {
8628 state.insert_extension::<crate::audit::AuditLogger>((*logger).clone());
8629 }
8630
8631 #[cfg(feature = "i18n")]
8632 let _custom_layers =
8633 install_i18n_bundle_layer(custom_layers, &state, i18n_bundle, &config.i18n);
8634
8635 #[cfg(feature = "storage")]
8636 let _storage_router = storage_bootstrap.and_then(|bootstrap| bootstrap.install(&state));
8637 run_state_initializers(state_initializers, &state);
8638 crate::cost::install(&state, &config);
8639 finalize_event_bus(listeners, &mut jobs, &state);
8640
8641 let task_shutdown = tokio_util::sync::CancellationToken::new();
8642 if let Err(error) = initialize_job_runtime(jobs, &state, &task_shutdown, &config.jobs, true)
8643 {
8644 eprintln!("job runtime initialization failed: {error}");
8645 #[cfg(feature = "managed-pg")]
8646 crate::managed_pg::emergency_stop_async().await;
8647 std::process::exit(1);
8648 }
8649
8650 #[cfg(feature = "db")]
8651 {
8652 #[cfg(feature = "ws")]
8653 crate::repository_commit_hooks::set_global_channels(state.channels().clone());
8654 }
8655
8656 #[cfg(all(feature = "db", not(feature = "sqlite")))]
8659 if let Some(pool) = state.pool().cloned() {
8660 #[cfg(feature = "ws")]
8661 {
8662 let channels = state.channels().clone();
8663 crate::repository_commit_hooks::start_repository_commit_hook_worker(
8664 pool,
8665 Some(channels),
8666 task_shutdown.child_token(),
8667 );
8668 }
8669 #[cfg(not(feature = "ws"))]
8670 crate::repository_commit_hooks::start_repository_commit_hook_worker(
8671 pool,
8672 task_shutdown.child_token(),
8673 );
8674 }
8675 #[cfg(all(feature = "db", not(feature = "sqlite")))]
8678 if let Some(shards) = state.shards() {
8679 for shard in shards.iter() {
8680 #[cfg(feature = "ws")]
8681 crate::repository_commit_hooks::start_repository_commit_hook_worker(
8682 shard.primary_pool().clone(),
8683 Some(state.channels().clone()),
8684 task_shutdown.child_token(),
8685 );
8686 #[cfg(not(feature = "ws"))]
8687 crate::repository_commit_hooks::start_repository_commit_hook_worker(
8688 shard.primary_pool().clone(),
8689 task_shutdown.child_token(),
8690 );
8691 }
8692 }
8693 #[cfg(all(feature = "db", feature = "sqlite"))]
8696 if let Some(pool) = state.pool().cloned() {
8697 #[cfg(feature = "ws")]
8698 {
8699 let channels = state.channels().clone();
8700 crate::repository_commit_hooks::start_repository_commit_hook_worker(
8701 pool,
8702 Some(channels),
8703 task_shutdown.child_token(),
8704 );
8705 }
8706 #[cfg(not(feature = "ws"))]
8707 crate::repository_commit_hooks::start_repository_commit_hook_worker(
8708 pool,
8709 task_shutdown.child_token(),
8710 );
8711 }
8712 #[cfg(feature = "sqlite")]
8713 let fleet_maintenance =
8714 start_fleet_runtime(&state, task_shutdown.child_token(), None).await;
8715
8716 if let Err(error) = run_startup_hooks(&startup_hooks, state.clone()).await {
8717 eprintln!("startup hook failed: {error}");
8718 task_shutdown.cancel();
8719 #[cfg(feature = "managed-pg")]
8720 crate::managed_pg::emergency_stop_async().await;
8721 std::process::exit(1);
8722 }
8723 state.probes().mark_startup_complete();
8724
8725 tracing::info!(task = %task_name, "Running one-off task");
8726 let span = tracing::info_span!("one_off_task", task = %task_name);
8727 #[cfg(feature = "oauth2")]
8728 let result = {
8729 use crate::interceptor::{ACTIVE_HTTP_INTERCEPTORS, HttpInterceptor};
8730 let interceptors: Vec<std::sync::Arc<dyn HttpInterceptor>> = state
8731 .extension::<std::sync::Arc<dyn HttpInterceptor>>()
8732 .map(|interceptor_arc| vec![(*interceptor_arc).clone()])
8733 .unwrap_or_default();
8734 ACTIVE_HTTP_INTERCEPTORS
8735 .scope(
8736 interceptors,
8737 (task_handler)(state.clone(), args).instrument(span),
8738 )
8739 .await
8740 };
8741 #[cfg(not(feature = "oauth2"))]
8742 let result = (task_handler)(state.clone(), args).instrument(span).await;
8743
8744 task_shutdown.cancel();
8745 #[cfg(feature = "sqlite")]
8748 if let Some(runtime) = fleet_maintenance {
8749 let _ = runtime.maintenance.await;
8750 if let Some(ownership) = runtime.ownership {
8751 ownership.finish().await;
8752 }
8753 }
8754 run_shutdown_hooks(&shutdown_hooks).await;
8755 #[cfg(feature = "managed-pg")]
8760 crate::managed_pg::emergency_stop_async().await;
8761
8762 match result {
8763 Ok(()) => {
8764 tracing::info!(task = %task_name, "One-off task completed");
8765 }
8766 Err(error) => {
8767 tracing::error!(task = %task_name, error = %error, "One-off task failed");
8768 eprintln!("Task '{task_name}' failed: {error}");
8769 for cause in error.source_chain() {
8770 eprintln!("Caused by: {cause}");
8771 }
8772 std::process::exit(1);
8773 }
8774 }
8775 }
8776
8777 #[cfg(feature = "reporting")]
8854 #[allow(clippy::too_many_lines)]
8855 async fn run_replay_mode(self, capsule_path: String) {
8856 let Self {
8857 routes,
8858 api_versions,
8859 listeners,
8860 exception_filters,
8861 scoped_groups,
8862 merge_routers,
8863 nest_routers,
8864 declared_routes,
8871 custom_layers,
8872 state_initializers,
8873 config_loader_factory,
8874 telemetry_provider,
8875 jobs,
8883 job_interceptor,
8884 session_store: _replay_ignores_custom_session_store,
8892 policy_registrations,
8893 #[cfg(feature = "db")]
8894 db_interceptor,
8895 #[cfg(feature = "ws")]
8902 channels_backend: _replay_ignores_custom_channels_backend,
8903 #[cfg(feature = "i18n")]
8904 i18n_bundle,
8905 #[cfg(feature = "i18n")]
8906 i18n_auto_load,
8907 #[cfg(feature = "maud")]
8908 error_page_renderer,
8909 #[cfg(feature = "embed-assets")]
8910 embedded_static,
8911 #[cfg(all(feature = "embed-assets", feature = "i18n"))]
8912 embedded_locales,
8913 plugin_config_roots,
8914 agent_skills,
8915 commerce_docs,
8916 #[cfg(feature = "mail")]
8919 mail_delivery_queue_factory,
8920 cache_backend,
8922 ..
8923 } = self;
8924 #[cfg(feature = "mail")]
8925 let builder_mail_queue = mail_delivery_queue_factory.is_some();
8926
8927 #[cfg(feature = "http-client")]
8931 crate::http_client::block_outbound_for_replay();
8932 crate::capsule::boundary::block_egress_for_replay();
8935 crate::cache::block_backends_for_replay();
8938
8939 let path = std::path::PathBuf::from(&capsule_path);
8940 let capsule = match crate::capsule::load_capsule(&path) {
8941 Ok(capsule) => capsule,
8942 Err(error) => {
8943 std::process::exit(crate::capsule::print_refusal(&error.to_string(), &path))
8944 }
8945 };
8946 if let Some(reason) = crate::capsule::refusal_reason(&capsule) {
8947 std::process::exit(crate::capsule::print_refusal(&reason, &path));
8948 }
8949
8950 let _replay_ignores_custom_telemetry_provider = telemetry_provider;
8956 let _replay_ignores_custom_config_loader = config_loader_factory;
8965 let (mut config, telemetry_guard) = load_config_and_telemetry(
8966 None,
8967 Some(Box::new(crate::telemetry::ReplayTelemetryProvider)),
8968 plugin_config_roots,
8969 )
8970 .await;
8971 force_offline_replay_config(&mut config);
8972
8973 if let Some(recorded_app) = capsule.app.name.as_deref() {
8979 let this_app = config.telemetry.service_name.as_str();
8980 if recorded_app != this_app {
8981 tracing::warn!(
8982 recorded_app,
8983 this_app,
8984 "the capsule was recorded by a different application; a verdict against \
8985 this one may be meaningless"
8986 );
8987 eprintln!(
8988 "warning: capsule was recorded by application {recorded_app:?}, but this \
8989 build is {this_app:?} — if this is not a renamed service, the verdict \
8990 below compares apples to oranges"
8991 );
8992 }
8993 }
8994
8995 #[cfg(feature = "embed-assets")]
8996 register_embedded_static_dir(embedded_static);
8997 #[cfg(all(feature = "embed-assets", feature = "i18n"))]
8998 let i18n_bundle = embedded_i18n_bundle(i18n_bundle, embedded_locales, &config);
8999 #[cfg(feature = "i18n")]
9000 let i18n_bundle =
9001 resolve_i18n_bundle(i18n_bundle, i18n_auto_load, &config, &crate::config::OsEnv);
9002
9003 let divergences = std::sync::Arc::new(crate::capsule::DivergenceLog::new());
9006 #[cfg(feature = "db")]
9007 let topology = replay_database_topology(&capsule, &divergences, &path);
9008
9009 let mut state = build_state(
9010 &config,
9011 #[cfg(feature = "db")]
9012 topology.as_ref(),
9013 #[cfg(feature = "db")]
9014 None,
9015 #[cfg(feature = "ws")]
9016 None,
9017 );
9018
9019 let fixtures = crate::capsule::ReplayFixtures::from_capsule(&capsule);
9023 state = state.with_clock(fixtures.clock());
9024 state = state.with_entropy(fixtures.entropy());
9025 if let Some(buf) = telemetry_guard.log_buffer.clone() {
9026 state.insert_extension(buf);
9027 }
9028 state.probes = crate::probe::ProbeState::default();
9032 state.apply_health_config(&config.health);
9033 state.insert_extension(RegisteredApiVersions(api_versions));
9034 state.insert_extension(crate::aeo::RegisteredAgentSkills(agent_skills));
9036 state.insert_extension(commerce_docs);
9037 #[cfg(feature = "db")]
9038 if let Some(interceptor) = db_interceptor {
9039 state.insert_extension(interceptor);
9040 }
9041 state.shared_cache = crate::cache::install_replay_cache(cache_backend.is_some());
9044
9045 for register in policy_registrations {
9046 register(state.policy_registry());
9047 }
9048
9049 #[cfg(feature = "i18n")]
9050 let custom_layers =
9051 install_i18n_bundle_layer(custom_layers, &state, i18n_bundle, &config.i18n);
9052
9053 install_webhook_registry(&state, &config);
9054 #[cfg(all(feature = "mail", feature = "db"))]
9058 let builder_mail_queue =
9059 builder_mail_queue || crate::outbox::installs_mail_queue(&state, &config.outbox);
9060 #[cfg(feature = "mail")]
9063 crate::mail::install_replay_mailer(&state, &config.mail, builder_mail_queue);
9064 run_state_initializers(state_initializers, &state);
9065 #[cfg(feature = "mail")]
9069 crate::mail::install_replay_mailer(&state, &config.mail, builder_mail_queue);
9070 crate::cache::install_late_replay_cache(&state, &capsule.effects);
9073 crate::cost::install(&state, &config);
9074 let sync_listeners: Vec<_> = listeners
9078 .into_iter()
9079 .filter(|listener| listener.mode == crate::events::DispatchMode::Sync)
9080 .collect();
9081 finalize_event_bus(sync_listeners, &mut Vec::new(), &state);
9082 state.insert_extension(config.clone());
9085
9086 let router_state = state.clone();
9089 crate::graph::install(
9092 &state,
9093 crate::graph::manifest::audit(
9094 &graph_mounted_routes(
9095 &routes,
9096 &scoped_groups,
9097 &declared_routes,
9098 &config,
9099 #[cfg(feature = "openapi")]
9101 None,
9102 ),
9103 omitted_router_count(
9104 merge_routers.len(),
9105 nest_routers.iter().map(|(prefix, _)| prefix.as_str()),
9106 &declared_routes,
9107 ),
9108 ),
9109 );
9110 let router = crate::router::try_build_router_inner(
9111 routes,
9112 &config,
9113 state,
9114 crate::router::RouterContext {
9115 exception_filters,
9116 scoped_groups,
9117 merge_routers,
9118 nest_routers,
9119 declared_routes,
9120 custom_layers,
9121 static_gate_layers: Vec::new(),
9122 #[cfg(feature = "maud")]
9123 error_page_renderer,
9124 session_store: None,
9125 #[cfg(feature = "openapi")]
9126 openapi: None,
9127 #[cfg(feature = "mcp")]
9128 mcp: None,
9129 #[cfg(feature = "webmcp")]
9130 webmcp: None,
9131 },
9132 )
9133 .unwrap_or_else(|error| {
9134 std::process::exit(crate::capsule::print_refusal(
9135 &format!("the application's router could not be rebuilt: {error}"),
9136 &path,
9137 ))
9138 });
9139
9140 let outcome = if let Some(job) = capsule.job.as_ref() {
9143 let Some(info) = jobs.iter().find(|info| info.name == job.name) else {
9144 std::process::exit(crate::capsule::print_refusal(
9145 &format!(
9146 "the capsule records a failure in job {:?}, which this build does not \
9147 register; replay it against the build that ran it, or add the job to \
9148 `AppBuilder::jobs()`",
9149 job.name
9150 ),
9151 &path,
9152 ));
9153 };
9154 let handler = info.handler;
9155 let job_state = router_state.clone();
9156 if let Some(interceptor) = job_interceptor {
9157 job_state.insert_extension(interceptor);
9158 }
9159 let job_name = job.name.clone();
9160 let dispatch: crate::capsule::JobDispatch = Box::new(move |payload| {
9161 Box::pin(async move {
9162 crate::job::run_handler_with_interceptor(&job_name, handler, job_state, payload)
9163 .await
9164 })
9165 });
9166 crate::capsule::execute_job(dispatch, &capsule, divergences, &fixtures).await
9167 } else {
9168 crate::capsule::execute(router, &capsule, divergences, &fixtures).await
9169 };
9170 std::process::exit(crate::capsule::print_verdict(&outcome, &path));
9171 }
9172}
9173
9174pub(crate) fn is_static_build_mode() -> bool {
9175 std::env::var("AUTUMN_BUILD_STATIC").as_deref() == Ok("1")
9176}
9177
9178pub(crate) const BUILD_STATIC_ALLOW_EMPTY_ENV: &str = "AUTUMN_BUILD_STATIC_ALLOW_EMPTY";
9181
9182fn clear_stale_static_output(dist_dir: &std::path::Path) -> std::io::Result<()> {
9185 match std::fs::remove_dir_all(dist_dir) {
9186 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
9187 other => other,
9188 }
9189}
9190
9191const fn no_static_routes_exit_code(allow_empty: bool) -> i32 {
9193 if allow_empty { 0 } else { 1 }
9194}
9195
9196#[allow(clippy::missing_const_for_fn)]
9207fn exit_stop_managed_pg() {
9208 #[cfg(feature = "managed-pg")]
9209 {
9210 let _ = std::thread::spawn(crate::managed_pg::emergency_stop).join();
9211 }
9212}
9213
9214pub(crate) fn is_dump_routes_mode() -> bool {
9215 std::env::var("AUTUMN_DUMP_ROUTES").as_deref() == Ok("1")
9216}
9217
9218pub(crate) fn is_dump_openapi_mode() -> bool {
9224 std::env::var("AUTUMN_DUMP_OPENAPI").as_deref() == Ok("1")
9225}
9226
9227pub(crate) fn is_dump_plugin_contract_mode() -> bool {
9234 std::env::var("AUTUMN_DUMP_PLUGIN_CONTRACT").as_deref() == Ok("1")
9235}
9236
9237pub(crate) fn is_dump_security_mode() -> bool {
9244 std::env::var("AUTUMN_DUMP_SECURITY").as_deref() == Ok("1")
9245}
9246
9247pub(crate) fn is_dump_jobs_mode() -> bool {
9248 std::env::var("AUTUMN_DUMP_JOBS").as_deref() == Ok("1")
9249}
9250
9251fn graph_mounted_routes(
9257 routes: &[Route],
9258 scoped_groups: &[ScopedGroup],
9259 declared_routes: &[crate::route_listing::RouteInfo],
9260 config: &crate::config::AutumnConfig,
9261 #[cfg(feature = "openapi")] openapi: Option<&crate::openapi::OpenApiConfig>,
9262) -> Vec<crate::graph::MountedRoute> {
9263 let mut framework: Vec<crate::route_listing::RouteInfo> = Vec::new();
9271 crate::route_listing::append_framework_routes(&mut framework, config);
9272 #[cfg(feature = "openapi")]
9274 if let Some(openapi) = openapi.filter(|_| config.openapi_runtime.enabled) {
9275 crate::route_listing::append_openapi_routes(&mut framework, openapi);
9276 }
9277
9278 routes
9279 .iter()
9280 .map(|route| graph_route_summary(route, None))
9281 .chain(scoped_groups.iter().flat_map(|group| {
9282 group
9283 .routes
9284 .iter()
9285 .map(move |route| graph_route_summary(route, Some(&group.prefix)))
9286 }))
9287 .chain(declared_routes.iter().map(graph_declared_route_summary))
9288 .chain(framework.iter().map(|route| crate::graph::MountedRoute {
9290 auth_declared: false,
9291 ..graph_declared_route_summary(route)
9292 }))
9293 .collect()
9294}
9295
9296fn graph_worker_mounted_routes(
9300 config: &crate::config::AutumnConfig,
9301) -> Vec<crate::graph::MountedRoute> {
9302 let mut framework: Vec<crate::route_listing::RouteInfo> = Vec::new();
9303 crate::route_listing::append_probe_and_actuator_routes(&mut framework, config);
9304 framework
9306 .iter()
9307 .map(|route| crate::graph::MountedRoute {
9308 auth_declared: false,
9309 ..graph_declared_route_summary(route)
9310 })
9311 .collect()
9312}
9313
9314fn graph_declared_route_summary(
9325 route: &crate::route_listing::RouteInfo,
9326) -> crate::graph::MountedRoute {
9327 let mut roles = route.roles.clone();
9328 roles.sort();
9329 roles.dedup();
9330 let mut scopes = route.scopes.clone();
9331 scopes.sort();
9332 scopes.dedup();
9333 crate::graph::MountedRoute {
9334 method: route.method.clone(),
9335 path: route.path.clone(),
9336 handler: route.handler.clone(),
9337 module_path: route.module.clone().unwrap_or_default(),
9338 auth: crate::graph::RouteAuth {
9339 secured: route.classification == crate::route_listing::RouteClassification::Gated,
9340 roles,
9341 scopes,
9342 policy: route.policy,
9343 repository_scope: false,
9349 public: route.classification == crate::route_listing::RouteClassification::Public,
9350 },
9351 repository_api: None,
9354 auth_declared: matches!(
9358 route.classification,
9359 crate::route_listing::RouteClassification::Gated
9360 | crate::route_listing::RouteClassification::Public
9361 ),
9362 }
9363}
9364
9365fn graph_route_summary(route: &Route, scope_prefix: Option<&str>) -> crate::graph::MountedRoute {
9378 let mut roles: Vec<String> = route
9379 .api_doc
9380 .required_roles
9381 .iter()
9382 .map(|r| (*r).to_owned())
9383 .collect();
9384 roles.sort();
9385 roles.dedup();
9386 let mut scopes: Vec<String> = route
9387 .api_doc
9388 .required_scopes
9389 .iter()
9390 .map(|s| (*s).to_owned())
9391 .collect();
9392 scopes.sort();
9393 scopes.dedup();
9394 crate::graph::MountedRoute {
9395 method: route.method.to_string(),
9396 path: scope_prefix.map_or_else(
9397 || route.path.to_string(),
9398 |prefix| crate::router::join_nested_path(prefix, route.path),
9399 ),
9400 handler: route.name.to_owned(),
9401 module_path: route.api_doc.module_path.to_owned(),
9402 auth: crate::graph::RouteAuth {
9411 secured: route.api_doc.secured,
9412 roles,
9413 scopes,
9414 policy: route.api_doc.has_policy
9415 || route
9416 .repository
9417 .as_ref()
9418 .is_some_and(|meta| meta.has_policy),
9419 repository_scope: route
9420 .repository
9421 .as_ref()
9422 .is_some_and(|meta| meta.scope_check.is_some()),
9423 public: route.api_doc.public,
9424 },
9425 repository_api: route
9428 .repository
9429 .as_ref()
9430 .map(|meta| crate::graph::RepositoryApiOwner {
9431 api: meta.api_path.to_owned(),
9432 model: meta.resource_type_name.to_owned(),
9433 }),
9434 auth_declared: false,
9435 }
9436}
9437
9438#[derive(Debug, Clone, Copy, Default)]
9440struct AgentExposure {
9441 mcp_expose_all: bool,
9443 webmcp_expose_all: bool,
9445}
9446
9447fn agent_authority_route_summary(
9462 route: &Route,
9463 scope_prefix: Option<&str>,
9464 exposure: AgentExposure,
9465) -> crate::agent_authority::manifest::RouteSummary {
9466 let method = route.method.to_string();
9469 let path = scope_prefix.map_or_else(
9473 || route.path.to_string(),
9474 |prefix| crate::router::join_nested_path(prefix, route.path),
9475 );
9476 let input = crate::agent_authority::manifest::McpExposureInput {
9477 method: &method,
9478 hidden: route.api_doc.hidden,
9479 mcp_tool: route.api_doc.mcp_tool,
9480 mcp_exclude: route.api_doc.mcp_exclude,
9481 mcp_stream: route.api_doc.mcp_stream,
9482 has_response_schema: route.api_doc.response.is_some(),
9483 success_status: route.api_doc.success_status,
9484 expose_all: exposure.mcp_expose_all,
9485 };
9486 let exposed_by = crate::agent_authority::manifest::mcp_exposure(&input);
9487 let webmcp_exposed_by = crate::agent_authority::manifest::webmcp_exposure(
9488 &input,
9489 route.api_doc.webmcp_tool,
9490 route.api_doc.webmcp_exclude,
9491 exposure.webmcp_expose_all,
9492 );
9493 crate::agent_authority::manifest::RouteSummary {
9494 method,
9495 path,
9496 handler: route.name,
9497 operation_id: route.api_doc.operation_id,
9500 module_path: route.api_doc.module_path,
9501 mcp_tool: exposed_by.is_some(),
9502 exposed_by,
9503 duplicate_operation_id: false,
9504 webmcp_tool: webmcp_exposed_by.is_some(),
9505 webmcp_exposed_by,
9506 agent_authority: route.api_doc.agent_authority,
9509 }
9510}
9511
9512pub(crate) fn is_list_one_off_tasks_mode() -> bool {
9513 std::env::var("AUTUMN_LIST_TASKS").as_deref() == Ok("1")
9514}
9515
9516pub(crate) fn is_migrate_only_mode() -> bool {
9521 std::env::var("AUTUMN_MIGRATE").as_deref() == Ok("1")
9522}
9523
9524pub(crate) fn framework_retention_mode_from_env() -> Option<FrameworkRetentionMode> {
9532 let raw = std::env::var(FRAMEWORK_RETENTION_ENV).ok()?;
9533 match raw.trim() {
9534 "" => None,
9535 "report" => Some(FrameworkRetentionMode::Report),
9536 "purge" => Some(FrameworkRetentionMode::Purge),
9537 other => {
9538 eprintln!(
9544 "Warning: {FRAMEWORK_RETENTION_ENV}={other:?} is not valid (expected \"report\" \
9545 or \"purge\"), ignoring"
9546 );
9547 None
9548 }
9549 }
9550}
9551
9552#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9554pub(crate) enum DataCapsuleMode {
9555 Export,
9557 Import,
9559 Verify,
9561}
9562
9563impl DataCapsuleMode {
9564 const fn as_str(self) -> &'static str {
9565 match self {
9566 Self::Export => "export",
9567 Self::Import => "import",
9568 Self::Verify => "verify",
9569 }
9570 }
9571}
9572
9573pub(crate) const DATA_CAPSULE_ENV: &str = "AUTUMN_DATA_CAPSULE";
9575pub(crate) const DATA_CAPSULE_SUBJECT_ENV: &str = "AUTUMN_DATA_CAPSULE_SUBJECT";
9577pub(crate) const DATA_CAPSULE_PATH_ENV: &str = "AUTUMN_DATA_CAPSULE_PATH";
9579pub(crate) const DATA_CAPSULE_JSON_PREFIX: &str = "AUTUMN_DATA_CAPSULE_REPORT=";
9581
9582pub(crate) fn data_capsule_mode_from_env() -> Option<DataCapsuleMode> {
9587 parse_data_capsule_mode(&std::env::var(DATA_CAPSULE_ENV).ok()?)
9588}
9589
9590fn parse_data_capsule_mode(raw: &str) -> Option<DataCapsuleMode> {
9591 match raw.trim() {
9592 "" => None,
9593 "export" => Some(DataCapsuleMode::Export),
9594 "import" => Some(DataCapsuleMode::Import),
9595 "verify" => Some(DataCapsuleMode::Verify),
9596 other => {
9597 eprintln!(
9598 "Warning: {DATA_CAPSULE_ENV}={other:?} is not valid (expected \"export\", \
9599 \"import\" or \"verify\"), ignoring"
9600 );
9601 None
9602 }
9603 }
9604}
9605
9606fn data_capsule_report_line(
9608 mode: DataCapsuleMode,
9609 result: &Result<serde_json::Value, crate::gdpr::portability::DataCapsuleError>,
9610) -> String {
9611 let body = match result {
9612 Ok(report) => serde_json::json!({"ok": true, "mode": mode.as_str(), "report": report}),
9613 Err(error) => {
9614 serde_json::json!({"ok": false, "mode": mode.as_str(), "error": error.to_string()})
9615 }
9616 };
9617 format!("{DATA_CAPSULE_JSON_PREFIX}{body}")
9618}
9619
9620fn emit_data_capsule_report(
9622 mode: DataCapsuleMode,
9623 result: &Result<serde_json::Value, crate::gdpr::portability::DataCapsuleError>,
9624) -> ! {
9625 println!("{}", data_capsule_report_line(mode, result));
9626 if let Err(error) = result {
9627 eprintln!("autumn data capsule {}: {error}", mode.as_str());
9628 }
9629 std::process::exit(i32::from(result.is_err()));
9630}
9631
9632#[cfg(feature = "db")]
9642async fn verify_database_parts(
9643 config: &AutumnConfig,
9644 pool_provider: Option<PoolProviderFactory>,
9645 shard_provider: Option<ShardProviderFactory>,
9646 shard_router: Option<Arc<dyn crate::sharding::ShardRouter>>,
9647 directory_shard_router: bool,
9648) -> (
9649 Option<crate::db::DatabaseTopology>,
9650 Option<crate::sharding::ShardSet>,
9651) {
9652 let topology = match pool_provider {
9653 Some(factory) => factory(config.database.clone()).await,
9654 None => crate::db::create_topology(&config.database),
9655 }
9656 .ok()
9657 .flatten();
9658 let use_directory_router = shard_router.is_none()
9659 && (directory_shard_router || config.database.directory_shard_router);
9660 let shards = resolve_shard_set(
9662 config,
9663 shard_router,
9664 shard_provider,
9665 use_directory_router,
9666 false,
9667 topology.as_ref(),
9668 )
9669 .await
9670 .ok()
9671 .flatten();
9672 (topology, shards)
9673}
9674
9675#[cfg(test)]
9676mod data_capsule_mode_tests {
9677 use super::*;
9678
9679 #[cfg(all(feature = "db", not(feature = "sqlite")))]
9680 #[tokio::test]
9681 async fn verify_gives_initializers_lazy_pools_and_shards() {
9682 let mut config = AutumnConfig::default();
9686 config.database.url = Some("postgres://u:p@127.0.0.1:9/none".to_owned());
9687 config.database.shards = vec![crate::config::ShardConfig {
9688 name: "s1".to_owned(),
9689 primary_url: crate::test_urls::primary("s1"),
9690 slots: None,
9691 replica_url: None,
9692 primary_pool_size: None,
9693 replica_pool_size: None,
9694 replica_fallback: None,
9695 }];
9696 let (topology, shards) = verify_database_parts(&config, None, None, None, false).await;
9697 assert!(topology.is_some(), "a lazy control pool");
9698 assert_eq!(shards.map(|s| s.len()), Some(1), "the configured shard");
9699 }
9700
9701 #[cfg(feature = "sqlite")]
9702 #[tokio::test]
9703 async fn verify_gives_sqlite_initializers_a_lazy_pool_without_a_file() {
9704 let dir = tempfile::tempdir().unwrap();
9707 let path = dir.path().join("app.db");
9708 let mut config = AutumnConfig::default();
9709 config.database.url = Some(format!("sqlite://{}", path.display()));
9710 let (topology, _) = verify_database_parts(&config, None, None, None, false).await;
9711 assert!(topology.is_some(), "a lazy pool");
9712 assert!(!path.exists(), "verify creates no database file");
9713 }
9714
9715 #[cfg(feature = "db")]
9716 #[tokio::test]
9717 async fn verify_uses_the_pool_provider_of_the_app() {
9718 let dir = tempfile::tempdir().unwrap();
9721 let url = if cfg!(feature = "sqlite") {
9722 format!("sqlite://{}", dir.path().join("app.db").display())
9723 } else {
9724 "postgres://u:p@127.0.0.1:9/none".to_owned()
9725 };
9726 let provider: PoolProviderFactory = Box::new(move |mut database| {
9727 Box::pin(async move {
9728 database.url = Some(url);
9729 crate::db::create_topology(&database)
9730 })
9731 });
9732 let config = AutumnConfig::default();
9733 let (topology, _) = verify_database_parts(&config, Some(provider), None, None, false).await;
9734 assert!(topology.is_some(), "the topology of the provider");
9735 }
9736
9737 #[test]
9738 fn mode_parses_the_three_values_and_ignores_others() {
9739 assert_eq!(
9740 parse_data_capsule_mode("export"),
9741 Some(DataCapsuleMode::Export)
9742 );
9743 assert_eq!(
9744 parse_data_capsule_mode(" import "),
9745 Some(DataCapsuleMode::Import)
9746 );
9747 assert_eq!(
9748 parse_data_capsule_mode("verify"),
9749 Some(DataCapsuleMode::Verify)
9750 );
9751 assert_eq!(parse_data_capsule_mode(""), None);
9752 assert_eq!(parse_data_capsule_mode("purge"), None);
9753 }
9754
9755 #[test]
9756 fn report_line_carries_the_prefix_and_the_outcome() {
9757 let ok = data_capsule_report_line(
9758 DataCapsuleMode::Export,
9759 &Ok(serde_json::json!({"records": 3})),
9760 );
9761 let json: serde_json::Value =
9762 serde_json::from_str(ok.strip_prefix(DATA_CAPSULE_JSON_PREFIX).unwrap()).unwrap();
9763 assert_eq!(json["ok"], true);
9764 assert_eq!(json["mode"], "export");
9765 assert_eq!(json["report"]["records"], 3);
9766
9767 let err = data_capsule_report_line(
9768 DataCapsuleMode::Verify,
9769 &Err(crate::gdpr::portability::DataCapsuleError::MissingSigningSecret),
9770 );
9771 let json: serde_json::Value =
9772 serde_json::from_str(err.strip_prefix(DATA_CAPSULE_JSON_PREFIX).unwrap()).unwrap();
9773 assert_eq!(json["ok"], false);
9774 assert!(json["error"].as_str().unwrap().contains("signing secret"));
9775 }
9776}
9777
9778pub(crate) const FRAMEWORK_RETENTION_ENV: &str = "AUTUMN_DB_RETENTION";
9780
9781fn framework_purge_refusal(
9784 mode: FrameworkRetentionMode,
9785 profile: Option<&str>,
9786 force_env: Option<&str>,
9787) -> Option<String> {
9788 if mode != FrameworkRetentionMode::Purge {
9789 return None;
9790 }
9791 crate::data_retention::purge_refusal(profile, force_env == Some("1"))
9792}
9793
9794pub(crate) const FRAMEWORK_RETENTION_FORCE_ENV: &str = "AUTUMN_DB_RETENTION_FORCE";
9796
9797pub(crate) const FRAMEWORK_RETENTION_DATASET_ENV: &str = "AUTUMN_DB_RETENTION_DATASET";
9799
9800pub(crate) const FRAMEWORK_RETENTION_JSON_PREFIX: &str = "AUTUMN_DB_RETENTION_REPORT=";
9803
9804#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9806pub(crate) enum FrameworkRetentionMode {
9807 Report,
9809 Purge,
9811}
9812
9813pub(crate) fn framework_retention_dataset_from_env() -> Option<String> {
9817 std::env::var(FRAMEWORK_RETENTION_DATASET_ENV)
9818 .ok()
9819 .map(|value| value.trim().to_owned())
9820 .filter(|value| !value.is_empty())
9821}
9822
9823pub(crate) fn is_retention_dry_run_mode() -> bool {
9828 std::env::var("AUTUMN_RETENTION_DRY_RUN").as_deref() == Ok("1")
9829}
9830
9831const RETENTION_DRY_RUN_JSON_PREFIX: &str = "AUTUMN_RETENTION_DRY_RUN_REPORT=";
9847
9848#[cfg(feature = "db")]
9851fn retention_dry_run_model_filter_from_env() -> Option<String> {
9852 std::env::var("AUTUMN_RETENTION_MODEL")
9853 .ok()
9854 .map(|value| value.trim().to_owned())
9855 .filter(|value| !value.is_empty())
9856}
9857
9858#[cfg(feature = "db")]
9869fn merge_and_validate_task_names(
9870 descriptors: &[&crate::retention::RetentionSweepDescriptor],
9871 tasks: Vec<crate::task::TaskInfo>,
9872) -> Result<(), String> {
9873 let mut merged: Vec<crate::task::TaskInfo> = descriptors
9874 .iter()
9875 .map(|descriptor| (descriptor.task_info)())
9876 .collect();
9877 merged.extend(tasks);
9878 crate::task::validate_unique_scheduled_task_names(&merged)
9879}
9880
9881pub(crate) fn is_replay_mode() -> bool {
9885 replay_capsule_from_env().is_some()
9886}
9887
9888fn replay_capsule_from_env() -> Option<String> {
9890 std::env::var("AUTUMN_REPLAY_CAPSULE")
9891 .ok()
9892 .as_deref()
9893 .and_then(normalize_replay_capsule)
9894}
9895
9896fn normalize_replay_capsule(value: &str) -> Option<String> {
9898 let trimmed = value.trim();
9899 (!trimmed.is_empty()).then(|| trimmed.to_owned())
9900}
9901
9902#[cfg(feature = "reporting")]
9910fn force_offline_replay_config(config: &mut AutumnConfig) {
9911 config.failure_capture.enabled = false;
9914 config.session.backend = crate::session::SessionBackend::Memory;
9917 config.session.allow_memory_in_production = true;
9918 config.channels.backend = crate::config::ChannelBackend::InProcess;
9922 config.security.rate_limit.backend = crate::security::config::RateLimitBackend::Memory;
9930 config.idempotency.backend = crate::config::IdempotencyBackend::Memory;
9931 config.idempotency.allow_memory_in_production = true;
9932 config.security.submit_token.backend = Some(crate::config::IdempotencyBackend::Memory);
9935 config.security.webhooks.replay.backend = crate::webhook::WebhookReplayBackend::Memory;
9936 config.security.webhooks.replay.allow_memory_in_production = true;
9937 config.cache.backend = crate::config::CacheBackend::Memory;
9940 "local".clone_into(&mut config.jobs.backend);
9944 config.server.timeouts.request_timeout_ms = None;
9953 config.shadow.enabled = false;
9955}
9956
9957#[cfg(all(feature = "reporting", feature = "db", not(feature = "sqlite")))]
9961fn replay_database_topology(
9962 capsule: &crate::capsule::Capsule,
9963 divergences: &std::sync::Arc<crate::capsule::DivergenceLog>,
9964 capsule_path: &std::path::Path,
9965) -> Option<crate::db::DatabaseTopology> {
9966 if capsule.db.is_none() && capsule.db_roles.is_empty() {
9974 return None;
9975 }
9976 let pool = crate::capsule::pool_from_capsule(capsule, std::sync::Arc::clone(divergences))
9977 .unwrap_or_else(|error| {
9978 std::process::exit(crate::capsule::print_refusal(
9979 &format!("the capsule's database tape could not be served: {error}"),
9980 capsule_path,
9981 ))
9982 });
9983 let replica =
9988 crate::capsule::replica_pool_from_capsule(capsule, std::sync::Arc::clone(divergences))
9989 .unwrap_or_else(|error| {
9990 std::process::exit(crate::capsule::print_refusal(
9991 &format!("the capsule's replica tape could not be served: {error}"),
9992 capsule_path,
9993 ))
9994 });
9995 Some(crate::db::DatabaseTopology::from_pools(pool, replica))
9996}
9997
9998#[cfg(all(feature = "reporting", feature = "db", feature = "sqlite"))]
10002fn replay_database_topology(
10003 capsule: &crate::capsule::Capsule,
10004 _divergences: &std::sync::Arc<crate::capsule::DivergenceLog>,
10005 capsule_path: &std::path::Path,
10006) -> Option<crate::db::DatabaseTopology> {
10007 if capsule.db.is_some() {
10008 std::process::exit(crate::capsule::print_refusal(
10009 "the capsule carries a PostgreSQL database tape, but this binary was built with the \
10010 `sqlite` backend, which has neither wire capture nor wire replay. Replay it with a \
10011 PostgreSQL build of the application.",
10012 capsule_path,
10013 ));
10014 }
10015 None
10016}
10017
10018fn one_off_task_name_from_env() -> Option<String> {
10019 std::env::var("AUTUMN_RUN_TASK")
10020 .ok()
10021 .map(|value| value.trim().to_owned())
10022 .filter(|value| !value.is_empty())
10023}
10024
10025fn one_off_task_args_from_env() -> Result<Vec<String>, String> {
10026 match std::env::var("AUTUMN_TASK_ARGS_JSON") {
10027 Ok(raw) if !raw.trim().is_empty() => serde_json::from_str(&raw)
10028 .map_err(|error| format!("AUTUMN_TASK_ARGS_JSON must be a JSON string array: {error}")),
10029 _ => Ok(Vec::new()),
10030 }
10031}
10032
10033fn print_available_one_off_tasks(tasks: &[crate::task::OneOffTaskInfo]) {
10034 let listing = crate::task::list_one_off_tasks(tasks);
10035 if listing.is_empty() {
10036 eprintln!("No one-off tasks are registered. Add .one_off_tasks(one_off_tasks![...]).");
10037 return;
10038 }
10039
10040 eprintln!("Available tasks:");
10041 for task in listing {
10042 if task.description.is_empty() {
10043 eprintln!(" {}", task.name);
10044 } else {
10045 eprintln!(" {:<24} {}", task.name, task.description);
10046 }
10047 }
10048}
10049
10050#[allow(clippy::cast_possible_truncation)]
10057#[allow(clippy::cognitive_complexity)]
10058#[allow(dead_code)]
10059fn start_task_scheduler(
10060 tasks: Vec<crate::task::TaskInfo>,
10061 state: &AppState,
10062 shutdown: &tokio_util::sync::CancellationToken,
10063) {
10064 if let Err(error) = start_task_scheduler_with_config(
10065 tasks,
10066 state,
10067 shutdown,
10068 &crate::config::SchedulerConfig::default(),
10069 ) {
10070 tracing::error!(error = %error, "scheduled task runtime initialization failed");
10071 }
10072}
10073
10074#[allow(clippy::cast_possible_truncation)]
10075#[allow(clippy::cognitive_complexity)]
10076pub(crate) fn start_task_scheduler_with_config(
10077 tasks: Vec<crate::task::TaskInfo>,
10078 state: &AppState,
10079 shutdown: &tokio_util::sync::CancellationToken,
10080 scheduler_config: &crate::config::SchedulerConfig,
10081) -> crate::AutumnResult<()> {
10082 tracing::info!(count = tasks.len(), "Starting scheduled tasks");
10083 let coordinator = crate::scheduler::coordinator_from_config(scheduler_config, state)?;
10084 let lease_ttl = std::time::Duration::from_secs(scheduler_config.lease_ttl_secs);
10085 for task_info in &tasks {
10086 let schedule_desc = task_info.schedule.to_string();
10087 tracing::info!(
10088 name = %task_info.name,
10089 schedule = %schedule_desc,
10090 coordination = %task_info.coordination,
10091 scheduler_backend = coordinator.backend(),
10092 replica_id = coordinator.replica_id(),
10093 lease_ttl_secs = scheduler_config.lease_ttl_secs,
10094 "Registered task"
10095 );
10096 }
10097
10098 let mut cron_tasks: Vec<CronTaskSpec> = Vec::new();
10099
10100 for task_info in tasks {
10101 let state = state.clone();
10102 let name = task_info.name.clone();
10103 let handler = task_info.handler;
10104 let coordination = task_info.coordination;
10105 let schedule_desc = task_info.schedule.to_string();
10106 state.task_registry.register_scheduled(
10107 &name,
10108 &schedule_desc,
10109 coordination,
10110 coordinator.backend(),
10111 coordinator.replica_id(),
10112 );
10113
10114 match task_info.schedule {
10115 crate::task::Schedule::FixedDelay(delay) => {
10116 let coordinator = Arc::clone(&coordinator);
10117 let shutdown = shutdown.child_token();
10118 crate::sim::spawn_app_task(&state.clone(), async move {
10119 loop {
10120 state.task_registry.record_next_run_at(
10121 &name,
10122 &format_next_task_run_after(state.clock().now(), delay),
10123 );
10124 tokio::select! {
10125 biased;
10127 () = shutdown.cancelled() => break,
10128 () = tokio::time::sleep(delay) => {
10129 execute_fixed_delay_task(
10130 name.clone(),
10131 state.clone(),
10132 handler,
10133 delay,
10134 coordination,
10135 Arc::clone(&coordinator),
10136 lease_ttl,
10137 shutdown.clone(),
10138 )
10139 .await;
10140 }
10141 }
10142 }
10143 });
10144 }
10145 crate::task::Schedule::Cron {
10146 expression,
10147 timezone,
10148 } => {
10149 cron_tasks.push(CronTaskSpec {
10150 name,
10151 expression,
10152 timezone,
10153 coordination,
10154 handler,
10155 });
10156 }
10157 }
10158 }
10159
10160 run_cron_scheduler(cron_tasks, state, shutdown, &coordinator, lease_ttl);
10161
10162 Ok(())
10163}
10164
10165fn in_process_fleet_warning(
10169 config: &AutumnConfig,
10170 tasks: &[crate::task::TaskInfo],
10171 env: &dyn crate::config::Env,
10172) -> Option<String> {
10173 let has_fleet_task = tasks
10174 .iter()
10175 .any(|task| task.coordination == crate::task::TaskCoordination::Fleet);
10176 if !has_fleet_task {
10177 return None;
10178 }
10179 let replicas = env.var("AUTUMN_REPLICAS").ok();
10180 let kubernetes = env.var("KUBERNETES_SERVICE_HOST").is_ok();
10181 let hint = crate::scheduler::in_process_fleet_hint(
10182 config.scheduler.backend,
10183 &config.jobs.backend,
10184 replicas.as_deref(),
10185 kubernetes,
10186 )?;
10187 let fix = if cfg!(feature = "sqlite") {
10188 "sqlite"
10189 } else {
10190 "postgres"
10191 };
10192 Some(format!(
10193 "scheduler.backend = \"in_process\" runs each fleet tick on every replica \
10194 ({hint}). If more than one replica runs, set scheduler.backend = \"{fix}\". \
10195 See docs/guide/scheduled-multi-replica.md"
10196 ))
10197}
10198
10199#[allow(unused_variables, clippy::needless_pass_by_value)]
10200fn send_ws_sys_task_msg(
10201 state: &AppState,
10202 event: &str,
10203 name: &str,
10204 extra: Vec<(&str, serde_json::Value)>,
10205) {
10206 #[cfg(feature = "ws")]
10207 {
10208 let mut msg = serde_json::json!({
10212 "event": event,
10213 "task": name,
10214 "timestamp": state.clock().now().to_rfc3339(),
10215 });
10216 if let Some(map) = msg.as_object_mut() {
10217 for (k, v) in extra {
10218 map.insert(k.to_string(), v);
10219 }
10220 }
10221 let _ = state.channels().sender("sys:tasks").send(msg.to_string());
10222 }
10223}
10224
10225fn task_duration_ms(state: &AppState, start: crate::time::MonotonicInstant) -> u64 {
10230 let elapsed = state.monotonic().saturating_duration_since(start);
10231 u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX)
10232}
10233
10234async fn execute_task_result(
10235 state: &AppState,
10236 handler: crate::task::TaskHandler,
10237 start: crate::time::MonotonicInstant,
10238 name: &str,
10239 schedule: &'static str,
10240 waited: bool,
10241) -> Result<u64, (u64, String)> {
10242 crate::sim::note_drain_progress();
10244 let task_span = tracing::info_span!(
10248 parent: None,
10249 "scheduled_task",
10250 otel.kind = "internal",
10251 task = %name,
10252 schedule = schedule,
10253 );
10254 let handler_state = state.clone();
10258 let future = async move { (handler)(handler_state).instrument(task_span).await };
10259 let run = crate::cost::WorkRun {
10260 tenant: None,
10261 waited,
10262 };
10263 let future = crate::cost::meter_work(state, crate::cost::WorkKind::Task, name, run, future);
10264 let result = std::panic::AssertUnwindSafe(future).catch_unwind().await;
10265 let duration_ms = task_duration_ms(state, start);
10266
10267 match result {
10268 Ok(Ok(())) => Ok(duration_ms),
10269 Ok(Err(e)) => Err((duration_ms, e.message())),
10272 Err(panic) => Err((duration_ms, format_scheduled_task_panic(panic.as_ref()))),
10273 }
10274}
10275
10276fn format_scheduled_task_panic(panic: &(dyn Any + Send)) -> String {
10277 let detail = panic
10278 .downcast_ref::<String>()
10279 .map(String::as_str)
10280 .or_else(|| panic.downcast_ref::<&'static str>().copied())
10281 .unwrap_or("non-string panic payload");
10282 format!("scheduled task handler panicked: {detail}")
10283}
10284
10285#[allow(
10288 clippy::too_many_arguments,
10289 reason = "one tick: its task, schedule, lease and cost wait"
10290)]
10291async fn execute_task_result_with_optional_lease_ttl(
10292 state: &AppState,
10293 handler: crate::task::TaskHandler,
10294 start: crate::time::MonotonicInstant,
10295 name: &str,
10296 schedule: &'static str,
10297 lease_ttl: Option<std::time::Duration>,
10298 tick: crate::scheduler::ScheduledTick,
10299 waited: bool,
10300) -> Result<u64, (u64, String)> {
10301 let run = execute_task_result(state, handler, start, name, schedule, waited);
10302 #[cfg(feature = "db")]
10305 let run = crate::db::scope_background_tx_timeouts(state, run);
10306 let run = crate::scheduler::with_tick(tick, run);
10307 let Some(lease_ttl) = lease_ttl else {
10308 return run.await;
10309 };
10310
10311 tokio::time::timeout(lease_ttl, run)
10312 .await
10313 .unwrap_or_else(|_| {
10314 let duration_ms = task_duration_ms(state, start);
10315 Err((
10316 duration_ms,
10317 format!(
10318 "scheduled task exceeded lease TTL of {}s",
10319 lease_ttl.as_secs()
10320 ),
10321 ))
10322 })
10323}
10324
10325#[allow(clippy::cognitive_complexity)]
10327#[allow(
10328 clippy::too_many_arguments,
10329 reason = "one tick: its task, schedule, lease and cost gate"
10330)]
10331async fn execute_fixed_delay_task(
10332 name: String,
10333 state: AppState,
10334 handler: crate::task::TaskHandler,
10335 delay: std::time::Duration,
10336 coordination: crate::task::TaskCoordination,
10337 coordinator: Arc<dyn crate::scheduler::SchedulerCoordinator>,
10338 lease_ttl: std::time::Duration,
10339 shutdown: tokio_util::sync::CancellationToken,
10340) {
10341 let gate = CostGate {
10343 shutdown,
10344 waiting: None,
10345 waited: std::sync::atomic::AtomicBool::new(false),
10346 };
10347 let wait_first = CostGate::waits_before_lease(&*coordinator);
10348 let (lease, tick_key) = loop {
10349 if wait_first && !gate.wait(&state, &name).await {
10350 return;
10351 }
10352 let tick_key = crate::scheduler::fixed_delay_tick_key(
10353 &name,
10354 delay,
10355 crate::time::clock_unix_duration(state.clock()),
10356 );
10357 let lease = match coordinator
10358 .try_acquire_for_period(&name, &tick_key, coordination, delay)
10360 .await
10361 {
10362 Ok(Some(lease)) => lease,
10363 Ok(None) => {
10364 tracing::debug!(task = %name, tick = %tick_key, "Scheduled task tick already claimed");
10365 return;
10366 }
10367 Err(error) => {
10368 tracing::warn!(task = %name, tick = %tick_key, error = %error, "Failed to acquire scheduled task lease");
10369 return;
10370 }
10371 };
10372 if wait_first && CostGate::must_wait(&state, &name) {
10375 CostGate::free_unrun_claim(lease, &name, &tick_key).await;
10376 continue;
10377 }
10378 break (lease, tick_key);
10379 };
10380 state
10381 .task_registry
10382 .record_leader(&name, lease.leader_id(), &tick_key);
10383 if !wait_first && !gate.wait(&state, &name).await {
10384 release_task_lease(lease, &name, &tick_key).await;
10385 return;
10386 }
10387 tracing::debug!(task = %name, "Running scheduled task");
10388 state.task_registry.record_start(&name);
10389
10390 send_ws_sys_task_msg(&state, "started", &name, vec![]);
10391
10392 let start = state.monotonic();
10393 let lease_ttl = lease_ttl_for_run(&lease, coordination, lease_ttl);
10394 let tick = crate::scheduler::ScheduledTick::new(&tick_key, &lease);
10395 match execute_task_result_with_optional_lease_ttl(
10396 &state,
10397 handler,
10398 start,
10399 &name,
10400 "fixed_delay",
10401 lease_ttl,
10402 tick,
10403 gate.waited(),
10404 )
10405 .await
10406 {
10407 Ok(duration_ms) => {
10408 state.task_registry.record_success(&name, duration_ms);
10409 crate::alerts::notify_scheduled_task_recovered(&state, &name);
10410 tracing::debug!(task = %name, "Task completed");
10411 send_ws_sys_task_msg(
10412 &state,
10413 "success",
10414 &name,
10415 vec![("duration_ms", serde_json::json!(duration_ms))],
10416 );
10417 }
10418 Err((duration_ms, error_str)) => {
10419 state
10420 .task_registry
10421 .record_failure(&name, duration_ms, &error_str);
10422 crate::alerts::notify_scheduled_task_failure(&state, &name, &error_str);
10423 tracing::warn!(task = %name, error = %error_str, "Task failed");
10424 send_ws_sys_task_msg(
10425 &state,
10426 "failure",
10427 &name,
10428 vec![
10429 ("duration_ms", serde_json::json!(duration_ms)),
10430 ("error", serde_json::json!(error_str)),
10431 ],
10432 );
10433 }
10434 }
10435
10436 if let Err(error) = lease.release().await {
10437 tracing::warn!(task = %name, tick = %tick_key, error = %error, "Failed to release scheduled task lease");
10438 }
10439}
10440
10441struct CostGate {
10443 shutdown: tokio_util::sync::CancellationToken,
10444 waiting: Option<Arc<std::sync::atomic::AtomicBool>>,
10446 waited: std::sync::atomic::AtomicBool,
10448}
10449
10450impl CostGate {
10451 async fn wait(&self, state: &AppState, name: &str) -> bool {
10453 crate::cost::wait_while_deferred(
10454 state,
10455 crate::cost::WorkKind::Task,
10456 name,
10457 &self.shutdown,
10458 self.waiting.as_deref(),
10459 &self.waited,
10460 )
10461 .await
10462 }
10463
10464 fn waited(&self) -> bool {
10465 self.waited.load(std::sync::atomic::Ordering::Acquire)
10466 }
10467
10468 async fn free_unrun_claim(lease: crate::scheduler::SchedulerLease, name: &str, tick_key: &str) {
10476 const ATTEMPTS: u32 = 5;
10477 for attempt in 1..=ATTEMPTS {
10478 match lease.free().await {
10479 Ok(()) => break,
10480 Err(error) if attempt < ATTEMPTS => {
10481 tracing::debug!(task = %name, tick = %tick_key, attempt, error = %error, "Retrying the free of a deferred tick's claim");
10482 tokio::time::sleep(std::time::Duration::from_millis(200 * u64::from(attempt)))
10483 .await;
10484 }
10485 Err(error) => {
10486 tracing::warn!(task = %name, tick = %tick_key, error = %error, "Failed to free the claim of a deferred tick; the tick may not run");
10487 }
10488 }
10489 }
10490 release_task_lease(lease, name, tick_key).await;
10491 }
10492
10493 fn waits_before_lease(coordinator: &dyn crate::scheduler::SchedulerCoordinator) -> bool {
10500 matches!(coordinator.backend(), "sqlite" | "postgres")
10501 }
10502
10503 fn must_wait(state: &AppState, name: &str) -> bool {
10506 crate::cost::deferral_signal(state, crate::cost::WorkKind::Task, name).is_some()
10507 }
10508
10509 fn release(&self) {
10512 if let Some(flag) = &self.waiting {
10513 flag.store(false, std::sync::atomic::Ordering::Release);
10514 }
10515 }
10516}
10517
10518async fn release_task_lease(lease: crate::scheduler::SchedulerLease, name: &str, tick_key: &str) {
10519 if let Err(error) = lease.release().await {
10520 tracing::warn!(task = %name, tick = %tick_key, error = %error, "Failed to release scheduled task lease");
10521 }
10522}
10523
10524#[derive(Debug, Clone, Copy)]
10526struct CronTick {
10527 unix_secs: u64,
10529 window: std::time::Duration,
10532}
10533
10534impl CronTick {
10535 const fn is_past_window(self, now_unix_secs: u64) -> bool {
10538 !self.window.is_zero()
10539 && now_unix_secs >= self.unix_secs.saturating_add(self.window.as_secs())
10540 }
10541
10542 fn is_late(self, state: &AppState, name: &str, tick_key: &str) -> bool {
10544 let late = self.is_past_window(crate::time::clock_unix_secs(state.clock()));
10545 if late {
10546 tracing::debug!(task = %name, tick = %tick_key, "Cron task tick is past its window");
10547 }
10548 late
10549 }
10550}
10551
10552#[allow(clippy::cognitive_complexity)]
10554#[allow(
10555 clippy::too_many_arguments,
10556 reason = "one tick: its task, schedule, lease and cost gate"
10557)]
10558async fn execute_cron_task(
10559 name: String,
10560 state: AppState,
10561 handler: crate::task::TaskHandler,
10562 coordination: crate::task::TaskCoordination,
10563 coordinator: Arc<dyn crate::scheduler::SchedulerCoordinator>,
10564 lease_ttl: std::time::Duration,
10565 occurrence: CronTick,
10566 gate: CostGate,
10567) {
10568 let scheduled_key = crate::scheduler::cron_tick_key(&name, occurrence.unix_secs);
10569 let wait_first = CostGate::waits_before_lease(&*coordinator);
10572 let tick_key = scheduled_key;
10573 let lease = loop {
10574 if wait_first && !gate.wait(&state, &name).await {
10575 gate.release();
10576 return;
10577 }
10578 if gate.waited() && occurrence.is_late(&state, &name, &tick_key) {
10582 gate.release();
10583 return;
10584 }
10585 let lease = match coordinator
10586 .try_acquire_for_period(&name, &tick_key, coordination, occurrence.window)
10587 .await
10588 {
10589 Ok(Some(lease)) => lease,
10590 Ok(None) => {
10591 tracing::debug!(task = %name, tick = %tick_key, "Cron task tick already claimed");
10592 gate.release();
10593 return;
10594 }
10595 Err(error) => {
10596 tracing::warn!(task = %name, tick = %tick_key, error = %error, "Failed to acquire cron task lease");
10597 gate.release();
10598 return;
10599 }
10600 };
10601 if wait_first && CostGate::must_wait(&state, &name) {
10604 CostGate::free_unrun_claim(lease, &name, &tick_key).await;
10605 continue;
10606 }
10607 break lease;
10608 };
10609 if occurrence.is_late(&state, &name, &tick_key) {
10613 gate.release();
10614 release_task_lease(lease, &name, &tick_key).await;
10615 return;
10616 }
10617 state
10618 .task_registry
10619 .record_leader(&name, lease.leader_id(), &tick_key);
10620 if !wait_first && !gate.wait(&state, &name).await {
10621 gate.release();
10622 release_task_lease(lease, &name, &tick_key).await;
10623 return;
10624 }
10625 gate.release();
10626 tracing::debug!(task = %name, "Running cron task");
10627 state.task_registry.record_start(&name);
10628
10629 send_ws_sys_task_msg(&state, "started", &name, vec![]);
10630
10631 let start = state.monotonic();
10632 let lease_ttl = lease_ttl_for_run(&lease, coordination, lease_ttl);
10633 let tick = crate::scheduler::ScheduledTick::new(&tick_key, &lease);
10634 match execute_task_result_with_optional_lease_ttl(
10635 &state,
10636 handler,
10637 start,
10638 &name,
10639 "cron",
10640 lease_ttl,
10641 tick,
10642 gate.waited(),
10643 )
10644 .await
10645 {
10646 Ok(duration_ms) => {
10647 state.task_registry.record_success(&name, duration_ms);
10648 crate::alerts::notify_scheduled_task_recovered(&state, &name);
10649 tracing::debug!(task = %name, "Cron task completed");
10650 send_ws_sys_task_msg(
10651 &state,
10652 "success",
10653 &name,
10654 vec![("duration_ms", serde_json::json!(duration_ms))],
10655 );
10656 }
10657 Err((duration_ms, error_str)) => {
10658 state
10659 .task_registry
10660 .record_failure(&name, duration_ms, &error_str);
10661 crate::alerts::notify_scheduled_task_failure(&state, &name, &error_str);
10662 tracing::warn!(task = %name, error = %error_str, "Cron task failed");
10663 send_ws_sys_task_msg(
10664 &state,
10665 "failure",
10666 &name,
10667 vec![
10668 ("duration_ms", serde_json::json!(duration_ms)),
10669 ("error", serde_json::json!(error_str)),
10670 ],
10671 );
10672 }
10673 }
10674
10675 if let Err(error) = lease.release().await {
10676 tracing::warn!(task = %name, tick = %tick_key, error = %error, "Failed to release cron task lease");
10677 }
10678}
10679
10680struct CronTaskSpec {
10681 name: String,
10682 expression: String,
10683 timezone: Option<String>,
10684 coordination: crate::task::TaskCoordination,
10685 handler: crate::task::TaskHandler,
10686}
10687
10688fn lease_ttl_for_run(
10689 lease: &crate::scheduler::SchedulerLease,
10690 coordination: crate::task::TaskCoordination,
10691 lease_ttl: std::time::Duration,
10692) -> Option<std::time::Duration> {
10693 (coordination == crate::task::TaskCoordination::Fleet && lease.backend() == "postgres")
10694 .then_some(lease_ttl)
10695}
10696
10697fn run_cron_scheduler(
10698 tasks: Vec<CronTaskSpec>,
10699 state: &AppState,
10700 shutdown: &tokio_util::sync::CancellationToken,
10701 coordinator: &Arc<dyn crate::scheduler::SchedulerCoordinator>,
10702 lease_ttl: std::time::Duration,
10703) {
10704 if tasks.is_empty() {
10705 return;
10706 }
10707
10708 tracing::info!(count = tasks.len(), "Cron scheduler started");
10709 for task in tasks {
10710 let state = state.clone();
10711 let coordinator = Arc::clone(coordinator);
10712 let shutdown = shutdown.child_token();
10713 crate::sim::spawn_app_task(&state.clone(), async move {
10714 run_cron_task_loop(task, state, shutdown, coordinator, lease_ttl).await;
10715 });
10716 }
10717}
10718
10719#[allow(clippy::cognitive_complexity)]
10720async fn run_cron_task_loop(
10721 task: CronTaskSpec,
10722 state: AppState,
10723 shutdown: tokio_util::sync::CancellationToken,
10724 coordinator: Arc<dyn crate::scheduler::SchedulerCoordinator>,
10725 lease_ttl: std::time::Duration,
10726) {
10727 let CronTaskSpec {
10728 name,
10729 expression,
10730 timezone,
10731 coordination,
10732 handler,
10733 } = task;
10734
10735 let cron = match parse_cron_expression(&expression) {
10736 Ok(cron) => cron,
10737 Err(error) => {
10738 tracing::error!(task = %name, expression = %expression, error = %error, "Failed to create cron job");
10739 return;
10740 }
10741 };
10742 let timezone = timezone
10743 .as_deref()
10744 .and_then(|timezone| {
10745 timezone.parse::<chrono_tz::Tz>().map_or_else(
10746 |_| {
10747 tracing::warn!(task = %name, timezone = %timezone, "Unrecognized timezone; falling back to UTC");
10748 None
10749 },
10750 Some,
10751 )
10752 })
10753 .unwrap_or(chrono_tz::UTC);
10754 let mut cursor = state.clock().now().with_timezone(&timezone);
10755 let deferring = Arc::new(std::sync::atomic::AtomicBool::new(false));
10756
10757 loop {
10758 let now = state.clock().now().with_timezone(&timezone);
10759 let scheduled_at = match next_cron_occurrence_after(&cron, &cursor, &now) {
10760 Ok(scheduled_at) => scheduled_at,
10761 Err(error) => {
10762 tracing::error!(task = %name, expression = %expression, error = %error, "Failed to compute next cron tick");
10763 return;
10764 }
10765 };
10766 state.task_registry.record_next_run_at(
10767 &name,
10768 &scheduled_at.with_timezone(&chrono::Utc).to_rfc3339(),
10769 );
10770 let sleep_for = cron_sleep_duration_until(state.clock().now(), &scheduled_at);
10771 tokio::select! {
10772 biased;
10774 () = shutdown.cancelled() => break,
10775 () = tokio::time::sleep(sleep_for) => {
10776 let woke_at = state.clock().now().with_timezone(&timezone);
10777 match cron_occurrence_is_overdue(&cron, &scheduled_at, &woke_at) {
10778 Ok(true) => {
10779 tracing::warn!(
10780 task = %name,
10781 scheduled_at = %scheduled_at,
10782 woke_at = %woke_at,
10783 "Skipping overdue cron task tick"
10784 );
10785 cursor = woke_at;
10786 continue;
10787 }
10788 Ok(false) => {}
10789 Err(error) => {
10790 tracing::error!(task = %name, expression = %expression, error = %error, "Failed to evaluate cron tick lateness");
10791 return;
10792 }
10793 }
10794 if deferring.load(std::sync::atomic::Ordering::Acquire) {
10797 tracing::debug!(task = %name, "cron tick folds into the tick that waits for the cost signal");
10798 cursor = scheduled_at;
10799 continue;
10800 }
10801 if crate::cost::is_deferrable(crate::cost::WorkKind::Task, &name) {
10806 deferring.store(true, std::sync::atomic::Ordering::Release);
10807 }
10808 let occurrence = CronTick {
10809 unix_secs: u64::try_from(scheduled_at.timestamp()).unwrap_or_default(),
10810 window: cron_occurrence_window(&cron, &scheduled_at),
10811 };
10812 crate::sim::spawn_app_task(&state, execute_cron_task(
10813 name.clone(),
10814 state.clone(),
10815 handler,
10816 coordination,
10817 Arc::clone(&coordinator),
10818 lease_ttl,
10819 occurrence,
10820 CostGate {
10821 shutdown: shutdown.clone(),
10822 waiting: Some(Arc::clone(&deferring)),
10823 waited: std::sync::atomic::AtomicBool::new(false),
10824 },
10825 ));
10826 cursor = scheduled_at;
10827 }
10828 }
10829 }
10830}
10831
10832fn format_next_task_run_after(
10847 now: chrono::DateTime<chrono::Utc>,
10848 delay: std::time::Duration,
10849) -> String {
10850 crate::job::due_at_from(now, delay).to_rfc3339()
10851}
10852
10853fn parse_cron_expression(expression: &str) -> Result<croner::Cron, croner::errors::CronError> {
10859 croner::parser::CronParser::builder()
10860 .sloppy_ranges(true)
10861 .build()
10862 .parse(expression)
10863}
10864
10865fn next_cron_occurrence_after<Tz: chrono::TimeZone>(
10866 cron: &croner::Cron,
10867 cursor: &chrono::DateTime<Tz>,
10868 now: &chrono::DateTime<Tz>,
10869) -> Result<chrono::DateTime<Tz>, croner::errors::CronError> {
10870 let anchor = if cursor < now { now } else { cursor };
10871 cron.find_next_occurrence(anchor, false)
10872}
10873
10874fn cron_occurrence_is_overdue<Tz: chrono::TimeZone>(
10875 cron: &croner::Cron,
10876 scheduled_at: &chrono::DateTime<Tz>,
10877 now: &chrono::DateTime<Tz>,
10878) -> Result<bool, croner::errors::CronError> {
10879 let next_after_scheduled = cron.find_next_occurrence(scheduled_at, false)?;
10880 Ok(&next_after_scheduled <= now)
10881}
10882
10883fn cron_occurrence_window<Tz: chrono::TimeZone>(
10887 cron: &croner::Cron,
10888 scheduled_at: &chrono::DateTime<Tz>,
10889) -> std::time::Duration {
10890 cron.find_next_occurrence(scheduled_at, false)
10891 .ok()
10892 .and_then(|next| next.signed_duration_since(scheduled_at).to_std().ok())
10893 .unwrap_or_default()
10894}
10895
10896fn cron_sleep_duration_until<Tz: chrono::TimeZone>(
10904 now: chrono::DateTime<chrono::Utc>,
10905 scheduled_at: &chrono::DateTime<Tz>,
10906) -> std::time::Duration {
10907 scheduled_at
10908 .with_timezone(&chrono::Utc)
10909 .signed_duration_since(now)
10910 .to_std()
10911 .unwrap_or_default()
10912}
10913
10914async fn run_startup_hooks(hooks: &[StartupHook], state: AppState) -> crate::AutumnResult<()> {
10915 for hook in hooks {
10916 hook(state.clone()).await?;
10917 }
10918 Ok(())
10919}
10920
10921fn install_live_state<T>(
10930 state: &AppState,
10931 initial: T,
10932 decode: fn(&crate::upgrade::StateEnvelope) -> Result<T, crate::upgrade::AdoptError>,
10933) where
10934 T: crate::upgrade::LiveState,
10935{
10936 if let Some(existing) = state.extension::<crate::upgrade::LiveStateRegistry>() {
10937 state.insert_extension(crate::upgrade::LiveStateInstallFailure(format!(
10938 "an app may designate only one block of live state for in-place upgrades, but \
10939 both {} and {} were designated; carrying just one of two designated blocks \
10940 would be the silent state loss this feature exists to prevent",
10941 existing.type_name(),
10942 std::any::type_name::<T>(),
10943 )));
10944 return;
10945 }
10946
10947 let value = match crate::upgrade::carried_snapshot_path() {
10948 None => initial,
10949 Some(path) => {
10950 match crate::upgrade::read_snapshot(&path).and_then(|envelope| {
10951 let decoded = decode(&envelope);
10952 if decoded.is_ok() {
10953 tracing::info!(
10954 state = std::any::type_name::<T>(),
10955 from_version = envelope.version,
10956 to_version = T::VERSION,
10957 generation = envelope.generation,
10958 "adopted the live state handed over by the previous build"
10959 );
10960 }
10961 decoded
10962 }) {
10963 Ok(value) => value,
10964 Err(error) => {
10965 state.insert_extension(crate::upgrade::LiveStateInstallFailure(format!(
10966 "the previous build's live state cannot be carried into this one \
10967 ({}): {error}. The previous build keeps serving; fix the \
10968 migration (autumn_web::state_migration!) and try the upgrade again",
10969 std::any::type_name::<T>(),
10970 )));
10971 return;
10972 }
10973 }
10974 }
10975 };
10976
10977 let handle = if crate::upgrade::handoff_requested() {
10985 crate::upgrade::LiveStateHandle::new_frozen(value)
10986 } else {
10987 crate::upgrade::LiveStateHandle::new(value)
10988 };
10989 state.insert_extension(crate::upgrade::LiveStateRegistry::new(&handle));
10990 state.insert_extension(handle);
10991}
10992
10993fn unfreeze_adopted_live_state(state: &AppState) {
10999 if !crate::upgrade::handoff_requested() {
11000 return;
11001 }
11002 if let Some(registry) = state.extension::<crate::upgrade::LiveStateRegistry>() {
11003 #[cfg(unix)]
11006 if let Some(path) = crate::upgrade::journal_path() {
11007 registry.start_journal(path);
11008 }
11009 registry.unfreeze();
11010 }
11011}
11012
11013fn run_state_initializers(initializers: Vec<StateInitializer>, state: &AppState) {
11014 for initializer in initializers {
11015 initializer(state);
11016 }
11017}
11018
11019fn synthesize_durable_listener_jobs(
11037 listeners: Vec<crate::events::ListenerInfo>,
11038 jobs: &mut Vec<crate::job::JobInfo>,
11039) -> crate::events::EventRegistry {
11040 let registry = crate::events::EventRegistry::from_listeners(listeners);
11041 jobs.extend(registry.durable_job_infos());
11042 registry
11043}
11044
11045fn finalize_event_bus(
11046 listeners: Vec<crate::events::ListenerInfo>,
11047 jobs: &mut Vec<crate::job::JobInfo>,
11048 state: &AppState,
11049) {
11050 let registry = synthesize_durable_listener_jobs(listeners, jobs);
11051 state.insert_extension(registry.clone());
11052 crate::events::init_global_event_bus(®istry, state, None);
11053}
11054
11055fn dump_jobs_manifest(
11070 cfg: &crate::config::JobQueuesConfig,
11071 mut jobs: Vec<crate::job::JobInfo>,
11072 listeners: Vec<crate::events::ListenerInfo>,
11073) -> String {
11074 synthesize_durable_listener_jobs(listeners, &mut jobs);
11075 crate::job::render_jobs_manifest(cfg, &jobs)
11076}
11077
11078fn initialize_job_runtime(
11079 jobs: Vec<crate::job::JobInfo>,
11080 state: &AppState,
11081 shutdown: &tokio_util::sync::CancellationToken,
11082 config: &crate::config::JobConfig,
11083 run_workers: bool,
11084) -> crate::AutumnResult<()> {
11085 crate::job::clear_global_job_client();
11086 if jobs.is_empty() {
11087 Ok(())
11088 } else {
11089 crate::job::start_runtime(jobs, state, shutdown, config, run_workers)
11090 }
11091}
11092
11093async fn bind_or_adopt_tcp_listener(addr: &str) -> tokio::net::TcpListener {
11103 #[cfg(unix)]
11104 {
11105 if let Some(inherited) = crate::upgrade::adopt_inherited_listener() {
11106 match tokio::net::TcpListener::from_std(inherited) {
11107 Ok(listener) => {
11108 if let Ok(bound) = listener.local_addr()
11113 && bound.to_string() != addr
11114 {
11115 tracing::warn!(
11116 inherited = %bound,
11117 configured = %addr,
11118 "the inherited listening socket does not match this build's \
11119 configured address; an in-place upgrade cannot change where the \
11120 app listens — restart the process to apply it"
11121 );
11122 }
11123 return listener;
11124 }
11125 Err(e) => {
11126 tracing::error!("Failed to adopt the inherited listening socket: {e}");
11127 #[cfg(feature = "managed-pg")]
11128 crate::managed_pg::emergency_stop_async().await;
11129 std::process::exit(1);
11130 }
11131 }
11132 }
11133 if crate::upgrade::handoff_requested() {
11139 tracing::error!(
11140 "refusing to start: this build was handed the previous build's listening \
11141 socket but could not adopt it (see the error above). The previous build \
11142 keeps serving"
11143 );
11144 #[cfg(feature = "managed-pg")]
11145 crate::managed_pg::emergency_stop_async().await;
11146 std::process::exit(1);
11147 }
11148 }
11149 match tokio::net::TcpListener::bind(addr).await {
11150 Ok(listener) => listener,
11151 Err(e) => {
11152 tracing::error!(addr = %addr, "Failed to bind: {e}");
11153 #[cfg(feature = "managed-pg")]
11156 crate::managed_pg::emergency_stop_async().await;
11157 std::process::exit(1);
11158 }
11159 }
11160}
11161
11162enum BoundListener {
11169 Tcp(tokio::net::TcpListener),
11171 #[cfg(unix)]
11173 Unix(tokio::net::UnixListener),
11174 #[cfg(feature = "tls")]
11176 Tls(crate::tls::TlsListener),
11177}
11178
11179#[cfg(feature = "tls")]
11182fn build_tls_listener(
11183 tcp: tokio::net::TcpListener,
11184 cfg: &crate::config::TlsConfig,
11185 shutdown: tokio_util::sync::CancellationToken,
11186) -> Result<
11187 (
11188 crate::tls::TlsListener,
11189 crate::tls::CertReloader,
11190 Option<crate::tls::client_auth::ClientTrustReloader>,
11191 ),
11192 crate::tls::TlsError,
11193> {
11194 let provider = crate::tls::crypto_provider();
11195 let cert_path = cfg
11199 .cert_path
11200 .as_deref()
11201 .expect("validated: static [server.tls] sets cert_path");
11202 let key_path = cfg
11203 .key_path
11204 .as_deref()
11205 .expect("validated: static [server.tls] sets key_path");
11206 let (resolver, reload) = crate::tls::CertReloader::load(
11211 cert_path.to_path_buf(),
11212 key_path.to_path_buf(),
11213 std::sync::Arc::clone(&provider),
11214 crate::tls::now_unix(),
11215 std::time::Duration::from_secs(cfg.reload_interval_secs.max(1)),
11217 )?;
11218 let (client_verifier, client_reload) = build_client_auth(cfg, &provider)?;
11219 let server_config = crate::tls::build_server_config_with_client_auth(
11220 std::sync::Arc::clone(&provider),
11221 std::sync::Arc::clone(&resolver) as std::sync::Arc<dyn rustls::server::ResolvesServerCert>,
11222 client_verifier,
11223 )?;
11224 let handshake_timeout = std::time::Duration::from_secs(cfg.handshake_timeout_secs.max(1));
11227 let listener = crate::tls::TlsListener::new(tcp, server_config, handshake_timeout, shutdown);
11228 Ok((listener, reload, client_reload))
11229}
11230
11231#[cfg(feature = "tls")]
11235type ClientAuthWiring = (
11236 Option<std::sync::Arc<dyn rustls::server::danger::ClientCertVerifier>>,
11237 Option<crate::tls::client_auth::ClientTrustReloader>,
11238);
11239
11240#[cfg(feature = "tls")]
11247fn build_client_auth(
11248 cfg: &crate::config::TlsConfig,
11249 provider: &std::sync::Arc<rustls::crypto::CryptoProvider>,
11250) -> Result<ClientAuthWiring, crate::tls::TlsError> {
11251 if !cfg.client_auth_active() {
11252 return Ok((None, None));
11253 }
11254 let client_auth = cfg
11258 .client_auth
11259 .as_ref()
11260 .expect("validated: an active client_auth section is present");
11261 let bundle = client_auth
11262 .ca_bundle_path
11263 .clone()
11264 .expect("validated: an active client_auth section sets ca_bundle_path");
11265 let (verifier, reloader) = crate::tls::client_auth::ClientTrustReloader::load(
11266 bundle,
11267 client_auth.crl_path.clone(),
11268 client_auth.mode,
11269 std::sync::Arc::clone(provider),
11270 std::time::Duration::from_secs(client_auth.reload_interval_secs.max(1)),
11272 )?;
11273 Ok((Some(verifier), Some(reloader)))
11274}
11275
11276#[cfg(feature = "acme")]
11279struct AcmeBindState {
11280 renewal_task: crate::acme::renewal::AcmeRenewalTask,
11281 tokens: crate::acme::challenge::Http01Tokens,
11282 http_challenge_port: u16,
11283 https_port: u16,
11284 dns01: bool,
11288 custom_domains: Option<CustomDomainBindState>,
11291 client_trust_reload: Option<crate::tls::client_auth::ClientTrustReloader>,
11295}
11296
11297#[cfg(feature = "acme")]
11300struct CustomDomainBindState {
11301 registry: std::sync::Arc<crate::custom_domain::CustomDomainRegistry>,
11302 cache: std::sync::Arc<crate::custom_domain::CustomDomainCertCache>,
11303 store: std::sync::Arc<crate::acme::store::FsAcmeStore>,
11304 provider: std::sync::Arc<rustls::crypto::CryptoProvider>,
11305 config: crate::config::CustomDomainsConfig,
11306 acme: crate::config::AcmeConfig,
11309}
11310
11311#[cfg(feature = "acme")]
11316#[allow(
11317 clippy::too_many_arguments,
11318 clippy::too_many_lines,
11319 reason = "one bind-time assembly of the ACME listener, its store, its \
11320 placeholder and the custom-domain registry; splitting it would \
11321 only scatter the ordering these steps depend on"
11322)]
11323async fn build_acme_tls_listener(
11324 tcp: tokio::net::TcpListener,
11325 tls_cfg: &crate::config::TlsConfig,
11326 acme_cfg: &crate::config::AcmeConfig,
11327 tenancy_base_domain: Option<&str>,
11328 credentials: &crate::credentials::CredentialsStore,
11329 https_port: u16,
11330 status: Option<crate::acme::renewal::AcmeStatus>,
11331 shutdown: tokio_util::sync::CancellationToken,
11332) -> Result<(crate::tls::TlsListener, AcmeBindState), String> {
11333 use crate::acme::store::{AcmeStore, CertId, FsAcmeStore};
11334
11335 let provider = crate::tls::crypto_provider();
11336 let cert_id = CertId::from_domains(&acme_cfg.domains);
11337 let directory_label = crate::acme::directory_label(&acme_cfg.directory);
11338 let store: std::sync::Arc<dyn AcmeStore> = std::sync::Arc::new(FsAcmeStore::new(
11339 acme_cfg.cache_dir.clone(),
11340 directory_label,
11341 ));
11342 let status = status.unwrap_or_default();
11343
11344 let (initial, serving_stored_cert) = match store.load_cert(&cert_id).await {
11347 Ok(Some(stored)) => match crate::tls::certified_key_from_pem(
11348 stored.chain_pem.as_bytes(),
11349 stored.key_pem.as_bytes(),
11350 &provider,
11351 ) {
11352 Ok(ck) => {
11353 if let Ok(not_after) =
11354 crate::tls::leaf_not_after_from_pem(stored.chain_pem.as_bytes())
11355 {
11356 status.set_cert_not_after(not_after);
11357 }
11358 (ck, true)
11359 }
11360 Err(e) => {
11361 tracing::warn!(
11362 "stored ACME certificate is unusable ({e}); serving a self-signed \
11363 placeholder until the renewal task issues a real one"
11364 );
11365 (acme_placeholder_key(&acme_cfg.domains, &provider)?, false)
11366 }
11367 },
11368 Ok(None) => (acme_placeholder_key(&acme_cfg.domains, &provider)?, false),
11369 Err(e) => {
11370 tracing::warn!(
11371 "failed to read the stored ACME certificate ({e}); serving a self-signed \
11372 placeholder"
11373 );
11374 (acme_placeholder_key(&acme_cfg.domains, &provider)?, false)
11375 }
11376 };
11377
11378 let resolver = std::sync::Arc::new(crate::tls::ReloadableCertResolver::new(initial));
11379
11380 let custom_domains = match acme_cfg.custom_domains.as_ref() {
11386 Some(cd_cfg) if cd_cfg.enabled => {
11387 let reserved = custom_domain_reserved_names(acme_cfg, tenancy_base_domain);
11388 let registry = std::sync::Arc::new(
11389 crate::custom_domain::CustomDomainRegistry::new(
11390 std::sync::Arc::new(crate::custom_domain::FsCustomDomainStore::new(
11391 cd_cfg.store_dir.clone(),
11392 )),
11393 cd_cfg.max_domains,
11394 )
11395 .with_reserved(reserved),
11396 );
11397 match registry.load().await {
11398 Ok(count) => tracing::info!(count, "loaded tenant custom domains"),
11399 Err(e) => tracing::error!(
11406 "failed to load the tenant custom-domain registry: {e}; connected domains will \
11407 not route and no new domain can be connected until this is fixed and the \
11408 process restarted"
11409 ),
11410 }
11411 Some(CustomDomainBindState {
11412 registry,
11413 cache: std::sync::Arc::new(crate::custom_domain::CustomDomainCertCache::new(
11414 cd_cfg.cert_cache_size,
11415 )),
11416 store: std::sync::Arc::new(FsAcmeStore::new(
11417 acme_cfg.cache_dir.clone(),
11418 crate::acme::directory_label(&acme_cfg.directory),
11419 )),
11420 provider: std::sync::Arc::clone(&provider),
11421 config: cd_cfg.clone(),
11422 acme: acme_cfg.clone(),
11423 })
11424 }
11425 _ => None,
11426 };
11427
11428 let cert_resolver: std::sync::Arc<dyn rustls::server::ResolvesServerCert> =
11429 custom_domains.as_ref().map_or_else(
11430 || {
11431 std::sync::Arc::clone(&resolver)
11432 as std::sync::Arc<dyn rustls::server::ResolvesServerCert>
11433 },
11434 |cd| {
11435 std::sync::Arc::new(
11436 crate::custom_domain::SniCertResolver::new(
11437 std::sync::Arc::clone(&resolver),
11438 acme_cfg.domains.clone(),
11439 std::sync::Arc::clone(&cd.registry),
11440 std::sync::Arc::clone(&cd.cache),
11441 )
11442 .with_source(std::sync::Arc::new(
11447 crate::acme::tenant_domains::FsSniCertSource::new(
11448 std::sync::Arc::clone(&cd.store),
11449 std::sync::Arc::clone(&provider),
11450 ),
11451 )),
11452 )
11453 },
11454 );
11455 let (client_verifier, client_reload) =
11461 build_client_auth(tls_cfg, &provider).map_err(|e| e.to_string())?;
11462 let server_config = crate::tls::build_server_config_with_client_auth(
11463 std::sync::Arc::clone(&provider),
11464 cert_resolver,
11465 client_verifier,
11466 )
11467 .map_err(|e| e.to_string())?;
11468 let handshake_timeout = std::time::Duration::from_secs(tls_cfg.handshake_timeout_secs.max(1));
11469 let listener = crate::tls::TlsListener::new(tcp, server_config, handshake_timeout, shutdown);
11470
11471 let tokens = crate::acme::challenge::Http01Tokens::new();
11472 let dns = build_dns_challenge(acme_cfg, credentials)?;
11477 let renewal_task = crate::acme::renewal::AcmeRenewalTask {
11478 resolver,
11479 provider,
11480 store,
11481 cert_id,
11482 tokens: tokens.clone(),
11483 status,
11484 config: acme_cfg.clone(),
11485 serving_stored_cert,
11486 leadership_degraded: false,
11489 renew_window_misconfigured: std::sync::atomic::AtomicBool::new(false),
11490 dns,
11491 recovery: None,
11494 };
11495 Ok((
11496 listener,
11497 AcmeBindState {
11498 renewal_task,
11499 tokens,
11500 http_challenge_port: acme_cfg.http_challenge_port,
11501 https_port,
11502 dns01: acme_cfg.dns.is_some(),
11503 custom_domains,
11504 client_trust_reload: client_reload,
11505 },
11506 ))
11507}
11508
11509#[cfg(feature = "acme")]
11535const fn acme_fleet_warning(
11536 backend: crate::config::SchedulerBackend,
11537 dns01: bool,
11538) -> Option<&'static str> {
11539 if !backend.is_fleet_distributed() {
11540 return None;
11541 }
11542 if matches!(backend, crate::config::SchedulerBackend::Sqlite) {
11543 if dns01 {
11544 return None;
11546 }
11547 return Some(
11548 "ACME HTTP-01 validation is not safe across the processes scheduler.backend = \
11549 \"sqlite\" coordinates: the token store is per-process, so a proxy may route the \
11550 CA's :80 challenge to a process without the token (404). The certificate store is \
11551 not at risk — every process on the host reads the same [server.tls.acme] \
11552 cache_dir. Serve ACME from one process, terminate TLS at the proxy, or use DNS-01 \
11553 (#1620)",
11554 );
11555 }
11556 Some(if dns01 {
11557 "ACME DNS-01 issuance is fleet-safe, but the on-disk certificate store is not: only \
11558 the replica holding the renewal lease writes the issued certificate, and the others \
11559 cannot adopt it from a non-shared store, so they keep serving the self-signed \
11560 placeholder. Run ACME on a single host, or point [server.tls.acme] cache_dir at \
11561 storage every replica shares (#1620)"
11562 } else {
11563 "ACME HTTP-01 validation is not fleet-safe with the local on-disk token store: behind \
11564 a load balancer the CA's :80 challenge may reach a replica without the token (404), \
11565 and non-leader replicas cannot adopt issued certificates from a non-shared store. Run \
11566 ACME on a single host, or terminate TLS at a shared proxy. DNS-01 removes the :80 \
11567 hazard but not the store one, so it needs a shared [server.tls.acme] cache_dir too \
11568 (#1620)"
11569 })
11570}
11571
11572#[cfg(feature = "acme")]
11580fn build_dns_challenge(
11581 acme_cfg: &crate::config::AcmeConfig,
11582 credentials: &crate::credentials::CredentialsStore,
11583) -> Result<Option<crate::acme::renewal::DnsChallenge>, String> {
11584 use crate::acme::dns;
11585
11586 let Some(dns_cfg) = acme_cfg.dns.as_ref() else {
11587 return Ok(None);
11588 };
11589 let resolvers = dns_cfg.resolver_addrs()?;
11590 let credential = dns::DnsCredential::resolve(dns_cfg, credentials, &dns::process_env);
11591 let transport: std::sync::Arc<dyn dns::http::HttpTransport> = std::sync::Arc::new(
11595 dns::http::ReqwestTransport::new(std::time::Duration::from_secs(30))?,
11596 );
11597 let provider = dns::build_provider(dns_cfg, &credential, transport)?;
11598 Ok(Some(crate::acme::renewal::DnsChallenge {
11599 provider,
11600 lookup: std::sync::Arc::new(dns::resolver::UdpDnsLookup::new(
11601 std::time::Duration::from_secs(5),
11602 )),
11603 resolvers,
11604 propagation_timeout: std::time::Duration::from_secs(dns_cfg.propagation_timeout_secs),
11605 poll_interval: std::time::Duration::from_secs(dns_cfg.poll_interval_secs),
11606 }))
11607}
11608
11609#[cfg(feature = "acme")]
11611fn acme_placeholder_key(
11612 domains: &[String],
11613 provider: &rustls::crypto::CryptoProvider,
11614) -> Result<std::sync::Arc<rustls::sign::CertifiedKey>, String> {
11615 let placeholder = crate::acme::renewal::self_signed_placeholder(domains)?;
11616 crate::tls::certified_key_from_pem(
11617 placeholder.chain_pem.as_bytes(),
11618 placeholder.key_pem.as_bytes(),
11619 provider,
11620 )
11621}
11622
11623#[cfg(all(feature = "acme", feature = "reporting"))]
11628fn make_acme_reporter(
11629 reporters: Vec<std::sync::Arc<dyn crate::reporting::ErrorReporter>>,
11630) -> crate::acme::renewal::ReporterFn {
11631 std::sync::Arc::new(move |message: String| {
11632 if reporters.is_empty() {
11633 return;
11634 }
11635 let reporters = reporters.clone();
11636 tokio::spawn(async move {
11637 let event = crate::reporting::ErrorEvent {
11638 status: axum::http::StatusCode::INTERNAL_SERVER_ERROR,
11639 message,
11640 problem_type: None,
11641 request_id: None,
11642 route: Some("acme-renewal".to_owned()),
11643 method: None,
11644 panic: None,
11645 capsule: None,
11646 };
11647 for reporter in &reporters {
11648 reporter.report(&event).await;
11649 }
11650 });
11651 })
11652}
11653
11654#[cfg(feature = "acme")]
11658const ACME_RENEWAL_TASK_NAME: &str = "acme-renewal";
11659
11660#[cfg(feature = "acme")]
11667fn compose_acme_alert_reporter(
11668 inner: crate::acme::renewal::ReporterFn,
11669 state: &AppState,
11670) -> crate::acme::renewal::ReporterFn {
11671 let state = state.clone();
11672 std::sync::Arc::new(move |message: String| {
11673 crate::alerts::notify_scheduled_task_failure(&state, ACME_RENEWAL_TASK_NAME, &message);
11674 inner(message);
11675 })
11676}
11677
11678#[cfg(feature = "acme")]
11685fn custom_domain_reserved_names(
11686 acme_cfg: &crate::config::AcmeConfig,
11687 tenancy_base_domain: Option<&str>,
11688) -> Vec<String> {
11689 let mut reserved = acme_cfg.domains.clone();
11695 reserved.extend(tenancy_base_domain.map(ToOwned::to_owned));
11696 if let Some(cd_cfg) = acme_cfg.custom_domains.as_ref() {
11705 reserved.extend(cd_cfg.ingress_hostname.clone());
11706 }
11707 reserved
11708}
11709
11710#[cfg(feature = "acme")]
11720async fn install_custom_domain_retention_pruner(
11721 acme: Option<&crate::config::AcmeConfig>,
11722 tenancy_base_domain: Option<&str>,
11723 state: &AppState,
11724 migrate: bool,
11725) {
11726 let Some(acme_cfg) = acme else {
11727 return;
11728 };
11729 let Some(cd_cfg) = acme_cfg.custom_domains.as_ref() else {
11730 return;
11731 };
11732 if !cd_cfg.enabled {
11733 return;
11734 }
11735 let registry = std::sync::Arc::new(
11736 crate::custom_domain::CustomDomainRegistry::new(
11737 std::sync::Arc::new(crate::custom_domain::FsCustomDomainStore::new(
11738 cd_cfg.store_dir.clone(),
11739 )),
11740 cd_cfg.max_domains,
11741 )
11742 .with_reserved(custom_domain_reserved_names(acme_cfg, tenancy_base_domain)),
11743 );
11744 let loaded = if migrate {
11749 registry.load().await
11750 } else {
11751 registry.load_without_migration().await
11752 };
11753 match loaded {
11754 Ok(count) => tracing::info!(count, "loaded tenant custom domains"),
11755 Err(e) => tracing::error!(
11761 "failed to load the tenant custom-domain registry: {e}; the custom_domains \
11762 retention dataset will refuse to prune until this is fixed"
11763 ),
11764 }
11765 let store = std::sync::Arc::new(crate::acme::store::FsAcmeStore::new(
11766 acme_cfg.cache_dir.clone(),
11767 crate::acme::directory_label(&acme_cfg.directory),
11768 ));
11769 let pruner = crate::acme::tenant_domains::PruneOnlyCustomDomainPruner {
11770 registry,
11771 cache: std::sync::Arc::new(crate::custom_domain::CustomDomainCertCache::new(
11772 cd_cfg.cert_cache_size,
11773 )),
11774 certs: std::sync::Arc::clone(&store) as std::sync::Arc<dyn crate::acme::store::AcmeStore>,
11775 limiter: std::sync::Arc::new(crate::custom_domain::IssuanceLimiter::new(
11776 cd_cfg.issuance_per_domain_per_day,
11777 cd_cfg.issuance_global_per_hour,
11778 cd_cfg.failure_backoff_secs,
11779 cd_cfg.max_failure_backoff_secs,
11780 )),
11781 cert_store_paths: Some(store),
11782 retained_cert_ids: std::iter::once(
11786 crate::acme::store::CertId::from_domains(&acme_cfg.domains)
11787 .as_str()
11788 .to_owned(),
11789 )
11790 .collect(),
11791 reporter: make_custom_domain_reporter(state),
11792 recovery: Some(make_custom_domain_recovery(state)),
11793 };
11794 state
11795 .insert_extension(std::sync::Arc::new(pruner)
11796 as std::sync::Arc<dyn crate::custom_domain::CustomDomainPruner>);
11797}
11798
11799#[cfg(feature = "acme")]
11806fn spawn_custom_domain_task(
11807 bind_state: CustomDomainBindState,
11808 tokens: crate::acme::challenge::Http01Tokens,
11809 coordinator: std::sync::Arc<dyn crate::scheduler::SchedulerCoordinator>,
11810 leadership_degraded: bool,
11811 state: &AppState,
11812 shutdown: tokio_util::sync::CancellationToken,
11813) {
11814 let CustomDomainBindState {
11815 registry,
11816 cache,
11817 store,
11818 provider,
11819 config,
11820 acme,
11821 } = bind_state;
11822
11823 state.insert_extension(std::sync::Arc::clone(®istry));
11824 if let Err(e) = state.health_indicator_registry.register(
11825 "custom_domains",
11826 crate::actuator::IndicatorGroup::HealthOnly,
11827 std::sync::Arc::new(crate::custom_domain::CustomDomainHealthIndicator::new(
11828 std::sync::Arc::clone(®istry),
11829 )),
11830 ) {
11831 tracing::warn!("{e}");
11832 }
11833
11834 let issuer = std::sync::Arc::new(crate::acme::tenant_domains::AcmeDomainIssuer::new(
11835 acme.clone(),
11836 std::sync::Arc::clone(&store) as std::sync::Arc<dyn crate::acme::store::AcmeStore>,
11837 tokens,
11838 ));
11839 let task = std::sync::Arc::new(crate::acme::tenant_domains::CustomDomainTask {
11840 registry,
11841 cache,
11842 certs: std::sync::Arc::clone(&store) as std::sync::Arc<dyn crate::acme::store::AcmeStore>,
11843 provider,
11844 verifier: std::sync::Arc::new(crate::custom_domain::SystemDomainVerifier::new(
11846 config.resolver_addrs().unwrap_or_default(),
11847 )),
11848 issuer,
11849 limiter: std::sync::Arc::new(crate::custom_domain::IssuanceLimiter::new(
11850 config.issuance_per_domain_per_day,
11851 config.issuance_global_per_hour,
11852 config.failure_backoff_secs,
11853 config.max_failure_backoff_secs,
11854 )),
11855 ingress: config.ingress(),
11856 renew_before_days: acme.renew_before_days,
11857 reporter: make_custom_domain_reporter(state),
11860 recovery: Some(make_custom_domain_recovery(state)),
11861 coordinator,
11862 leadership_degraded,
11863 cert_store_paths: Some(store),
11864 retained_cert_ids: std::iter::once(
11868 crate::acme::store::CertId::from_domains(&acme.domains)
11869 .as_str()
11870 .to_owned(),
11871 )
11872 .collect(),
11873 });
11874
11875 state.insert_extension(std::sync::Arc::new(task.pruner())
11878 as std::sync::Arc<dyn crate::custom_domain::CustomDomainPruner>);
11879
11880 let interval = std::time::Duration::from_secs(config.poll_interval_secs.max(1));
11881 tokio::spawn(async move {
11882 task.run(interval, shutdown).await;
11883 });
11884}
11885
11886#[cfg(feature = "acme")]
11888fn make_custom_domain_reporter(state: &AppState) -> crate::acme::tenant_domains::ReporterFn {
11889 let state = state.clone();
11890 std::sync::Arc::new(move |message: String| {
11891 crate::alerts::notify_scheduled_task_failure(&state, CUSTOM_DOMAIN_TASK_NAME, &message);
11892 })
11893}
11894
11895#[cfg(feature = "acme")]
11897fn make_custom_domain_recovery(state: &AppState) -> crate::acme::tenant_domains::RecoveryFn {
11898 let state = state.clone();
11899 std::sync::Arc::new(move || {
11900 crate::alerts::notify_scheduled_task_recovered(&state, CUSTOM_DOMAIN_TASK_NAME);
11901 })
11902}
11903
11904#[cfg(feature = "acme")]
11906const CUSTOM_DOMAIN_TASK_NAME: &str = "custom_domain_certificates";
11907
11908#[cfg(feature = "acme")]
11911fn make_acme_alert_recovery(state: &AppState) -> crate::acme::renewal::RecoveryFn {
11912 let state = state.clone();
11913 std::sync::Arc::new(move || {
11914 crate::alerts::notify_scheduled_task_recovered(&state, ACME_RENEWAL_TASK_NAME);
11915 })
11916}
11917
11918#[cfg(all(feature = "acme", not(feature = "reporting")))]
11921fn make_acme_reporter() -> crate::acme::renewal::ReporterFn {
11922 std::sync::Arc::new(|_message: String| {})
11923}
11924
11925#[cfg(unix)]
11932#[derive(Clone, Debug)]
11933struct UdsConnectInfo;
11934
11935#[cfg(unix)]
11936impl
11937 axum::extract::connect_info::Connected<
11938 axum::serve::IncomingStream<'_, tokio::net::UnixListener>,
11939 > for UdsConnectInfo
11940{
11941 fn connect_info(_stream: axum::serve::IncomingStream<'_, tokio::net::UnixListener>) -> Self {
11942 Self
11943 }
11944}
11945
11946#[cfg(unix)]
11949impl
11950 axum::extract::connect_info::Connected<
11951 axum::serve::IncomingStream<
11952 '_,
11953 crate::accept_drain::StopAcceptingOnShutdown<tokio::net::UnixListener>,
11954 >,
11955 > for UdsConnectInfo
11956{
11957 fn connect_info(
11958 _stream: axum::serve::IncomingStream<
11959 '_,
11960 crate::accept_drain::StopAcceptingOnShutdown<tokio::net::UnixListener>,
11961 >,
11962 ) -> Self {
11963 Self
11964 }
11965}
11966
11967#[cfg(unix)]
11968impl
11969 axum::extract::connect_info::Connected<
11970 crate::http_server::IncomingStream<
11971 '_,
11972 crate::accept_drain::StopAcceptingOnShutdown<tokio::net::UnixListener>,
11973 >,
11974 > for UdsConnectInfo
11975{
11976 fn connect_info(
11977 _stream: crate::http_server::IncomingStream<
11978 '_,
11979 crate::accept_drain::StopAcceptingOnShutdown<tokio::net::UnixListener>,
11980 >,
11981 ) -> Self {
11982 Self
11983 }
11984}
11985
11986#[cfg(feature = "db")]
11988fn replica_lag_check_interval(max_lag: std::time::Duration) -> std::time::Duration {
11989 (max_lag / 2).clamp(
11990 std::time::Duration::from_millis(250),
11991 std::time::Duration::from_secs(5),
11992 )
11993}
11994
11995#[cfg(feature = "db")]
11997async fn sample_replica_lag(
11998 replica: &diesel_async::pooled_connection::deadpool::Pool<crate::db::RuntimeConnection>,
11999 probes: &crate::probe::ProbeState,
12000 max_lag: std::time::Duration,
12001 budget: std::time::Duration,
12002) {
12003 match replica.get().await {
12004 Ok(conn) => {
12005 crate::probe::refresh_replica_lag_bounded(probes, conn, budget, |conn| {
12006 Box::pin(crate::db::measure_replica_lag(conn, max_lag))
12007 })
12008 .await;
12009 }
12010 Err(error) => tracing::debug!(
12013 target: "autumn::db",
12014 error = %crate::db_url::redact_targets_in_message(&error.to_string()),
12015 "replica lag check skipped: no replica connection"
12016 ),
12017 }
12018}
12019
12020#[cfg(all(feature = "db", feature = "test-support"))]
12021impl AppState {
12022 #[doc(hidden)]
12026 pub async fn sample_replica_lag_for_test(&self) {
12027 let (Some(replica), Some(max_lag)) = (self.replica_pool(), self.probes().replica_max_lag())
12028 else {
12029 return;
12030 };
12031 let budget = crate::probe::replica_lag_query_budget(max_lag);
12032 sample_replica_lag(replica, self.probes(), max_lag, budget).await;
12033 }
12034}
12035
12036#[cfg(feature = "db")]
12039fn spawn_replica_lag_monitor(
12040 state: &AppState,
12041 shutdown: tokio_util::sync::CancellationToken,
12042) -> Option<tokio::task::JoinHandle<()>> {
12043 let replica = state.replica_pool()?.clone();
12044 let max_lag = state.probes().replica_max_lag()?;
12045 let probes = state.probes().clone();
12046 let interval = replica_lag_check_interval(max_lag);
12047 let budget = crate::probe::replica_lag_query_budget(max_lag);
12050 Some(tokio::spawn(async move {
12051 loop {
12052 sample_replica_lag(&replica, &probes, max_lag, budget).await;
12053 tokio::select! {
12054 () = tokio::time::sleep(interval) => {}
12055 () = shutdown.cancelled() => break,
12056 }
12057 }
12058 }))
12059}
12060
12061#[cfg(unix)]
12071async fn stamp_loopback_connect_info(
12072 mut req: axum::extract::Request,
12073 next: axum::middleware::Next,
12074) -> axum::response::Response {
12075 if req
12076 .extensions()
12077 .get::<axum::extract::ConnectInfo<std::net::SocketAddr>>()
12078 .is_none()
12079 {
12080 let loopback =
12081 std::net::SocketAddr::new(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST), 0);
12082 req.extensions_mut()
12083 .insert(axum::extract::ConnectInfo(loopback));
12084 }
12085 next.run(req).await
12086}
12087
12088fn signal_serve_ready(drain_budget_secs: u64, bound_endpoint: &str) {
12115 let Some(path) = std::env::var_os("AUTUMN_SERVE_READY_FILE") else {
12116 return;
12117 };
12118 if path.is_empty() {
12119 return;
12120 }
12121 let path = std::path::PathBuf::from(path);
12122 let mut tmp = path.clone();
12127 tmp.as_mut_os_string().push(".tmp");
12128 if let Err(e) = std::fs::write(&tmp, serve_ready_payload(drain_budget_secs, bound_endpoint))
12129 .and_then(|()| std::fs::rename(&tmp, &path))
12130 {
12131 let _ = std::fs::remove_file(&tmp);
12132 tracing::warn!(error = %e, path = %path.display(),
12133 "could not write serve readiness file");
12134 }
12135}
12136
12137fn dialable_endpoint(addr: &str) -> String {
12152 let Ok(parsed) = addr.parse::<std::net::SocketAddr>() else {
12153 return addr.to_owned();
12154 };
12155 if !parsed.ip().is_unspecified() {
12156 return addr.to_owned();
12157 }
12158 match parsed {
12159 std::net::SocketAddr::V4(_) => format!("127.0.0.1:{}", parsed.port()),
12160 std::net::SocketAddr::V6(_) => format!("[::1]:{}", parsed.port()),
12161 }
12162}
12163
12164#[must_use]
12180pub fn serve_ready_payload(drain_budget_secs: u64, bound_endpoint: &str) -> String {
12181 let endpoint = bound_endpoint.trim();
12182 if endpoint.is_empty() {
12183 return format!("{drain_budget_secs}\n{SHUTDOWN_SIGNAL_CAPABILITY_LINE}");
12184 }
12185 format!("{drain_budget_secs}\n{endpoint}\n{SHUTDOWN_SIGNAL_CAPABILITY_LINE}")
12186}
12187
12188const SHUTDOWN_SIGNAL_CAPABILITY_LINE: &str = "capability shutdown-signal";
12196
12197#[must_use]
12199pub fn serve_ready_advertises_shutdown_signal(payload: &str) -> bool {
12200 payload
12201 .lines()
12202 .any(|line| line.trim() == SHUTDOWN_SIGNAL_CAPABILITY_LINE)
12203}
12204
12205#[cfg(unix)]
12216fn prepare_unix_socket_path(path: &std::path::Path) -> std::io::Result<()> {
12217 use std::os::unix::fs::FileTypeExt;
12218 match std::fs::symlink_metadata(path) {
12219 Ok(meta) if meta.file_type().is_socket() => {
12220 match std::os::unix::net::UnixStream::connect(path) {
12221 Ok(_) => Err(std::io::Error::new(
12223 std::io::ErrorKind::AddrInUse,
12224 format!(
12225 "refusing to bind unix socket: {} is already in use by a \
12226 live listener",
12227 path.display()
12228 ),
12229 )),
12230 Err(e)
12233 if matches!(
12234 e.kind(),
12235 std::io::ErrorKind::ConnectionRefused | std::io::ErrorKind::NotFound
12236 ) =>
12237 {
12238 std::fs::remove_file(path)
12239 }
12240 Err(e) => Err(std::io::Error::new(
12245 std::io::ErrorKind::AddrInUse,
12246 format!(
12247 "refusing to bind unix socket: cannot determine whether {} \
12248 is live ({e}); not removing it",
12249 path.display()
12250 ),
12251 )),
12252 }
12253 }
12254 Ok(_) => Err(std::io::Error::new(
12255 std::io::ErrorKind::AlreadyExists,
12256 format!(
12257 "refusing to bind unix socket: {} exists and is not a socket",
12258 path.display()
12259 ),
12260 )),
12261 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
12262 Err(e) => Err(e),
12263 }
12264}
12265
12266#[cfg(feature = "sqlite")]
12272fn fleet_flush_budget(
12273 shutdown_budget: std::time::Duration,
12274 injected_elapsed: std::time::Duration,
12275 real_elapsed: std::time::Duration,
12276) -> std::time::Duration {
12277 shutdown_budget.saturating_sub(injected_elapsed.max(real_elapsed))
12278}
12279
12280fn cluster_departure_wait(
12294 shutdown_budget: std::time::Duration,
12295 drain_elapsed: std::time::Duration,
12296) -> std::time::Duration {
12297 crate::cluster::LEAVE_BUDGET.min(shutdown_budget.saturating_sub(drain_elapsed))
12298}
12299
12300async fn run_shutdown_hooks(hooks: &[ShutdownHook]) {
12301 for hook in hooks.iter().rev() {
12302 hook().await;
12303 }
12304}
12305
12306async fn run_shutdown_hooks_with_timeout(
12315 hooks: &[ShutdownHook],
12316 per_hook_budget: std::time::Duration,
12317 total_budget: std::time::Duration,
12318) {
12319 let deadline = tokio::time::Instant::now() + total_budget;
12320 for hook in hooks.iter().rev() {
12321 let remaining = deadline.saturating_duration_since(tokio::time::Instant::now());
12322 if remaining.is_zero() {
12323 tracing::warn!("shutdown: total hook budget exhausted; skipping remaining hooks");
12324 break;
12325 }
12326 let timeout = remaining.min(per_hook_budget);
12327 if tokio::time::timeout(timeout, hook()).await.is_err() {
12330 tracing::warn!(
12331 per_hook_budget_ms = timeout.as_millis(),
12332 "shutdown: hook overran per-hook timeout; continuing with remaining budget"
12333 );
12334 }
12335 }
12336}
12337
12338#[allow(clippy::cognitive_complexity)]
12345fn log_startup_transparency(
12346 routes: &[Route],
12347 tasks: &[crate::task::TaskInfo],
12348 scoped_groups: &[ScopedGroup],
12349 config: &AutumnConfig,
12350) {
12351 tracing::info!(
12352 "Registered routes:{}",
12353 format_route_lines(routes, scoped_groups, config)
12354 );
12355
12356 if let Some(task_lines) = format_task_lines(tasks) {
12357 tracing::info!("Scheduled tasks:{task_lines}");
12358 }
12359
12360 tracing::info!("Active middleware: {}", format_middleware_list(config));
12361
12362 tracing::info!("Configuration:{}", format_config_summary(config));
12363}
12364
12365fn fail_fast_on_invalid_session_config(config: &AutumnConfig, has_custom_session_store: bool) {
12378 if has_custom_session_store {
12379 return;
12380 }
12381 if let Err(error) = config.session.backend_plan(config.profile.as_deref()) {
12382 eprintln!("Invalid session backend config: {error}");
12383 std::process::exit(1);
12384 }
12385}
12386
12387fn fail_fast_on_missing_encryption_keys(config: &AutumnConfig) {
12397 if let Err(diagnostic) = crate::encryption::init_attribute_encryption(config.credentials()) {
12398 let is_production = matches!(config.profile.as_deref(), Some("prod" | "production"));
12399 if is_production {
12400 eprintln!("Attribute encryption misconfiguration: {diagnostic}");
12401 std::process::exit(1);
12402 }
12403 eprintln!(
12404 "warning: attribute encryption is not fully configured (dev): {diagnostic}\n \
12405 note: encrypted-column reads/writes will fail until keys are set; \
12406 this is a hard error in production."
12407 );
12408 }
12409}
12410
12411fn fail_fast_on_invalid_signing_secret(config: &AutumnConfig) {
12417 use crate::security::config::validate_signing_secret;
12418
12419 let is_production = matches!(config.profile.as_deref(), Some("prod" | "production"));
12420 let secret = config.security.signing_secret.secret.as_deref();
12421
12422 if let Err(error) = validate_signing_secret(secret, is_production) {
12423 eprintln!("Invalid signing secret configuration: {error}");
12424 eprintln!(
12425 " hint: generate a secret with `openssl rand -hex 32` and set \
12426 AUTUMN_SECURITY__SIGNING_SECRET"
12427 );
12428 std::process::exit(1);
12429 }
12430
12431 if is_production {
12434 for (i, prev) in config
12435 .security
12436 .signing_secret
12437 .previous_secrets
12438 .iter()
12439 .enumerate()
12440 {
12441 if let Err(error) = validate_signing_secret(Some(prev.as_str()), true) {
12442 eprintln!("Invalid signing secret configuration: previous_secrets[{i}]: {error}");
12443 eprintln!(
12444 " hint: every previous secret must meet the same entropy requirement \
12445 as the current secret"
12446 );
12447 std::process::exit(1);
12448 }
12449 }
12450 }
12451}
12452
12453fn fail_fast_on_invalid_webhook_config(config: &AutumnConfig) {
12454 let is_production = matches!(config.profile.as_deref(), Some("prod" | "production"));
12455 if let Err(error) = config.security.webhooks.validate(is_production) {
12456 eprintln!("Invalid signed webhook configuration: {error}");
12457 std::process::exit(1);
12458 }
12459}
12460
12461fn fail_fast_on_invalid_trusted_hosts(config: &AutumnConfig) {
12462 let is_production = matches!(config.profile.as_deref(), Some("prod" | "production"));
12463 if !is_production {
12464 return;
12465 }
12466 let hosts: Vec<String> = config
12467 .security
12468 .trusted_hosts
12469 .hosts
12470 .iter()
12471 .map(|h| h.trim().to_owned())
12472 .filter(|h| !h.is_empty())
12473 .collect();
12474 if hosts.is_empty() {
12475 eprintln!(
12476 "[security.trusted_hosts] is required in production; set hosts = [\"example.com\"] or explicit entries"
12477 );
12478 std::process::exit(1);
12479 }
12480 if hosts.iter().any(|h| h == "*") {
12481 tracing::warn!("trusted host validation disabled via wildcard '*' in production");
12482 }
12483}
12484
12485fn fail_fast_on_invalid_idempotency_config(config: &AutumnConfig) {
12486 if !config.idempotency.enabled.unwrap_or(false) {
12487 return;
12488 }
12489 let is_production = matches!(config.profile.as_deref(), Some("prod" | "production"));
12490 if is_production
12491 && config.idempotency.backend == crate::config::IdempotencyBackend::Memory
12492 && !config.idempotency.allow_memory_in_production
12493 {
12494 eprintln!(
12495 "The in-memory idempotency backend is not safe for multi-replica production use.\n\
12496 Set `[idempotency] backend = \"redis\"` in autumn.toml, or set \
12497 `allow_memory_in_production = true` to suppress this check."
12498 );
12499 std::process::exit(1);
12500 }
12501 #[cfg(feature = "redis")]
12502 if config.idempotency.backend == crate::config::IdempotencyBackend::Redis {
12503 let url_missing = config
12504 .idempotency
12505 .redis
12506 .url
12507 .as_deref()
12508 .is_none_or(|u| u.trim().is_empty());
12509 if url_missing {
12510 eprintln!(
12511 "Redis idempotency backend requires a connection URL.\n\
12512 Set AUTUMN_IDEMPOTENCY__REDIS__URL or `[idempotency.redis] url` in autumn.toml."
12513 );
12514 std::process::exit(1);
12515 }
12516 }
12517}
12518
12519pub(crate) fn install_webhook_registry(state: &AppState, config: &AutumnConfig) {
12520 if let Err(error) =
12521 crate::webhook::install_registry_from_config(state, &config.security.webhooks)
12522 {
12523 eprintln!("Invalid signed webhook configuration: {error}");
12524 std::process::exit(1);
12525 }
12526}
12527
12528#[cfg(feature = "storage")]
12534struct StorageBootstrap {
12535 store: crate::storage::SharedBlobStore,
12536 serving: Option<axum::Router<AppState>>,
12537}
12538
12539#[cfg(feature = "storage")]
12540impl StorageBootstrap {
12541 fn install(self, state: &AppState) -> Option<axum::Router<AppState>> {
12545 state.insert_extension::<crate::storage::BlobStoreState>(
12546 crate::storage::BlobStoreState::new(self.store),
12547 );
12548 self.serving
12549 }
12550}
12551
12552#[cfg(feature = "storage")]
12560#[allow(clippy::too_many_lines)] fn preflight_storage(config: &AutumnConfig) -> Option<StorageBootstrap> {
12562 use crate::storage::StorageBackendPlan;
12563
12564 let plan = config
12565 .storage
12566 .backend_plan(config.profile.as_deref())
12567 .unwrap_or_else(|error| {
12568 tracing::error!(%error, "invalid storage backend config; aborting startup");
12574 std::process::exit(1);
12575 });
12576
12577 match plan {
12578 StorageBackendPlan::Disabled => None,
12579 StorageBackendPlan::Local {
12580 provider_id,
12581 root,
12582 mount_path,
12583 default_url_expiry_secs,
12584 warn_in_production,
12585 } => Some(bootstrap_local_storage(
12586 config,
12587 &provider_id,
12588 &root,
12589 &mount_path,
12590 default_url_expiry_secs,
12591 warn_in_production,
12592 )),
12593 StorageBackendPlan::S3 { .. } => {
12594 tracing::error!(
12599 "storage.backend=s3 requires the `autumn-storage-s3` plugin. \
12600 Add it to your Cargo.toml, build an S3BlobStore from your config, \
12601 and call `.with_blob_store(store)` on your AppBuilder. \
12602 Aborting startup."
12603 );
12604 std::process::exit(1);
12605 }
12606 }
12607}
12608
12609#[cfg(feature = "storage")]
12610fn bootstrap_local_storage(
12611 config: &AutumnConfig,
12612 provider_id: &str,
12613 root: &std::path::Path,
12614 mount_path: &str,
12615 default_url_expiry_secs: u64,
12616 warn_in_production: bool,
12617) -> StorageBootstrap {
12618 use crate::storage::{LocalBlobStore, SharedBlobStore, local::SigningKey};
12619
12620 if warn_in_production {
12621 tracing::warn!(
12622 "prod profile is using the local-disk blob store; \
12623 bytes won't survive replica turnover. Set \
12624 storage.backend=s3 or storage.allow_local_in_production=true \
12625 to acknowledge"
12626 );
12627 }
12628
12629 let (signing_key, previous_signing_keys) = config
12634 .security
12635 .signing_secret
12636 .secret
12637 .as_deref()
12638 .filter(|s| !s.is_empty())
12639 .map_or_else(
12640 || {
12641 config
12642 .storage
12643 .local
12644 .signing_key
12645 .as_deref()
12646 .filter(|s| !s.is_empty())
12647 .map_or_else(
12648 || {
12649 if matches!(config.profile.as_deref(), Some("prod" | "production")) {
12650 tracing::warn!(
12651 "no signing secret configured in prod; blob URL signatures \
12652 won't survive a process restart. Set \
12653 AUTUMN_SECURITY__SIGNING_SECRET."
12654 );
12655 }
12656 (SigningKey::random(), vec![])
12657 },
12658 |legacy| (SigningKey::new(legacy.as_bytes().to_vec()), vec![]),
12659 )
12660 },
12661 |secret| {
12662 let current = SigningKey::new(secret.as_bytes().to_vec());
12663 let previous = config
12664 .security
12665 .signing_secret
12666 .previous_secrets
12667 .iter()
12668 .map(|s| SigningKey::new(s.as_bytes().to_vec()))
12669 .collect::<Vec<_>>();
12670 (current, previous)
12671 },
12672 );
12673
12674 let store = match LocalBlobStore::new(
12675 provider_id.to_string(),
12676 root.to_path_buf(),
12677 mount_path.to_string(),
12678 std::time::Duration::from_secs(default_url_expiry_secs),
12679 signing_key,
12680 previous_signing_keys,
12681 ) {
12682 Ok(store) => store,
12683 Err(err) => {
12684 tracing::error!(
12689 error = %err,
12690 root = %root.display(),
12691 "failed to initialize local blob store; aborting startup"
12692 );
12693 std::process::exit(1);
12694 }
12695 };
12696
12697 let serving = crate::storage::local::serve_router(&store);
12698 let arc: SharedBlobStore = std::sync::Arc::new(store);
12699
12700 tracing::info!(
12701 provider = %provider_id,
12702 root = %root.display(),
12703 mount = %mount_path,
12704 "Local blob store mounted"
12705 );
12706
12707 StorageBootstrap {
12708 store: arc,
12709 serving: Some(serving),
12710 }
12711}
12712async fn load_config_and_telemetry(
12713 config_loader: Option<ConfigLoaderFactory>,
12714 telemetry_provider: Option<Box<dyn crate::telemetry::TelemetryProvider>>,
12715 plugin_config_roots: BTreeSet<String>,
12716) -> (AutumnConfig, crate::telemetry::TelemetryGuard) {
12717 let config = load_config_only(config_loader, plugin_config_roots).await;
12718
12719 let provider: Box<dyn crate::telemetry::TelemetryProvider> = telemetry_provider
12722 .unwrap_or_else(|| Box::new(crate::telemetry::TracingOtlpTelemetryProvider::new()));
12723 let telemetry_guard = provider
12724 .init(&config.log, &config.telemetry, config.profile.as_deref())
12725 .unwrap_or_else(|error| {
12726 eprintln!("Failed to initialize telemetry: {error}");
12727 std::process::exit(1);
12728 });
12729
12730 (config, telemetry_guard)
12731}
12732
12733async fn load_config_only(
12746 config_loader: Option<ConfigLoaderFactory>,
12747 plugin_config_roots: BTreeSet<String>,
12748) -> AutumnConfig {
12749 try_load_config(config_loader, plugin_config_roots)
12750 .await
12751 .unwrap_or_else(|e| {
12752 eprintln!("Failed to load configuration: {e}");
12753 std::process::exit(1);
12754 })
12755}
12756
12757async fn try_load_config(
12759 config_loader: Option<ConfigLoaderFactory>,
12760 plugin_config_roots: BTreeSet<String>,
12761) -> Result<AutumnConfig, crate::config::ConfigError> {
12762 let mut config = match config_loader {
12772 Some(factory) => factory().await,
12773 None => {
12774 crate::config::TomlEnvConfigLoader::new()
12775 .with_plugin_config_roots(plugin_config_roots)
12776 .load()
12777 .await
12778 }
12779 }?;
12780
12781 if let Ok(forced) = std::env::var("AUTUMN_SERVE_FORCE_UNIX_SOCKET")
12789 && !forced.is_empty()
12790 {
12791 config.server.unix_socket = Some(forced);
12792 }
12793
12794 config.apply_retention_caps();
12801
12802 crate::metrics::set_limits(config.metrics.limits());
12816
12817 Ok(config)
12818}
12819
12820#[cfg(feature = "embed-assets")]
12825fn register_embedded_static_dir(embedded_static: Option<crate::assets::EmbeddedStaticDir>) {
12826 if let Some(dir) = embedded_static {
12827 crate::assets::register_embedded_static(dir);
12828 }
12829}
12830
12831#[cfg(all(feature = "embed-assets", feature = "i18n"))]
12835fn embedded_i18n_bundle(
12836 explicit: Option<Arc<crate::i18n::Bundle>>,
12837 embedded_locales: Option<&'static include_dir::Dir<'static>>,
12838 config: &AutumnConfig,
12839) -> Option<Arc<crate::i18n::Bundle>> {
12840 explicit.or_else(|| {
12841 embedded_locales.map(|dir| {
12842 Arc::new(
12843 crate::i18n::Bundle::load_from_embedded(dir, &config.i18n)
12844 .unwrap_or_else(|e| panic!("embedded_locales: {e}")),
12845 )
12846 })
12847 })
12848}
12849
12850#[cfg(feature = "i18n")]
12874fn exclude_static_routes_from_locale_prefix(
12875 config: &mut AutumnConfig,
12876 static_metas: &[crate::static_gen::StaticRouteMeta],
12877) {
12878 if config.i18n.locale_prefix_enabled {
12879 config
12880 .i18n
12881 .locale_prefix_exclude_exact
12882 .extend(static_metas.iter().map(|meta| meta.path.to_owned()));
12883 }
12884}
12885
12886#[cfg(not(feature = "i18n"))]
12887const fn exclude_static_routes_from_locale_prefix(
12888 _config: &mut AutumnConfig,
12889 _static_metas: &[crate::static_gen::StaticRouteMeta],
12890) {
12891}
12892
12893#[cfg(feature = "i18n")]
12898fn sitemap_locale_config(config: &AutumnConfig) -> Option<crate::seo::SitemapLocaleConfig<'_>> {
12899 config
12900 .i18n
12901 .locale_prefix_enabled
12902 .then_some(crate::seo::SitemapLocaleConfig {
12903 supported_locales: &config.i18n.supported_locales,
12904 exclude_prefixes: &config.i18n.locale_prefix_exclude,
12905 exclude_exact: &config.i18n.locale_prefix_exclude_exact,
12906 })
12907}
12908
12909#[cfg(not(feature = "i18n"))]
12910const fn sitemap_locale_config(
12911 _config: &AutumnConfig,
12912) -> Option<crate::seo::SitemapLocaleConfig<'_>> {
12913 None
12914}
12915
12916#[cfg(feature = "i18n")]
12917fn resolve_i18n_bundle(
12918 explicit_bundle: Option<Arc<crate::i18n::Bundle>>,
12919 auto_load: bool,
12920 config: &AutumnConfig,
12921 env: &dyn crate::config::Env,
12922) -> Option<Arc<crate::i18n::Bundle>> {
12923 if explicit_bundle.is_some() {
12924 return explicit_bundle;
12925 }
12926 if !auto_load {
12927 return None;
12928 }
12929
12930 let dir = project_dir(&config.i18n.dir, env);
12931 Some(Arc::new(
12932 crate::i18n::Bundle::load_from_dir(&dir, &config.i18n)
12933 .unwrap_or_else(|e| panic!("i18n_auto: {e}")),
12934 ))
12935}
12936
12937#[cfg(feature = "i18n")]
12938fn install_i18n_bundle_layer(
12939 mut custom_layers: Vec<CustomLayerRegistration>,
12940 state: &AppState,
12941 bundle: Option<Arc<crate::i18n::Bundle>>,
12942 i18n: &crate::i18n::I18nConfig,
12943) -> Vec<CustomLayerRegistration> {
12944 crate::i18n::install_locale_defaults(&i18n.default_locale, i18n.resolved_fallback_chain());
12954
12955 let Some(bundle) = bundle else {
12956 return custom_layers;
12957 };
12958
12959 tracing::info!(
12960 locales = ?bundle.locales(),
12961 default = bundle.default_locale(),
12962 "i18n bundle loaded"
12963 );
12964 crate::i18n::install_locale_defaults(bundle.default_locale(), bundle.fallback_chain().to_vec());
12969 state.insert_extension::<Arc<crate::i18n::Bundle>>(bundle.clone());
12970 let ambient_layer = crate::i18n::AmbientLocaleLayer::new(&bundle);
12974 let ext_layer = axum::Extension(bundle);
12975 custom_layers.push(CustomLayerRegistration {
12976 type_id: TypeId::of::<axum::Extension<Arc<crate::i18n::Bundle>>>(),
12977 type_name: std::any::type_name::<axum::Extension<Arc<crate::i18n::Bundle>>>(),
12978 layer: tower::util::BoxCloneSyncServiceLayer::new(ext_layer),
12979 });
12980 custom_layers.push(CustomLayerRegistration {
12986 type_id: TypeId::of::<crate::i18n::AmbientLocaleLayer>(),
12987 type_name: std::any::type_name::<crate::i18n::AmbientLocaleLayer>(),
12988 layer: tower::util::BoxCloneSyncServiceLayer::new(ambient_layer),
12989 });
12990 custom_layers
12991}
12992
12993#[cfg(feature = "db")]
12994struct DatabaseBootstrap {
12995 topology: Option<crate::db::DatabaseTopology>,
12996 shards: Option<crate::sharding::ShardSet>,
12997 replica_readiness: Option<crate::migrate::ReplicaMigrationReadiness>,
12998 replica_migration_check: Option<(String, String)>,
12999}
13000
13001#[cfg(feature = "db")]
13014async fn resolve_shard_set(
13015 config: &AutumnConfig,
13016 shard_router: Option<Arc<dyn crate::sharding::ShardRouter>>,
13017 shard_provider: Option<ShardProviderFactory>,
13018 directory_routing_enabled: bool,
13019 spawn_directory_listener: bool,
13020 topology: Option<&crate::db::DatabaseTopology>,
13021) -> Result<Option<crate::sharding::ShardSet>, String> {
13022 if !config.database.has_shards() {
13023 return Ok(None);
13024 }
13025 let router: Arc<dyn crate::sharding::ShardRouter> = match shard_router {
13026 Some(explicit) => explicit,
13027 None if directory_routing_enabled => {
13028 let control_primary = topology
13029 .map(crate::db::DatabaseTopology::primary)
13030 .ok_or_else(|| {
13031 "directory_shard_router is enabled but no control database is configured. \
13032 The directory router needs a control `database.primary_url`/`url` to read \
13033 the tenant→shard directory. Set one, or disable directory routing to use \
13034 the hash router."
13035 .to_owned()
13036 })?;
13037 let control_max = control_primary.status().max_size;
13045 if control_max < 2 {
13046 return Err(format!(
13047 "directory_shard_router requires a control database pool of at least 2 \
13048 connections, but the configured maximum is {control_max}. Directory \
13049 routing checks out a second control connection during extraction to \
13050 resolve the tenant→shard key, which deadlocks a pool sized to 1 when a \
13051 handler already holds a control connection (e.g. `Db` + `ShardedDb`). \
13052 Increase the control pool size (database.pool.max_size), or disable \
13053 directory routing to use the hash router."
13054 ));
13055 }
13056 let timeout_ms = config.database.statement_timeout.map_or(0, |d| {
13059 u64::try_from(d.as_millis())
13060 .unwrap_or(i32::MAX as u64)
13061 .min(i32::MAX as u64)
13062 });
13063 let dir_router = Arc::new(
13064 crate::sharding::DirectoryShardRouter::new(control_primary.clone())
13065 .with_statement_timeout_ms(timeout_ms),
13066 );
13067 if spawn_directory_listener {
13073 if let Some(control_url) = topology
13079 .and_then(crate::db::DatabaseTopology::migration_url)
13080 .or_else(|| config.database.effective_primary_url())
13081 {
13082 #[cfg(feature = "postgres")]
13086 drop(
13087 crate::sharding::DirectoryShardRouter::spawn_invalidation_listener(
13088 Arc::clone(&dir_router),
13089 control_url.to_owned(),
13090 crate::sharding::DEFAULT_DIRECTORY_INVALIDATION_SWEEP_INTERVAL,
13091 ),
13092 );
13093 #[cfg(not(feature = "postgres"))]
13095 let _ = control_url;
13096 } else {
13097 tracing::warn!(
13107 "directory shard routing is enabled but no control database URL is \
13108 configured (database.primary_url/url is unset, e.g. a custom \
13109 DatabasePoolProvider supplied the control pool); the cache-\
13110 invalidation LISTEN/NOTIFY task cannot be started, so directory \
13111 re-pins will only take effect after the cache TTL expires rather \
13112 than fleet-wide on commit"
13113 );
13114 }
13115 }
13116 dir_router
13117 }
13118 None => Arc::new(crate::sharding::HashShardRouter),
13119 };
13120 let set = match shard_provider {
13121 Some(factory) => {
13122 let topologies = factory(config.database.clone())
13123 .await
13124 .map_err(|e| format!("Failed to create shard pools: {e}"))?;
13125 #[cfg(feature = "sqlite")]
13134 {
13135 let target = topologies
13141 .iter()
13142 .zip(&config.database.shards)
13143 .map(|(topology, shard)| {
13144 topology
13145 .migration_url()
13146 .unwrap_or(shard.primary_url.as_str())
13147 })
13148 .find(|url| crate::db::sqlite_target_is_shared_cache(url))
13149 .unwrap_or_default();
13150 crate::db::reject_sqlite_unsupported_timeouts(&config.database, target)
13151 .map_err(|e| format!("Failed to create shard pools: {e}"))?;
13152 }
13153 crate::sharding::build_shard_set(&config.database, topologies, router)
13154 }
13155 None => crate::sharding::create_shard_set(&config.database, router)
13156 .map(|set| set.expect("has_shards() checked above")),
13157 }
13158 .map_err(|e| format!("Failed to configure shards: {e}"))?;
13159 Ok(Some(set))
13160}
13161
13162#[cfg(feature = "db")]
13163#[allow(clippy::too_many_lines)]
13166async fn setup_database(
13167 config: &AutumnConfig,
13168 migrations: Vec<(&'static str, crate::migrate::EmbeddedMigrations)>,
13169 pool_provider: Option<PoolProviderFactory>,
13170 shard_provider: Option<ShardProviderFactory>,
13171 shard_router: Option<Arc<dyn crate::sharding::ShardRouter>>,
13172 directory_shard_router: bool,
13173 hook_queue_migration_mode: RepositoryCommitHookQueueMigrationMode,
13174) -> Result<DatabaseBootstrap, String> {
13175 if config.replication.as_ref().is_some_and(|r| r.enabled) {
13180 crate::db::set_sqlite_replication_active();
13181 }
13182 let migrations = Arc::new(migrations_with_repository_framework_migrations(
13186 migrations,
13187 crate::repository_commit_hooks::has_repository_commit_hook_descriptors(),
13188 crate::version_history::has_versioned_repository_descriptors(),
13189 crate::derivation::has_derivation_descriptors(),
13190 hook_queue_migration_mode,
13191 ));
13192 let use_directory_router = shard_router.is_none()
13200 && (directory_shard_router || config.database.directory_shard_router);
13201 #[cfg(feature = "sqlite")]
13203 if config.database.fleet.is_some() && shard_router.is_some() {
13204 tracing::warn!(
13205 "with_shard_router is ignored: database.fleet routes each key to its own \
13206 database by fleet mode (ADR 0019)"
13207 );
13208 }
13209 let directory_migration_required = directory_migration_is_required(
13217 use_directory_router,
13218 config.database.has_shards(),
13219 hook_queue_migration_mode,
13220 );
13221 let shard_map_migration_required =
13222 shard_map_migration_is_required(config.database.has_shards(), hook_queue_migration_mode);
13223 let check_replica_migrations = !migrations.is_empty();
13224 #[cfg(feature = "sqlite")]
13225 let custom_pool_provider = pool_provider.is_some();
13226 let topology = match pool_provider {
13227 Some(factory) => factory(config.database.clone()).await,
13228 None => crate::db::create_topology(&config.database),
13229 }
13230 .map_err(|e| format!("Failed to create database pool: {e}"))?;
13231 #[cfg(feature = "sqlite")]
13247 if let Some(topology) = topology.as_ref() {
13248 let target = topology
13252 .migration_url()
13253 .or_else(|| config.database.effective_primary_url())
13254 .unwrap_or_default();
13255 crate::db::reject_sqlite_unsupported_timeouts(&config.database, target)
13256 .map_err(|e| format!("Failed to create database pool: {e}"))?;
13257 }
13258
13259 let runtime_boot = hook_queue_migration_mode == RepositoryCommitHookQueueMigrationMode::Runtime;
13262 let shards = match resolve_shard_set(
13263 config,
13264 shard_router,
13265 shard_provider,
13266 use_directory_router,
13267 runtime_boot,
13268 topology.as_ref(),
13269 )
13270 .await
13271 {
13272 Ok(shards) => shards,
13273 Err(e) => {
13274 #[cfg(feature = "managed-pg")]
13280 crate::managed_pg::emergency_stop_async().await;
13281 return Err(e);
13282 }
13283 };
13284
13285 let provider_migration_url = topology
13294 .as_ref()
13295 .and_then(|t| t.migration_url())
13296 .map(str::to_owned);
13297
13298 #[cfg(feature = "sqlite")]
13308 let sqlite_guard_shard_urls: Vec<&str> = if shards.is_some() {
13309 config
13310 .database
13311 .shards
13312 .iter()
13313 .map(|shard| shard.primary_url.as_str())
13314 .collect()
13315 } else {
13316 Vec::new()
13317 };
13318 #[cfg(feature = "sqlite")]
13319 #[allow(clippy::question_mark)] if let Err(e) = sqlite_sharding_unsupported_guard(
13321 if topology.is_some() {
13322 provider_migration_url
13323 .as_deref()
13324 .or_else(|| config.database.effective_primary_url())
13325 } else {
13326 None
13327 },
13328 directory_migration_required
13329 || shard_map_migration_required
13330 || config.database.has_shards(),
13331 &sqlite_guard_shard_urls,
13332 ) {
13333 #[cfg(feature = "managed-pg")]
13334 crate::managed_pg::emergency_stop_async().await;
13335 return Err(e);
13336 }
13337
13338 run_startup_migrations(
13339 config,
13340 topology.is_some(),
13341 shards.is_some(),
13342 provider_migration_url,
13343 Arc::clone(&migrations),
13344 directory_migration_required,
13345 shard_map_migration_required,
13346 )
13347 .await;
13348
13349 #[cfg(feature = "sqlite")]
13354 let shards = match (runtime_boot, topology.as_ref()) {
13355 (true, Some(topology)) => {
13356 if config.database.fleet.is_some() {
13357 fleet_control_target_identified(custom_pool_provider, topology.migration_url())?;
13358 }
13359 crate::db::fleet::build_for_app(config, topology.migration_url(), &migrations, |set| {
13360 !migration_set_is_control_framework(set)
13361 })?
13362 .map_or(shards, |fleet| {
13363 tracing::info!(
13364 mode = %fleet.mode(),
13365 root = %fleet.root().display(),
13366 max_open = fleet.max_open(),
13367 "SQLite database fleet ready"
13368 );
13369 if config
13370 .database
13371 .fleet
13372 .as_ref()
13373 .is_some_and(|f| f.restore_missing)
13374 && !config.replication.as_ref().is_some_and(|r| r.enabled)
13375 {
13376 tracing::warn!(
13377 "database.fleet.restore_missing is set but [replication] is off; there is \
13378 no replica to restore a missing database from"
13379 );
13380 }
13381 Some(crate::sharding::ShardSet::from_fleet(
13382 fleet,
13383 topology.primary().clone(),
13384 ))
13385 })
13386 }
13387 _ => shards,
13388 };
13389
13390 #[allow(clippy::question_mark)]
13398 if runtime_boot
13399 && crate::derivation::has_derivation_descriptors()
13400 && let Err(e) = start_derivation_backfill(
13401 topology.as_ref(),
13402 shards.as_ref(),
13403 crate::db::TxTimeouts::from_config(&config.database),
13404 )
13405 .await
13406 {
13407 #[cfg(feature = "managed-pg")]
13408 crate::managed_pg::emergency_stop_async().await;
13409 return Err(e);
13410 }
13411
13412 let (replica_readiness, replica_migration_check) = if topology
13413 .as_ref()
13414 .is_some_and(|topology| check_replica_migrations && topology.replica().is_some())
13415 {
13416 match (
13417 config.database.effective_primary_url(),
13418 config.database.replica_url.as_deref(),
13419 ) {
13420 (Some(primary_url), Some(replica_url)) => {
13421 let primary_url = primary_url.to_owned();
13422 let replica_url = replica_url.to_owned();
13423 let readiness = crate::migrate::check_replica_migration_readiness_blocking(
13424 primary_url.clone(),
13425 replica_url.clone(),
13426 )
13427 .await;
13428 (Some(readiness), Some((primary_url, replica_url)))
13429 }
13430 _ => (None, None),
13431 }
13432 } else {
13433 (None, None)
13434 };
13435
13436 if check_replica_migrations && let Some(set) = &shards {
13437 check_shard_replica_migration_parity(config, set).await;
13438 }
13439
13440 #[allow(clippy::question_mark)]
13445 if let Err(e) = Box::pin(enforce_shard_map_guard(
13446 config,
13447 topology.as_ref(),
13448 runtime_boot,
13449 ))
13450 .await
13451 {
13452 #[cfg(feature = "managed-pg")]
13455 crate::managed_pg::emergency_stop_async().await;
13456 return Err(e);
13457 }
13458
13459 Ok(DatabaseBootstrap {
13460 topology,
13461 shards,
13462 replica_readiness,
13463 replica_migration_check,
13464 })
13465}
13466
13467#[cfg(feature = "db")]
13473const BOOT_BACKFILL_BATCHES: usize = 8;
13474
13475#[cfg(feature = "db")]
13501async fn start_derivation_backfill(
13502 topology: Option<&crate::db::DatabaseTopology>,
13503 shards: Option<&crate::sharding::ShardSet>,
13504 timeouts: crate::db::TxTimeouts,
13505) -> Result<(), String> {
13506 crate::derivation::check_registered_derivations()
13508 .map_err(|error| format!("Invalid `#[derivation]` registry: {error}"))?;
13509
13510 let mut targets: Vec<(String, crate::db::Pool<crate::db::RuntimeConnection>)> = Vec::new();
13511 if let Some(topology) = topology {
13512 targets.push(("control".to_owned(), topology.primary().clone()));
13513 }
13514 if let Some(shards) = shards {
13515 for shard in shards.iter() {
13516 targets.push((
13517 format!("shard {}", shard.name()),
13518 shard.primary_pool().clone(),
13519 ));
13520 }
13521 }
13522
13523 for (label, pool) in targets {
13524 let mut conn = match pool.get().await {
13525 Ok(conn) => conn,
13526 Err(error) => {
13527 tracing::warn!(
13528 %error,
13529 database = %label,
13530 "no connection to reconcile derivation definitions"
13531 );
13532 continue;
13533 }
13534 };
13535 match timeouts
13536 .scope(crate::derivation::ensure_derivations(&mut conn))
13537 .await
13538 {
13539 Ok(enqueued) => {
13540 if !enqueued.is_empty() {
13541 tracing::info!(
13542 database = %label,
13543 derivations = ?enqueued,
13544 "derivation definitions changed; backfill enqueued"
13545 );
13546 }
13547 }
13548 Err(error) => {
13549 tracing::warn!(
13550 %error,
13551 database = %label,
13552 "could not reconcile derivation definitions; \
13553 see /actuator/derivations"
13554 );
13555 continue;
13556 }
13557 }
13558 drop(conn);
13559 spawn_derivation_backfill(label, pool, timeouts);
13560 }
13561 Ok(())
13562}
13563
13564#[cfg(feature = "db")]
13573fn spawn_derivation_backfill(
13574 label: String,
13575 pool: crate::db::Pool<crate::db::RuntimeConnection>,
13576 timeouts: crate::db::TxTimeouts,
13577) {
13578 tokio::spawn(async move {
13579 let options = crate::derivation::BackfillOptions {
13580 max_batches: Some(BOOT_BACKFILL_BATCHES),
13581 ..crate::derivation::BackfillOptions::default()
13582 };
13583 let mut completed: Vec<String> = Vec::new();
13584 let mut rows_repaired = 0usize;
13585 let mut rounds = 0usize;
13586 loop {
13587 let mut conn = match pool.get().await {
13588 Ok(conn) => conn,
13589 Err(error) => {
13590 tracing::warn!(
13591 %error,
13592 database = %label,
13593 "no connection to backfill derivations"
13594 );
13595 return;
13596 }
13597 };
13598 let report = match timeouts
13599 .scope(crate::derivation::run_backfill(&mut conn, &options))
13600 .await
13601 {
13602 Ok(report) => report,
13603 Err(error) => {
13604 tracing::warn!(%error, database = %label, "derivation backfill failed");
13605 return;
13606 }
13607 };
13608 drop(conn);
13609 completed.extend(report.completed);
13610 rows_repaired += report.rows_repaired;
13611 if report.in_progress.is_empty() {
13612 break;
13613 }
13614 rounds += 1;
13615 if report.batches_run == 0 {
13622 tracing::warn!(
13623 database = %label,
13624 pending = ?report.in_progress,
13625 rounds,
13626 "derivation backfill made no progress; see /actuator/derivations"
13627 );
13628 break;
13629 }
13630 }
13631 if !completed.is_empty() || rows_repaired > 0 {
13632 tracing::info!(
13633 database = %label,
13634 completed = ?completed,
13635 rows_repaired,
13636 "derivation backfill finished"
13637 );
13638 }
13639 });
13640}
13641
13642#[cfg(feature = "postgres")]
13660fn apply_pending_or_exit(
13661 database_url: &str,
13662 migrations: impl diesel::migration::MigrationSource<diesel::pg::Pg> + Send,
13663 target: &str,
13664 lock_policy: crate::migrate::MigrationLockPolicy,
13665) -> usize {
13666 match crate::migrate::run_pending_locked_with_policy(
13667 database_url,
13668 migrations,
13669 None,
13670 lock_policy,
13671 ) {
13672 Ok(result) => result.applied.len(),
13673 Err(error) => {
13674 let reason = match error {
13675 crate::migrate::MigrationError::Connection(_) => {
13676 "could not connect to the database"
13677 }
13678 crate::migrate::MigrationError::Migration(_) => "a migration failed to apply",
13679 crate::migrate::MigrationError::LockContention { .. } => {
13680 "a migration timed out on a table lock on every attempt"
13681 }
13682 _ => "migration error",
13683 };
13684 eprintln!("autumn migrate: {reason} (target {target})");
13685 #[cfg(feature = "managed-pg")]
13688 crate::managed_pg::emergency_stop();
13689 std::process::exit(1);
13690 }
13691 }
13692}
13693
13694#[cfg(all(feature = "db", not(feature = "postgres")))]
13697fn apply_pending_or_exit(
13698 _database_url: &str,
13699 _migrations: impl diesel::migration::MigrationSource<crate::db::RuntimeBackend> + Send,
13700 target: &str,
13701 _lock_policy: crate::migrate::MigrationLockPolicy,
13702) -> usize {
13703 eprintln!(
13704 "autumn migrate: {} (target {target})",
13705 crate::migrate::NO_POSTGRES_MSG
13706 );
13707 std::process::exit(1);
13708}
13709
13710#[cfg(feature = "sqlite")]
13721fn apply_pending_sqlite_or_exit(
13722 database_url: &str,
13723 migrations: impl diesel::migration::MigrationSource<diesel::sqlite::Sqlite> + Send,
13724 target: &str,
13725) -> usize {
13726 if let Some(err) = crate::migrate::reject_in_memory_migrations(database_url, &migrations) {
13733 eprintln!("autumn migrate: {err} (target {target})");
13734 std::process::exit(1);
13735 }
13736 match crate::migrate::run_pending_sqlite(database_url, migrations) {
13737 Ok(result) => result.applied.len(),
13738 Err(error) => {
13739 let reason = match error {
13740 crate::migrate::MigrationError::Connection(_) => {
13741 "could not connect to the database"
13742 }
13743 crate::migrate::MigrationError::Migration(_) => "a migration failed to apply",
13744 _ => "migration error",
13745 };
13746 eprintln!("autumn migrate: {reason} (target {target})");
13747 std::process::exit(1);
13748 }
13749 }
13750}
13751
13752#[cfg(feature = "sqlite")]
13754struct FleetRuntime {
13755 maintenance: tokio::task::JoinHandle<()>,
13757 sweeper: Option<tokio::task::JoinHandle<()>>,
13759 ownership: Option<FleetOwnershipRuntime>,
13761}
13762
13763#[cfg(feature = "sqlite")]
13766struct FleetOwnershipRuntime {
13767 fleet: crate::db::fleet::DatabaseFleet,
13768 stop: tokio_util::sync::CancellationToken,
13769 handle: tokio::task::JoinHandle<()>,
13770}
13771
13772#[cfg(feature = "sqlite")]
13773impl FleetOwnershipRuntime {
13774 async fn finish(self) {
13778 self.stop.cancel();
13779 let _ = self.handle.await;
13780 if self.fleet.release_ownership().await {
13781 tracing::info!("shutdown: fleet slot ranges released");
13782 }
13783 }
13784
13785 fn abandon(self) -> tokio::task::JoinHandle<()> {
13789 self.stop.cancel();
13790 self.handle
13791 }
13792}
13793
13794#[cfg(feature = "sqlite")]
13799async fn start_fleet_ownership(
13800 state: &AppState,
13801 fleet: &crate::db::fleet::DatabaseFleet,
13802) -> Result<Option<FleetOwnershipRuntime>, String> {
13803 use crate::db::fleet_ownership::{
13804 FleetOwnership, OWNERSHIP_INDICATOR, OwnerId, SlotRange, SqlSlotOwnership,
13805 };
13806 let Some(config) = state.config().database.fleet.and_then(|f| f.ownership) else {
13807 return Ok(None);
13808 };
13809 let control = state
13810 .pool()
13811 .cloned()
13812 .ok_or("fleet ownership needs a control database")?;
13813 let store = SqlSlotOwnership::new(control, state.clock_arc());
13814 store.ensure_schema().await.map_err(|e| e.to_string())?;
13815 let ranges = config
13816 .ranges()?
13817 .into_iter()
13818 .map(|(start, end)| SlotRange::new(start, end))
13819 .collect::<Result<Vec<_>, _>>()
13820 .map_err(|e| e.to_string())?;
13821 let owner = OwnerId::new(config.owner.as_str()).map_err(|e| e.to_string())?;
13822 let ownership = Arc::new(
13823 FleetOwnership::new(
13824 Arc::new(store),
13825 owner,
13826 ranges,
13827 std::time::Duration::from_secs(config.lease_secs),
13828 )
13829 .map_err(|e| e.to_string())?,
13830 );
13831 let report = fleet
13834 .start_ownership(Arc::clone(&ownership))
13835 .await
13836 .map_err(|e| e.to_string())?;
13837 tracing::info!(
13838 owner = %ownership.owner(),
13839 claimed = report.claimed.len(),
13840 waiting = report.waiting.len(),
13841 failed = report.failed.len(),
13842 "fleet slot ownership installed"
13843 );
13844 if let Err(e) = state.health_indicator_registry.register(
13845 OWNERSHIP_INDICATOR,
13846 crate::actuator::IndicatorGroup::HealthOnly,
13847 ownership.indicator(),
13848 ) {
13849 tracing::warn!("{e}");
13850 }
13851 let stop = tokio_util::sync::CancellationToken::new();
13852 let handle = fleet.spawn_ownership(stop.clone());
13853 Ok(Some(FleetOwnershipRuntime {
13854 fleet: fleet.clone(),
13855 stop,
13856 handle,
13857 }))
13858}
13859
13860#[cfg(feature = "sqlite")]
13878async fn start_fleet_runtime(
13879 state: &AppState,
13880 shutdown: tokio_util::sync::CancellationToken,
13881 sweep: Option<tokio_util::sync::CancellationToken>,
13882) -> Option<FleetRuntime> {
13883 let fleet = state.shards()?.fleet()?.clone();
13884 let ownership = match start_fleet_ownership(state, &fleet).await {
13886 Ok(ownership) => ownership,
13887 Err(error) => {
13888 tracing::error!(%error, "SQLite fleet slot ownership could not start");
13889 eprintln!("SQLite fleet slot ownership could not start: {error}");
13890 exit_stop_managed_pg();
13891 std::process::exit(1);
13892 }
13893 };
13894 fleet.add_lifecycle(Arc::new(FleetCommitHookWorkers {
13895 #[cfg(feature = "ws")]
13896 channels: state.channels().clone(),
13897 }));
13898 let options = crate::db::fleet_sweeper::FleetSweeperOptions {
13899 relay: state.role().runs_workers(),
13900 ..crate::db::fleet_sweeper::FleetSweeperOptions::default()
13901 };
13902 let sweeper = match crate::db::fleet_sweeper::FleetSweeper::install_with(state, options).await {
13905 Ok(sweeper) => sweeper,
13906 Err(error) => {
13907 tracing::error!(%error, "the SQLite fleet sweeper could not start");
13908 eprintln!("the SQLite fleet sweeper could not start: {error}");
13909 exit_stop_managed_pg();
13910 std::process::exit(1);
13911 }
13912 };
13913 let sweeper = sweeper
13914 .zip(sweep)
13915 .map(|(sweeper, token)| sweeper.spawn(token));
13916 Some(FleetRuntime {
13917 maintenance: fleet.spawn_maintenance(shutdown),
13918 sweeper,
13919 ownership,
13920 })
13921}
13922
13923#[cfg(feature = "sqlite")]
13928fn start_fleet_replication(
13929 state: &AppState,
13930 config: &AutumnConfig,
13931 runtime: &crate::replication::ReplicationRuntime,
13932 lag_alert_after: std::time::Duration,
13933) -> Result<(), String> {
13934 let Some(fleet) = state.shards().and_then(crate::sharding::ShardSet::fleet) else {
13935 return Ok(());
13936 };
13937 let restore_missing = config
13938 .database
13939 .fleet
13940 .as_ref()
13941 .is_some_and(|f| f.restore_missing);
13942 if restore_missing
13943 && config
13944 .database
13945 .fleet
13946 .as_ref()
13947 .is_some_and(|f| f.ownership.is_none())
13948 {
13949 tracing::warn!(
13950 "database.fleet.restore_missing is on without [database.fleet.ownership]: nothing \
13951 fences the previous owner, so two hosts can write one database and diverge"
13952 );
13953 }
13954 let replication = Arc::new(
13955 crate::db::fleet_replication::FleetReplication::new(
13956 Arc::clone(runtime.replicator.destination()),
13957 runtime.settings.clone(),
13958 lag_alert_after,
13959 restore_missing,
13960 state.clock_arc(),
13961 )
13962 .with_final_flush(std::time::Duration::from_secs(
13964 config.server.shutdown_timeout_secs,
13965 )),
13966 );
13967 fleet
13968 .install_replication(Arc::clone(&replication))
13969 .map_err(|_| "the fleet already replicates".to_owned())?;
13970 if let Err(e) = state.health_indicator_registry.register(
13971 crate::db::fleet_replication::FLEET_REPLICATION_INDICATOR,
13972 crate::actuator::IndicatorGroup::HealthOnly,
13973 replication.indicator(),
13974 ) {
13975 tracing::warn!("{e}");
13976 }
13977 replication
13978 .spawn_loop()
13979 .map_err(|e| format!("could not start the fleet replication thread: {e}"))?;
13980 tracing::info!(
13981 prefix = %format!("{}/fleet/", runtime.settings.root),
13982 restore_missing,
13983 "SQLite fleet replication is enabled"
13984 );
13985 Ok(())
13986}
13987
13988#[cfg(feature = "sqlite")]
13992struct FleetCommitHookWorkers {
13993 #[cfg(feature = "ws")]
13994 channels: crate::channels::Channels,
13995}
13996
13997#[cfg(feature = "sqlite")]
13998impl crate::db::fleet::FleetLifecycle for FleetCommitHookWorkers {
13999 fn on_open(&self, db: &crate::db::fleet::FleetDatabase) -> Result<(), String> {
14000 let stop = tokio_util::sync::CancellationToken::new();
14003 #[cfg(feature = "ws")]
14004 let handle = crate::repository_commit_hooks::start_repository_commit_hook_worker(
14005 db.pool().clone(),
14006 Some(self.channels.clone()),
14007 stop.clone(),
14008 );
14009 #[cfg(not(feature = "ws"))]
14010 let handle = crate::repository_commit_hooks::start_repository_commit_hook_worker(
14011 db.pool().clone(),
14012 stop.clone(),
14013 );
14014 if let Some(handle) = handle {
14015 db.register_worker(stop, handle);
14016 }
14017 Ok(())
14018 }
14019
14020 fn on_close(&self, db: &crate::db::fleet::FleetDatabase) {
14021 crate::repository_commit_hooks::forget_sqlite_repository_commit_hook_kick(db.pool());
14022 }
14023}
14024
14025#[cfg(feature = "sqlite")]
14032fn fleet_control_target_identified(
14033 custom_pool_provider: bool,
14034 migration_url: Option<&str>,
14035) -> Result<(), String> {
14036 if !custom_pool_provider {
14037 return Ok(());
14038 }
14039 match migration_url {
14040 None => Err(
14041 "database.fleet with a custom DatabasePoolProvider: the provider's topology must \
14042 name its control database with DatabaseTopology::with_migration_url(...), so the \
14043 fleet can refuse a control file inside database.fleet.root"
14044 .to_owned(),
14045 ),
14046 Some(url)
14047 if crate::config::DatabaseBackend::detect(url)
14048 != Some(crate::config::DatabaseBackend::Sqlite)
14049 || crate::config::sqlite_url_file(url).is_none() =>
14050 {
14051 Err(format!(
14052 "database.fleet with a custom DatabasePoolProvider: the provider names its control \
14053 database as {}, which is not a file-backed sqlite: database; a fleet keeps \
14054 framework state (sessions, jobs, flags) there, so it must survive a restart",
14055 crate::db_url::redact_target(url)
14056 ))
14057 }
14058 Some(_) => Ok(()),
14059 }
14060}
14061
14062#[cfg(feature = "sqlite")]
14066async fn migrate_fleet_or_exit(
14067 config: &AutumnConfig,
14068 migrations: Arc<Vec<(&'static str, crate::migrate::EmbeddedMigrations)>>,
14069) -> usize {
14070 let fleet = match crate::db::fleet::build_for_app(config, None, &migrations, |set| {
14071 !migration_set_is_control_framework(set)
14072 }) {
14073 Ok(Some(fleet)) => fleet,
14074 Ok(None) => return 0,
14075 Err(error) => {
14076 eprintln!("autumn migrate: {error}");
14077 std::process::exit(1);
14078 }
14079 };
14080 match fleet
14081 .migrate_all(crate::db::fleet::DEFAULT_FLEET_CONCURRENCY)
14082 .await
14083 {
14084 Ok(report) if report.failed.is_empty() => {
14085 eprintln!(
14086 "autumn migrate: fleet: {} database(s), {} migration(s) applied",
14087 report.databases, report.applied
14088 );
14089 report.applied
14090 }
14091 Ok(report) => {
14092 for (key, error) in &report.failed {
14093 eprintln!("autumn migrate: fleet database {key}: {error}");
14094 }
14095 eprintln!(
14096 "autumn migrate: fleet: {} of {} database(s) failed; the others are migrated",
14097 report.failed.len(),
14098 report.databases
14099 );
14100 std::process::exit(1);
14101 }
14102 Err(error) => {
14103 eprintln!("autumn migrate: fleet: {error}");
14104 std::process::exit(1);
14105 }
14106 }
14107}
14108
14109#[cfg(feature = "sqlite")]
14140fn sqlite_sharding_unsupported_guard(
14141 control_url: Option<&str>,
14142 control_sharding_required: bool,
14143 shard_urls: &[&str],
14144) -> Result<(), String> {
14145 fn is_sqlite(url: &str) -> bool {
14146 crate::config::DatabaseBackend::detect(url) == Some(crate::config::DatabaseBackend::Sqlite)
14147 }
14148 if control_url.is_some_and(is_sqlite) && control_sharding_required {
14149 return Err(
14150 "SQLite deployments do not support sharding. The configured sqlite:// control target \
14151 has sharding enabled (shards and/or the directory/shard-map control migrations), \
14152 which is a Postgres-only capability \u{2014} remove the shard configuration to run \
14153 on SQLite, or use a Postgres control database. Tracking: #1614."
14154 .to_owned(),
14155 );
14156 }
14157 if shard_urls.iter().copied().any(is_sqlite) {
14158 return Err(
14159 "SQLite deployments do not support sharding. A configured shard targets a SQLite \
14160 database, and per-shard migration/fan-out is a Postgres-only capability \u{2014} \
14161 remove the SQLite shard configuration to run on SQLite, or use Postgres shard \
14162 targets. Tracking: #1614."
14163 .to_owned(),
14164 );
14165 }
14166 Ok(())
14167}
14168
14169#[cfg(all(test, feature = "sqlite"))]
14170mod sqlite_sharding_unsupported_guard_tests {
14171 use super::sqlite_sharding_unsupported_guard;
14172
14173 #[test]
14174 fn sqlite_control_target_with_sharding_fails_fast() {
14175 for url in [
14178 "sqlite:///var/lib/app.db",
14179 "sqlite://./relative.db",
14180 "sqlite::memory:",
14181 ] {
14182 let err = sqlite_sharding_unsupported_guard(Some(url), true, &[])
14183 .expect_err("sqlite control target + sharding must be rejected");
14184 assert!(
14185 err.contains("do not support sharding"),
14186 "message must name the sharding situation clearly: {err}"
14187 );
14188 assert!(
14189 err.contains("#1614"),
14190 "message must point at the tracking issue: {err}"
14191 );
14192 }
14193 }
14194
14195 #[test]
14196 fn sqlite_control_target_without_sharding_boots() {
14197 for url in [
14200 "sqlite:///var/lib/app.db",
14201 "sqlite://./relative.db",
14202 "sqlite::memory:",
14203 ] {
14204 assert!(
14205 sqlite_sharding_unsupported_guard(Some(url), false, &[]).is_ok(),
14206 "sqlite target without sharding must boot (migrations now applied): {url}"
14207 );
14208 }
14209 }
14210
14211 #[test]
14212 fn postgres_target_is_unchanged() {
14213 for url in [
14215 "postgres://u@h/db",
14216 "postgresql://user:pass@db:5432/app",
14217 "host=db user=app sslmode=require",
14218 ] {
14219 assert!(
14220 sqlite_sharding_unsupported_guard(Some(url), true, &[]).is_ok(),
14221 "postgres target must never be gated: {url}"
14222 );
14223 assert!(
14224 sqlite_sharding_unsupported_guard(Some(url), false, &[]).is_ok(),
14225 "postgres target must never be gated: {url}"
14226 );
14227 }
14228 }
14229
14230 #[test]
14231 fn absent_control_url_boots() {
14232 assert!(sqlite_sharding_unsupported_guard(None, true, &[]).is_ok());
14234 assert!(sqlite_sharding_unsupported_guard(None, false, &[]).is_ok());
14235 }
14236
14237 #[test]
14238 fn sqlite_shard_target_fails_fast() {
14239 for shards in [
14244 &["sqlite:///var/lib/shard0.db"][..],
14245 &["sqlite:///var/lib/shard0.db", "postgres://u@h/shard1"][..],
14246 ] {
14247 let err = sqlite_sharding_unsupported_guard(None, false, shards)
14248 .expect_err("a sqlite shard target must be rejected");
14249 assert!(
14250 err.contains("do not support sharding") && err.contains("shard"),
14251 "message must name the SQLite shard situation clearly: {err}"
14252 );
14253 assert!(
14254 err.contains("#1614"),
14255 "message must point at the tracking issue: {err}"
14256 );
14257 }
14258 }
14259
14260 #[test]
14261 fn postgres_shard_targets_are_unchanged() {
14262 assert!(
14264 sqlite_sharding_unsupported_guard(
14265 Some("postgres://u@h/control"),
14266 true,
14267 &["postgres://u@h/shard0", "postgres://u@h/shard1"],
14268 )
14269 .is_ok(),
14270 "all-postgres shard targets must never be gated"
14271 );
14272 }
14273
14274 #[test]
14275 fn migrate_only_mode_reuses_the_boot_guard_for_sqlite_targets() {
14276 let err = sqlite_sharding_unsupported_guard(Some("sqlite:///var/lib/app.db"), true, &[])
14284 .expect_err("sqlite migrate control target with sharding must be rejected");
14285 assert!(
14286 err.contains("do not support sharding") && err.contains("#1614"),
14287 "migrate-only sqlite control error must be the actionable sharding message: {err}"
14288 );
14289
14290 let shard_err = sqlite_sharding_unsupported_guard(
14292 Some("postgres://u@h/control"),
14293 true,
14294 &["sqlite:///var/lib/shard0.db"],
14295 )
14296 .expect_err("sqlite migrate shard target must be rejected");
14297 assert!(
14298 shard_err.contains("do not support sharding") && shard_err.contains("shard"),
14299 "migrate-only sqlite shard error must be the actionable sharding message: {shard_err}"
14300 );
14301
14302 assert!(
14304 sqlite_sharding_unsupported_guard(
14305 Some("postgres://u@h/control"),
14306 true,
14307 &["postgres://u@h/shard0"],
14308 )
14309 .is_ok(),
14310 "an all-postgres migrate configuration must proceed unchanged"
14311 );
14312
14313 assert!(
14316 sqlite_sharding_unsupported_guard(Some("sqlite:///var/lib/app.db"), false, &[]).is_ok(),
14317 "sqlite control target without sharding must never be gated"
14318 );
14319 }
14320}
14321
14322#[cfg(feature = "db")]
14353fn migration_sets_for_disambiguation<'a>(
14354 migrations: &'a [(&'static str, crate::migrate::EmbeddedMigrations)],
14355 shards_configured: bool,
14356) -> Vec<(&'a str, &'a crate::migrate::EmbeddedMigrations)> {
14357 let owned = migrations.iter().map(|(name, set)| (*name, set));
14358 if shards_configured {
14359 owned
14360 .chain([
14361 (
14362 "shard-directory",
14363 &crate::sharding::SHARD_DIRECTORY_MIGRATIONS,
14364 ),
14365 ("shard-map", &crate::sharding::SHARD_MAP_MIGRATIONS),
14366 ])
14367 .collect()
14368 } else {
14369 owned.collect()
14370 }
14371}
14372
14373#[cfg(feature = "db")]
14374#[allow(
14375 clippy::too_many_arguments,
14376 clippy::fn_params_excessive_bools,
14377 clippy::too_many_lines
14378)]
14379async fn run_startup_migrations(
14380 config: &AutumnConfig,
14381 control_configured: bool,
14382 shards_configured: bool,
14383 provider_migration_url: Option<String>,
14384 migrations: Arc<Vec<(&'static str, crate::migrate::EmbeddedMigrations)>>,
14385 directory_migration_required: bool,
14386 shard_map_migration_required: bool,
14387) {
14388 let control_url = if control_configured {
14389 provider_migration_url
14395 .or_else(|| config.database.effective_primary_url().map(str::to_owned))
14396 } else {
14397 None
14398 };
14399 let shard_targets: Vec<(String, String)> = if shards_configured {
14400 config
14401 .database
14402 .shards
14403 .iter()
14404 .map(|shard| (format!("shard:{}", shard.name), shard.primary_url.clone()))
14405 .collect()
14406 } else {
14407 Vec::new()
14408 };
14409 let profile = config.profile.clone();
14410 let auto_migrate = config.database.auto_migrate;
14411 let auto_in_prod = config.database.auto_migrate_in_production;
14412 let lock_policy = crate::migrate::MigrationLockPolicy::from_config(&config.database);
14413 let disambiguation_sets =
14419 migration_sets_for_disambiguation(&migrations, config.database.has_shards());
14420 let disambiguated = crate::migrate::compute_migration_disambiguation(&disambiguation_sets);
14421 #[cfg(feature = "sqlite")]
14422 let sqlite_history_sets = crate::migrate::sqlite_collision_pairs(&disambiguation_sets);
14423 let migration_result = crate::time::spawn_blocking(move || {
14424 #[cfg(feature = "sqlite")]
14432 if let Some(url) = control_url.as_deref()
14433 && crate::config::DatabaseBackend::detect(url)
14434 == Some(crate::config::DatabaseBackend::Sqlite)
14435 {
14436 if crate::migrate::should_auto_apply(profile.as_deref(), auto_migrate, auto_in_prod)
14442 && let Err(error) = crate::migrate::adopt_sqlite_collision_history(
14443 url,
14444 &sqlite_history_sets,
14445 &disambiguated,
14446 )
14447 {
14448 tracing::error!(
14449 error = %error,
14450 target = "control",
14451 "Could not move an already-applied migration's version record"
14452 );
14453 std::process::exit(1);
14454 }
14455 for (_, mig) in migrations.iter() {
14456 crate::migrate::auto_migrate_sqlite(
14457 url,
14458 profile.as_deref(),
14459 auto_migrate,
14460 auto_in_prod,
14461 crate::migrate::DisambiguatedMigrations::new(mig, &disambiguated),
14462 "control",
14463 );
14464 }
14465 return;
14466 }
14467
14468 if let Some(url) = control_url {
14469 for (_, mig) in migrations.iter() {
14470 crate::migrate::auto_migrate(
14471 &url,
14472 profile.as_deref(),
14473 auto_migrate,
14474 auto_in_prod,
14475 crate::migrate::DisambiguatedMigrations::new(mig, &disambiguated),
14476 "control",
14477 lock_policy,
14478 );
14479 }
14480 if directory_migration_required {
14483 crate::migrate::auto_migrate(
14484 &url,
14485 profile.as_deref(),
14486 auto_migrate,
14487 auto_in_prod,
14488 crate::migrate::DisambiguatedMigrations::new(
14489 &crate::sharding::SHARD_DIRECTORY_MIGRATIONS,
14490 &disambiguated,
14491 ),
14492 "control",
14493 lock_policy,
14494 );
14495 }
14496 if shard_map_migration_required {
14504 crate::migrate::auto_migrate(
14505 &url,
14506 profile.as_deref(),
14507 auto_migrate,
14508 auto_in_prod,
14509 crate::migrate::DisambiguatedMigrations::new(
14510 &crate::sharding::SHARD_MAP_MIGRATIONS,
14511 &disambiguated,
14512 ),
14513 "control",
14514 lock_policy,
14515 );
14516 }
14517 }
14518 for (target, url) in &shard_targets {
14525 for (_, mig) in migrations
14526 .iter()
14527 .filter(|(_, mig)| !migration_set_is_control_framework(mig))
14528 {
14529 crate::migrate::auto_migrate(
14530 url,
14531 profile.as_deref(),
14532 auto_migrate,
14533 auto_in_prod,
14534 crate::migrate::DisambiguatedMigrations::new(mig, &disambiguated),
14535 target,
14536 lock_policy,
14537 );
14538 }
14539 }
14540 })
14541 .await;
14542 if let Err(e) = migration_result {
14543 tracing::error!(error = %e, "Migration task panicked");
14544 #[cfg(feature = "managed-pg")]
14549 crate::managed_pg::emergency_stop_async().await;
14550 std::process::exit(1);
14551 }
14552}
14553
14554#[cfg(feature = "db")]
14558async fn check_shard_replica_migration_parity(
14559 config: &AutumnConfig,
14560 set: &crate::sharding::ShardSet,
14561) {
14562 for (shard_config, shard) in config.database.shards.iter().zip(set.iter()) {
14563 let Some(replica_url) = shard_config.replica_url.as_deref() else {
14564 continue;
14565 };
14566 shard
14570 .runtime()
14571 .configure_migration_check(shard_config.primary_url.clone(), replica_url.to_owned());
14572 let _ = shard.runtime().parity_check_due();
14573 let readiness = crate::migrate::check_replica_migration_readiness_blocking(
14574 shard_config.primary_url.clone(),
14575 replica_url.to_owned(),
14576 )
14577 .await;
14578 if readiness.is_ready() {
14579 shard.runtime().mark_replica_migrations_ready();
14580 } else if let Some(detail) = readiness.detail() {
14581 tracing::warn!(
14582 shard = %shard.name(),
14583 detail = %detail,
14584 "shard replica migrations are not ready"
14585 );
14586 shard.runtime().mark_replica_migrations_unready(detail);
14587 }
14588 }
14589}
14590
14591#[cfg(feature = "db")]
14592const REPOSITORY_COMMIT_HOOK_QUEUE_MIGRATION: &str =
14593 "20260515000000_create_repository_commit_hook_queue";
14594
14595#[cfg(feature = "db")]
14596const VERSION_HISTORY_MIGRATION: &str = "20260526000000_create_version_history";
14597
14598#[cfg(feature = "db")]
14599const DERIVATION_MIGRATION: &str = "20260907101530_create_derivations";
14600
14601#[cfg(feature = "db")]
14608const fn directory_migration_is_required(
14609 directory_routing_enabled: bool,
14610 has_shards: bool,
14611 mode: RepositoryCommitHookQueueMigrationMode,
14612) -> bool {
14613 directory_routing_enabled
14614 && has_shards
14615 && matches!(mode, RepositoryCommitHookQueueMigrationMode::Runtime)
14616}
14617
14618#[cfg(feature = "db")]
14626const fn shard_map_migration_is_required(
14627 has_shards: bool,
14628 mode: RepositoryCommitHookQueueMigrationMode,
14629) -> bool {
14630 has_shards && matches!(mode, RepositoryCommitHookQueueMigrationMode::Runtime)
14631}
14632
14633#[cfg(feature = "postgres")]
14635#[derive(diesel::QueryableByName)]
14636struct ShardMapRow {
14637 #[diesel(sql_type = diesel::sql_types::Text)]
14638 shard_name: String,
14639 #[diesel(sql_type = diesel::sql_types::Text)]
14640 slots: String,
14641}
14642
14643#[cfg(feature = "postgres")]
14656pub async fn run_shard_map_guard(
14657 control_pool: &deadpool::managed::Pool<
14658 diesel_async::pooled_connection::AsyncDieselConnectionManager<
14659 diesel_async::AsyncPgConnection,
14660 >,
14661 >,
14662 computed: &[crate::config::ShardSlotAssignment],
14663 auto_split: bool,
14664) -> Result<(), String> {
14665 use diesel_async::RunQueryDsl as _;
14666
14667 if !auto_split {
14668 return Ok(());
14669 }
14670
14671 let mut conn = match control_pool.get().await {
14672 Ok(conn) => conn,
14673 Err(e) => {
14674 return Err(format!(
14675 "shard-map guard could not acquire a control connection: {e} — \
14676 ensure the control database is reachable to enforce topology \
14677 change detection"
14678 ));
14679 }
14680 };
14681
14682 let rows: Vec<ShardMapRow> = match diesel::sql_query(
14683 "SELECT shard_name, slots FROM _autumn_shard_map ORDER BY shard_name",
14684 )
14685 .load::<ShardMapRow>(&mut conn)
14686 .await
14687 {
14688 Ok(rows) => rows,
14689 Err(e) => {
14690 return Err(format!(
14691 "shard-map guard could not read _autumn_shard_map: {e} — \
14692 run `autumn migrate` to create the control schema before \
14693 starting with auto-split shards"
14694 ));
14695 }
14696 };
14697
14698 let stored: Vec<crate::config::ShardSlotAssignment> = rows
14699 .into_iter()
14700 .map(|r| crate::config::ShardSlotAssignment {
14701 name: r.shard_name,
14702 ranges: r.slots,
14703 })
14704 .collect();
14705 let stored_opt = if stored.is_empty() {
14706 None
14707 } else {
14708 Some(stored.as_slice())
14709 };
14710
14711 crate::config::check_stored_slot_map(auto_split, computed, stored_opt)?;
14712
14713 if stored.is_empty() {
14717 use diesel_async::AsyncConnection as _;
14718 let assignments: Vec<_> = computed.to_vec();
14719 conn.transaction::<(), diesel::result::Error, _>(async move |conn| {
14720 for assignment in &assignments {
14721 diesel::sql_query(
14722 "INSERT INTO _autumn_shard_map (shard_name, slots) VALUES ($1, $2) \
14723 ON CONFLICT (shard_name) DO UPDATE \
14724 SET slots = EXCLUDED.slots, updated_at = NOW()",
14725 )
14726 .bind::<diesel::sql_types::Text, _>(&assignment.name)
14727 .bind::<diesel::sql_types::Text, _>(&assignment.ranges)
14728 .execute(conn)
14729 .await?;
14730 }
14731 Ok(())
14732 })
14733 .await
14734 .map_err(|e| format!("shard-map guard could not persist map: {e}"))?;
14735 }
14736
14737 Ok(())
14738}
14739
14740#[cfg(all(feature = "db", feature = "sqlite"))]
14754#[allow(clippy::unused_async)]
14755async fn enforce_shard_map_guard(
14756 config: &AutumnConfig,
14757 topology: Option<&crate::db::DatabaseTopology>,
14758 runtime_boot: bool,
14759) -> Result<(), String> {
14760 let _ = (config, topology, runtime_boot);
14761 Ok(())
14762}
14763
14764#[cfg(all(feature = "db", not(feature = "sqlite")))]
14765async fn enforce_shard_map_guard(
14766 config: &AutumnConfig,
14767 topology: Option<&crate::db::DatabaseTopology>,
14768 runtime_boot: bool,
14769) -> Result<(), String> {
14770 if !runtime_boot || !config.database.has_shards() {
14771 return Ok(());
14772 }
14773 let Some(topology) = topology else {
14774 return Ok(());
14775 };
14776 if !config.database.shards_auto_split() {
14777 return Ok(());
14778 }
14779 let computed = config
14780 .database
14781 .resolved_shard_assignments()
14782 .map_err(|e| format!("shard-map guard: {e}"))?;
14783 run_shard_map_guard(topology.primary(), &computed, true).await
14784}
14785
14786#[cfg(feature = "db")]
14787#[derive(Clone, Copy, Debug, Eq, PartialEq)]
14788enum RepositoryCommitHookQueueMigrationMode {
14789 Runtime,
14790 StaticBuild,
14791}
14792
14793#[cfg(feature = "db")]
14794fn migrations_with_repository_framework_migrations(
14795 mut migrations: Vec<(&'static str, crate::migrate::EmbeddedMigrations)>,
14796 hook_queue_required: bool,
14797 version_history_required: bool,
14798 derivations_required: bool,
14799 mode: RepositoryCommitHookQueueMigrationMode,
14800) -> Vec<(&'static str, crate::migrate::EmbeddedMigrations)> {
14801 if hook_queue_required
14802 && mode == RepositoryCommitHookQueueMigrationMode::Runtime
14803 && !shard_applied_sets_include(&migrations, REPOSITORY_COMMIT_HOOK_QUEUE_MIGRATION)
14804 {
14805 migrations.push((
14806 "repository-commit-hooks",
14807 crate::repository_commit_hooks::REPOSITORY_COMMIT_HOOK_MIGRATIONS,
14808 ));
14809 }
14810 if version_history_required
14811 && mode == RepositoryCommitHookQueueMigrationMode::Runtime
14812 && !shard_applied_sets_include(&migrations, VERSION_HISTORY_MIGRATION)
14813 {
14814 migrations.push((
14815 "version-history",
14816 crate::version_history::VERSION_HISTORY_MIGRATIONS,
14817 ));
14818 }
14819 if derivations_required
14824 && mode == RepositoryCommitHookQueueMigrationMode::Runtime
14825 && !shard_applied_sets_include(&migrations, DERIVATION_MIGRATION)
14826 {
14827 migrations.push(("derivations", crate::derivation::DERIVATION_MIGRATIONS));
14828 }
14829 migrations
14830}
14831
14832#[cfg(feature = "db")]
14847fn shard_applied_sets_include(
14848 migrations: &[(&'static str, crate::migrate::EmbeddedMigrations)],
14849 migration_name: &str,
14850) -> bool {
14851 use crate::db::RuntimeBackend;
14852 use diesel::migration::{Migration, MigrationSource as _};
14853
14854 migrations
14855 .iter()
14856 .filter(|(_, set)| !migration_set_is_control_framework(set))
14857 .any(|(_, source)| {
14858 let Ok(source_migrations): Result<Vec<Box<dyn Migration<RuntimeBackend>>>, _> =
14859 source.migrations()
14860 else {
14861 return false;
14862 };
14863
14864 source_migrations
14865 .iter()
14866 .any(|migration| migration.name().to_string() == migration_name)
14867 })
14868}
14869
14870#[cfg(feature = "db")]
14881fn migration_set_is_control_framework(set: &crate::migrate::EmbeddedMigrations) -> bool {
14882 use crate::db::RuntimeBackend;
14883 use diesel::migration::{Migration, MigrationSource as _};
14884
14885 fn names(set: &crate::migrate::EmbeddedMigrations) -> std::collections::HashSet<String> {
14886 let migrations: Vec<Box<dyn Migration<RuntimeBackend>>> =
14887 set.migrations().unwrap_or_default();
14888 migrations.iter().map(|m| m.name().to_string()).collect()
14889 }
14890
14891 let mut control_only = names(&crate::migrate::FRAMEWORK_MIGRATIONS);
14892 for shard_required in [
14893 &crate::version_history::VERSION_HISTORY_MIGRATIONS,
14894 &crate::repository_commit_hooks::REPOSITORY_COMMIT_HOOK_MIGRATIONS,
14895 &crate::derivation::DERIVATION_MIGRATIONS,
14896 ] {
14897 for name in names(shard_required) {
14898 control_only.remove(&name);
14899 }
14900 }
14901
14902 names(set).iter().any(|name| control_only.contains(name))
14903}
14904
14905#[cfg(feature = "db")]
14906fn apply_replica_migration_readiness(
14907 state: &AppState,
14908 readiness: Option<crate::migrate::ReplicaMigrationReadiness>,
14909) {
14910 let Some(readiness) = readiness else {
14911 return;
14912 };
14913
14914 if readiness.is_ready() {
14915 state.probes().mark_replica_migrations_ready();
14916 } else if let Some(detail) = readiness.detail() {
14917 state.probes().mark_replica_migrations_unready(detail);
14918 }
14919}
14920
14921#[cfg(feature = "db")]
14922fn configure_replica_migration_check(state: &AppState, check: Option<(String, String)>) {
14923 let Some((primary_url, replica_url)) = check else {
14924 return;
14925 };
14926
14927 state
14928 .probes()
14929 .configure_replica_migration_check(primary_url, replica_url);
14930}
14931
14932fn collect_unguarded_repository_writes(
14955 routes: &[Route],
14956 scoped_groups: &[ScopedGroup],
14957) -> Vec<(String, String)> {
14958 let mut offenders: Vec<(String, String)> = Vec::new();
14959 let mut seen: std::collections::HashSet<(&'static str, &'static str)> =
14960 std::collections::HashSet::new();
14961 let mut record_route = |route: &Route| {
14962 if let Some(meta) = route.repository
14963 && !meta.has_policy
14964 && is_mutating_method(&route.method)
14965 && seen.insert((meta.resource_type_name, meta.api_path))
14966 {
14967 offenders.push((meta.resource_type_name.to_owned(), meta.api_path.to_owned()));
14968 }
14969 };
14970 for route in routes {
14971 record_route(route);
14972 }
14973 for group in scoped_groups {
14974 for route in &group.routes {
14975 record_route(route);
14976 }
14977 }
14978 offenders
14979}
14980
14981fn format_unguarded_repository_listing(offenders: &[(String, String)]) -> String {
14985 use std::fmt::Write;
14986 let mut s = String::new();
14987 let mut first = true;
14988 for (name, path) in offenders {
14989 if !first {
14990 s.push('\n');
14991 }
14992 first = false;
14993 write!(s, " - #[repository({name}, api = \"{path}\")]").unwrap();
14994 }
14995 s
14996}
14997
14998#[cfg(feature = "mail")]
15009const fn apply_mail_builder_overrides(config: &mut AutumnConfig, mount_unsubscribe_endpoint: bool) {
15010 if mount_unsubscribe_endpoint {
15011 config.mail.mount_unsubscribe_endpoint = true;
15012 }
15013}
15014
15015#[cfg(feature = "openapi")]
15030struct ExportPreflight<'a> {
15031 routes: &'a [Route],
15032 scoped_groups: &'a [ScopedGroup],
15033 api_versions: &'a [ApiVersion],
15034 openapi_config: &'a crate::openapi::OpenApiConfig,
15035 merge_routers: &'a [axum::Router<AppState>],
15036 nest_routers: &'a [(String, axum::Router<AppState>)],
15037 declared_routes: &'a [crate::route_listing::RouteInfo],
15038 config: &'a AutumnConfig,
15039 mcp_mount_path: Option<&'a str>,
15041}
15042
15043#[cfg(feature = "openapi")]
15044fn export_preflight(ctx: &ExportPreflight<'_>) -> Result<(), String> {
15045 let &ExportPreflight {
15046 routes,
15047 scoped_groups,
15048 api_versions,
15049 openapi_config,
15050 merge_routers,
15051 nest_routers,
15052 declared_routes,
15053 config,
15054 mcp_mount_path,
15055 } = ctx;
15056
15057 validate_pre_router_preconditions(routes, scoped_groups, config)?;
15062
15063 if !config.role.serves_http() {
15075 return Ok(());
15076 }
15077
15078 let mounted_openapi = config.openapi_runtime.enabled.then_some(openapi_config);
15082
15083 let registered_versions: std::collections::HashSet<&str> =
15084 api_versions.iter().map(|av| av.version.as_str()).collect();
15085 let preflight = crate::router::reject_unregistered_api_versions(
15086 routes,
15087 scoped_groups,
15088 ®istered_versions,
15089 )
15090 .and_then(|()| {
15091 crate::router::reject_duplicate_user_routes(
15092 routes,
15093 scoped_groups,
15094 merge_routers,
15095 nest_routers,
15096 declared_routes,
15097 config,
15098 )
15099 })
15100 .and_then(|()| {
15101 mounted_openapi.map_or(Ok(()), crate::router::validate_openapi_mount_paths)
15111 })
15112 .and_then(|()| {
15113 crate::router::reject_openapi_path_collisions(
15114 mounted_openapi,
15115 routes,
15116 scoped_groups,
15117 merge_routers,
15118 nest_routers,
15119 config,
15120 )
15121 })
15122 .and_then(|()| {
15123 crate::router::reject_declared_routes_on_context_mounts(
15126 declared_routes,
15127 mounted_openapi,
15128 mcp_mount_path,
15129 )
15130 });
15131
15132 #[cfg(feature = "mcp")]
15140 let preflight = preflight.and_then(|()| {
15141 let Some(path) = mcp_mount_path else {
15142 return Ok(());
15143 };
15144 crate::router::validate_mcp_mount_path(path).and_then(|()| {
15145 crate::router::reject_mcp_path_collisions(
15149 path,
15150 routes,
15151 scoped_groups,
15152 config,
15153 mounted_openapi,
15154 merge_routers,
15155 nest_routers,
15156 )
15157 })
15158 });
15159
15160 preflight.map_err(|error| error.to_string())
15161}
15162
15163#[cfg(feature = "openapi")]
15178fn merge_framework_scheduled_tasks(
15179 mut tasks: Vec<crate::task::TaskInfo>,
15180 config: &AutumnConfig,
15181) -> Vec<crate::task::TaskInfo> {
15182 #[cfg(feature = "db")]
15183 tasks.extend(crate::retention::collect_retention_tasks());
15184 tasks.extend(crate::data_retention::framework_retention_tasks(
15185 &config.retention,
15186 ));
15187 tasks
15188}
15189
15190fn validate_config_preconditions(
15216 config: &AutumnConfig,
15217 tasks: &[crate::task::TaskInfo],
15218) -> Result<(), String> {
15219 if crate::config::split_role_requires_durable_backend(config.role, &config.jobs.backend) {
15220 return Err(format!(
15221 "process role '{role}' requires a durable jobs backend: backend '{backend}' is not \
15222 a recognized durable backend and falls through to the in-process 'local' runtime, \
15223 which cannot be shared across a split web/worker topology. Set jobs.backend = \
15224 \"postgres\" or \"redis\", or run the combined role.",
15225 role = config.role.as_str(),
15226 backend = config.jobs.backend,
15227 ));
15228 }
15229
15230 crate::task::validate_unique_task_names(tasks.iter().map(|task| task.name.as_str()))?;
15231
15232 Ok(())
15233}
15234
15235fn validate_pre_router_preconditions(
15251 routes: &[Route],
15252 scoped_groups: &[ScopedGroup],
15253 config: &AutumnConfig,
15254) -> Result<(), String> {
15255 if routes.is_empty() {
15256 return Err("No routes registered. Did you forget to call .routes()?".to_owned());
15257 }
15258 validate_repository_api_policies(routes, scoped_groups, config);
15259 Ok(())
15260}
15261
15262fn validate_repository_api_policies(
15263 routes: &[Route],
15264 scoped_groups: &[ScopedGroup],
15265 config: &AutumnConfig,
15266) {
15267 let profile = config.profile.as_deref().unwrap_or("default");
15268 let strict =
15269 is_production_profile(profile) && !config.security.allow_unauthorized_repository_api;
15270
15271 let offenders = collect_unguarded_repository_writes(routes, scoped_groups);
15272 if offenders.is_empty() {
15273 return;
15274 }
15275
15276 let listing = format_unguarded_repository_listing(&offenders);
15277
15278 if strict {
15279 tracing::error!(
15280 "refusing to start: the following #[repository(api = ...)] mutating endpoints have no paired `policy = ...` argument:\n{listing}\n\
15281 Add `policy = SomePolicy` to each, or set `[security] allow_unauthorized_repository_api = true` to opt out explicitly."
15282 );
15283 std::process::exit(1);
15284 }
15285 tracing::warn!(
15286 "the following #[repository(api = ...)] mutating endpoints have no paired `policy = ...` argument; \
15287 auto-generated POST/PUT/PATCH/DELETE handlers will accept writes from any authenticated user:\n{listing}\n\
15288 This will become a startup-time error in `prod` profile builds."
15289 );
15290}
15291
15292type MissingRepositoryRegistration = (String, String);
15307
15308fn collect_unregistered_repository_handlers(
15318 routes: &[Route],
15319 scoped_groups: &[ScopedGroup],
15320 registry: &crate::authorization::PolicyRegistry,
15321) -> (
15322 Vec<MissingRepositoryRegistration>,
15323 Vec<MissingRepositoryRegistration>,
15324) {
15325 let mut missing_policies: Vec<(String, String)> = Vec::new();
15326 let mut missing_scopes: Vec<(String, String)> = Vec::new();
15327 let mut seen_policies: std::collections::HashSet<(&'static str, &'static str)> =
15328 std::collections::HashSet::new();
15329 let mut seen_scopes: std::collections::HashSet<(&'static str, &'static str)> =
15330 std::collections::HashSet::new();
15331 let mut record_route = |route: &Route| {
15332 if let Some(meta) = route.repository {
15333 if let Some(check) = meta.policy_check
15334 && !check(registry)
15335 && seen_policies.insert((meta.resource_type_name, meta.api_path))
15336 {
15337 missing_policies
15338 .push((meta.resource_type_name.to_owned(), meta.api_path.to_owned()));
15339 }
15340 if let Some(check) = meta.scope_check
15341 && !check(registry)
15342 && seen_scopes.insert((meta.resource_type_name, meta.api_path))
15343 {
15344 missing_scopes.push((meta.resource_type_name.to_owned(), meta.api_path.to_owned()));
15345 }
15346 }
15347 };
15348 for route in routes {
15349 record_route(route);
15350 }
15351 for group in scoped_groups {
15352 for route in &group.routes {
15353 record_route(route);
15354 }
15355 }
15356 (missing_policies, missing_scopes)
15357}
15358
15359fn format_missing_policy_listing(missing: &[(String, String)]) -> String {
15362 use std::fmt::Write;
15363 let mut s = String::new();
15364 let mut first = true;
15365 for (name, path) in missing {
15366 if !first {
15367 s.push('\n');
15368 }
15369 first = false;
15370 write!(s, " - #[repository({name}, api = \"{path}\", policy = ...)]: call `.policy::<{name}, _>(...)` on the app builder").unwrap();
15371 }
15372 s
15373}
15374
15375fn format_missing_scope_listing(missing: &[(String, String)]) -> String {
15378 use std::fmt::Write;
15379 let mut s = String::new();
15380 let mut first = true;
15381 for (name, path) in missing {
15382 if !first {
15383 s.push('\n');
15384 }
15385 first = false;
15386 write!(s, " - #[repository({name}, api = \"{path}\", scope = ...)]: call `.scope::<{name}, _>(...)` on the app builder").unwrap();
15387 }
15388 s
15389}
15390
15391#[allow(clippy::cognitive_complexity)]
15392fn validate_repository_policies_registered(
15399 routes: &[Route],
15400 scoped_groups: &[ScopedGroup],
15401 registry: &crate::authorization::PolicyRegistry,
15402 config: &AutumnConfig,
15403) {
15404 let profile = config.profile.as_deref().unwrap_or("default");
15405 let strict = is_production_profile(profile);
15406
15407 let (missing_policies, missing_scopes) =
15408 collect_unregistered_repository_handlers(routes, scoped_groups, registry);
15409
15410 if missing_policies.is_empty() && missing_scopes.is_empty() {
15411 return;
15412 }
15413
15414 if !missing_policies.is_empty() {
15415 let listing = format_missing_policy_listing(&missing_policies);
15416
15417 if strict {
15418 tracing::error!(
15419 "refusing to start: the following #[repository] routes declare a `policy = ...` argument, but no policy is registered for the resource type. Without registration, every protected request would fail at runtime with `500 no policy registered`:\n{listing}"
15420 );
15421 } else {
15422 tracing::warn!(
15423 "the following #[repository] routes declare `policy = ...` but no matching `.policy::<R, _>(...)` registration is on the app builder. Protected requests will 500 at runtime:\n{listing}\n\
15424 This will become a startup-time error in `prod` profile builds."
15425 );
15426 }
15427 }
15428
15429 if !missing_scopes.is_empty() {
15430 let listing = format_missing_scope_listing(&missing_scopes);
15431
15432 if strict {
15433 tracing::error!(
15434 "refusing to start: the following #[repository] routes declare a `scope = ...` argument, but no scope is registered for the resource type. Without registration, every list request would fail at runtime with `500 missing scope registration`:\n{listing}"
15435 );
15436 } else {
15437 tracing::warn!(
15438 "the following #[repository] routes declare `scope = ...` but no matching `.scope::<R, _>(...)` registration is on the app builder. List requests will 500 at runtime:\n{listing}\n\
15439 This will become a startup-time error in `prod` profile builds."
15440 );
15441 }
15442 }
15443
15444 if strict {
15445 std::process::exit(1);
15446 }
15447}
15448
15449const fn is_mutating_method(method: &http::Method) -> bool {
15450 matches!(
15451 *method,
15452 http::Method::POST | http::Method::PUT | http::Method::PATCH | http::Method::DELETE
15453 )
15454}
15455
15456fn is_production_profile(profile: &str) -> bool {
15462 matches!(profile, "prod" | "production")
15463}
15464
15465#[cfg(test)]
15466mod agent_authority_route_summary_tests {
15467
15468 use super::*;
15469
15470 fn route_with(path: &'static str, mcp_tool: bool) -> Route {
15471 let mut api_doc = crate::openapi::ApiDoc {
15472 method: "GET",
15473 path,
15474 operation_id: "list_items",
15475 mcp_tool,
15476 ..crate::openapi::ApiDoc::default()
15477 };
15478 api_doc.response = Some(crate::openapi::SchemaEntry {
15481 name: "Item",
15482 kind: crate::openapi::SchemaKind::Ref,
15483 identity: None,
15484 });
15485 Route {
15486 method: http::Method::GET,
15487 path,
15488 handler: axum::routing::any(|| async { "" }),
15489 name: "list_items",
15490 api_doc,
15491 repository: None,
15492 idempotency: crate::route::RouteIdempotency::Direct,
15493 timeout: crate::route::RouteTimeout::Inherit,
15494 criticality: crate::admission::Criticality::Default,
15495 seo: crate::seo::SeoRouteDefaults::EMPTY,
15496 api_version: None,
15497 sunset_opt_out: false,
15498 }
15499 }
15500
15501 #[test]
15502 fn a_scoped_route_records_the_path_an_agent_actually_calls() {
15503 let route = route_with("/items", true);
15508 let summary =
15509 agent_authority_route_summary(&route, Some("/api/v1"), AgentExposure::default());
15510 assert_eq!(summary.path, "/api/v1/items");
15511 assert_eq!(summary.method, "GET");
15512 assert!(summary.mcp_tool);
15513
15514 let top = agent_authority_route_summary(&route, None, AgentExposure::default());
15516 assert_eq!(top.path, "/items");
15517 }
15518
15519 #[test]
15520 fn a_scoped_root_route_joins_the_way_the_openapi_collector_does() {
15521 let root = route_with("/", true);
15524 let summary = agent_authority_route_summary(&root, Some("/api"), AgentExposure::default());
15525 assert_eq!(
15526 summary.path,
15527 crate::router::join_nested_path("/api", "/"),
15528 "the manifest must join paths exactly as the spec does"
15529 );
15530 }
15531
15532 #[test]
15533 fn the_operation_id_names_the_tool_and_exposure_says_why() {
15534 use crate::agent_authority::manifest::McpExposedBy;
15535
15536 let tagged = agent_authority_route_summary(
15537 &route_with("/items", true),
15538 None,
15539 AgentExposure::default(),
15540 );
15541 assert_eq!(tagged.operation_id, "list_items");
15542 assert_eq!(tagged.exposed_by, Some(McpExposedBy::Attribute));
15543
15544 let plain = agent_authority_route_summary(
15546 &route_with("/items", false),
15547 None,
15548 AgentExposure::default(),
15549 );
15550 assert!(!plain.mcp_tool);
15551 assert_eq!(plain.exposed_by, None);
15552
15553 let hatched = agent_authority_route_summary(
15555 &route_with("/items", false),
15556 None,
15557 AgentExposure {
15558 mcp_expose_all: true,
15559 webmcp_expose_all: false,
15560 },
15561 );
15562 assert!(hatched.mcp_tool);
15563 assert_eq!(hatched.exposed_by, Some(McpExposedBy::Hatch));
15564 assert!(
15565 !hatched.webmcp_tool,
15566 "the MCP hatch is not the WebMCP hatch"
15567 );
15568 }
15569
15570 #[test]
15571 fn webmcp_exposure_is_reported_apart_from_mcp() {
15572 use crate::agent_authority::manifest::McpExposedBy;
15573
15574 let mut route = route_with("/items", true);
15575 route.api_doc.webmcp_tool = true;
15576 let both = agent_authority_route_summary(&route, None, AgentExposure::default());
15577 assert!(both.mcp_tool && both.webmcp_tool);
15578 assert_eq!(both.webmcp_exposed_by, Some(McpExposedBy::Attribute));
15579
15580 let web_hatch = agent_authority_route_summary(
15581 &route_with("/items", false),
15582 None,
15583 AgentExposure {
15584 mcp_expose_all: false,
15585 webmcp_expose_all: true,
15586 },
15587 );
15588 assert!(!web_hatch.mcp_tool);
15589 assert_eq!(web_hatch.webmcp_exposed_by, Some(McpExposedBy::Hatch));
15590
15591 let mut excluded = route_with("/items", false);
15592 excluded.api_doc.webmcp_exclude = true;
15593 let excluded = agent_authority_route_summary(
15594 &excluded,
15595 None,
15596 AgentExposure {
15597 mcp_expose_all: false,
15598 webmcp_expose_all: true,
15599 },
15600 );
15601 assert!(!excluded.webmcp_tool);
15602 }
15603}
15604
15605#[cfg(all(test, feature = "sqlite"))]
15606mod fleet_boot_tests {
15607 use super::*;
15608
15609 async fn table_exists(db: &crate::db::fleet::FleetDatabase, table: &str) -> bool {
15610 use diesel_async::RunQueryDsl as _;
15611 #[derive(diesel::QueryableByName)]
15612 struct Count {
15613 #[diesel(sql_type = diesel::sql_types::BigInt)]
15614 n: i64,
15615 }
15616 let mut conn = db.pool().get().await.unwrap();
15617 diesel::sql_query(
15618 "SELECT COUNT(*) AS n FROM sqlite_master WHERE type = 'table' AND name = ?",
15619 )
15620 .bind::<diesel::sql_types::Text, _>(table)
15621 .get_result::<Count>(&mut *conn)
15622 .await
15623 .unwrap()
15624 .n == 1
15625 }
15626
15627 #[cfg(unix)]
15631 #[test]
15632 #[allow(
15633 clippy::literal_string_with_formatting_args,
15634 reason = "fleet path templates use {placeholders}"
15635 )]
15636 fn boot_refuses_a_fleet_root_that_aliases_the_control_database() {
15637 let tmp = tempfile::tempdir().unwrap();
15638 let data = tmp.path().join("data");
15639 std::fs::create_dir(&data).unwrap();
15640 let link = tmp.path().join("fleet-link");
15641 std::os::unix::fs::symlink(&data, &link).unwrap();
15642 let mut config = AutumnConfig {
15643 profile: Some("dev".into()),
15644 ..AutumnConfig::default()
15645 };
15646 config.database.url = Some(format!("sqlite://{}", data.join("control.db").display()));
15647 config.database.fleet = Some(crate::config::DatabaseFleetConfig {
15648 mode: crate::fleet_layout::FleetMode::Tenant,
15649 root: link.display().to_string(),
15650 path: Some("{tenant}.db".to_owned()),
15651 max_open: 4,
15652 pool_size: 1,
15653 create_on_demand: None,
15654 idle_close_secs: 0,
15655 restore_missing: false,
15656 ownership: None,
15657 });
15658 config
15659 .database
15660 .validate()
15661 .expect("lexically, the paths differ");
15662 let Err(err) =
15663 crate::db::fleet::build_for_app(&config, None, &Arc::new(Vec::new()), |_| true)
15664 else {
15665 panic!("a fleet root aliasing the control database must be refused");
15666 };
15667 assert!(err.contains("control database"), "{err}");
15668 }
15669
15670 #[test]
15675 #[allow(
15676 clippy::literal_string_with_formatting_args,
15677 reason = "fleet path templates use {placeholders}"
15678 )]
15679 fn boot_refuses_a_provider_resolved_control_database_inside_the_root() {
15680 let tmp = tempfile::tempdir().unwrap();
15681 let root = tmp.path().join("fleet");
15682 std::fs::create_dir(&root).unwrap();
15683 let mut config = AutumnConfig {
15684 profile: Some("dev".into()),
15685 ..AutumnConfig::default()
15686 };
15687 config.database.url = Some(format!(
15688 "sqlite://{}",
15689 tmp.path().join("control.db").display()
15690 ));
15691 config.database.fleet = Some(crate::config::DatabaseFleetConfig {
15692 mode: crate::fleet_layout::FleetMode::Tenant,
15693 root: root.display().to_string(),
15694 path: Some("{tenant}.db".to_owned()),
15695 max_open: 4,
15696 pool_size: 1,
15697 create_on_demand: None,
15698 idle_close_secs: 0,
15699 restore_missing: false,
15700 ownership: None,
15701 });
15702 let resolved = format!("sqlite://{}", root.join("control.db").display());
15703 let Err(err) = crate::db::fleet::build_for_app(
15704 &config,
15705 Some(&resolved),
15706 &Arc::new(Vec::new()),
15707 |_| true,
15708 ) else {
15709 panic!("a resolved control database inside the root must be refused");
15710 };
15711 assert!(err.contains("control database"), "{err}");
15712 crate::db::fleet::build_for_app(&config, None, &Arc::new(Vec::new()), |_| true)
15713 .expect("the configured control database is outside the root");
15714 }
15715
15716 #[test]
15720 fn a_fleet_requires_a_custom_provider_to_name_its_control_database() {
15721 let err = fleet_control_target_identified(true, None).unwrap_err();
15722 assert!(err.contains("with_migration_url"), "{err}");
15723 fleet_control_target_identified(true, Some("sqlite:///srv/control.db"))
15724 .expect("named: checked against the root");
15725 for target in [
15727 "sqlite::memory:",
15728 "file::memory:?cache=shared",
15729 "postgres://db/app",
15730 ] {
15731 let err = fleet_control_target_identified(true, Some(target)).unwrap_err();
15732 assert!(err.contains("file-backed"), "{target}: {err}");
15733 }
15734 fleet_control_target_identified(false, None)
15735 .expect("the built-in provider opens the configured url, already checked");
15736 }
15737
15738 #[tokio::test]
15742 async fn the_boot_fleet_gets_the_shard_sets_and_never_the_control_set() {
15743 let tmp = tempfile::tempdir().unwrap();
15744 let mut config = AutumnConfig {
15745 profile: Some("dev".into()),
15746 ..AutumnConfig::default()
15747 };
15748 config.database.url = Some(format!(
15749 "sqlite://{}",
15750 tmp.path().join("control.db").display()
15751 ));
15752 config.database.fleet = Some(crate::config::DatabaseFleetConfig {
15753 mode: crate::fleet_layout::FleetMode::Slot,
15754 root: tmp.path().join("fleet").display().to_string(),
15755 path: None,
15756 max_open: 4,
15757 pool_size: 1,
15758 create_on_demand: None,
15759 idle_close_secs: 0,
15760 restore_missing: false,
15761 ownership: None,
15762 });
15763 let all = Arc::new(vec![
15764 ("framework", crate::migrate::FRAMEWORK_MIGRATIONS),
15765 (
15766 "version-history",
15767 crate::version_history::VERSION_HISTORY_MIGRATIONS,
15768 ),
15769 ]);
15770 let fleet = crate::db::fleet::build_for_app(&config, None, &all, |set| {
15771 !migration_set_is_control_framework(set)
15772 })
15773 .unwrap()
15774 .expect("a fleet is configured");
15775 let db = fleet.open_for("acme").await.unwrap();
15776 assert!(table_exists(&db, "_autumn_version_history").await);
15777 assert!(!table_exists(&db, "api_tokens").await);
15778 assert!(!table_exists(&db, "autumn_jobs").await);
15779
15780 config.database.fleet = None;
15781 assert!(
15782 crate::db::fleet::build_for_app(&config, None, &Arc::new(Vec::new()), |_| true)
15783 .unwrap()
15784 .is_none()
15785 );
15786 }
15787
15788 #[allow(
15791 clippy::literal_string_with_formatting_args,
15792 reason = "fleet path templates use {placeholders}"
15793 )]
15794 fn booted(dir: &std::path::Path) -> (AutumnConfig, crate::db::fleet::DatabaseFleet, AppState) {
15795 let mut config = AutumnConfig {
15796 profile: Some("dev".into()),
15797 ..AutumnConfig::default()
15798 };
15799 config.database.url = Some(format!("sqlite://{}", dir.join("control.db").display()));
15800 config.database.fleet = Some(crate::config::DatabaseFleetConfig {
15801 mode: crate::fleet_layout::FleetMode::Slot,
15802 root: dir.join("fleet").display().to_string(),
15803 path: None,
15804 max_open: 4,
15805 pool_size: 1,
15806 create_on_demand: None,
15807 idle_close_secs: 0,
15808 restore_missing: false,
15809 ownership: None,
15810 });
15811 let all = Arc::new(vec![("framework", crate::migrate::FRAMEWORK_MIGRATIONS)]);
15812 let fleet = crate::db::fleet::build_for_app(&config, None, &all, |set| {
15813 !migration_set_is_control_framework(set)
15814 })
15815 .unwrap()
15816 .expect("a fleet is configured");
15817 let control = crate::db::create_pool(&config.database)
15818 .unwrap()
15819 .expect("a control url is configured");
15820 let state = AppState::for_test().with_pool(control.clone()).with_shards(
15821 crate::sharding::ShardSet::from_fleet(fleet.clone(), control),
15822 );
15823 (config, fleet, state)
15824 }
15825
15826 #[tokio::test]
15830 async fn the_fleet_runtime_closes_every_database_on_shutdown() {
15831 let tmp = tempfile::tempdir().unwrap();
15832 let (_config, fleet, state) = booted(tmp.path());
15833 let (shutdown, sweep) = (
15834 tokio_util::sync::CancellationToken::new(),
15835 tokio_util::sync::CancellationToken::new(),
15836 );
15837 let runtime = start_fleet_runtime(&state, shutdown.clone(), Some(sweep.clone()))
15838 .await
15839 .expect("the app has a fleet");
15840 let names: Vec<String> = state
15841 .health_indicator_registry()
15842 .run_all()
15843 .await
15844 .into_iter()
15845 .map(|result| result.name)
15846 .collect();
15847 assert!(
15848 names.iter().any(|name| name == "fleet:sweeper"),
15849 "{names:?}"
15850 );
15851 let db = fleet.open_for("acme").await.unwrap();
15852 assert!(!db.closed().is_cancelled());
15854 drop(db);
15855 assert_eq!(fleet.stats().open, 1);
15856 sweep.cancel();
15857 runtime
15858 .sweeper
15859 .expect("a server runs the sweeper loop")
15860 .await
15861 .unwrap();
15862 shutdown.cancel();
15863 runtime.maintenance.await.unwrap();
15864 assert_eq!(fleet.stats().open, 0, "shutdown closed every database");
15865
15866 assert!(
15867 start_fleet_runtime(&AppState::for_test(), shutdown, None)
15868 .await
15869 .is_none(),
15870 "no fleet, no runtime"
15871 );
15872 }
15873
15874 #[tokio::test]
15878 async fn the_fleet_runtime_claims_and_releases_slot_ownership() {
15879 use crate::db::fleet_ownership::{SlotOwnershipStore as _, SqlSlotOwnership};
15880 let tmp = tempfile::tempdir().unwrap();
15881 let (mut config, fleet, state) = booted(tmp.path());
15882 if let Some(fleet_config) = config.database.fleet.as_mut() {
15883 fleet_config.ownership = Some(crate::config::FleetOwnershipConfig {
15884 owner: "host-a".to_owned(),
15885 slots: vec![crate::config::SlotSpec::Range("0-16383".to_owned())],
15886 lease_secs: 30,
15887 });
15888 }
15889 state.insert_extension(config);
15890 let shutdown = tokio_util::sync::CancellationToken::new();
15891 let runtime = start_fleet_runtime(&state, shutdown.clone(), None)
15892 .await
15893 .expect("the app has a fleet");
15894 assert!(fleet.ownership().is_some());
15895 assert!(
15896 state
15897 .health_indicator_registry()
15898 .contains(crate::db::fleet_ownership::OWNERSHIP_INDICATOR)
15899 );
15900 let db = fleet.open_for("acme").await.unwrap();
15901 assert_eq!(db.epoch().map(crate::lock::FencingToken::get), Some(1));
15902 drop(db);
15903
15904 shutdown.cancel();
15905 runtime.maintenance.await.unwrap();
15906 runtime.ownership.expect("ownership runs").finish().await;
15907 let store = SqlSlotOwnership::new(state.pool().cloned().unwrap(), state.clock_arc());
15908 let rows = store.rows().await.unwrap();
15909 assert_eq!(rows.len(), 1);
15910 assert!(
15911 rows[0].lease_expires_ms <= state.clock().now().timestamp_millis(),
15912 "released at shutdown"
15913 );
15914 }
15915
15916 #[tokio::test]
15919 async fn a_fleet_that_did_not_close_stops_renewing_without_a_release() {
15920 use crate::db::fleet_ownership::{SlotOwnershipStore as _, SqlSlotOwnership};
15921 let tmp = tempfile::tempdir().unwrap();
15922 let (mut config, _fleet, state) = booted(tmp.path());
15923 if let Some(fleet_config) = config.database.fleet.as_mut() {
15924 fleet_config.ownership = Some(crate::config::FleetOwnershipConfig {
15925 owner: "host-a".to_owned(),
15926 slots: vec![crate::config::SlotSpec::Range("0-16383".to_owned())],
15927 lease_secs: 30,
15928 });
15929 }
15930 state.insert_extension(config);
15931 let shutdown = tokio_util::sync::CancellationToken::new();
15932 let runtime = start_fleet_runtime(&state, shutdown.clone(), None)
15933 .await
15934 .expect("the app has a fleet");
15935 let renewals = runtime.ownership.expect("ownership runs").abandon();
15936 tokio::time::timeout(std::time::Duration::from_secs(5), renewals)
15937 .await
15938 .expect("the renew loop stops")
15939 .unwrap();
15940 let store = SqlSlotOwnership::new(state.pool().cloned().unwrap(), state.clock_arc());
15941 let rows = store.rows().await.unwrap();
15942 assert!(
15943 rows[0].lease_expires_ms > state.clock().now().timestamp_millis(),
15944 "not released: the leases lapse"
15945 );
15946 shutdown.cancel();
15947 runtime.maintenance.await.unwrap();
15948 }
15949
15950 #[tokio::test]
15953 async fn fleet_replication_installs_beside_the_control_database() {
15954 let tmp = tempfile::tempdir().unwrap();
15955 let (config, fleet, state) = booted(tmp.path());
15956 let replicas = tmp.path().join("replicas");
15957 std::fs::create_dir(&replicas).unwrap();
15958 let replication_config = crate::config::ReplicationConfig {
15959 enabled: true,
15960 path: Some(replicas.display().to_string()),
15961 ..crate::config::ReplicationConfig::default()
15962 };
15963 let control_url = config.database.url.clone().unwrap();
15964 let runtime = crate::time::spawn_blocking(move || {
15965 crate::replication::build(
15966 &replication_config,
15967 &control_url,
15968 "dev",
15969 None,
15970 Arc::new(crate::time::SystemClock),
15971 )
15972 })
15973 .await
15974 .unwrap()
15975 .expect("a file destination builds");
15976 let lag = std::time::Duration::from_secs(30);
15977
15978 start_fleet_replication(&state, &config, &runtime, lag).unwrap();
15979 assert!(fleet.replicating());
15980 assert!(
15981 state
15982 .health_indicator_registry()
15983 .contains(crate::db::fleet_replication::FLEET_REPLICATION_INDICATOR)
15984 );
15985 let again = start_fleet_replication(&state, &config, &runtime, lag).unwrap_err();
15986 assert!(again.contains("already replicates"), "{again}");
15987 if let Some(replication) = fleet.replication() {
15988 replication.stop();
15989 }
15990
15991 start_fleet_replication(&AppState::for_test(), &config, &runtime, lag)
15992 .expect("no fleet, nothing to replicate");
15993 }
15994
15995 #[tokio::test]
15998 async fn migrate_mode_migrates_every_fleet_database() {
15999 let tmp = tempfile::tempdir().unwrap();
16000 let (mut config, fleet, _state) = booted(tmp.path());
16001 for tenant in ["acme", "globex"] {
16002 fleet.open_for(tenant).await.unwrap();
16003 }
16004 fleet.close_all().await;
16005 let all = Arc::new(vec![("framework", crate::migrate::FRAMEWORK_MIGRATIONS)]);
16006 assert_eq!(migrate_fleet_or_exit(&config, Arc::clone(&all)).await, 0);
16008 config.database.fleet = None;
16009 assert_eq!(migrate_fleet_or_exit(&config, all).await, 0);
16010 }
16011}
16012
16013#[cfg(test)]
16014mod validate_repository_api_policies_tests {
16015 use super::*;
16016 use crate::RepositoryApiMeta;
16017
16018 fn build_route(
16019 method: http::Method,
16020 path: &'static str,
16021 meta: Option<RepositoryApiMeta>,
16022 ) -> Route {
16023 Route {
16024 method,
16025 path,
16026 handler: axum::routing::any(|| async { "" }),
16027 name: "test_route",
16028 api_doc: crate::openapi::ApiDoc::default(),
16029 repository: meta,
16030 idempotency: crate::route::RouteIdempotency::Direct,
16031 timeout: crate::route::RouteTimeout::Inherit,
16032 criticality: crate::admission::Criticality::Default,
16033 seo: crate::seo::SeoRouteDefaults::EMPTY,
16034 api_version: None,
16035 sunset_opt_out: false,
16036 }
16037 }
16038
16039 fn unguarded(path: &'static str, type_name: &'static str) -> RepositoryApiMeta {
16040 RepositoryApiMeta {
16041 resource_type_name: type_name,
16042 api_path: path,
16043 has_policy: false,
16044 policy_check: None,
16045 scope_check: None,
16046 }
16047 }
16048
16049 fn collect_offenders(routes: &[Route]) -> Vec<(String, String)> {
16053 collect_unguarded_repository_writes(routes, &[])
16054 }
16055
16056 #[test]
16057 fn read_only_mount_without_policy_is_not_an_offender() {
16058 let routes = vec![
16059 build_route(
16060 http::Method::GET,
16061 "/api/posts",
16062 Some(unguarded("/api/posts", "Post")),
16063 ),
16064 build_route(
16065 http::Method::GET,
16066 "/api/posts/{id}",
16067 Some(unguarded("/api/posts", "Post")),
16068 ),
16069 ];
16070 let offenders = collect_offenders(&routes);
16071 assert!(
16072 offenders.is_empty(),
16073 "read-only mounts should not trigger the unauthorized-repo guard"
16074 );
16075 }
16076
16077 #[test]
16078 fn write_mount_without_policy_is_an_offender() {
16079 let routes = vec![build_route(
16080 http::Method::POST,
16081 "/api/posts",
16082 Some(unguarded("/api/posts", "Post")),
16083 )];
16084 let offenders = collect_offenders(&routes);
16085 assert_eq!(offenders.len(), 1);
16086 assert_eq!(offenders[0].0, "Post");
16087 assert_eq!(offenders[0].1, "/api/posts");
16088 }
16089
16090 #[test]
16091 fn mixed_mount_only_dedups_one_offender_per_repository() {
16092 let routes = vec![
16093 build_route(
16094 http::Method::GET,
16095 "/api/posts",
16096 Some(unguarded("/api/posts", "Post")),
16097 ),
16098 build_route(
16099 http::Method::POST,
16100 "/api/posts",
16101 Some(unguarded("/api/posts", "Post")),
16102 ),
16103 build_route(
16104 http::Method::PUT,
16105 "/api/posts/{id}",
16106 Some(unguarded("/api/posts", "Post")),
16107 ),
16108 build_route(
16109 http::Method::DELETE,
16110 "/api/posts/{id}",
16111 Some(unguarded("/api/posts", "Post")),
16112 ),
16113 ];
16114 let offenders = collect_offenders(&routes);
16115 assert_eq!(offenders.len(), 1);
16116 }
16117
16118 #[test]
16119 fn is_mutating_method_classifies_methods() {
16120 assert!(is_mutating_method(&http::Method::POST));
16121 assert!(is_mutating_method(&http::Method::PUT));
16122 assert!(is_mutating_method(&http::Method::PATCH));
16123 assert!(is_mutating_method(&http::Method::DELETE));
16124 assert!(!is_mutating_method(&http::Method::GET));
16125 assert!(!is_mutating_method(&http::Method::HEAD));
16126 assert!(!is_mutating_method(&http::Method::OPTIONS));
16127 }
16128
16129 use crate::authorization::{Policy, PolicyRegistry};
16132
16133 #[derive(Debug, Clone, PartialEq)]
16134 struct TestPost;
16135
16136 #[derive(Default)]
16137 struct TestPostPolicy;
16138 impl Policy<TestPost> for TestPostPolicy {}
16139
16140 fn guarded_with_check(path: &'static str, type_name: &'static str) -> RepositoryApiMeta {
16141 RepositoryApiMeta {
16142 resource_type_name: type_name,
16143 api_path: path,
16144 has_policy: true,
16145 policy_check: Some(|registry: &PolicyRegistry| registry.has_policy::<TestPost>()),
16146 scope_check: None,
16147 }
16148 }
16149
16150 fn collect_missing(routes: &[Route], registry: &PolicyRegistry) -> Vec<(String, String)> {
16151 let (missing_policies, _) = collect_unregistered_repository_handlers(routes, &[], registry);
16152 missing_policies
16153 }
16154
16155 #[test]
16156 fn registry_check_flags_routes_missing_their_policy_registration() {
16157 let registry = PolicyRegistry::default();
16160 let routes = vec![build_route(
16161 http::Method::POST,
16162 "/api/posts",
16163 Some(guarded_with_check("/api/posts", "TestPost")),
16164 )];
16165 let missing = collect_missing(&routes, ®istry);
16166 assert_eq!(missing.len(), 1);
16167 assert_eq!(missing[0].0, "TestPost");
16168 assert_eq!(missing[0].1, "/api/posts");
16169 }
16170
16171 #[test]
16172 fn registry_check_passes_when_policy_is_registered() {
16173 let registry = PolicyRegistry::default();
16174 registry.register_policy::<TestPost, _>(TestPostPolicy);
16175 let routes = vec![build_route(
16176 http::Method::POST,
16177 "/api/posts",
16178 Some(guarded_with_check("/api/posts", "TestPost")),
16179 )];
16180 let missing = collect_missing(&routes, ®istry);
16181 assert!(missing.is_empty(), "policy is registered, no offenders");
16182 }
16183
16184 #[test]
16185 fn registry_check_skips_routes_without_policy_check_fn() {
16186 let registry = PolicyRegistry::default();
16191 let routes = vec![build_route(
16192 http::Method::POST,
16193 "/api/posts",
16194 Some(unguarded("/api/posts", "TestPost")),
16195 )];
16196 let missing = collect_missing(&routes, ®istry);
16197 assert!(missing.is_empty());
16198 }
16199
16200 #[test]
16201 fn registry_check_dedups_one_offender_per_repository() {
16202 let registry = PolicyRegistry::default();
16203 let routes = vec![
16204 build_route(
16205 http::Method::GET,
16206 "/api/posts",
16207 Some(guarded_with_check("/api/posts", "TestPost")),
16208 ),
16209 build_route(
16210 http::Method::POST,
16211 "/api/posts",
16212 Some(guarded_with_check("/api/posts", "TestPost")),
16213 ),
16214 build_route(
16215 http::Method::DELETE,
16216 "/api/posts/{id}",
16217 Some(guarded_with_check("/api/posts", "TestPost")),
16218 ),
16219 ];
16220 let missing = collect_missing(&routes, ®istry);
16221 assert_eq!(missing.len(), 1);
16222 }
16223
16224 use crate::authorization::{BoxFuture, PolicyContext, Scope};
16227
16228 #[derive(Default)]
16229 struct TestPostScope;
16230 impl Scope<TestPost> for TestPostScope {
16231 fn list<'a>(
16232 &'a self,
16233 _ctx: &'a PolicyContext,
16234 _conn: &'a mut crate::db::RuntimeConnection,
16235 ) -> BoxFuture<'a, crate::AutumnResult<Vec<TestPost>>> {
16236 Box::pin(async { Ok(Vec::new()) })
16237 }
16238 }
16239
16240 fn scope_only_meta(path: &'static str, type_name: &'static str) -> RepositoryApiMeta {
16241 RepositoryApiMeta {
16242 resource_type_name: type_name,
16243 api_path: path,
16244 has_policy: false,
16245 policy_check: None,
16246 scope_check: Some(|registry: &PolicyRegistry| registry.scope::<TestPost>().is_some()),
16247 }
16248 }
16249
16250 fn collect_missing_scopes(
16251 routes: &[Route],
16252 registry: &PolicyRegistry,
16253 ) -> Vec<(String, String)> {
16254 let (_, missing_scopes) = collect_unregistered_repository_handlers(routes, &[], registry);
16255 missing_scopes
16256 }
16257
16258 #[test]
16259 fn scope_check_flags_unregistered_scope() {
16260 let registry = PolicyRegistry::default();
16261 let routes = vec![build_route(
16262 http::Method::GET,
16263 "/api/posts",
16264 Some(scope_only_meta("/api/posts", "TestPost")),
16265 )];
16266 let missing = collect_missing_scopes(&routes, ®istry);
16267 assert_eq!(missing.len(), 1);
16268 assert_eq!(missing[0].0, "TestPost");
16269 }
16270
16271 #[test]
16272 fn scope_check_passes_when_scope_is_registered() {
16273 let registry = PolicyRegistry::default();
16274 registry.register_scope::<TestPost, _>(TestPostScope);
16275 let routes = vec![build_route(
16276 http::Method::GET,
16277 "/api/posts",
16278 Some(scope_only_meta("/api/posts", "TestPost")),
16279 )];
16280 let missing = collect_missing_scopes(&routes, ®istry);
16281 assert!(missing.is_empty());
16282 }
16283
16284 #[test]
16285 fn scope_check_skips_routes_without_scope_check_fn() {
16286 let registry = PolicyRegistry::default();
16287 let routes = vec![build_route(
16288 http::Method::POST,
16289 "/api/posts",
16290 Some(unguarded("/api/posts", "TestPost")),
16291 )];
16292 let missing = collect_missing_scopes(&routes, ®istry);
16293 assert!(missing.is_empty());
16294 }
16295
16296 #[test]
16299 fn purge_guard_uses_the_resolved_profile_and_the_force_flag() {
16300 use FrameworkRetentionMode::{Purge, Report};
16301 assert!(framework_purge_refusal(Purge, Some("prod"), None).is_some());
16302 assert!(framework_purge_refusal(Purge, None, None).is_some());
16303 assert!(framework_purge_refusal(Purge, Some("prod"), Some("0")).is_some());
16304 assert!(framework_purge_refusal(Purge, Some("prod"), Some("1")).is_none());
16305 assert!(framework_purge_refusal(Purge, Some("dev"), None).is_none());
16306 assert!(framework_purge_refusal(Report, Some("prod"), None).is_none());
16307 }
16308
16309 #[test]
16310 fn is_production_profile_matches_both_aliases() {
16311 assert!(is_production_profile("prod"));
16312 assert!(is_production_profile("production"));
16313 assert!(!is_production_profile("dev"));
16314 assert!(!is_production_profile("staging"));
16315 assert!(!is_production_profile("test"));
16316 assert!(!is_production_profile("default"));
16317 assert!(!is_production_profile("Prod"));
16321 assert!(!is_production_profile("Production"));
16322 }
16323
16324 #[test]
16327 fn format_unguarded_listing_renders_one_bullet_per_offender() {
16328 let offenders = vec![
16329 ("Post".to_owned(), "/api/posts".to_owned()),
16330 ("Comment".to_owned(), "/api/comments".to_owned()),
16331 ];
16332 let listing = format_unguarded_repository_listing(&offenders);
16333 assert!(listing.contains("Post"));
16334 assert!(listing.contains("/api/posts"));
16335 assert!(listing.contains("Comment"));
16336 assert!(listing.contains("/api/comments"));
16337 assert_eq!(listing.matches("\n - ").count() + 1, 2);
16338 }
16339
16340 #[test]
16341 fn format_unguarded_listing_empty_input_yields_empty_string() {
16342 let listing = format_unguarded_repository_listing(&[]);
16343 assert!(listing.is_empty());
16344 }
16345
16346 #[test]
16347 fn format_missing_policy_listing_includes_policy_call_hint() {
16348 let missing = vec![("Post".to_owned(), "/api/posts".to_owned())];
16349 let listing = format_missing_policy_listing(&missing);
16350 assert!(listing.contains("Post"));
16351 assert!(listing.contains("/api/posts"));
16352 assert!(listing.contains(".policy::<Post, _>"));
16353 assert!(listing.contains("policy = ..."));
16354 }
16355
16356 #[test]
16357 fn format_missing_scope_listing_includes_scope_call_hint() {
16358 let missing = vec![("Post".to_owned(), "/api/posts".to_owned())];
16359 let listing = format_missing_scope_listing(&missing);
16360 assert!(listing.contains("Post"));
16361 assert!(listing.contains("/api/posts"));
16362 assert!(listing.contains(".scope::<Post, _>"));
16363 assert!(listing.contains("scope = ..."));
16364 }
16365
16366 #[test]
16369 fn collect_unguarded_walks_scoped_groups() {
16370 let group_route = build_route(
16375 http::Method::POST,
16376 "/api/posts",
16377 Some(unguarded("/api/posts", "Post")),
16378 );
16379 let group = ScopedGroup {
16380 prefix: "/scoped".to_owned(),
16381 routes: vec![group_route],
16382 source: crate::route_listing::RouteSource::User,
16383 layer_guarded: false,
16384 apply_layer: Box::new(|r| r),
16385 };
16386 let offenders = collect_unguarded_repository_writes(&[], std::slice::from_ref(&group));
16387 assert_eq!(offenders.len(), 1);
16388 assert_eq!(offenders[0].0, "Post");
16389 }
16390
16391 #[test]
16392 fn scoped_guarded_marks_the_group_and_scoped_does_not() {
16393 let app = app()
16394 .scoped(
16395 "/open",
16396 tower::layer::util::Identity::new(),
16397 vec![build_route(http::Method::GET, "/a", None)],
16398 )
16399 .scoped_guarded(
16400 "/api",
16401 tower::layer::util::Identity::new(),
16402 vec![build_route(http::Method::GET, "/b", None)],
16403 );
16404 let flags: Vec<(&str, bool)> = app
16405 .scoped_groups
16406 .iter()
16407 .map(|g| (g.prefix.as_str(), g.layer_guarded))
16408 .collect();
16409 assert_eq!(flags, vec![("/open", false), ("/api", true)]);
16410 }
16411
16412 #[test]
16413 fn collect_unregistered_walks_scoped_groups() {
16414 let group_route = build_route(
16415 http::Method::POST,
16416 "/api/posts",
16417 Some(guarded_with_check("/api/posts", "TestPost")),
16418 );
16419 let group = ScopedGroup {
16420 prefix: "/scoped".to_owned(),
16421 routes: vec![group_route],
16422 source: crate::route_listing::RouteSource::User,
16423 layer_guarded: false,
16424 apply_layer: Box::new(|r| r),
16425 };
16426 let registry = PolicyRegistry::default();
16427 let (missing, _) =
16428 collect_unregistered_repository_handlers(&[], std::slice::from_ref(&group), ®istry);
16429 assert_eq!(missing.len(), 1);
16430 assert_eq!(missing[0].0, "TestPost");
16431 }
16432}
16433
16434#[cfg(feature = "maud")]
16439fn install_story_registry(state: &AppState, story_gallery: Option<crate::stories::StoryGallery>) {
16440 if let Some(gallery) = story_gallery {
16441 state.insert_extension(gallery.into_registry());
16442 }
16443}
16444
16445fn build_state(
16446 config: &AutumnConfig,
16447 #[cfg(feature = "db")] database_topology: Option<&crate::db::DatabaseTopology>,
16448 #[cfg(feature = "db")] shards: Option<crate::sharding::ShardSet>,
16449 #[cfg(feature = "ws")] channels_backend: Option<Arc<dyn crate::channels::ChannelsBackend>>,
16450) -> AppState {
16451 let probes = crate::probe::ProbeState::pending_startup();
16452 #[cfg(feature = "ws")]
16453 let channels = channels_backend.map_or_else(
16454 || {
16455 crate::channels::Channels::from_config(
16456 &config.channels,
16457 probes.shutdown_signal().child_token(),
16458 )
16459 .unwrap_or_else(|error| {
16460 tracing::error!(error = %error, "Failed to configure channels backend");
16461 std::process::exit(1);
16462 })
16463 },
16464 crate::channels::Channels::with_shared_backend,
16465 );
16466
16467 #[cfg(feature = "presence")]
16471 let presence = crate::presence::Presence::new(channels.clone());
16472
16473 let state = AppState {
16474 extensions: std::sync::Arc::new(std::sync::RwLock::new(std::collections::HashMap::new())),
16475 #[cfg(feature = "db")]
16476 pool: database_topology.map(|topology| topology.primary().clone()),
16477 #[cfg(feature = "db")]
16478 replica_pool: database_topology.and_then(|topology| topology.replica().cloned()),
16479 #[cfg(feature = "db")]
16480 shards,
16481 #[cfg(all(feature = "db", feature = "reporting"))]
16482 db_capture_gap: database_topology
16483 .and_then(|topology| topology.capture_gap().map(std::sync::Arc::from)),
16484 profile: config.profile.as_deref().map(Arc::from),
16485 role: config.role,
16486 started_at: crate::time::monotonic_now(),
16487 health_detailed: config.health.detailed,
16488 probes,
16489 metrics: crate::middleware::MetricsCollector::new(),
16490 log_levels: crate::actuator::LogLevels::new(&config.log.level),
16491 task_registry: crate::actuator::TaskRegistry::new(),
16492 job_registry: crate::actuator::JobRegistry::new(),
16493 config_props: crate::actuator::ConfigProperties::from_config(config),
16494 metrics_source_registry: crate::actuator::MetricsSourceRegistry::new(),
16495 health_indicator_registry: crate::actuator::HealthIndicatorRegistry::new(),
16496 #[cfg(all(feature = "collab", feature = "presence"))]
16497 collab: crate::collab::CollabHub::new(channels.clone(), presence.clone()),
16498 #[cfg(feature = "presence")]
16499 presence,
16500 #[cfg(feature = "ws")]
16501 channels,
16502 policy_registry: crate::authorization::PolicyRegistry::default(),
16503 forbidden_response: config.security.forbidden_response,
16504 auth_session_key: Arc::from(config.auth.session_key.as_str()),
16505 shared_cache: None,
16506 clock: std::sync::Arc::new(crate::time::SystemClock),
16507 entropy: std::sync::Arc::new(crate::entropy::OsEntropy),
16508 app_id: AppState::next_app_id(),
16509 };
16510 state.apply_health_config(&config.health);
16511 #[cfg(feature = "db")]
16512 if state.replica_pool.is_some() {
16513 state
16514 .probes()
16515 .configure_replica_dependency(config.database.replica_fallback);
16516 state.probes().configure_replica_max_lag(
16517 config
16518 .database
16519 .replica_max_lag_ms
16520 .filter(|ms| *ms > 0)
16521 .map(std::time::Duration::from_millis),
16522 );
16523 }
16524 #[cfg(feature = "db")]
16527 if let Some(set) = state.shards() {
16528 crate::sharding::register_shard_health_indicators(
16529 set,
16530 &state.health_indicator_registry,
16531 config.health.ping_timeout(),
16532 );
16533 }
16534 state.insert_extension(config.clone());
16535 state.insert_extension(crate::step_up::StepUpGlobalConfig {
16536 default_max_age_secs: config.auth.step_up.default_max_age_secs,
16537 });
16538 #[cfg(feature = "http-client")]
16539 state.insert_extension(crate::http_client::SharedReqwestClient {
16540 client: crate::http_client::Client::build_inner(&config.http.client),
16541 timeout_secs: config.http.client.timeout_secs,
16542 });
16543 #[cfg(feature = "http-client")]
16544 crate::http_client::install_shared_throttle(&state, &config.http.client);
16545 state
16546}
16547
16548fn format_route_lines(
16550 routes: &[Route],
16551 scoped_groups: &[ScopedGroup],
16552 config: &AutumnConfig,
16553) -> String {
16554 use std::fmt::Write as _;
16555
16556 let mut out = String::new();
16557 for route in routes {
16558 let _ = write!(
16559 out,
16560 "\n {} {:<8} -> {}",
16561 route.path, route.method, route.name
16562 );
16563 }
16564 for group in scoped_groups {
16565 for route in &group.routes {
16566 let _ = write!(
16567 out,
16568 "\n {}{} {:<8} -> {} (scoped)",
16569 group.prefix, route.path, route.method, route.name
16570 );
16571 }
16572 }
16573 let mut probe_paths = std::collections::HashSet::new();
16574 for (path, name) in [
16575 (config.health.live_path.as_str(), "live"),
16576 (config.health.ready_path.as_str(), "ready"),
16577 (config.health.startup_path.as_str(), "startup"),
16578 (config.health.path.as_str(), "health"),
16579 ] {
16580 if probe_paths.insert(path) {
16581 let _ = write!(out, "\n {} {:<8} -> {}", path, "GET", name);
16582 }
16583 }
16584 let _ = write!(
16585 out,
16586 "\n {} {:<8} -> actuator",
16587 crate::actuator::actuator_route_glob(&config.actuator.prefix),
16588 "GET"
16589 );
16590 #[cfg(feature = "htmx")]
16591 {
16592 out.push_str("\n /static/js/htmx.min.js GET -> htmx");
16593 out.push_str("\n /static/js/autumn-htmx-csrf.js GET -> htmx csrf");
16594 }
16595 out
16596}
16597
16598fn format_task_lines(tasks: &[crate::task::TaskInfo]) -> Option<String> {
16600 use std::fmt::Write as _;
16601
16602 if tasks.is_empty() {
16603 return None;
16604 }
16605
16606 let mut out = String::new();
16607 for task in tasks {
16608 let schedule = task.schedule.to_string();
16609 let _ = write!(out, "\n {} ({schedule})", task.name);
16610 }
16611 Some(out)
16612}
16613
16614fn format_middleware_list(config: &AutumnConfig) -> String {
16616 let mut items = vec![
16617 "RequestId",
16618 "SecurityHeaders",
16619 "Session (in-memory)",
16620 "ErrorPages",
16621 ];
16622 if !config.cors.allowed_origins.is_empty() {
16623 items.push("CORS");
16624 }
16625 if config.security.csrf.enabled {
16626 items.push("CSRF");
16627 }
16628 items.push("Metrics");
16629 items.join(", ")
16630}
16631
16632fn mask_database_url(url: &str, pool_size: usize) -> String {
16639 format!(
16640 "{} (pool_size={pool_size})",
16641 crate::db_url::redact_target(url)
16642 )
16643}
16644
16645fn format_config_summary(config: &AutumnConfig) -> String {
16647 let profile = config.profile.as_deref().unwrap_or("none");
16648 let db_status = config.database.effective_primary_url().map_or_else(
16649 || "not configured".to_owned(),
16650 |url| {
16651 let primary = mask_database_url(url, config.database.effective_primary_pool_size());
16652 if config.database.replica_url.is_some() {
16653 format!(
16654 "primary={primary}, replica=configured (pool_size={})",
16655 config.database.effective_replica_pool_size()
16656 )
16657 } else {
16658 primary
16659 }
16660 },
16661 );
16662 let telemetry_status = if config.telemetry.enabled {
16663 let endpoint = config
16664 .telemetry
16665 .otlp_endpoint
16666 .as_deref()
16667 .unwrap_or("<missing endpoint>");
16668 format!("{:?} -> {endpoint}", config.telemetry.protocol)
16669 } else {
16670 "disabled".to_owned()
16671 };
16672 format!(
16673 "\
16674 \n profile: {profile}\
16675 \n server: {}:{}\
16676 \n database: {db_status}\
16677 \n log_level: {}\
16678 \n log_format: {:?}\
16679 \n telemetry: {telemetry_status}\
16680 \n health: {} (detailed={})\
16681 \n actuator: sensitive={}\
16682 \n shutdown: prestop={}s drain={}s",
16683 config.server.host,
16684 config.server.port,
16685 config.log.level,
16686 config.log.format,
16687 config.health.path,
16688 config.health.detailed,
16689 config.actuator.sensitive,
16690 config.server.prestop_grace_secs,
16691 config.server.shutdown_timeout_secs,
16692 )
16693}
16694
16695pub(crate) fn project_dir(subdir: &str, env: &dyn crate::config::Env) -> std::path::PathBuf {
16698 env.var("AUTUMN_MANIFEST_DIR").map_or_else(
16699 |_| std::path::PathBuf::from(subdir),
16700 |d| std::path::PathBuf::from(d).join(subdir),
16701 )
16702}
16703
16704#[cfg(unix)]
16710async fn watch_for_in_place_upgrade(
16711 config: &crate::config::UpgradeConfig,
16712 signal: Option<tokio::signal::unix::Signal>,
16713 socket: Option<crate::upgrade::HandoffSocket>,
16714 state: AppState,
16715 cutover: tokio_util::sync::CancellationToken,
16716) {
16717 let Some(mut signal) = signal else {
16720 return;
16721 };
16722 let ready_timeout = std::time::Duration::from_secs(config.ready_timeout_secs.max(1));
16725
16726 while signal.recv().await.is_some() {
16727 if !config.enabled {
16728 tracing::warn!(
16729 "SIGUSR2 received but in-place upgrade is disabled \
16730 ([server.upgrade] enabled = false); ignoring"
16731 );
16732 continue;
16733 }
16734 #[cfg(feature = "managed-pg")]
16738 if crate::managed_pg::is_supervising() {
16739 let error = crate::upgrade::UpgradeError::Unsupported(
16740 "it supervises a managed Postgres cluster, which cannot be handed over with \
16741 the socket — the successor would start a second postmaster over the same \
16742 data directory",
16743 );
16744 tracing::error!(error = %error, "in-place upgrade refused");
16745 continue;
16746 }
16747 let Some(socket) = socket.as_ref() else {
16748 let error = crate::upgrade::UpgradeError::UnsupportedListener(
16749 "the server is not bound to a plain TCP listener",
16750 );
16751 tracing::error!(error = %error, "in-place upgrade refused");
16752 continue;
16753 };
16754 tracing::info!("SIGUSR2 received, starting an in-place upgrade");
16755 let plan = crate::upgrade::UpgradePlan {
16756 socket,
16757 registry: state.extension::<crate::upgrade::LiveStateRegistry>(),
16758 ready_timeout,
16759 clock: state.clock_arc(),
16760 entropy: state.entropy_arc(),
16761 };
16762 match crate::upgrade::upgrade_in_place(plan).await {
16763 Ok(handover) => {
16764 tracing::info!(
16765 successor_pid = handover.successor_pid,
16766 generation = handover.generation,
16767 elapsed_ms = u64::try_from(handover.elapsed.as_millis()).unwrap_or(u64::MAX),
16768 "in-place upgrade complete: the successor is serving on this socket, \
16769 draining this build"
16770 );
16771 cutover.cancel();
16772 return;
16773 }
16774 Err(e) => {
16775 tracing::error!(
16776 error = %e,
16777 "in-place upgrade abandoned; this build is still serving and its live \
16778 state is writable again"
16779 );
16780 }
16781 }
16782 }
16783}
16784
16785enum DrainCause {
16786 Signal,
16789 UpgradeCutover,
16792}
16793
16794pub const SHUTDOWN_SIGNAL_FILE_ENV: &str = "AUTUMN_SHUTDOWN_SIGNAL_FILE";
16813
16814#[cfg_attr(
16821 unix,
16822 allow(
16823 dead_code,
16824 reason = "the non-Unix drain arm, compiled and tested on every platform"
16825 )
16826)]
16827fn external_shutdown_path_from(value: Option<&std::ffi::OsStr>) -> Option<std::path::PathBuf> {
16828 let trimmed = value?.to_string_lossy().trim().to_string();
16829 if trimmed.is_empty() {
16830 return None;
16831 }
16832 Some(std::path::PathBuf::from(trimmed))
16838}
16839
16840#[cfg_attr(
16848 unix,
16849 allow(
16850 dead_code,
16851 reason = "the non-Unix drain arm, compiled and tested on every platform"
16852 )
16853)]
16854async fn external_shutdown_signal(path: Option<std::path::PathBuf>) {
16855 let Some(path) = path else {
16856 std::future::pending::<()>().await;
16857 return;
16858 };
16859 let interval = std::time::Duration::from_millis(100);
16860 loop {
16861 if tokio::fs::metadata(&path).await.is_ok_and(|m| m.is_file()) {
16866 return;
16867 }
16868 tokio::time::sleep(interval).await;
16869 }
16870}
16871
16872async fn shutdown_signal(upgrade_cutover: tokio_util::sync::CancellationToken) -> DrainCause {
16887 let ctrl_c = async {
16888 tokio::signal::ctrl_c()
16889 .await
16890 .expect("Failed to install Ctrl+C handler");
16891 tracing::info!("Received Ctrl+C, starting graceful shutdown");
16892 };
16893
16894 #[cfg(unix)]
16895 let terminate = async {
16896 tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
16897 .expect("Failed to install SIGTERM handler")
16898 .recv()
16899 .await;
16900 tracing::info!("Received SIGTERM, starting graceful shutdown");
16901 };
16902
16903 #[cfg(windows)]
16927 let terminate = async {
16928 use tokio::signal::windows;
16929 let mut close = windows::ctrl_close().expect("Failed to install CTRL_CLOSE handler");
16930 let mut logoff = windows::ctrl_logoff().expect("Failed to install CTRL_LOGOFF handler");
16931 let mut shutdown =
16932 windows::ctrl_shutdown().expect("Failed to install CTRL_SHUTDOWN handler");
16933 let mut brk = windows::ctrl_break().expect("Failed to install CTRL_BREAK handler");
16934 let event = tokio::select! {
16935 _ = close.recv() => "CTRL_CLOSE",
16936 _ = logoff.recv() => "CTRL_LOGOFF",
16937 _ = shutdown.recv() => "CTRL_SHUTDOWN",
16938 _ = brk.recv() => "CTRL_BREAK",
16939 };
16940 tracing::info!(
16941 event,
16942 "Received a console control event, starting graceful shutdown"
16943 );
16944 };
16945
16946 #[cfg(not(any(unix, windows)))]
16947 let terminate = std::future::pending::<()>();
16948
16949 let canary_rollback = async {
16950 canary_rollback_signal(std::path::Path::new(
16951 crate::canary::CANARY_ROLLBACK_FLAG_FILE,
16952 ))
16953 .await;
16954 tracing::info!("Canary rollback signalled, starting graceful shutdown");
16955 };
16956
16957 #[cfg(not(unix))]
16967 let external_stop = async {
16968 let path =
16969 external_shutdown_path_from(std::env::var_os(SHUTDOWN_SIGNAL_FILE_ENV).as_deref());
16970 external_shutdown_signal(path.clone()).await;
16971 tracing::warn!(
16972 path = ?path,
16973 "Cooperative shutdown requested via {SHUTDOWN_SIGNAL_FILE_ENV}, starting graceful shutdown"
16974 );
16975 };
16976
16977 #[cfg(unix)]
16978 let external_stop = std::future::pending::<()>();
16979
16980 let upgrade = async {
16981 upgrade_cutover.cancelled().await;
16982 tracing::info!("Successor is serving after an in-place upgrade, draining this build");
16983 };
16984
16985 tokio::select! {
16986 () = ctrl_c => DrainCause::Signal,
16987 () = terminate => DrainCause::Signal,
16988 () = canary_rollback => DrainCause::Signal,
16989 () = external_stop => DrainCause::Signal,
16990 () = upgrade => DrainCause::UpgradeCutover,
16991 }
16992}
16993
16994async fn canary_rollback_signal(path: &std::path::Path) {
17007 let interval = std::time::Duration::from_millis(500);
17008 loop {
17009 if tokio::fs::metadata(path).await.is_ok() {
17010 return;
17011 }
17012 tokio::time::sleep(interval).await;
17013 }
17014}
17015
17016#[cfg(test)]
17017mod tests {
17018 #[cfg(feature = "mcp")]
17021 #[test]
17022 fn mcp_catalog_and_instructions_set_the_runtime() {
17023 use crate::mcp::McpCatalog;
17024
17025 let builder = super::app()
17026 .mcp_catalog(McpCatalog::Progressive)
17027 .mcp_instructions("Todo tools.");
17028 let rt = builder.mcp.as_ref().expect("an MCP runtime");
17029 assert_eq!(rt.mount_path, "/mcp");
17030 assert_eq!(rt.catalog, McpCatalog::Progressive);
17031 assert_eq!(rt.instructions.as_deref(), Some("Todo tools."));
17032
17033 let builder = builder.mount_mcp("/agent");
17034 let rt = builder.mcp.as_ref().expect("an MCP runtime");
17035 assert_eq!(rt.mount_path, "/agent");
17036 assert_eq!(rt.catalog, McpCatalog::Progressive);
17037 assert_eq!(rt.instructions.as_deref(), Some("Todo tools."));
17038
17039 let rt = super::app().mcp_instructions("x").mcp.expect("a runtime");
17040 assert_eq!(rt.catalog, McpCatalog::Full, "the default catalog is full");
17041 }
17042
17043 #[test]
17046 fn on_session_rotation_registers_hooks_on_the_state() {
17047 let builder = super::app()
17048 .on_session_rotation(|_| async { Ok(()) })
17049 .on_session_rotation(|_| async { Ok(()) });
17050 let state = crate::state::AppState::test_default();
17051 assert!(state.session_rotation_hooks().is_empty());
17052
17053 super::run_state_initializers(builder.state_initializers, &state);
17054
17055 assert_eq!(state.session_rotation_hooks().len(), 2);
17056 }
17057
17058 #[cfg(feature = "acme")]
17066 #[test]
17067 fn acme_fleet_warning_covers_the_certificate_store_under_dns01() {
17068 use crate::config::SchedulerBackend;
17069
17070 assert!(super::acme_fleet_warning(SchedulerBackend::InProcess, false).is_none());
17072 assert!(super::acme_fleet_warning(SchedulerBackend::InProcess, true).is_none());
17073
17074 let http01 = super::acme_fleet_warning(SchedulerBackend::Postgres, false)
17076 .expect("a distributed backend under HTTP-01 must warn");
17077 assert!(
17078 http01.contains("token"),
17079 "HTTP-01 warning must name the token store: {http01}"
17080 );
17081
17082 let dns01 = super::acme_fleet_warning(SchedulerBackend::Postgres, true).expect(
17084 "a distributed backend under DNS-01 must still warn: DNS-01 proves domain \
17085 control but does not share the certificate store",
17086 );
17087 assert!(
17088 dns01.contains("certificate store"),
17089 "DNS-01 warning must name the certificate store as the hazard: {dns01}"
17090 );
17091 assert!(
17092 dns01.contains("cache_dir"),
17093 "DNS-01 warning must name the config key an operator would change: {dns01}"
17094 );
17095 assert!(
17097 !dns01.contains("token") && !dns01.contains("404"),
17098 "DNS-01 warning must not blame the HTTP-01 token map: {dns01}"
17099 );
17100 }
17101
17102 #[cfg(feature = "acme")]
17106 #[test]
17107 fn acme_fleet_warning_for_sqlite_names_only_the_token_hazard() {
17108 use crate::config::SchedulerBackend;
17109
17110 let http01 = super::acme_fleet_warning(SchedulerBackend::Sqlite, false)
17111 .expect("multi-process HTTP-01 must warn about the per-process token store");
17112 assert!(
17113 http01.contains("token"),
17114 "the warning must name the token store: {http01}"
17115 );
17116 assert!(
17117 !http01.contains("Run ACME on a single host"),
17118 "a single-host deployment must not be told to move to a single host: {http01}"
17119 );
17120 assert!(
17121 http01.contains("cache_dir"),
17122 "the warning must say the certificate store is already shared: {http01}"
17123 );
17124
17125 assert!(
17128 super::acme_fleet_warning(SchedulerBackend::Sqlite, true).is_none(),
17129 "DNS-01 on one host clears both hazards"
17130 );
17131 }
17132
17133 #[test]
17141 fn next_task_run_saturates_instead_of_panicking_at_the_end_of_time() {
17142 let max = chrono::DateTime::<chrono::Utc>::MAX_UTC;
17143
17144 let rendered = super::format_next_task_run_after(max, std::time::Duration::from_secs(60));
17146 assert_eq!(
17147 rendered,
17148 max.to_rfc3339(),
17149 "a fixed delay past the end of time must clamp, not panic"
17150 );
17151
17152 let absurd = super::format_next_task_run_after(
17155 chrono::Utc::now(),
17156 std::time::Duration::from_secs(u64::MAX),
17157 );
17158 assert_eq!(absurd, max.to_rfc3339());
17159
17160 let epoch = chrono::DateTime::from_timestamp(0, 0).unwrap();
17162 assert_eq!(
17163 super::format_next_task_run_after(epoch, std::time::Duration::from_secs(60)),
17164 (epoch + chrono::TimeDelta::seconds(60)).to_rfc3339()
17165 );
17166 }
17167
17168 use super::*;
17169 use axum::body::Body;
17170 use axum::http::{Request, StatusCode};
17171 use std::sync::atomic::{AtomicUsize, Ordering};
17172 use tower::ServiceExt;
17173
17174 #[test]
17181 fn plugin_declares_config_section_via_build() {
17182 struct DummyMediaPlugin;
17183 impl crate::plugin::Plugin for DummyMediaPlugin {
17184 fn build(self, app: AppBuilder) -> AppBuilder {
17185 app.config_section("media")
17186 }
17187 }
17188
17189 let builder = app().plugin(DummyMediaPlugin);
17190 assert!(
17191 builder.has_config_section("media"),
17192 "a plugin's build() must declare its [media] config section"
17193 );
17194 assert!(
17195 !builder.has_config_section("definitely_not_a_root"),
17196 "only explicitly-declared roots are registered — the seam is fail-closed"
17197 );
17198 }
17199
17200 fn omitted_for(builder: &AppBuilder) -> usize {
17204 omitted_router_count(
17205 builder.merge_routers.len(),
17206 builder
17207 .nest_routers
17208 .iter()
17209 .map(|(prefix, _)| prefix.as_str()),
17210 &builder.declared_routes,
17211 )
17212 }
17213
17214 #[test]
17224 fn documented_nest_then_declare_is_not_counted_as_omitted() {
17225 let raw =
17226 axum::Router::<AppState>::new().route("/ping", axum::routing::get(|| async { "pong" }));
17227 let declared = vec![crate::route_listing::RouteInfo {
17228 method: "GET".to_owned(),
17229 path: "/admin/ping".to_owned(),
17230 handler: "admin::ping".to_owned(),
17231 ..Default::default()
17232 }];
17233
17234 let builder = app().nest("/admin", raw).declare_plugin_routes(declared);
17237
17238 assert_eq!(builder.nest_routers.len(), 1);
17240 assert_eq!(builder.declared_routes.len(), 1);
17242
17243 assert_eq!(
17247 omitted_for(&builder),
17248 0,
17249 "a nest whose endpoints are declared is enumerable and must not count as omitted",
17250 );
17251 }
17252
17253 #[test]
17257 fn undeclared_nest_and_merge_still_count_as_omitted() {
17258 let raw_nest =
17259 axum::Router::<AppState>::new().route("/x", axum::routing::get(|| async { "x" }));
17260 let raw_merge =
17261 axum::Router::<AppState>::new().route("/y", axum::routing::get(|| async { "y" }));
17262
17263 let builder = app().nest("/v2", raw_nest).merge(raw_merge);
17264
17265 assert_eq!(builder.nest_routers.len(), 1);
17266 assert_eq!(builder.merge_routers.len(), 1);
17267 assert!(builder.declared_routes.is_empty());
17269
17270 assert_eq!(
17271 omitted_for(&builder),
17272 2,
17273 "an undeclared nest and a merge are both opaque and must be reported",
17274 );
17275 }
17276
17277 #[test]
17281 fn declared_routes_do_not_cover_a_rootless_merge() {
17282 let raw_merge =
17283 axum::Router::<AppState>::new().route("/y", axum::routing::get(|| async { "y" }));
17284
17285 let builder =
17286 app()
17287 .merge(raw_merge)
17288 .declare_plugin_routes(vec![crate::route_listing::RouteInfo {
17289 method: "GET".to_owned(),
17290 path: "/admin/ok".to_owned(),
17291 handler: "admin::ok".to_owned(),
17292 ..Default::default()
17293 }]);
17294
17295 assert_eq!(
17296 omitted_for(&builder),
17297 1,
17298 "a merge has no prefix to match declarations against and must always count",
17299 );
17300 }
17301
17302 #[test]
17307 fn mixed_declared_and_undeclared_nests_count_only_the_undeclared() {
17308 let declared_raw =
17309 axum::Router::<AppState>::new().route("/ok", axum::routing::get(|| async { "ok" }));
17310 let undeclared_raw = axum::Router::<AppState>::new()
17311 .route("/opaque", axum::routing::get(|| async { "opaque" }));
17312
17313 let builder = app()
17314 .nest("/admin", declared_raw)
17315 .declare_plugin_routes(vec![crate::route_listing::RouteInfo {
17316 method: "GET".to_owned(),
17317 path: "/admin/ok".to_owned(),
17318 handler: "admin::ok".to_owned(),
17319 ..Default::default()
17320 }])
17321 .nest("/raw", undeclared_raw);
17322
17323 assert_eq!(builder.nest_routers.len(), 2);
17324 assert_eq!(builder.declared_routes.len(), 1);
17325 assert_eq!(
17326 omitted_for(&builder),
17327 1,
17328 "only the bare nest() is omitted; the declared mount is covered",
17329 );
17330 }
17331
17332 #[test]
17337 fn prefix_match_respects_path_segment_boundaries() {
17338 let raw = axum::Router::<AppState>::new().route("/x", axum::routing::get(|| async { "x" }));
17339
17340 let builder = app().nest("/admin", raw).declare_plugin_routes(vec![
17341 crate::route_listing::RouteInfo {
17342 method: "GET".to_owned(),
17343 path: "/administrators".to_owned(),
17344 handler: "other::index".to_owned(),
17345 ..Default::default()
17346 },
17347 ]);
17348
17349 assert_eq!(
17350 omitted_for(&builder),
17351 1,
17352 "`/administrators` is not under the `/admin` nest prefix; the nest stays omitted",
17353 );
17354 }
17355
17356 #[test]
17357 fn is_dump_jobs_mode_only_true_for_exactly_one() {
17358 temp_env::with_var("AUTUMN_DUMP_JOBS", Some("1"), || {
17362 assert!(is_dump_jobs_mode(), "`1` must select the jobs-dump path");
17363 });
17364 temp_env::with_var("AUTUMN_DUMP_JOBS", Some("0"), || {
17365 assert!(!is_dump_jobs_mode(), "`0` must not select the dump path");
17366 });
17367 temp_env::with_var("AUTUMN_DUMP_JOBS", Some("true"), || {
17368 assert!(
17369 !is_dump_jobs_mode(),
17370 "only the literal `1` enables the mode"
17371 );
17372 });
17373 temp_env::with_var("AUTUMN_DUMP_JOBS", None::<&str>, || {
17374 assert!(!is_dump_jobs_mode(), "unset must not select the dump path");
17375 });
17376 }
17377
17378 #[test]
17379 fn is_retention_dry_run_mode_only_true_for_exactly_one() {
17380 temp_env::with_var("AUTUMN_RETENTION_DRY_RUN", Some("1"), || {
17384 assert!(
17385 is_retention_dry_run_mode(),
17386 "`1` must select the retention dry-run path"
17387 );
17388 });
17389 temp_env::with_var("AUTUMN_RETENTION_DRY_RUN", Some("0"), || {
17390 assert!(
17391 !is_retention_dry_run_mode(),
17392 "`0` must not select the dry-run path"
17393 );
17394 });
17395 temp_env::with_var("AUTUMN_RETENTION_DRY_RUN", Some("true"), || {
17396 assert!(
17397 !is_retention_dry_run_mode(),
17398 "only the literal `1` enables the mode"
17399 );
17400 });
17401 temp_env::with_var("AUTUMN_RETENTION_DRY_RUN", None::<&str>, || {
17402 assert!(
17403 !is_retention_dry_run_mode(),
17404 "unset must not select the dry-run path"
17405 );
17406 });
17407 }
17408
17409 #[cfg(feature = "db")]
17410 #[test]
17411 fn retention_dry_run_model_filter_from_env_trims_and_ignores_blank() {
17412 temp_env::with_var("AUTUMN_RETENTION_MODEL", Some(" Widget "), || {
17413 assert_eq!(
17414 retention_dry_run_model_filter_from_env().as_deref(),
17415 Some("Widget")
17416 );
17417 });
17418 temp_env::with_var("AUTUMN_RETENTION_MODEL", Some(""), || {
17419 assert_eq!(retention_dry_run_model_filter_from_env(), None);
17420 });
17421 temp_env::with_var("AUTUMN_RETENTION_MODEL", None::<&str>, || {
17422 assert_eq!(retention_dry_run_model_filter_from_env(), None);
17423 });
17424 }
17425
17426 #[cfg(feature = "db")]
17427 #[test]
17428 fn merge_and_validate_task_names_rejects_collision_with_hand_declared_task() {
17429 fn dry_run_stub(
17437 _state: AppState,
17438 ) -> std::pin::Pin<
17439 Box<
17440 dyn std::future::Future<
17441 Output = crate::AutumnResult<crate::retention::RetentionSweepReport>,
17442 > + Send,
17443 >,
17444 > {
17445 Box::pin(async {
17446 Ok(crate::retention::RetentionSweepReport {
17447 model: "Widget".to_string(),
17448 table: "widgets".to_string(),
17449 rows_swept: 0,
17450 duration_ms: 0,
17451 dry_run: true,
17452 })
17453 })
17454 }
17455 fn task_info_stub() -> crate::task::TaskInfo {
17456 crate::task::TaskInfo {
17457 name: "retention-sweep-widgets".to_string(),
17458 schedule: crate::task::Schedule::FixedDelay(std::time::Duration::from_secs(3600)),
17459 coordination: crate::task::TaskCoordination::Fleet,
17460 handler: |_state| Box::pin(async { Ok(()) }),
17461 }
17462 }
17463 let descriptor = crate::retention::RetentionSweepDescriptor {
17464 model_name: "Widget",
17465 table_name: "widgets",
17466 task_info: task_info_stub,
17467 dry_run: dry_run_stub,
17468 };
17469 let hand_declared_collision = crate::task::TaskInfo {
17470 name: "retention-sweep-widgets".to_string(),
17471 schedule: crate::task::Schedule::FixedDelay(std::time::Duration::from_secs(60)),
17472 coordination: crate::task::TaskCoordination::PerReplica,
17473 handler: |_state| Box::pin(async { Ok(()) }),
17474 };
17475
17476 let error = merge_and_validate_task_names(&[&descriptor], vec![hand_declared_collision])
17477 .expect_err("a hand-declared task colliding with a generated task name must error");
17478
17479 assert!(
17480 error.contains("retention-sweep-widgets"),
17481 "the error must name the colliding task: {error}"
17482 );
17483 }
17484
17485 #[cfg(feature = "db")]
17486 #[test]
17487 fn merge_and_validate_task_names_accepts_distinct_names() {
17488 fn dry_run_stub(
17489 _state: AppState,
17490 ) -> std::pin::Pin<
17491 Box<
17492 dyn std::future::Future<
17493 Output = crate::AutumnResult<crate::retention::RetentionSweepReport>,
17494 > + Send,
17495 >,
17496 > {
17497 Box::pin(async {
17498 Ok(crate::retention::RetentionSweepReport {
17499 model: "Widget".to_string(),
17500 table: "widgets".to_string(),
17501 rows_swept: 0,
17502 duration_ms: 0,
17503 dry_run: true,
17504 })
17505 })
17506 }
17507 fn task_info_stub() -> crate::task::TaskInfo {
17508 crate::task::TaskInfo {
17509 name: "retention-sweep-widgets".to_string(),
17510 schedule: crate::task::Schedule::FixedDelay(std::time::Duration::from_secs(3600)),
17511 coordination: crate::task::TaskCoordination::Fleet,
17512 handler: |_state| Box::pin(async { Ok(()) }),
17513 }
17514 }
17515 let descriptor = crate::retention::RetentionSweepDescriptor {
17516 model_name: "Widget",
17517 table_name: "widgets",
17518 task_info: task_info_stub,
17519 dry_run: dry_run_stub,
17520 };
17521 let unrelated_task = crate::task::TaskInfo {
17522 name: "nightly-report".to_string(),
17523 schedule: crate::task::Schedule::FixedDelay(std::time::Duration::from_secs(60)),
17524 coordination: crate::task::TaskCoordination::PerReplica,
17525 handler: |_state| Box::pin(async { Ok(()) }),
17526 };
17527
17528 assert!(merge_and_validate_task_names(&[&descriptor], vec![unrelated_task]).is_ok());
17529 }
17530
17531 #[test]
17532 fn retention_dry_run_one_shot_dispatches_before_server_start() {
17533 let source = include_str!("app.rs").replace("\r\n", "\n");
17538 let run_start = source.find("pub async fn run(self)").expect("run() exists");
17539 let run_end = source
17540 .find("async fn run_build_mode(self)")
17541 .expect("build mode follows run()");
17542 let run_body = &source[run_start..run_end];
17543
17544 let dispatch = run_body
17545 .find("if is_retention_dry_run_mode() {")
17546 .expect("run() dispatches the retention dry-run one-shot");
17547 let server_start = run_body
17548 .find("let Self {")
17549 .expect("run() destructures self to start the server");
17550 assert!(
17551 dispatch < server_start,
17552 "AUTUMN_RETENTION_DRY_RUN must be handled before the server-start path"
17553 );
17554 let dry_run_branch = &run_body[dispatch..server_start];
17555 assert!(
17556 dry_run_branch.contains("self.run_retention_dry_run_mode().await;")
17557 && dry_run_branch.contains("return;"),
17558 "the retention dry-run one-shot must run then return before server start"
17559 );
17560 }
17561
17562 #[test]
17563 fn dump_jobs_manifest_includes_synthesized_durable_listener_default_queue() {
17564 fn listener_handler(
17572 _state: AppState,
17573 _payload: serde_json::Value,
17574 ) -> std::pin::Pin<
17575 Box<dyn std::future::Future<Output = crate::AutumnResult<()>> + Send + 'static>,
17576 > {
17577 Box::pin(async move { Ok(()) })
17578 }
17579 let durable = crate::events::ListenerInfo {
17580 event_name: "UserSignedUp",
17581 listener_name: "app::send_welcome_email".to_string(),
17582 mode: crate::events::DispatchMode::Durable,
17583 job_name: Some("__event_listener::send_welcome_email".to_string()),
17584 max_attempts: 4,
17585 initial_backoff_ms: 250,
17586 handler: listener_handler,
17587 };
17588 let cfg = crate::config::JobQueuesConfig::strict_list(["critical"]);
17589
17590 let manifest = dump_jobs_manifest(&cfg, Vec::new(), vec![durable]);
17594 assert_eq!(manifest, "queues = [\"critical\", \"default\"]\n");
17595 }
17596
17597 #[cfg(feature = "db")]
17598 const APP_TEST_MIGRATIONS: crate::migrate::EmbeddedMigrations =
17599 diesel_migrations::embed_migrations!("test_migrations");
17600
17601 #[cfg(feature = "mail")]
17604 struct MailTestNoopQueue;
17605
17606 #[cfg(feature = "mail")]
17607 impl crate::mail::MailDeliveryQueue for MailTestNoopQueue {
17608 fn enqueue<'a>(
17609 &'a self,
17610 _mail: crate::mail::Mail,
17611 ) -> std::pin::Pin<
17612 Box<dyn std::future::Future<Output = Result<(), crate::mail::MailError>> + Send + 'a>,
17613 > {
17614 Box::pin(async { Ok(()) })
17615 }
17616 }
17617
17618 #[cfg(feature = "mail")]
17619 fn test_mail() -> crate::mail::Mail {
17620 crate::mail::Mail::builder()
17621 .to("test@example.com")
17622 .subject("hi")
17623 .text("hello")
17624 .build()
17625 .expect("test mail should build")
17626 }
17627
17628 pub fn test_router(routes: Vec<Route>) -> axum::Router {
17630 let config = AutumnConfig::default();
17631 let state = AppState {
17632 health_detailed: true,
17633 ..AppState::test_default()
17634 };
17635 crate::router::build_router(routes, &config, state)
17636 }
17637
17638 #[test]
17646 fn a_wildcard_bind_is_published_as_a_dialable_loopback_address() {
17647 assert_eq!(dialable_endpoint("0.0.0.0:3000"), "127.0.0.1:3000");
17652 assert_eq!(dialable_endpoint("[::]:3000"), "[::1]:3000");
17653 }
17654
17655 #[test]
17656 fn a_specific_bind_is_published_exactly_as_bound() {
17657 assert_eq!(dialable_endpoint("192.168.1.10:3000"), "192.168.1.10:3000");
17660 assert_eq!(dialable_endpoint("127.0.0.1:8080"), "127.0.0.1:8080");
17661 assert_eq!(dialable_endpoint("[::1]:8080"), "[::1]:8080");
17662 }
17663
17664 #[test]
17665 fn the_resolved_port_survives_the_rewrite() {
17666 assert_eq!(dialable_endpoint("0.0.0.0:49152"), "127.0.0.1:49152");
17669 }
17670
17671 #[test]
17672 fn an_unparseable_address_is_published_unchanged() {
17673 assert_eq!(dialable_endpoint("not-an-address"), "not-an-address");
17675 }
17676
17677 #[test]
17678 fn serve_ready_payload_leads_with_the_drain_budget() {
17679 let payload = serve_ready_payload(35, "tcp 127.0.0.1:3000");
17682 assert_eq!(payload.lines().next(), Some("35"));
17683 }
17684
17685 #[test]
17686 fn serve_ready_payload_carries_the_bound_endpoint_on_line_two() {
17687 let payload = serve_ready_payload(35, "tcp 127.0.0.1:3000");
17688 assert_eq!(payload.lines().nth(1), Some("tcp 127.0.0.1:3000"));
17689 }
17690
17691 #[test]
17692 fn serve_ready_payload_survives_a_socket_path_containing_spaces() {
17693 let payload = serve_ready_payload(1, "unix /home/a b/serve.sock");
17696 let line = payload.lines().nth(1).expect("endpoint line");
17697 assert_eq!(line.split_once(' '), Some(("unix", "/home/a b/serve.sock")));
17698 }
17699
17700 #[test]
17701 fn serve_ready_payload_never_emits_a_bare_newline_for_an_unknown_endpoint() {
17702 let payload = serve_ready_payload(7, "");
17705 assert!(payload.lines().all(|l| !l.trim().is_empty()), "{payload:?}");
17706 assert_eq!(
17707 payload.lines().nth(1),
17708 Some(SHUTDOWN_SIGNAL_CAPABILITY_LINE)
17709 );
17710 }
17711
17712 #[test]
17713 fn serve_ready_payload_advertises_the_shutdown_signal_capability() {
17714 for endpoint in ["tcp 127.0.0.1:3000", ""] {
17715 let payload = serve_ready_payload(35, endpoint);
17716 assert!(
17717 serve_ready_advertises_shutdown_signal(&payload),
17718 "{payload:?}"
17719 );
17720 }
17721 }
17722
17723 #[test]
17724 fn the_capability_line_never_displaces_the_endpoint_line() {
17725 let payload = serve_ready_payload(35, "tcp 127.0.0.1:3000");
17726 assert_eq!(payload.lines().nth(1), Some("tcp 127.0.0.1:3000"));
17727 }
17728
17729 #[test]
17730 fn a_budget_only_ready_file_does_not_advertise_the_capability() {
17731 for old in ["35", "35\n", "35\ntcp 127.0.0.1:3000", ""] {
17733 assert!(!serve_ready_advertises_shutdown_signal(old), "{old:?}");
17734 }
17735 }
17736
17737 #[test]
17738 fn an_unrelated_capability_is_not_the_shutdown_signal() {
17739 assert!(!serve_ready_advertises_shutdown_signal(
17740 "35\ncapability other"
17741 ));
17742 }
17743
17744 #[test]
17753 fn external_shutdown_path_is_none_when_the_env_var_is_absent() {
17754 assert_eq!(external_shutdown_path_from(None), None);
17755 }
17756
17757 #[test]
17758 fn external_shutdown_path_ignores_an_empty_env_value() {
17759 assert_eq!(
17763 external_shutdown_path_from(Some(std::ffi::OsStr::new(""))),
17764 None
17765 );
17766 assert_eq!(
17767 external_shutdown_path_from(Some(std::ffi::OsStr::new(" "))),
17768 None
17769 );
17770 }
17771
17772 #[test]
17773 fn external_shutdown_path_trims_surrounding_whitespace() {
17774 assert_eq!(
17781 external_shutdown_path_from(Some(std::ffi::OsStr::new(" /tmp/autumn-stop "))),
17782 Some(std::path::PathBuf::from("/tmp/autumn-stop"))
17783 );
17784 }
17785
17786 #[tokio::test]
17787 async fn external_shutdown_signal_ignores_a_directory_at_the_path() {
17788 let tmp = tempfile::TempDir::new().unwrap();
17793 let path = tmp.path().join("not-a-signal");
17794 std::fs::create_dir(&path).unwrap();
17795
17796 let resolved = tokio::time::timeout(
17797 std::time::Duration::from_millis(400),
17798 external_shutdown_signal(Some(path)),
17799 )
17800 .await;
17801 assert!(resolved.is_err(), "a directory must not count as a signal");
17802 }
17803
17804 #[test]
17805 fn external_shutdown_path_uses_the_env_value_verbatim() {
17806 assert_eq!(
17807 external_shutdown_path_from(Some(std::ffi::OsStr::new("/tmp/autumn-stop"))),
17808 Some(std::path::PathBuf::from("/tmp/autumn-stop"))
17809 );
17810 }
17811
17812 #[tokio::test]
17813 async fn external_shutdown_signal_never_resolves_without_a_path() {
17814 let pending = tokio::time::timeout(
17817 std::time::Duration::from_millis(300),
17818 external_shutdown_signal(None),
17819 )
17820 .await;
17821 assert!(
17822 pending.is_err(),
17823 "an unconfigured signal must never resolve"
17824 );
17825 }
17826
17827 #[tokio::test]
17828 async fn external_shutdown_signal_resolves_when_the_file_appears() {
17829 let tmp = tempfile::TempDir::new().unwrap();
17830 let path = tmp.path().join("dev-shutdown.signal");
17831
17832 let writer_path = path.clone();
17833 let writer = tokio::spawn(async move {
17834 tokio::time::sleep(std::time::Duration::from_millis(150)).await;
17835 std::fs::write(&writer_path, b"stop").unwrap();
17836 });
17837
17838 let signalled = tokio::time::timeout(
17839 std::time::Duration::from_secs(5),
17840 external_shutdown_signal(Some(path)),
17841 )
17842 .await;
17843 assert!(signalled.is_ok(), "writing the file must drain the app");
17844 writer.await.unwrap();
17845 }
17846
17847 #[tokio::test]
17848 async fn external_shutdown_signal_resolves_immediately_when_present_at_boot() {
17849 let tmp = tempfile::TempDir::new().unwrap();
17853 let path = tmp.path().join("dev-shutdown.signal");
17854 std::fs::write(&path, b"stop").unwrap();
17855
17856 let signalled = tokio::time::timeout(
17857 std::time::Duration::from_secs(5),
17858 external_shutdown_signal(Some(path)),
17859 )
17860 .await;
17861 assert!(
17862 signalled.is_ok(),
17863 "a file present at boot must drain the app"
17864 );
17865 }
17866
17867 #[test]
17868 fn external_shutdown_env_var_is_the_name_the_cli_writes() {
17869 assert_eq!(SHUTDOWN_SIGNAL_FILE_ENV, "AUTUMN_SHUTDOWN_SIGNAL_FILE");
17873 }
17874
17875 #[tokio::test]
17876 async fn canary_rollback_signal_resolves_when_flag_newly_written() {
17877 let tmp = tempfile::TempDir::new().unwrap();
17878 let path = tmp.path().join("canary-rollback.json");
17879
17880 let writer_path = path.clone();
17882 let writer = tokio::spawn(async move {
17883 tokio::time::sleep(std::time::Duration::from_millis(150)).await;
17884 crate::canary::CanaryState::write_rollback_flag(
17885 &writer_path,
17886 &crate::canary::RollbackSignal::default(),
17887 )
17888 .unwrap();
17889 });
17890
17891 let signalled = tokio::time::timeout(
17892 std::time::Duration::from_secs(5),
17893 canary_rollback_signal(&path),
17894 )
17895 .await;
17896 assert!(signalled.is_ok(), "rollback signal should resolve");
17897 writer.await.unwrap();
17898 }
17899
17900 #[tokio::test]
17901 async fn canary_rollback_signal_resolves_immediately_when_flag_present_at_boot() {
17902 let tmp = tempfile::TempDir::new().unwrap();
17903 let path = tmp.path().join("canary-rollback.json");
17904 crate::canary::CanaryState::write_rollback_flag(
17907 &path,
17908 &crate::canary::RollbackSignal::default(),
17909 )
17910 .unwrap();
17911
17912 let signalled = tokio::time::timeout(
17913 std::time::Duration::from_secs(5),
17914 canary_rollback_signal(&path),
17915 )
17916 .await;
17917 assert!(
17918 signalled.is_ok(),
17919 "a flag present at boot must trigger rollback (sticky across restarts)"
17920 );
17921 }
17922
17923 #[cfg(not(feature = "sqlite"))]
17933 #[cfg(feature = "db")]
17934 #[test]
17935 fn build_state_applies_replica_fallback_policy_to_read_routing() {
17936 let mut config = AutumnConfig::default();
17937 config.database.primary_url = Some("postgres://localhost/primary".to_owned());
17938 config.database.primary_pool_size = Some(5);
17939 config.database.replica_url = Some("postgres://localhost/replica".to_owned());
17940 config.database.replica_pool_size = Some(2);
17941 config.database.replica_fallback = crate::config::ReplicaFallback::Primary;
17942 let topology = crate::db::create_topology(&config.database)
17943 .expect("topology should build")
17944 .expect("database should be configured");
17945
17946 let state = build_state(
17947 &config,
17948 Some(&topology),
17949 None,
17950 #[cfg(feature = "ws")]
17951 None,
17952 );
17953 state
17954 .probes()
17955 .mark_replica_unready("replica migrations lag primary");
17956
17957 assert_eq!(state.read_pool().expect("read pool").status().max_size, 5);
17958 }
17959
17960 #[test]
17961 fn build_state_exposes_resolved_process_role() {
17962 use crate::config::ProcessRole;
17963
17964 let mut config = AutumnConfig::default();
17967 let state = build_state(
17968 &config,
17969 #[cfg(feature = "db")]
17970 None,
17971 #[cfg(feature = "db")]
17972 None,
17973 #[cfg(feature = "ws")]
17974 None,
17975 );
17976 assert_eq!(state.role(), ProcessRole::Combined);
17977 assert!(state.role().serves_http());
17978 assert!(state.role().runs_workers());
17979
17980 config.role = ProcessRole::Worker;
17983 let state = build_state(
17984 &config,
17985 #[cfg(feature = "db")]
17986 None,
17987 #[cfg(feature = "db")]
17988 None,
17989 #[cfg(feature = "ws")]
17990 None,
17991 );
17992 assert_eq!(state.role(), ProcessRole::Worker);
17993 assert!(state.role().runs_workers());
17994 assert!(!state.role().serves_http());
17995 }
17996
17997 #[cfg(not(feature = "sqlite"))]
18007 #[cfg(feature = "db")]
18008 #[tokio::test]
18009 async fn custom_pool_provider_preserves_configured_replica_topology() {
18010 struct PassthroughPoolProvider;
18011
18012 impl crate::db::DatabasePoolProvider for PassthroughPoolProvider {
18013 async fn create_pool(
18014 &self,
18015 config: &crate::config::DatabaseConfig,
18016 ) -> Result<
18017 Option<
18018 diesel_async::pooled_connection::deadpool::Pool<crate::db::RuntimeConnection>,
18019 >,
18020 crate::db::PoolError,
18021 > {
18022 crate::db::create_pool(config)
18023 }
18024 }
18025
18026 let mut config = AutumnConfig::default();
18027 config.database.primary_url = Some("postgres://localhost/primary".to_owned());
18028 config.database.primary_pool_size = Some(5);
18029 config.database.replica_url = Some("postgres://localhost/replica".to_owned());
18030 config.database.replica_pool_size = Some(2);
18031 config.database.replica_fallback = crate::config::ReplicaFallback::FailReadiness;
18032 let AppBuilder {
18033 pool_provider_factory,
18034 ..
18035 } = app().with_pool_provider(PassthroughPoolProvider);
18036
18037 let database = setup_database(
18038 &config,
18039 Vec::new(),
18040 pool_provider_factory,
18041 None,
18042 None,
18043 false,
18044 RepositoryCommitHookQueueMigrationMode::Runtime,
18045 )
18046 .await
18047 .expect("custom provider should build database topology");
18048 let topology = database.topology.expect("database should be configured");
18049
18050 assert_eq!(topology.primary().status().max_size, 5);
18051 assert_eq!(
18052 topology
18053 .replica()
18054 .expect("custom provider should create replica pool")
18055 .status()
18056 .max_size,
18057 2
18058 );
18059
18060 let state = build_state(
18061 &config,
18062 Some(&topology),
18063 None,
18064 #[cfg(feature = "ws")]
18065 None,
18066 );
18067 state
18068 .probes()
18069 .mark_replica_connection_unready("replica connection failed");
18070
18071 assert!(state.read_pool().is_none());
18072 let (status, _) = crate::probe::readiness_response(&state).await;
18073 assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE);
18074 }
18075
18076 #[cfg(feature = "sqlite")]
18093 #[tokio::test]
18094 async fn custom_pool_provider_with_established_sqlite_pool_fails_closed_on_statement_timeout() {
18095 struct RealSqlitePoolProvider;
18098
18099 impl crate::db::DatabasePoolProvider for RealSqlitePoolProvider {
18100 async fn create_pool(
18101 &self,
18102 config: &crate::config::DatabaseConfig,
18103 ) -> Result<
18104 Option<
18105 diesel_async::pooled_connection::deadpool::Pool<crate::db::RuntimeConnection>,
18106 >,
18107 crate::db::PoolError,
18108 > {
18109 let mut relaxed = config.clone();
18114 relaxed.statement_timeout = None;
18115 crate::db::create_pool(&relaxed)
18116 }
18117 }
18118
18119 let mut config = AutumnConfig::default();
18120 config.database.primary_url = Some("sqlite::memory:".to_owned());
18121 config.database.statement_timeout = Some(std::time::Duration::from_secs(30));
18122 let AppBuilder {
18123 pool_provider_factory,
18124 shard_provider_factory,
18125 ..
18126 } = app().with_pool_provider(RealSqlitePoolProvider);
18127
18128 let Err(err) = setup_database(
18129 &config,
18130 Vec::new(),
18131 pool_provider_factory,
18132 shard_provider_factory,
18133 None,
18134 false,
18135 RepositoryCommitHookQueueMigrationMode::Runtime,
18136 )
18137 .await
18138 else {
18139 panic!(
18140 "sqlite + statement_timeout must fail closed once the provider establishes a pool"
18141 );
18142 };
18143 assert!(
18144 err.contains("database.statement_timeout") && err.contains("SQLite"),
18145 "dispatch guard error must name the config key and SQLite, got: {err}"
18146 );
18147 }
18148
18149 #[cfg(feature = "sqlite")]
18154 #[tokio::test]
18155 async fn custom_pool_provider_no_database_mode_boots_with_statement_timeout() {
18156 struct NoDatabaseProvider;
18157
18158 impl crate::db::DatabasePoolProvider for NoDatabaseProvider {
18159 async fn create_pool(
18160 &self,
18161 _config: &crate::config::DatabaseConfig,
18162 ) -> Result<
18163 Option<
18164 diesel_async::pooled_connection::deadpool::Pool<crate::db::RuntimeConnection>,
18165 >,
18166 crate::db::PoolError,
18167 > {
18168 Ok(None)
18170 }
18171 }
18172
18173 let mut config = AutumnConfig::default();
18174 config.database.primary_url = Some("sqlite::memory:".to_owned());
18176 config.database.statement_timeout = Some(std::time::Duration::from_secs(30));
18177 let AppBuilder {
18178 pool_provider_factory,
18179 shard_provider_factory,
18180 ..
18181 } = app().with_pool_provider(NoDatabaseProvider);
18182
18183 let bootstrap = setup_database(
18184 &config,
18185 Vec::new(),
18186 pool_provider_factory,
18187 shard_provider_factory,
18188 None,
18189 false,
18190 RepositoryCommitHookQueueMigrationMode::Runtime,
18191 )
18192 .await
18193 .expect("no-database provider must boot even with a nonzero statement_timeout");
18194 assert!(
18195 bootstrap.topology.is_none(),
18196 "no-database mode must yield no control topology"
18197 );
18198 assert!(
18199 bootstrap.shards.is_none(),
18200 "no-database mode must yield no shard set"
18201 );
18202 }
18203
18204 #[cfg(all(feature = "db", not(feature = "sqlite")))]
18206 fn sharded_test_config() -> AutumnConfig {
18207 let mut config = AutumnConfig::default();
18208 config.database.primary_url = Some("postgres://localhost/control".to_owned());
18209 config.database.shards = vec![
18210 crate::config::ShardConfig {
18211 name: "shard0".to_owned(),
18212 primary_url: "postgres://localhost/shard0".to_owned(),
18213 slots: Some(vec![crate::config::SlotSpec::Range("0-8191".to_owned())]),
18214 replica_url: None,
18215 primary_pool_size: Some(3),
18216 replica_pool_size: None,
18217 replica_fallback: None,
18218 },
18219 crate::config::ShardConfig {
18220 name: "shard1".to_owned(),
18221 primary_url: "postgres://localhost/shard1".to_owned(),
18222 slots: Some(vec![crate::config::SlotSpec::Range(
18223 "8192-16383".to_owned(),
18224 )]),
18225 replica_url: Some("postgres://localhost/shard1_ro".to_owned()),
18226 primary_pool_size: None,
18227 replica_pool_size: Some(2),
18228 replica_fallback: None,
18229 },
18230 ];
18231 config
18232 }
18233
18234 #[cfg(not(feature = "sqlite"))]
18244 #[cfg(feature = "db")]
18245 #[tokio::test]
18246 async fn setup_database_builds_shard_set_from_config() {
18247 let config = sharded_test_config();
18248
18249 let database = setup_database(
18250 &config,
18251 Vec::new(),
18252 None,
18253 None,
18254 None,
18255 false,
18256 RepositoryCommitHookQueueMigrationMode::Runtime,
18257 )
18258 .await
18259 .expect("sharded config should bootstrap");
18260
18261 assert!(database.topology.is_some(), "control role configured");
18262 let shards = database.shards.expect("shards configured");
18263 assert_eq!(shards.len(), 2);
18264 assert_eq!(
18265 shards
18266 .by_name("shard0")
18267 .expect("shard0")
18268 .primary_pool()
18269 .status()
18270 .max_size,
18271 3
18272 );
18273 assert_eq!(
18274 shards
18275 .by_name("shard1")
18276 .expect("shard1")
18277 .replica_pool()
18278 .expect("shard1 replica")
18279 .status()
18280 .max_size,
18281 2
18282 );
18283
18284 let state = build_state(
18285 &config,
18286 database.topology.as_ref(),
18287 Some(shards),
18288 #[cfg(feature = "ws")]
18289 None,
18290 );
18291 let state_shards = state.shards().expect("state should expose shards");
18292 assert_eq!(state_shards.len(), 2);
18293 let routed = state_shards.route("tenant-1").await.expect("route");
18295 assert!(["shard0", "shard1"].contains(&routed.name()));
18296 }
18297
18298 #[cfg(not(feature = "sqlite"))]
18308 #[cfg(feature = "db")]
18309 #[tokio::test]
18310 async fn custom_pool_provider_builds_shard_topologies() {
18311 struct CountingProvider(std::sync::Arc<std::sync::atomic::AtomicUsize>);
18312
18313 impl crate::db::DatabasePoolProvider for CountingProvider {
18314 async fn create_pool(
18315 &self,
18316 config: &crate::config::DatabaseConfig,
18317 ) -> Result<
18318 Option<
18319 diesel_async::pooled_connection::deadpool::Pool<crate::db::RuntimeConnection>,
18320 >,
18321 crate::db::PoolError,
18322 > {
18323 crate::db::create_pool(config)
18324 }
18325
18326 async fn create_shard_topology(
18327 &self,
18328 shard: &crate::config::ShardConfig,
18329 defaults: &crate::config::DatabaseConfig,
18330 ) -> Result<crate::db::DatabaseTopology, crate::db::PoolError> {
18331 self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
18332 crate::db::create_shard_topology(shard, defaults)
18333 }
18334 }
18335
18336 let config = sharded_test_config();
18337 let calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
18338 let AppBuilder {
18339 pool_provider_factory,
18340 shard_provider_factory,
18341 ..
18342 } = app().with_pool_provider(CountingProvider(calls.clone()));
18343
18344 let database = setup_database(
18345 &config,
18346 Vec::new(),
18347 pool_provider_factory,
18348 shard_provider_factory,
18349 None,
18350 false,
18351 RepositoryCommitHookQueueMigrationMode::Runtime,
18352 )
18353 .await
18354 .expect("provider should build shard topologies");
18355
18356 assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 2);
18357 assert_eq!(database.shards.expect("shards").len(), 2);
18358 }
18359
18360 #[cfg(feature = "db")]
18361 #[test]
18362 fn repository_commit_hook_worker_starts_after_job_runtime_initialization() {
18363 let source = include_str!("app.rs").replace("\r\n", "\n");
18364 let server_init = "initialize_job_runtime(
18365 jobs,
18366 &state,
18367 &server_shutdown,";
18368 let server_worker = "start_repository_commit_hook_worker(\n pool,\n server_shutdown.child_token(),\n );";
18369 let task_init = "initialize_job_runtime(jobs, &state, &task_shutdown, &config.jobs, true)";
18370 let task_worker = "start_repository_commit_hook_worker(\n pool,\n task_shutdown.child_token(),\n );";
18371
18372 assert!(
18373 source
18374 .find(server_init)
18375 .expect("normal server path should initialize jobs")
18376 < source
18377 .find(server_worker)
18378 .expect("normal server path should start repository hook worker"),
18379 "normal server startup must initialize jobs before repository commit hooks can enqueue them"
18380 );
18381 assert!(
18382 source
18383 .find(task_init)
18384 .expect("task runner path should initialize jobs")
18385 < source
18386 .find(task_worker)
18387 .expect("task runner path should start repository hook worker"),
18388 "task runner startup must initialize jobs before repository commit hooks can enqueue them"
18389 );
18390 }
18391
18392 #[cfg(feature = "db")]
18393 #[test]
18394 fn repository_commit_hook_workers_are_gated_on_worker_role() {
18395 let source = include_str!("app.rs").replace("\r\n", "\n");
18400 let primary_gate =
18401 "if role.runs_workers()\n && let Some(pool) = state.pool().cloned()";
18402 let shard_gate = "if role.runs_workers()\n && let Some(shards) = state.shards()";
18403 assert!(
18404 source.contains(primary_gate),
18405 "primary-pool commit-hook worker must be gated on role.runs_workers()"
18406 );
18407 assert!(
18408 source.contains(shard_gate),
18409 "shard commit-hook workers must be gated on role.runs_workers()"
18410 );
18411 }
18412
18413 #[cfg(feature = "db")]
18432 #[test]
18433 fn the_replication_final_flush_is_waited_for_after_the_drain() {
18434 let source = include_str!("app.rs").replace("\r\n", "\n");
18435 let server_start = source
18436 .find("pub async fn run(self)")
18437 .expect("normal server path should exist");
18438 let build_mode_start = source
18441 .find("async fn run_build_mode(self)")
18442 .expect("static build path should follow server path");
18443 let server_source = &source[server_start..build_mode_start];
18444
18445 let spawn = server_source
18446 .find("Ok(_handle) => replication_done = Some(waiter),")
18447 .expect("the replication thread's completion signal must be kept, not dropped");
18448 let drain = server_source
18449 .find("let server_result = server_task.await")
18450 .expect("the normal server path should await the drain");
18451 let wait = server_source
18452 .find("if let Some(waiter) = replication_done.take()")
18453 .expect("shutdown must wait for the replication thread");
18454 let cancel = server_source
18455 .find("replication_shutdown.cancel();")
18456 .expect("shutdown must cancel the replication token explicitly");
18457
18458 assert!(
18459 spawn < drain && drain < wait && drain < cancel,
18460 "the final flush must be triggered AND awaited AFTER the request drain \
18461 (spawn={spawn}, drain={drain}, wait={wait}, cancel={cancel})"
18462 );
18463 assert!(
18466 server_source
18467 .contains("let replication_shutdown = tokio_util::sync::CancellationToken::new();"),
18468 "the replication token must be independent of `server_shutdown`, not a child of it"
18469 );
18470 assert!(
18471 !server_source[..wait].contains("let replication_shutdown = server_shutdown"),
18472 "the replication token must not be derived from `server_shutdown`"
18473 );
18474 assert!(
18476 !server_source[wait..].contains("spawn_blocking"),
18477 "the bounded wait must not block on a join it cannot cancel"
18478 );
18479 }
18480
18481 #[test]
18492 fn cluster_departs_after_the_request_drain() {
18493 let source = include_str!("app.rs").replace("\r\n", "\n");
18494 let server_start = source
18495 .find("pub async fn run(self)")
18496 .expect("normal server path should exist");
18497 let build_mode_start = source
18500 .find("async fn run_build_mode(self)")
18501 .expect("static build path should follow server path");
18502 let server_source = &source[server_start..build_mode_start];
18503
18504 let install = server_source
18505 .find("crate::cluster::install_from_config(&state, &config.cluster, &cluster_shutdown)")
18506 .expect("the cluster must be installed on its own token, not on server_shutdown");
18507 let drain = server_source
18508 .find("let server_result = server_task.await")
18509 .expect("the normal server path should await the drain");
18510 let depart = server_source
18511 .find("cluster_shutdown.cancel();")
18512 .expect("the cluster token should be cancelled explicitly");
18513
18514 assert!(
18515 install < drain && drain < depart,
18516 "the cluster must be installed before the drain and cancelled only \
18517 after it completes, so an increment accepted while requests drain \
18518 still has a push loop to replicate it"
18519 );
18520 assert!(
18521 !server_source.contains("&config.cluster, &server_shutdown"),
18522 "the cluster must not ride server_shutdown: that token fires when \
18523 the listener closes, with the request drain still ahead of it"
18524 );
18525 }
18526
18527 #[cfg(feature = "sqlite")]
18532 #[test]
18533 fn the_fleet_flush_budget_counts_real_time_a_pinned_clock_misses() {
18534 use std::time::Duration;
18535
18536 let budget = Duration::from_secs(30);
18537 assert_eq!(
18539 fleet_flush_budget(budget, Duration::ZERO, Duration::from_secs(25)),
18540 Duration::from_secs(5)
18541 );
18542 assert_eq!(
18544 fleet_flush_budget(budget, Duration::from_secs(20), Duration::from_secs(1)),
18545 Duration::from_secs(10)
18546 );
18547 assert_eq!(
18549 fleet_flush_budget(budget, Duration::ZERO, budget),
18550 Duration::ZERO
18551 );
18552 }
18553
18554 #[test]
18563 fn the_cluster_departure_wait_fits_inside_the_shutdown_budget() {
18564 use std::time::Duration;
18565
18566 let budget = Duration::from_secs(30);
18567
18568 let quick = Duration::from_secs(1);
18570 assert_eq!(
18571 cluster_departure_wait(budget, quick),
18572 crate::cluster::LEAVE_BUDGET,
18573 "a drain that finished early must leave room for the full departure"
18574 );
18575
18576 assert!(
18578 cluster_departure_wait(budget, Duration::ZERO) <= crate::cluster::LEAVE_BUDGET,
18579 "waiting longer than LEAVE_BUDGET would idle past the node's own bound"
18580 );
18581
18582 assert_eq!(
18585 cluster_departure_wait(budget, budget),
18586 Duration::ZERO,
18587 "a drain that consumed the budget must not buy extra shutdown time"
18588 );
18589 assert_eq!(
18590 cluster_departure_wait(budget, budget.saturating_add(Duration::from_secs(5))),
18591 Duration::ZERO,
18592 "an overrun drain must saturate, not wrap into a fresh wait"
18593 );
18594
18595 for elapsed_ms in [0_u64, 250, 29_800, 29_999, 30_000, 45_000] {
18600 let drained = Duration::from_millis(elapsed_ms);
18601 let departure = cluster_departure_wait(budget, drained);
18602 let hooks = budget.saturating_sub(drained.saturating_add(departure));
18603 let left_by_the_drain = budget.saturating_sub(drained);
18604 assert!(
18605 departure.saturating_add(hooks) <= left_by_the_drain,
18606 "after a {drained:?} drain only {left_by_the_drain:?} of the \
18607 {budget:?} budget is left, but the departure ({departure:?}) \
18608 and hooks ({hooks:?}) would spend more"
18609 );
18610 }
18611 }
18612
18613 #[test]
18634 fn accept_loop_defers_only_for_upgrade_successor() {
18635 let source = include_str!("app.rs").replace("\r\n", "\n");
18636 let server_start = source
18637 .find("pub async fn run(self)")
18638 .expect("normal server path should exist");
18639 let build_mode_start = source
18642 .find("async fn run_build_mode(self)")
18643 .expect("static build path should follow server path");
18644 let server_source = &source[server_start..build_mode_start];
18645
18646 let bound = server_source
18647 .find("let server_future:")
18648 .expect("the accept-loop future must be built from the bound listener");
18649 let gate = server_source
18652 .find("let defer_accept_loop = crate::upgrade::handoff_requested();")
18653 .expect("the deferral must be gated on the upgrade handoff");
18654 let cold_spawn = server_source
18655 .find("server_task = Some(tokio::spawn(server_future));")
18656 .expect("a cold start must spawn the accept loop up front, as before");
18657 let hooks = server_source
18658 .find("run_startup_hooks(&startup_hooks, state.clone())")
18659 .expect("startup hooks must run on the normal server path");
18660 let deferred_spawn = server_source
18661 .find("tokio::spawn(future)")
18662 .expect("an upgrade successor must spawn its deferred accept loop after the hooks");
18663
18664 assert!(
18665 bound < gate && gate < cold_spawn && cold_spawn < hooks && hooks < deferred_spawn,
18666 "ordering must be: bound listener -> deferral gate -> cold-start spawn \
18667 -> startup hooks -> successor's deferred spawn \
18668 (bound={bound}, gate={gate}, cold_spawn={cold_spawn}, hooks={hooks}, \
18669 deferred_spawn={deferred_spawn})"
18670 );
18671 assert!(
18674 server_source.contains("if let Some(task) = server_task.take()"),
18675 "the hook-failure path must abort the cold-start accept loop"
18676 );
18677 assert!(
18679 !server_source.contains("let server_task = match bound_listener"),
18680 "the accept loop must not be spawned from the bound-listener match"
18681 );
18682 assert!(
18683 !server_source.contains("let server_task = tokio::spawn(server_future);"),
18684 "the accept loop must not be unconditionally deferred past the hooks"
18685 );
18686 }
18687
18688 #[test]
18689 fn state_initializers_run_before_job_runtime_initialization() {
18690 let source = include_str!("app.rs").replace("\r\n", "\n");
18691 let server_start = source
18692 .find("pub async fn run(self)")
18693 .expect("normal server path should exist");
18694 let build_mode_start = source
18695 .find("async fn run_build_mode(self)")
18696 .expect("static build path should follow server path");
18697 let task_start = source
18698 .find("async fn run_one_off_task_mode(self, requested_name: String)")
18699 .expect("task runner path should exist");
18700 let server_source = &source[server_start..build_mode_start];
18701 let task_source = &source[task_start..];
18702 let server_init = "initialize_job_runtime(
18703 jobs,
18704 &state,
18705 &server_shutdown,";
18706 let task_init = "initialize_job_runtime(jobs, &state, &task_shutdown, &config.jobs, true)";
18707 let server_initializer = server_source
18708 .find("run_state_initializers(state_initializers, &state);")
18709 .expect("normal server path should run state initializers");
18710 let task_initializer = task_source
18711 .find("run_state_initializers(state_initializers, &state);")
18712 .expect("task runner path should run state initializers");
18713 let server_job = server_source
18714 .find(server_init)
18715 .expect("normal server path should initialize jobs");
18716 let task_job = task_source
18717 .find(task_init)
18718 .expect("task runner path should initialize jobs");
18719
18720 assert!(
18721 server_initializer < server_job,
18722 "normal server startup must install state-initialized resources before job workers start"
18723 );
18724 assert!(
18725 task_initializer < task_job,
18726 "task runner startup must install state-initialized resources before job workers start"
18727 );
18728 }
18729
18730 #[test]
18731 fn optional_interceptor_wiring_matches_across_all_three_run_modes() {
18732 let source = include_str!("app.rs").replace("\r\n", "\n");
18741 let start_marker =
18742 "#[cfg(feature = \"mail\")]\n if let Some(interceptor) = mail_interceptor {";
18743 let end_marker = "if let Some(interceptor) = http_interceptor {\n \
18744 state.insert_extension(interceptor);\n }\n";
18745
18746 let run_start = source.find("pub async fn run(self)").expect("run() exists");
18747 let build_start = source
18748 .find("async fn run_build_mode(self)")
18749 .expect("run_build_mode() follows run()");
18750 let task_start = source
18751 .find("async fn run_one_off_task_mode(self, requested_name: String)")
18752 .expect("run_one_off_task_mode() follows run_build_mode()");
18753
18754 let extract = |region: &str| -> String {
18755 let s = region
18756 .find(start_marker)
18757 .expect("the interceptor-wiring block starts with the mail arm");
18758 let e = region[s..]
18759 .find(end_marker)
18760 .map(|offset| s + offset + end_marker.len())
18761 .expect("the interceptor-wiring block ends with the http arm");
18762 region[s..e].to_owned()
18763 };
18764
18765 let run_block = extract(&source[run_start..build_start]);
18766 let build_block = extract(&source[build_start..task_start]);
18767 let task_block = extract(&source[task_start..(task_start + 20_000).min(source.len())]);
18770
18771 assert_eq!(
18772 run_block, build_block,
18773 "run() and run_build_mode() must wire the five optional interceptors identically"
18774 );
18775 assert_eq!(
18776 run_block, task_block,
18777 "run() and run_one_off_task_mode() must wire the five optional interceptors \
18778 identically (this exact drift shipped once, fixed by #862's \"wire interceptors \
18779 into one-off task mode\")"
18780 );
18781 }
18782
18783 #[test]
18784 fn flag_service_preloads_before_later_state_initializers() {
18785 let store = Arc::new(PreloadProbe::default());
18788 let seen = Arc::new(std::sync::atomic::AtomicUsize::new(usize::MAX));
18789 let (seen_in, probe) = (Arc::clone(&seen), Arc::clone(&store));
18790 let builder = app()
18791 .with_flag_store(Arc::clone(&store))
18792 .state_initializer(move |_| {
18793 let preloads = probe.preloads.load(std::sync::atomic::Ordering::SeqCst);
18794 seen_in.store(preloads, std::sync::atomic::Ordering::SeqCst);
18795 });
18796 run_state_initializers(builder.state_initializers, &AppState::for_test());
18797 assert_eq!(seen.load(std::sync::atomic::Ordering::SeqCst), 1);
18798 }
18799
18800 #[test]
18801 fn experiment_store_preloads_before_later_state_initializers() {
18802 let store = Arc::new(crate::experiments::ProbeStore::default());
18803 let seen = Arc::new(std::sync::atomic::AtomicUsize::new(usize::MAX));
18804 let (seen_in, probe) = (Arc::clone(&seen), Arc::clone(&store));
18805 let builder = app()
18806 .with_experiment_store(Arc::clone(&store))
18807 .state_initializer(move |_| {
18808 let preloads = probe.preloads.load(std::sync::atomic::Ordering::SeqCst);
18809 seen_in.store(preloads, std::sync::atomic::Ordering::SeqCst);
18810 });
18811 run_state_initializers(builder.state_initializers, &AppState::for_test());
18812 assert_eq!(seen.load(std::sync::atomic::Ordering::SeqCst), 1);
18813 }
18814
18815 #[test]
18816 fn experiment_store_with_sink_preloads_too() {
18817 let store = Arc::new(crate::experiments::ProbeStore::default());
18818 let builder = app().with_experiment_store_and_sink(
18819 Arc::clone(&store),
18820 Arc::new(crate::experiments::NoOpExposureSink),
18821 );
18822 run_state_initializers(builder.state_initializers, &AppState::for_test());
18823 assert_eq!(store.preloads.load(std::sync::atomic::Ordering::SeqCst), 1);
18824 }
18825
18826 #[test]
18827 fn static_builds_run_state_initializers_before_router_build() {
18828 let source = include_str!("app.rs").replace("\r\n", "\n");
18829 let build_mode_start = source
18830 .find("async fn run_build_mode(self)")
18831 .expect("static build path should exist");
18832 let dump_mode_start = source
18833 .find("async fn run_dump_routes_mode(self)")
18834 .expect("route dump path should follow static build path");
18835 let build_mode_source = &source[build_mode_start..dump_mode_start];
18836 let state_initializer = build_mode_source
18837 .find("run_state_initializers(state_initializers, &state);")
18838 .expect("static build path should run state initializers");
18839 let router_build = build_mode_source
18840 .find("let router = crate::router::try_build_router_inner(")
18841 .expect("static build path should build a router");
18842
18843 assert!(
18844 state_initializer < router_build,
18845 "static builds must install state-initialized resources before rendering routes"
18846 );
18847 }
18848
18849 #[test]
18850 fn graph_installed_before_every_router_build() {
18851 let whole = include_str!("app.rs").replace("\r\n", "\n");
18859 let source = whole
18861 .split_once("\n#[cfg(test)]\nmod tests {")
18862 .map_or(whole.as_str(), |(before, _)| before)
18863 .to_owned();
18864 let builds: Vec<usize> = source
18865 .match_indices("crate::router::try_build_router")
18866 .map(|(i, _)| i)
18867 .filter(|i| source[*i..].starts_with("crate::router::try_build_router"))
18870 .filter(|i| {
18871 let tail = &source[*i..*i + 80];
18872 tail.contains("_inner(") || tail.contains("_with_static_inner(")
18873 })
18874 .collect();
18875 assert!(
18876 builds.len() >= 3,
18877 "expected the serving, static-build and replay router builds: {}",
18878 builds.len()
18879 );
18880 let installs: Vec<usize> = source
18881 .match_indices("crate::graph::install(")
18882 .map(|(i, _)| i)
18883 .collect();
18884 assert_eq!(
18885 installs.len(),
18886 3,
18887 "every path that builds an application router must publish the graph \
18888 it is about to serve"
18889 );
18890 for install in &installs {
18892 assert!(
18893 builds.iter().any(|build| build > install),
18894 "an install with no router build after it is dead code"
18895 );
18896 }
18897 for build in &builds {
18898 assert!(
18901 installs.iter().any(|install| install < build),
18902 "a router built with no graph published before it answers 503 at \
18903 /actuator/graph"
18904 );
18905 }
18906 }
18907
18908 fn census_paths(mounted: &[crate::graph::MountedRoute]) -> Vec<&str> {
18911 mounted.iter().map(|m| m.path.as_str()).collect()
18912 }
18913
18914 #[cfg(feature = "openapi")]
18917 #[test]
18918 fn graph_census_names_openapi_routes_when_enabled() {
18919 let config = AutumnConfig::default();
18920 let openapi =
18921 crate::openapi::OpenApiConfig::new("Demo", "1.0.0").openapi_json_path("/spec.json");
18922 let mounted = graph_mounted_routes(&[], &[], &[], &config, Some(&openapi));
18923 assert!(
18924 census_paths(&mounted).contains(&"/spec.json"),
18925 "{:?}",
18926 census_paths(&mounted)
18927 );
18928 }
18929
18930 #[cfg(feature = "openapi")]
18931 #[test]
18932 fn graph_census_omits_openapi_routes_when_runtime_disabled() {
18933 let mut config = AutumnConfig::default();
18934 config.openapi_runtime.enabled = false;
18935 let openapi =
18936 crate::openapi::OpenApiConfig::new("Demo", "1.0.0").openapi_json_path("/spec.json");
18937 let mounted = graph_mounted_routes(&[], &[], &[], &config, Some(&openapi));
18938 assert!(
18939 !census_paths(&mounted).contains(&"/spec.json"),
18940 "{:?}",
18941 census_paths(&mounted)
18942 );
18943 }
18944
18945 #[test]
18948 fn graph_census_lists_no_probe_when_health_is_off() {
18949 let mut config = AutumnConfig::default();
18950 config.health.enabled = false;
18951 let mounted = graph_mounted_routes(
18952 &[],
18953 &[],
18954 &[],
18955 &config,
18956 #[cfg(feature = "openapi")]
18957 None,
18958 );
18959 let paths = census_paths(&mounted);
18960 for probe in [
18961 &config.health.path,
18962 &config.health.live_path,
18963 &config.health.ready_path,
18964 &config.health.startup_path,
18965 ] {
18966 assert!(!paths.contains(&probe.as_str()), "{probe} in {paths:?}");
18967 }
18968 }
18969
18970 #[test]
18972 fn graph_worker_census_is_probes_and_actuator_only() {
18973 let mut config = AutumnConfig::default();
18974 config.role = crate::config::ProcessRole::Worker;
18975 config.actuator.sensitive = true;
18976 let mounted = graph_worker_mounted_routes(&config);
18977 let paths = census_paths(&mounted);
18978 assert!(
18979 paths.contains(&config.health.live_path.as_str()),
18980 "{paths:?}"
18981 );
18982 assert!(
18983 paths.iter().any(|p| p.ends_with("/graph")),
18984 "the actuator graph endpoint is served: {paths:?}"
18985 );
18986 assert!(!paths.contains(&"/static/{*path}"), "{paths:?}");
18987 assert!(
18988 !paths.iter().any(|p| p.starts_with("/static/")),
18989 "no asset routes on a worker: {paths:?}"
18990 );
18991 assert!(
18992 mounted.iter().all(|route| !route.auth_declared),
18993 "framework endpoints are not declared plugin mounts"
18994 );
18995 }
18996
18997 #[test]
19000 fn graph_declared_summary_declares_auth_only_when_classified() {
19001 use crate::route_listing::{RouteClassification, RouteInfo};
19002 let summary = |classification| {
19003 let mut info = RouteInfo::framework_get("/plugin/ping".to_owned(), "ping");
19004 info.classification = classification;
19005 graph_declared_route_summary(&info).auth_declared
19006 };
19007 assert!(summary(RouteClassification::Gated));
19008 assert!(summary(RouteClassification::Public));
19009 assert!(!summary(RouteClassification::Unclassified));
19010 }
19011
19012 #[derive(Clone)]
19013 struct CensusLoader(Result<AutumnConfig, String>);
19014
19015 impl crate::config::ConfigLoader for CensusLoader {
19016 async fn load(&self) -> Result<AutumnConfig, crate::config::ConfigError> {
19017 self.0
19018 .clone()
19019 .map_err(crate::config::ConfigError::Validation)
19020 }
19021 }
19022
19023 #[tokio::test]
19026 #[allow(
19027 clippy::await_holding_lock,
19028 reason = "the guard spans the config load, which resets the global metric limits"
19029 )]
19030 async fn dump_graph_surface_uses_builder_loader_and_worker_role() {
19031 let _limits_lock = crate::metrics::LIMITS_TEST_LOCK
19034 .lock()
19035 .unwrap_or_else(std::sync::PoisonError::into_inner);
19036 let mut config = AutumnConfig::default();
19037 config.role = crate::config::ProcessRole::Worker;
19038 config.health.live_path = "/custom-live".to_owned();
19039 let builder = app().with_config_loader(CensusLoader(Ok(config)));
19040 let (mounted, opaque) = builder.dump_graph_surface().await;
19041 let paths = census_paths(&mounted);
19042 assert!(paths.contains(&"/custom-live"), "{paths:?}");
19043 assert!(!paths.contains(&"/static/{*path}"), "{paths:?}");
19044 assert_eq!(opaque, 0);
19045 }
19046
19047 #[tokio::test]
19048 #[allow(
19049 clippy::await_holding_lock,
19050 reason = "the guard spans the config load, which resets the global metric limits"
19051 )]
19052 async fn dump_graph_surface_falls_back_to_defaults_when_loader_fails() {
19053 let _limits_lock = crate::metrics::LIMITS_TEST_LOCK
19056 .lock()
19057 .unwrap_or_else(std::sync::PoisonError::into_inner);
19058 let builder = app().with_config_loader(CensusLoader(Err("boom".to_owned())));
19059 let (mounted, _) = builder.dump_graph_surface().await;
19060 assert!(
19061 census_paths(&mounted).contains(&"/static/{*path}"),
19062 "{:?}",
19063 census_paths(&mounted)
19064 );
19065 }
19066
19067 #[test]
19068 fn migrate_only_one_shot_applies_and_exits_without_serving() {
19069 let source = include_str!("app.rs").replace("\r\n", "\n");
19077 let run_start = source.find("pub async fn run(self)").expect("run() exists");
19078 let run_end = source
19079 .find("async fn run_build_mode(self)")
19080 .expect("build mode follows run()");
19081 let run_body = &source[run_start..run_end];
19082
19083 let dispatch = run_body
19087 .find("if is_migrate_only_mode() {")
19088 .expect("run() dispatches the migrate one-shot");
19089 let server_start = run_body
19090 .find("let Self {")
19091 .expect("run() destructures self to start the server");
19092 assert!(
19093 dispatch < server_start,
19094 "AUTUMN_MIGRATE must be handled before the server-start path"
19095 );
19096 let migrate_branch = &run_body[dispatch..server_start];
19097 assert!(
19098 migrate_branch.contains("self.run_migrate_only_mode().await;")
19099 && migrate_branch.contains("return;"),
19100 "the migrate one-shot must run then return before server start"
19101 );
19102
19103 let handler_start = source
19105 .find("async fn run_migrate_only_mode(self)")
19106 .expect("migrate handler exists");
19107 let handler_end = source
19108 .find("async fn run_one_off_task_mode(self, requested_name: String)")
19109 .expect("one-off task handler follows the migrate handler");
19110 let handler = &source[handler_start..handler_end];
19111 assert!(
19112 handler.contains("apply_pending_or_exit"),
19113 "the migrate handler applies pending migrations per target"
19114 );
19115 assert!(
19116 handler.contains("std::process::exit(0)"),
19117 "the migrate handler exits after applying"
19118 );
19119
19120 let guard_call = handler
19126 .find("sqlite_sharding_unsupported_guard(")
19127 .expect("migrate handler applies the SQLite sharding guard");
19128 let first_apply = handler
19129 .find("apply_pending_or_exit")
19130 .expect("migrate handler applies per target");
19131 assert!(
19132 guard_call < first_apply,
19133 "the SQLite guard must run BEFORE the migration loop / apply_pending_or_exit"
19134 );
19135 assert!(
19136 !handler.contains("initialize_job_runtime")
19137 && !handler.contains("try_build_router_inner"),
19138 "the migrate one-shot must not start the server"
19139 );
19140
19141 let helper_start = source
19145 .find("fn apply_pending_or_exit(")
19146 .expect("apply_pending_or_exit exists");
19147 let helper_end = source[helper_start..]
19148 .find("\n}\n")
19149 .map_or(source.len(), |end| helper_start + end);
19150 let helper = &source[helper_start..helper_end];
19151 assert!(
19152 helper.contains("crate::migrate::run_pending_locked_with_policy("),
19153 "must reuse the shared locked applier, not duplicate migration logic"
19154 );
19155 assert!(
19156 helper.contains("std::process::exit(1)"),
19157 "a failed migration must exit non-zero (abort before cutover)"
19158 );
19159 }
19160
19161 #[cfg(feature = "reporting")]
19164 fn replay_mode_source(source: &str) -> &str {
19165 let start = source
19166 .find("async fn run_replay_mode(self, capsule_path: String)")
19167 .expect("replay handler exists");
19168 let end = source
19169 .find("pub(crate) fn is_static_build_mode()")
19170 .expect("the mode predicates follow the AppBuilder impl block");
19171 source
19172 .get(start..end)
19173 .expect("the replay handler precedes the mode predicates")
19174 }
19175
19176 #[cfg(feature = "reporting")]
19177 #[test]
19178 fn replay_one_shot_runs_before_server_start() {
19179 let source = include_str!("app.rs").replace("\r\n", "\n");
19186 let run_start = source.find("pub async fn run(self)").expect("run() exists");
19187 let run_end = source
19188 .find("async fn run_build_mode(self)")
19189 .expect("build mode follows run()");
19190 let run_body = source
19191 .get(run_start..run_end)
19192 .expect("run() precedes build mode");
19193
19194 let dispatch = run_body
19195 .find("if is_replay_mode()")
19196 .expect("run() dispatches the replay one-shot");
19197 let server_start = run_body
19198 .find("let Self {")
19199 .expect("run() destructures self to start the server");
19200 assert!(
19201 dispatch < server_start,
19202 "AUTUMN_REPLAY_CAPSULE must be handled before the server-start path"
19203 );
19204 let replay_branch = run_body
19205 .get(dispatch..server_start)
19206 .expect("the dispatch precedes server start");
19207 assert!(
19208 replay_branch.contains("self.run_replay_mode(capsule_path).await;")
19209 && replay_branch.contains("return;"),
19210 "the replay one-shot must run then return before server start"
19211 );
19212
19213 let handler = replay_mode_source(&source);
19214 assert!(
19215 handler.contains("std::process::exit("),
19216 "the replay handler exits with the verdict's code"
19217 );
19218 assert!(
19219 !handler.contains("axum::serve") && !handler.contains("http_server::serve"),
19220 "the replay one-shot must never start the server"
19221 );
19222
19223 assert!(
19227 handler.contains("crate::capsule::load_capsule(")
19228 && handler.contains("crate::capsule::execute(")
19229 && handler.contains("crate::capsule::print_verdict("),
19230 "the replay handler loads the capsule, executes it and prints a verdict"
19231 );
19232 assert!(
19233 handler.contains("crate::router::try_build_router_inner(")
19234 && handler.contains("run_state_initializers(state_initializers, &state);")
19235 && handler.contains("register(state.policy_registry());"),
19236 "the replay handler must rebuild the app's real router and state"
19237 );
19238
19239 assert!(
19243 !handler.contains("setup_database(")
19244 && !handler.contains("initialize_job_runtime")
19245 && !handler.contains("start_task_scheduler")
19246 && !handler.contains("preflight_storage("),
19247 "replay must not run migrations, job workers or storage preflight"
19248 );
19249 assert!(
19250 handler.contains("force_offline_replay_config(&mut config);"),
19251 "the replay handler must force the offline configuration knobs"
19252 );
19253
19254 let forcer_start = source
19255 .find("fn force_offline_replay_config(")
19256 .expect("the offline-knob helper exists");
19257 let forcer = source
19258 .get(forcer_start..forcer_start.saturating_add(2_000))
19259 .unwrap_or_default();
19260 assert!(
19261 forcer.contains("SessionBackend::Memory"),
19262 "replay must force the in-memory session store (no Redis)"
19263 );
19264 assert!(
19265 forcer.contains("failure_capture.enabled = false"),
19266 "replay must not capture a capsule of the replay itself"
19267 );
19268 }
19269
19270 #[cfg(feature = "reporting")]
19279 #[test]
19280 fn replay_reaches_nothing_outside_the_capsule() {
19281 let source = include_str!("app.rs").replace("\r\n", "\n");
19282 let handler = replay_mode_source(&source);
19283
19284 assert!(
19285 handler.contains("crate::http_client::block_outbound_for_replay();"),
19286 "the replay handler must block outbound HTTP before rebuilding the app"
19287 );
19288 assert!(
19289 handler.contains("crate::capsule::boundary::block_egress_for_replay();"),
19290 "the replay handler must block egress outside the HTTP client in every build"
19291 );
19292 assert!(
19293 handler.contains("crate::cache::block_backends_for_replay();"),
19294 "the replay handler must keep every cache backend offline"
19295 );
19296 assert!(
19297 handler.contains("channels_backend: _replay_ignores_custom_channels_backend,"),
19298 "the replay handler must drop a custom channels backend, which outranks the \
19299 forced in-process one"
19300 );
19301 assert!(
19302 !handler.contains(" channels_backend,\n"),
19303 "the replay handler must not forward the builder's channels backend"
19304 );
19305
19306 let forcer_start = source
19307 .find("fn force_offline_replay_config(")
19308 .expect("the offline-knob helper exists");
19309 let forcer = source
19313 .get(forcer_start..forcer_start.saturating_add(6_000))
19314 .unwrap_or_default();
19315 assert!(
19316 forcer.contains("config.channels.backend = crate::config::ChannelBackend::InProcess;"),
19317 "replay must force the in-process channels backend (no Redis fan-out)"
19318 );
19319 assert!(
19320 forcer.contains("config.server.timeouts.request_timeout_ms = None;"),
19321 "replay must clear the wall-clock request deadline"
19322 );
19323 assert!(
19326 forcer.contains("config.shadow.enabled = false;"),
19327 "replay must not mirror traffic to a shadow target"
19328 );
19329 }
19330
19331 #[cfg(all(feature = "reporting", feature = "mail"))]
19334 #[test]
19335 fn replay_installs_a_tape_backed_mailer() {
19336 let source = include_str!("app.rs").replace("\r\n", "\n");
19337 let handler = replay_mode_source(&source);
19338 let install =
19339 "crate::mail::install_replay_mailer(&state, &config.mail, builder_mail_queue);";
19340 assert!(
19341 handler.contains(install),
19342 "the replay handler must install the tape-backed mailer"
19343 );
19344 let initializers = handler
19348 .find("run_state_initializers(state_initializers, &state);")
19349 .expect("the replay handler runs the state initializers");
19350 let first = handler.find(install).expect("installed");
19351 let last = handler.rfind(install).expect("installed");
19352 assert!(first < initializers, "installed before the initializers");
19353 assert!(last > initializers, "and again after them");
19354 #[cfg(feature = "db")]
19357 {
19358 let outbox = handler
19359 .find("crate::outbox::installs_mail_queue(&state, &config.outbox)")
19360 .expect("the replay handler counts the outbox's mail queue");
19361 assert!(outbox < first, "counted before the mailer is installed");
19362 }
19363 }
19364
19365 #[cfg(feature = "reporting")]
19369 #[test]
19370 fn replay_installs_each_cache_where_production_did() {
19371 let source = include_str!("app.rs").replace("\r\n", "\n");
19372 let handler = replay_mode_source(&source);
19373 let initializers = handler
19374 .find("run_state_initializers(state_initializers, &state);")
19375 .expect("the replay handler runs the state initializers");
19376 let builder = handler
19377 .find("crate::cache::install_replay_cache(cache_backend.is_some())")
19378 .expect("installs a builder's cache");
19379 let late = handler
19380 .find("crate::cache::install_late_replay_cache(&state, &capsule.effects)")
19381 .expect("installs a startup hook's cache");
19382 assert!(builder < initializers && late > initializers);
19383 }
19384
19385 #[cfg(feature = "reporting")]
19388 #[test]
19389 fn the_offline_replay_config_reaches_no_live_service() {
19390 let mut config = AutumnConfig::default();
19391 config.failure_capture.enabled = true;
19392 config.session.backend = crate::session::SessionBackend::Redis;
19393 config.channels.backend = crate::config::ChannelBackend::Redis;
19394 config.server.timeouts.request_timeout_ms = Some(30_000);
19395 config.security.rate_limit.backend = crate::security::config::RateLimitBackend::Redis;
19398 config.idempotency.backend = crate::config::IdempotencyBackend::Redis;
19399 config.security.submit_token.backend = Some(crate::config::IdempotencyBackend::Redis);
19400 config.security.webhooks.replay.backend = crate::webhook::WebhookReplayBackend::Redis;
19401 config.cache.backend = crate::config::CacheBackend::Redis;
19402 config.jobs.backend = "redis".to_owned();
19403
19404 force_offline_replay_config(&mut config);
19405
19406 assert!(
19407 !config.failure_capture.enabled,
19408 "a replay must not capture a capsule of itself"
19409 );
19410 assert_eq!(
19411 config.session.backend,
19412 crate::session::SessionBackend::Memory
19413 );
19414 assert!(config.session.allow_memory_in_production);
19415 assert_eq!(
19416 config.channels.backend,
19417 crate::config::ChannelBackend::InProcess,
19418 "a replayed app must not dial the application's Redis fan-out"
19419 );
19420 assert_eq!(
19421 config.server.timeouts.request_timeout_ms, None,
19422 "a deterministic offline replay has no wall-clock deadline — and a \
19423 breakpoint held in a debugger must not cancel the handler"
19424 );
19425 assert_eq!(
19429 config.security.rate_limit.backend,
19430 crate::security::config::RateLimitBackend::Memory,
19431 "a replay must not consume the deployment's shared rate-limit budget"
19432 );
19433 assert_eq!(
19434 config.idempotency.backend,
19435 crate::config::IdempotencyBackend::Memory,
19436 "a replay must not take an idempotency key or its in-flight lock"
19437 );
19438 assert!(config.idempotency.allow_memory_in_production);
19439 assert_eq!(
19440 config.security.submit_token.backend,
19441 Some(crate::config::IdempotencyBackend::Memory),
19442 "submit tokens inherit the idempotency backend when unset, so a replay \
19443 must pin them explicitly rather than rely on that inheritance"
19444 );
19445 assert_eq!(
19446 config.security.webhooks.replay.backend,
19447 crate::webhook::WebhookReplayBackend::Memory,
19448 "a replay must not insert or delete webhook replay-protection keys"
19449 );
19450 assert!(config.security.webhooks.replay.allow_memory_in_production);
19451 assert_eq!(
19452 config.cache.backend,
19453 crate::config::CacheBackend::Memory,
19454 "a `#[cached]` handler must not read or populate the shared cache"
19455 );
19456 assert_eq!(
19457 config.jobs.backend, "local",
19458 "a job enqueued during a replay must not reach a queue a live worker drains"
19459 );
19460 }
19461
19462 #[cfg(feature = "reporting")]
19469 #[test]
19470 fn replay_never_uses_the_applications_custom_session_store() {
19471 let source = include_str!("app.rs").replace("\r\n", "\n");
19472 let handler = replay_mode_source(&source);
19473
19474 assert!(
19475 handler.contains("session_store: None,"),
19476 "the replay router context must be built with no custom session store"
19477 );
19478 assert!(
19479 !handler.contains(" session_store,"),
19480 "the replay handler must not forward the builder's session store; \
19481 a custom store outranks the forced memory backend"
19482 );
19483 }
19484
19485 #[cfg(feature = "reporting")]
19486 #[test]
19487 fn replay_never_invokes_a_custom_config_loader() {
19488 let source = include_str!("app.rs").replace("\r\n", "\n");
19492 let handler = replay_mode_source(&source);
19493
19494 assert!(
19495 handler.contains("_replay_ignores_custom_config_loader = config_loader_factory"),
19496 "the replay handler must drop the builder's config loader — the documented \
19497 implementations call live services (Secrets Manager, Vault, Consul, HTTP)"
19498 );
19499 assert!(
19500 handler.contains("load_config_and_telemetry(\n None,"),
19501 "replay must load configuration through the default local path, never a \
19502 custom loader"
19503 );
19504 }
19505
19506 #[cfg(feature = "reporting")]
19507 #[test]
19508 fn replay_mode_reads_capsule_env_var() {
19509 temp_env::with_var(
19510 "AUTUMN_REPLAY_CAPSULE",
19511 Some("tmp/capsules/abc.json"),
19512 || {
19513 assert!(is_replay_mode(), "a capsule path selects the replay path");
19514 assert_eq!(
19515 replay_capsule_from_env().as_deref(),
19516 Some("tmp/capsules/abc.json")
19517 );
19518 },
19519 );
19520 temp_env::with_var("AUTUMN_REPLAY_CAPSULE", Some(" spaced.json "), || {
19521 assert_eq!(
19522 replay_capsule_from_env().as_deref(),
19523 Some("spaced.json"),
19524 "a shell-quoted path must be trimmed"
19525 );
19526 });
19527 temp_env::with_var("AUTUMN_REPLAY_CAPSULE", Some(" "), || {
19528 assert!(
19529 !is_replay_mode(),
19530 "a blank value must fall through to the normal boot path"
19531 );
19532 });
19533 temp_env::with_var("AUTUMN_REPLAY_CAPSULE", None::<&str>, || {
19534 assert!(!is_replay_mode(), "unset must not select the replay path");
19535 });
19536 }
19537
19538 #[cfg(feature = "db")]
19539 #[test]
19540 fn hooked_repository_apps_include_hook_queue_framework_migration() {
19541 let migrations = migrations_with_repository_framework_migrations(
19542 vec![("app", APP_TEST_MIGRATIONS)],
19543 true,
19544 false,
19545 false,
19546 RepositoryCommitHookQueueMigrationMode::Runtime,
19547 );
19548 let names = migration_names(&migrations);
19549
19550 assert!(
19551 names
19552 .iter()
19553 .any(|name| name == REPOSITORY_COMMIT_HOOK_QUEUE_MIGRATION),
19554 "hooked repository apps must auto-register the durable hook queue migration"
19555 );
19556 assert!(
19557 names.iter().all(|name| !name.contains("api_tokens")),
19558 "hooked repository apps must not auto-register unrelated framework migrations: {names:?}"
19559 );
19560 }
19561
19562 #[cfg(feature = "db")]
19563 #[test]
19564 fn runtime_hooked_apps_include_hook_queue_framework_migration_without_app_migrations() {
19565 let migrations = migrations_with_repository_framework_migrations(
19566 Vec::new(),
19567 true,
19568 false,
19569 false,
19570 RepositoryCommitHookQueueMigrationMode::Runtime,
19571 );
19572 let names = migration_names(&migrations);
19573
19574 assert!(
19575 names
19576 .iter()
19577 .any(|name| name == REPOSITORY_COMMIT_HOOK_QUEUE_MIGRATION),
19578 "runtime hooked repository apps must install the durable hook queue even when app migrations are managed elsewhere"
19579 );
19580 }
19581
19582 #[cfg(feature = "db")]
19583 #[test]
19584 fn versioned_repository_apps_include_version_history_framework_migration() {
19585 let migrations = migrations_with_repository_framework_migrations(
19586 vec![("app", APP_TEST_MIGRATIONS)],
19587 false,
19588 true,
19589 false,
19590 RepositoryCommitHookQueueMigrationMode::Runtime,
19591 );
19592 let names = migration_names(&migrations);
19593
19594 assert!(
19595 names.iter().any(|name| name == VERSION_HISTORY_MIGRATION),
19596 "versioned repository apps must auto-register the version-history migration"
19597 );
19598 assert!(
19599 names
19600 .iter()
19601 .all(|name| !name.contains("repository_commit_hook_queue")),
19602 "versioned-only repository apps must not auto-register the durable hook queue: {names:?}"
19603 );
19604 }
19605
19606 #[cfg(feature = "db")]
19607 #[test]
19608 fn runtime_versioned_apps_include_version_history_framework_migration_without_app_migrations() {
19609 let migrations = migrations_with_repository_framework_migrations(
19610 Vec::new(),
19611 false,
19612 true,
19613 false,
19614 RepositoryCommitHookQueueMigrationMode::Runtime,
19615 );
19616 let names = migration_names(&migrations);
19617
19618 assert!(
19619 names.iter().any(|name| name == VERSION_HISTORY_MIGRATION),
19620 "runtime versioned repository apps must install version history even when app migrations are managed elsewhere"
19621 );
19622 }
19623
19624 #[cfg(feature = "db")]
19625 #[test]
19626 fn static_builds_do_not_auto_add_hook_queue_when_no_migrations_registered() {
19627 let migrations = migrations_with_repository_framework_migrations(
19628 Vec::new(),
19629 true,
19630 true,
19631 true,
19632 RepositoryCommitHookQueueMigrationMode::StaticBuild,
19633 );
19634
19635 assert!(
19636 migrations.is_empty(),
19637 "static/export builds that pass no migrations must not mutate the database"
19638 );
19639 }
19640
19641 #[cfg(feature = "db")]
19642 #[test]
19643 fn directory_migration_required_only_at_runtime_with_shards_and_routing() {
19644 use RepositoryCommitHookQueueMigrationMode::{Runtime, StaticBuild};
19645
19646 assert!(directory_migration_is_required(true, true, Runtime));
19648
19649 assert!(!directory_migration_is_required(true, true, StaticBuild));
19652
19653 assert!(!directory_migration_is_required(false, true, Runtime));
19655 assert!(!directory_migration_is_required(true, false, Runtime));
19656 }
19657
19658 #[test]
19659 fn shard_map_migration_required_only_at_runtime_with_shards() {
19660 use RepositoryCommitHookQueueMigrationMode::{Runtime, StaticBuild};
19661
19662 assert!(shard_map_migration_is_required(true, Runtime));
19664
19665 assert!(!shard_map_migration_is_required(true, StaticBuild));
19667
19668 assert!(!shard_map_migration_is_required(false, Runtime));
19670 }
19671
19672 #[cfg(feature = "db")]
19673 #[test]
19674 fn unhooked_apps_do_not_auto_add_hook_queue_framework_migration() {
19675 let migrations = migrations_with_repository_framework_migrations(
19676 Vec::new(),
19677 false,
19678 false,
19679 false,
19680 RepositoryCommitHookQueueMigrationMode::Runtime,
19681 );
19682
19683 assert!(
19684 migrations.is_empty(),
19685 "unhooked apps should not get durable hook queue migrations for free"
19686 );
19687 }
19688
19689 #[cfg(feature = "db")]
19690 #[test]
19691 fn apps_with_a_derivation_include_the_derivation_state_migration() {
19692 let migrations = migrations_with_repository_framework_migrations(
19693 vec![("app", APP_TEST_MIGRATIONS)],
19694 false,
19695 false,
19696 true,
19697 RepositoryCommitHookQueueMigrationMode::Runtime,
19698 );
19699 let names = migration_names(&migrations);
19700
19701 assert!(
19702 names.iter().any(|name| name == DERIVATION_MIGRATION),
19703 "an app that declares a `#[derivation]` must auto-register its \
19704 backfill state table: {names:?}"
19705 );
19706 assert!(
19707 names.iter().all(|name| !name.contains("version_history")),
19708 "a derivation alone must not drag in unrelated framework tables: {names:?}"
19709 );
19710 }
19711
19712 #[cfg(feature = "db")]
19713 #[test]
19714 fn apps_without_a_derivation_do_not_get_the_state_table() {
19715 let migrations = migrations_with_repository_framework_migrations(
19718 vec![("app", APP_TEST_MIGRATIONS)],
19719 false,
19720 false,
19721 false,
19722 RepositoryCommitHookQueueMigrationMode::Runtime,
19723 );
19724 assert!(
19725 !migration_names(&migrations)
19726 .iter()
19727 .any(|name| name == DERIVATION_MIGRATION)
19728 );
19729 }
19730
19731 #[cfg(feature = "db")]
19732 fn migration_names(migrations: &[(&str, crate::migrate::EmbeddedMigrations)]) -> Vec<String> {
19733 use diesel::migration::{Migration, MigrationSource as _};
19734 use diesel::pg::Pg;
19735
19736 migrations
19737 .iter()
19738 .flat_map(|(_, source)| {
19739 let migrations: Vec<Box<dyn Migration<Pg>>> = source.migrations().unwrap();
19740 migrations
19741 })
19742 .map(|migration| migration.name().to_string())
19743 .collect()
19744 }
19745
19746 #[cfg(feature = "db")]
19747 #[test]
19748 fn control_framework_filter_skips_control_but_keeps_shard_required_sets() {
19749 assert!(migration_set_is_control_framework(
19751 &crate::migrate::FRAMEWORK_MIGRATIONS
19752 ));
19753 assert!(!migration_set_is_control_framework(
19757 &crate::version_history::VERSION_HISTORY_MIGRATIONS
19758 ));
19759 assert!(!migration_set_is_control_framework(
19760 &crate::repository_commit_hooks::REPOSITORY_COMMIT_HOOK_MIGRATIONS
19761 ));
19762 assert!(!migration_set_is_control_framework(
19763 &crate::derivation::DERIVATION_MIGRATIONS
19764 ));
19765 }
19766
19767 #[cfg(feature = "db")]
19768 #[test]
19769 fn sharded_app_with_full_framework_still_gets_shard_required_sets() {
19770 use diesel::migration::{Migration, MigrationSource as _};
19771 use diesel::pg::Pg;
19772
19773 let migrations = migrations_with_repository_framework_migrations(
19780 vec![("app", crate::migrate::FRAMEWORK_MIGRATIONS)],
19781 true,
19782 true,
19783 true,
19784 RepositoryCommitHookQueueMigrationMode::Runtime,
19785 );
19786
19787 let shard_names: Vec<String> = migrations
19790 .iter()
19791 .filter(|(_, set)| !migration_set_is_control_framework(set))
19792 .flat_map(|(_, set)| {
19793 let ms: Vec<Box<dyn Migration<Pg>>> = set.migrations().unwrap_or_default();
19794 ms.into_iter()
19795 .map(|m| m.name().to_string())
19796 .collect::<Vec<_>>()
19797 })
19798 .collect();
19799
19800 assert!(
19801 shard_names
19802 .iter()
19803 .any(|name| name == REPOSITORY_COMMIT_HOOK_QUEUE_MIGRATION),
19804 "shards must receive the commit-hook queue migration even when the full \
19805 control framework set is also registered: {shard_names:?}"
19806 );
19807 assert!(
19808 shard_names
19809 .iter()
19810 .any(|name| name == VERSION_HISTORY_MIGRATION),
19811 "shards must receive the version-history migration even when the full \
19812 control framework set is also registered: {shard_names:?}"
19813 );
19814 assert!(
19815 shard_names.iter().any(|name| name == DERIVATION_MIGRATION),
19816 "shards maintain derivations too, so they need the state table: \
19817 {shard_names:?}"
19818 );
19819 }
19820
19821 #[cfg(feature = "db")]
19822 #[test]
19823 fn plugin_migrations_registers_alongside_app_migrations() {
19824 const APP_MIGRATIONS: crate::migrate::EmbeddedMigrations =
19825 diesel_migrations::embed_migrations!("../examples/todo-app/migrations");
19826 const PLUGIN_MIGRATIONS: crate::migrate::EmbeddedMigrations =
19827 diesel_migrations::embed_migrations!("tests/fixtures/plugin_migrations_ok");
19828
19829 let builder = app()
19830 .migrations(APP_MIGRATIONS)
19831 .plugin_migrations("test-plugin", PLUGIN_MIGRATIONS);
19832
19833 let names = migration_names(&builder.migrations);
19834 assert!(
19835 names.iter().any(|n| n == "00000000000000_create_todos"),
19836 "app-registered migrations must still be applied: {names:?}"
19837 );
19838 assert!(
19839 names.iter().any(|n| n == "20260101000000_create_gizmos"),
19840 "plugin-registered migrations must be applied too: {names:?}"
19841 );
19842 }
19843
19844 #[cfg(feature = "db")]
19845 #[test]
19846 fn plugin_migrations_registration_never_panics_on_version_collision() {
19847 const APP_MIGRATIONS: crate::migrate::EmbeddedMigrations =
19857 diesel_migrations::embed_migrations!("../examples/todo-app/migrations");
19858 const COLLIDING_PLUGIN_MIGRATIONS: crate::migrate::EmbeddedMigrations =
19859 diesel_migrations::embed_migrations!("tests/fixtures/plugin_migrations_collision");
19860
19861 let builder = app()
19862 .migrations(APP_MIGRATIONS)
19863 .plugin_migrations("test-plugin", COLLIDING_PLUGIN_MIGRATIONS);
19864
19865 let names = migration_names(&builder.migrations);
19866 assert!(
19867 names.iter().any(|n| n == "00000000000000_create_todos"),
19868 "the app's own colliding migration must still be registered: {names:?}"
19869 );
19870 assert!(
19871 names.iter().any(|n| n == "00000000000000_create_gadgets"),
19872 "the plugin's colliding migration must still be registered: {names:?}"
19873 );
19874 }
19875
19876 #[cfg(feature = "db")]
19877 #[test]
19878 fn plugin_migrations_does_not_panic_on_identical_resubmission() {
19879 const PLUGIN_MIGRATIONS: crate::migrate::EmbeddedMigrations =
19884 diesel_migrations::embed_migrations!("tests/fixtures/plugin_migrations_ok");
19885
19886 let _ = app()
19887 .plugin_migrations("plugin-a", PLUGIN_MIGRATIONS)
19888 .plugin_migrations("plugin-b", PLUGIN_MIGRATIONS);
19889 }
19890
19891 #[cfg(not(feature = "sqlite"))]
19901 #[cfg(feature = "db")]
19902 #[test]
19903 fn configure_replica_migration_check_stores_recheck_urls() {
19904 let mut config = AutumnConfig::default();
19905 config.database.primary_url = Some("postgres://localhost/primary".to_owned());
19906 config.database.replica_url = Some("postgres://localhost/replica".to_owned());
19907 let topology = crate::db::create_topology(&config.database)
19908 .expect("topology should build")
19909 .expect("database should be configured");
19910
19911 let state = build_state(
19912 &config,
19913 Some(&topology),
19914 None,
19915 #[cfg(feature = "ws")]
19916 None,
19917 );
19918
19919 assert!(
19920 state.probes().replica_migration_check().is_none(),
19921 "build_state should not enable migration checks without registered migrations"
19922 );
19923
19924 configure_replica_migration_check(
19925 &state,
19926 Some((
19927 "postgres://localhost/primary".to_owned(),
19928 "postgres://localhost/replica".to_owned(),
19929 )),
19930 );
19931
19932 let check = state
19933 .probes()
19934 .replica_migration_check()
19935 .expect("replica migration check should be configured");
19936
19937 assert_eq!(check.primary_url, "postgres://localhost/primary");
19938 assert_eq!(check.replica_url, "postgres://localhost/replica");
19939 }
19940
19941 #[cfg(not(feature = "sqlite"))]
19951 #[cfg(feature = "db")]
19952 #[tokio::test]
19953 async fn replica_migration_readiness_marks_ready_endpoint_degraded() {
19954 let mut config = AutumnConfig::default();
19955 config.database.primary_url = Some("postgres://localhost/primary".to_owned());
19956 config.database.primary_pool_size = Some(5);
19957 config.database.replica_url = Some("postgres://localhost/replica".to_owned());
19958 config.database.replica_pool_size = Some(2);
19959 config.database.replica_fallback = crate::config::ReplicaFallback::FailReadiness;
19960 let topology = crate::db::create_topology(&config.database)
19961 .expect("topology should build")
19962 .expect("database should be configured");
19963 let state = build_state(
19964 &config,
19965 Some(&topology),
19966 None,
19967 #[cfg(feature = "ws")]
19968 None,
19969 );
19970
19971 apply_replica_migration_readiness(
19972 &state,
19973 Some(crate::migrate::ReplicaMigrationReadiness::Stale {
19974 primary_latest: Some("00000000000002".to_owned()),
19975 replica_latest: Some("00000000000001".to_owned()),
19976 }),
19977 );
19978
19979 let (status, _) = crate::probe::readiness_response(&state).await;
19980
19981 assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE);
19982 }
19983
19984 #[cfg(feature = "db")]
19985 #[tokio::test]
19986 async fn blocking_replica_migration_readiness_reports_unknown_connection_errors() {
19987 let readiness = crate::migrate::check_replica_migration_readiness_blocking(
19988 "not-a-primary-url".to_owned(),
19989 "not-a-replica-url".to_owned(),
19990 )
19991 .await;
19992
19993 assert!(matches!(
19994 readiness,
19995 crate::migrate::ReplicaMigrationReadiness::Unknown(_)
19996 ));
19997 }
19998
19999 #[cfg(feature = "ws")]
20000 #[test]
20001 fn with_channels_backend_overrides_config_driven_backend_selection() {
20002 let builder = app().with_channels_backend(crate::channels::LocalChannelsBackend::new(4));
20003 let AppBuilder {
20004 channels_backend, ..
20005 } = builder;
20006 assert!(channels_backend.is_some());
20007
20008 let mut config = AutumnConfig::default();
20009 config.channels.backend = crate::config::ChannelBackend::Redis;
20010 config.channels.redis.url = None;
20011
20012 let state = build_state(
20013 &config,
20014 #[cfg(feature = "db")]
20015 None,
20016 #[cfg(feature = "db")]
20017 None,
20018 #[cfg(feature = "ws")]
20019 channels_backend,
20020 );
20021 let mut rx = state.channels().subscribe("override");
20022
20023 state
20024 .broadcast()
20025 .publish("override", "ok")
20026 .expect("custom local backend should publish");
20027
20028 assert_eq!(rx.try_recv().expect("message should arrive").as_str(), "ok");
20029 }
20030
20031 pub fn test_get_route(path: &'static str, name: &'static str) -> Route {
20033 Route {
20034 method: http::Method::GET,
20035 path,
20036 handler: axum::routing::get(|| async { "ok" }),
20037 name,
20038 api_doc: crate::openapi::ApiDoc {
20039 method: "GET",
20040 path,
20041 operation_id: name,
20042 success_status: 200,
20043 ..Default::default()
20044 },
20045 repository: None,
20046 idempotency: crate::route::RouteIdempotency::Direct,
20047 timeout: crate::route::RouteTimeout::Inherit,
20048 criticality: crate::admission::Criticality::Default,
20049 seo: crate::seo::SeoRouteDefaults::EMPTY,
20050 api_version: None,
20051 sunset_opt_out: false,
20052 }
20053 }
20054
20055 #[cfg(feature = "i18n")]
20056 fn test_i18n_bundle(key: &str, value: &str) -> Arc<crate::i18n::Bundle> {
20057 let mut messages = std::collections::HashMap::new();
20058 let mut en = std::collections::HashMap::new();
20059 en.insert(key.to_owned(), value.to_owned());
20060 messages.insert("en".to_owned(), en);
20061 Arc::new(crate::i18n::Bundle::from_messages(
20062 messages,
20063 &crate::i18n::I18nConfig::default(),
20064 ))
20065 }
20066
20067 #[cfg(feature = "i18n")]
20068 #[test]
20069 fn i18n_auto_defers_loading_until_runtime_config_is_available() {
20070 let builder = app().i18n_auto();
20071
20072 assert!(builder.i18n_bundle.is_none());
20073 assert!(builder.i18n_auto_load);
20074 }
20075
20076 #[cfg(feature = "i18n")]
20084 fn static_meta(path: &'static str) -> crate::static_gen::StaticRouteMeta {
20085 crate::static_gen::StaticRouteMeta {
20086 path,
20087 name: "test_static_route",
20088 revalidate: None,
20089 params_fn: None,
20090 seo: crate::seo::SeoRouteDefaults::EMPTY,
20091 content_type: None,
20092 }
20093 }
20094
20095 #[cfg(feature = "i18n")]
20096 #[test]
20097 fn static_routes_are_excluded_when_locale_prefix_is_enabled() {
20098 let mut config = AutumnConfig::default();
20099 config.i18n.locale_prefix_enabled = true;
20100 let metas = vec![static_meta("/about"), static_meta("/pricing")];
20101
20102 exclude_static_routes_from_locale_prefix(&mut config, &metas);
20103
20104 assert_eq!(
20105 config.i18n.locale_prefix_exclude_exact,
20106 vec!["/about".to_owned(), "/pricing".to_owned()],
20107 "static routes must be tracked as EXACT exclusions, not prefix \
20108 exclusions — see the sibling `static_route_exclusion_does_not_leak_into_a_dynamic_sibling` test"
20109 );
20110 assert!(
20111 config.i18n.locale_prefix_exclude.is_empty(),
20112 "must not write static route paths into the prefix-matched exclude list"
20113 );
20114 }
20115
20116 #[cfg(feature = "i18n")]
20117 #[test]
20118 fn static_route_exclusion_is_a_noop_when_locale_prefix_is_disabled() {
20119 let mut config = AutumnConfig::default();
20120 assert!(!config.i18n.locale_prefix_enabled);
20121 let metas = vec![static_meta("/about")];
20122
20123 exclude_static_routes_from_locale_prefix(&mut config, &metas);
20124
20125 assert!(
20126 config.i18n.locale_prefix_exclude_exact.is_empty(),
20127 "must not touch the exact-exclude list when the feature is off"
20128 );
20129 }
20130
20131 #[cfg(feature = "i18n")]
20132 #[test]
20133 fn static_route_exclusion_preserves_existing_exclude_entries() {
20134 let mut config = AutumnConfig::default();
20135 config.i18n.locale_prefix_enabled = true;
20136 config.i18n.locale_prefix_exclude = vec!["/api".to_owned()];
20137 config.i18n.locale_prefix_exclude_exact = vec!["/contact".to_owned()];
20138 let metas = vec![static_meta("/about")];
20139
20140 exclude_static_routes_from_locale_prefix(&mut config, &metas);
20141
20142 assert_eq!(
20143 config.i18n.locale_prefix_exclude,
20144 vec!["/api".to_owned()],
20145 "must not touch the user-configured prefix-exclude list"
20146 );
20147 assert_eq!(
20148 config.i18n.locale_prefix_exclude_exact,
20149 vec!["/contact".to_owned(), "/about".to_owned()]
20150 );
20151 }
20152
20153 #[cfg(feature = "i18n")]
20154 #[test]
20155 fn static_route_exclusion_preserves_a_root_static_route_path_verbatim() {
20156 let mut config = AutumnConfig::default();
20157 config.i18n.locale_prefix_enabled = true;
20158 let metas = vec![static_meta("/")];
20159
20160 exclude_static_routes_from_locale_prefix(&mut config, &metas);
20161
20162 assert_eq!(
20163 config.i18n.locale_prefix_exclude_exact,
20164 vec!["/".to_owned()]
20165 );
20166 }
20167
20168 #[cfg(feature = "i18n")]
20176 #[test]
20177 fn appstate_config_extension_reflects_static_route_exclusions_after_refresh() {
20178 let mut config = AutumnConfig::default();
20179 config.i18n.locale_prefix_enabled = true;
20180 let state = crate::state::AppState::for_test();
20181 state.insert_extension(config.clone());
20183
20184 let metas = vec![static_meta("/about")];
20185 exclude_static_routes_from_locale_prefix(&mut config, &metas);
20186 state.insert_extension(config.clone());
20189
20190 let stored = state
20191 .extension::<AutumnConfig>()
20192 .expect("config extension must be installed");
20193 assert_eq!(
20194 stored.i18n.locale_prefix_exclude_exact,
20195 vec!["/about".to_owned()],
20196 "AppState's config snapshot must reflect the static-route exclusion, \
20197 not the stale pre-mutation copy"
20198 );
20199 }
20200
20201 #[cfg(feature = "i18n")]
20202 #[derive(Clone)]
20203 struct StaticConfigLoader {
20204 config: AutumnConfig,
20205 }
20206
20207 #[cfg(feature = "i18n")]
20208 impl crate::config::ConfigLoader for StaticConfigLoader {
20209 async fn load(&self) -> Result<AutumnConfig, crate::config::ConfigError> {
20210 Ok(self.config.clone())
20211 }
20212 }
20213
20214 #[cfg(feature = "i18n")]
20215 struct NoopTelemetryProvider;
20216
20217 #[cfg(feature = "i18n")]
20218 impl crate::telemetry::TelemetryProvider for NoopTelemetryProvider {
20219 fn init(
20220 &self,
20221 _log: &crate::config::LogConfig,
20222 _telemetry: &crate::config::TelemetryConfig,
20223 _profile: Option<&str>,
20224 ) -> Result<crate::telemetry::TelemetryGuard, crate::telemetry::TelemetryInitError>
20225 {
20226 Ok(crate::telemetry::TelemetryGuard::disabled())
20227 }
20228 }
20229
20230 #[cfg(feature = "i18n")]
20231 #[tokio::test]
20232 #[allow(
20233 clippy::await_holding_lock,
20234 reason = "the guard must span the `load_config_and_telemetry` await — that \
20235 await is what mutates the limits, so dropping the lock before it \
20236 would serialize nothing. Same shape, and the same reason, as \
20237 `config_runtime_drift_actuator_prefix_is_mounted`'s circuit-breaker \
20238 guard: the contending holders are sibling libtest threads, not \
20239 tasks on this runtime, so blocking here cannot starve the future \
20240 that would release it."
20241 )]
20242 async fn i18n_auto_uses_config_loader_output_for_bundle_dir() {
20243 let _limits_lock = crate::metrics::LIMITS_TEST_LOCK
20250 .lock()
20251 .unwrap_or_else(std::sync::PoisonError::into_inner);
20252
20253 let project = tempfile::tempdir().expect("project dir");
20254 let i18n_dir = project.path().join("custom-i18n");
20255 std::fs::create_dir_all(&i18n_dir).expect("i18n dir");
20256 std::fs::write(i18n_dir.join("en.ftl"), "nav.home = Loader Home\n").expect("bundle");
20257
20258 let mut config = AutumnConfig::default();
20259 config.i18n.dir = "custom-i18n".to_owned();
20260 let builder = app()
20261 .with_config_loader(StaticConfigLoader { config })
20262 .with_telemetry_provider(NoopTelemetryProvider)
20263 .i18n_auto();
20264 let AppBuilder {
20265 config_loader_factory,
20266 telemetry_provider,
20267 i18n_bundle,
20268 i18n_auto_load,
20269 plugin_config_roots,
20270 ..
20271 } = builder;
20272
20273 let (loaded_config, _guard) = load_config_and_telemetry(
20274 config_loader_factory,
20275 telemetry_provider,
20276 plugin_config_roots,
20277 )
20278 .await;
20279 let env = crate::config::MockEnv::new().with(
20280 "AUTUMN_MANIFEST_DIR",
20281 project.path().to_str().expect("utf-8 path"),
20282 );
20283 let bundle = resolve_i18n_bundle(i18n_bundle, i18n_auto_load, &loaded_config, &env)
20284 .expect("bundle loaded from configured dir");
20285
20286 assert_eq!(bundle.translate("en", "nav.home", &[]), "Loader Home");
20287 }
20288
20289 #[cfg(feature = "i18n")]
20297 #[test]
20298 fn locale_defaults_are_installed_even_with_no_bundle() {
20299 let mut config = AutumnConfig::default();
20300 config.i18n.default_locale = "fr".to_owned();
20301 config.i18n.supported_locales = vec!["fr".to_owned(), "en".to_owned()];
20302 let state = AppState::for_test();
20303
20304 let layers = install_i18n_bundle_layer(Vec::new(), &state, None, &config.i18n);
20305
20306 assert!(
20307 layers.is_empty(),
20308 "no bundle means no Extension and no ambient layer registration"
20309 );
20310 assert_eq!(
20311 &*crate::i18n::default_locale_snapshot(),
20312 "fr",
20313 "the configured default must reach column decoding without a bundle"
20314 );
20315 let legacy = crate::i18n::Translated::decode_column(
20318 "Bonjour le monde",
20319 &crate::i18n::default_locale_snapshot(),
20320 );
20321 assert_eq!(legacy.get("fr"), Some("Bonjour le monde"));
20322
20323 crate::i18n::install_locale_defaults("en", vec!["en".to_owned()]);
20325 }
20326
20327 #[cfg(feature = "i18n")]
20338 #[tokio::test]
20339 async fn ambient_locale_layer_resolves_translatable_content_through_the_real_stack() {
20340 use tower::ServiceExt as _;
20341
20342 async fn show() -> String {
20343 let title =
20344 crate::i18n::Translated::from_pairs([("en", "Hello world"), ("es", "Hola mundo")]);
20345 format!(
20347 "{}|{}",
20348 title,
20349 crate::i18n::ambient_locale().unwrap_or_default()
20350 )
20351 }
20352
20353 let mut config = AutumnConfig::default();
20354 config.i18n.supported_locales = vec!["en".to_owned(), "es".to_owned()];
20355 let bundle = Arc::new(crate::i18n::Bundle::from_messages(
20356 std::collections::HashMap::new(),
20357 &config.i18n,
20358 ));
20359 let state = AppState::for_test();
20360 let custom_layers =
20361 install_i18n_bundle_layer(Vec::new(), &state, Some(bundle), &config.i18n);
20362
20363 let router = crate::router::try_build_router_inner(
20364 vec![Route {
20365 method: http::Method::GET,
20366 path: "/show",
20367 handler: axum::routing::get(show),
20368 name: "show",
20369 api_doc: crate::openapi::ApiDoc {
20370 method: "GET",
20371 path: "/show",
20372 operation_id: "show",
20373 success_status: 200,
20374 ..Default::default()
20375 },
20376 repository: None,
20377 idempotency: crate::route::RouteIdempotency::Direct,
20378 timeout: crate::route::RouteTimeout::Inherit,
20379 criticality: crate::admission::Criticality::Default,
20380 seo: crate::seo::SeoRouteDefaults::EMPTY,
20381 api_version: None,
20382 sunset_opt_out: false,
20383 }],
20384 &config,
20385 state,
20386 crate::router::RouterContext {
20387 exception_filters: Vec::new(),
20388 scoped_groups: Vec::new(),
20389 merge_routers: Vec::new(),
20390 nest_routers: Vec::new(),
20391 declared_routes: Vec::new(),
20392 custom_layers,
20393 static_gate_layers: Vec::new(),
20394 #[cfg(feature = "maud")]
20395 error_page_renderer: None,
20396 session_store: None,
20397 #[cfg(feature = "openapi")]
20398 openapi: None,
20399 #[cfg(feature = "mcp")]
20400 mcp: None,
20401 #[cfg(feature = "webmcp")]
20402 webmcp: None,
20403 },
20404 )
20405 .expect("router builds");
20406
20407 for (accept_language, expected) in [
20408 ("es", "Hola mundo|es"),
20409 ("fr", "Hello world|en"),
20411 ] {
20412 let request = axum::http::Request::builder()
20413 .uri("/show")
20414 .header(http::header::ACCEPT_LANGUAGE, accept_language)
20415 .body(axum::body::Body::empty())
20416 .expect("request");
20417 let response = router.clone().oneshot(request).await.expect("response");
20418 let bytes = axum::body::to_bytes(response.into_body(), 4096)
20419 .await
20420 .expect("body");
20421 assert_eq!(
20422 String::from_utf8(bytes.to_vec()).expect("utf-8"),
20423 expected,
20424 "Accept-Language: {accept_language}"
20425 );
20426 }
20427 }
20428
20429 #[cfg(feature = "edge")]
20440 #[tokio::test]
20441 async fn an_uninjected_edge_seam_never_leaks_the_fallthrough_sentinel_to_a_real_client() {
20442 async fn note(_cache: autumn_edge::EdgeCache) -> &'static str {
20443 "never reached: extraction fails first"
20444 }
20445
20446 let custom_layers = app().custom_layers;
20449
20450 let router = crate::router::try_build_router_inner(
20451 vec![Route {
20452 method: http::Method::GET,
20453 path: "/edge/note",
20454 handler: axum::routing::get(note),
20455 name: "note",
20456 api_doc: crate::openapi::ApiDoc {
20457 method: "GET",
20458 path: "/edge/note",
20459 operation_id: "note",
20460 success_status: 200,
20461 ..Default::default()
20462 },
20463 repository: None,
20464 idempotency: crate::route::RouteIdempotency::Direct,
20465 timeout: crate::route::RouteTimeout::Inherit,
20466 criticality: crate::admission::Criticality::Default,
20467 seo: crate::seo::SeoRouteDefaults::EMPTY,
20468 api_version: None,
20469 sunset_opt_out: false,
20470 }],
20471 &AutumnConfig::default(),
20472 AppState::for_test(),
20473 crate::router::RouterContext {
20474 exception_filters: Vec::new(),
20475 scoped_groups: Vec::new(),
20476 merge_routers: Vec::new(),
20477 nest_routers: Vec::new(),
20478 declared_routes: Vec::new(),
20479 custom_layers,
20480 static_gate_layers: Vec::new(),
20481 #[cfg(feature = "maud")]
20482 error_page_renderer: None,
20483 session_store: None,
20484 #[cfg(feature = "openapi")]
20485 openapi: None,
20486 #[cfg(feature = "mcp")]
20487 mcp: None,
20488 #[cfg(feature = "webmcp")]
20489 webmcp: None,
20490 },
20491 )
20492 .expect("router builds");
20493
20494 let request = axum::http::Request::builder()
20495 .uri("/edge/note")
20496 .body(axum::body::Body::empty())
20497 .expect("request");
20498 let response = router.oneshot(request).await.expect("response");
20499
20500 assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
20502 assert!(
20504 !response
20505 .headers()
20506 .contains_key(autumn_edge::FALLTHROUGH_SENTINEL),
20507 "the origin must strip the internal fallthrough sentinel: {:?}",
20508 response.headers()
20509 );
20510
20511 let body = axum::body::to_bytes(response.into_body(), usize::MAX)
20512 .await
20513 .expect("body");
20514 let body = String::from_utf8_lossy(&body);
20515 assert!(
20516 body.contains("with_edge_kv"),
20517 "the actionable message must survive: {body}"
20518 );
20519 }
20520
20521 #[cfg(feature = "edge")]
20531 #[test]
20532 fn the_sentinel_strip_layer_is_recognized_as_idempotency_transparent() {
20533 let registration = &app().custom_layers[0];
20534 assert_eq!(
20535 registration.type_id,
20536 std::any::TypeId::of::<StripEdgeFallthroughSentinelLayer>(),
20537 "the real registration's type_id no longer matches what \
20538 router::is_idempotency_transparent_app_layer looks for"
20539 );
20540 }
20541
20542 #[cfg(feature = "edge")]
20551 #[test]
20552 fn get_layer_types_excludes_the_framework_owned_sentinel_strip_layer() {
20553 #[derive(Clone, Copy)]
20554 struct UserLayer;
20555 impl<S> tower::Layer<S> for UserLayer {
20556 type Service = S;
20557 fn layer(&self, inner: S) -> S {
20558 inner
20559 }
20560 }
20561
20562 let builder = app().layer(UserLayer);
20563 assert_eq!(
20564 builder.get_layer_types(),
20565 vec![std::any::TypeId::of::<UserLayer>()],
20566 "the framework's own sentinel-strip registration must not appear \
20567 in the user-facing layer list"
20568 );
20569 }
20570
20571 #[cfg(feature = "edge")]
20574 #[tokio::test]
20575 async fn the_sentinel_strip_service_removes_the_header_and_keeps_the_body() {
20576 use axum::response::IntoResponse as _;
20577 use tower::{Layer as _, Service as _, ServiceExt as _};
20578
20579 let inner = tower::service_fn(|_req: axum::extract::Request| async move {
20580 Ok::<_, std::convert::Infallible>(
20581 (
20582 [(autumn_edge::FALLTHROUGH_SENTINEL, "missing_capability")],
20583 "actionable message",
20584 )
20585 .into_response(),
20586 )
20587 });
20588 let mut service = StripEdgeFallthroughSentinelLayer.layer(inner);
20589 let request = axum::extract::Request::builder()
20590 .uri("/")
20591 .body(axum::body::Body::empty())
20592 .expect("request");
20593 let response: axum::response::Response = service
20594 .ready()
20595 .await
20596 .expect("ready")
20597 .call(request)
20598 .await
20599 .expect("infallible");
20600
20601 assert!(
20602 !response
20603 .headers()
20604 .contains_key(autumn_edge::FALLTHROUGH_SENTINEL)
20605 );
20606 let body = axum::body::to_bytes(response.into_body(), usize::MAX)
20607 .await
20608 .expect("body");
20609 assert_eq!(body, b"actionable message".as_slice());
20610 }
20611
20612 #[cfg(feature = "i18n")]
20613 #[tokio::test]
20614 async fn i18n_bundle_layer_is_applied_to_static_route_rendering() {
20615 async fn localized(locale: crate::i18n::Locale) -> String {
20616 locale.t("nav.home")
20617 }
20618
20619 let config = AutumnConfig::default();
20620 let state = AppState::for_test();
20621 let custom_layers = install_i18n_bundle_layer(
20622 Vec::new(),
20623 &state,
20624 Some(test_i18n_bundle("nav.home", "Home")),
20625 &config.i18n,
20626 );
20627 let router = crate::router::try_build_router_inner(
20628 vec![Route {
20629 method: http::Method::GET,
20630 path: "/about",
20631 handler: axum::routing::get(localized),
20632 name: "localized",
20633 api_doc: crate::openapi::ApiDoc {
20634 method: "GET",
20635 path: "/about",
20636 operation_id: "localized",
20637 success_status: 200,
20638 ..Default::default()
20639 },
20640 repository: None,
20641 idempotency: crate::route::RouteIdempotency::Direct,
20642 timeout: crate::route::RouteTimeout::Inherit,
20643 criticality: crate::admission::Criticality::Default,
20644 seo: crate::seo::SeoRouteDefaults::EMPTY,
20645 api_version: None,
20646 sunset_opt_out: false,
20647 }],
20648 &config,
20649 state,
20650 crate::router::RouterContext {
20651 exception_filters: Vec::new(),
20652 scoped_groups: Vec::new(),
20653 merge_routers: Vec::new(),
20654 nest_routers: Vec::new(),
20655 declared_routes: Vec::new(),
20656 custom_layers,
20657 static_gate_layers: Vec::new(),
20658 #[cfg(feature = "maud")]
20659 error_page_renderer: None,
20660 session_store: None,
20661 #[cfg(feature = "openapi")]
20662 openapi: None,
20663 #[cfg(feature = "mcp")]
20664 mcp: None,
20665 #[cfg(feature = "webmcp")]
20666 webmcp: None,
20667 },
20668 )
20669 .expect("router builds");
20670 let tmp = tempfile::tempdir().expect("dist parent");
20671 let dist = tmp.path().join("dist");
20672
20673 crate::static_gen::render_static_routes(
20674 router,
20675 &[crate::static_gen::StaticRouteMeta {
20676 path: "/about",
20677 name: "localized",
20678 revalidate: None,
20679 params_fn: None,
20680 seo: crate::seo::SeoRouteDefaults::EMPTY,
20681 content_type: None,
20682 }],
20683 &dist,
20684 )
20685 .await
20686 .expect("static render succeeds");
20687
20688 let html = std::fs::read_to_string(dist.join("about/index.html")).expect("rendered html");
20689 assert_eq!(html, "Home");
20690 }
20691
20692 #[tokio::test]
20714 async fn static_get_route_reading_tenant_fails_closed_for_every_tenancy_source() {
20715 async fn tenant_page(tenant: crate::tenancy::Tenant) -> String {
20716 tenant.0
20717 }
20718
20719 for source in ["header", "subdomain", "session", "jwt"] {
20720 let mut config = AutumnConfig::default();
20721 config.tenancy.enabled = true;
20722 config.tenancy.source = source.to_owned();
20723 config.tenancy.public_paths = vec!["/storefront".to_owned()];
20735
20736 let state = AppState::for_test();
20737 state.insert_extension(config.clone());
20738
20739 let router = crate::router::try_build_router_inner(
20740 vec![Route {
20741 method: http::Method::GET,
20742 path: "/storefront",
20743 handler: axum::routing::get(tenant_page),
20744 name: "tenant_page",
20745 api_doc: crate::openapi::ApiDoc {
20746 method: "GET",
20747 path: "/storefront",
20748 operation_id: "tenant_page",
20749 success_status: 200,
20750 ..Default::default()
20751 },
20752 repository: None,
20753 idempotency: crate::route::RouteIdempotency::Direct,
20754 timeout: crate::route::RouteTimeout::Inherit,
20755 criticality: crate::admission::Criticality::Default,
20756 seo: crate::seo::SeoRouteDefaults::EMPTY,
20757 api_version: None,
20758 sunset_opt_out: false,
20759 }],
20760 &config,
20761 state,
20762 crate::router::RouterContext {
20763 exception_filters: Vec::new(),
20764 scoped_groups: Vec::new(),
20765 merge_routers: Vec::new(),
20766 nest_routers: Vec::new(),
20767 declared_routes: Vec::new(),
20768 custom_layers: Vec::new(),
20769 static_gate_layers: Vec::new(),
20770 #[cfg(feature = "maud")]
20771 error_page_renderer: None,
20772 session_store: None,
20773 #[cfg(feature = "openapi")]
20774 openapi: None,
20775 #[cfg(feature = "mcp")]
20776 mcp: None,
20777 #[cfg(feature = "webmcp")]
20778 webmcp: None,
20779 },
20780 )
20781 .unwrap_or_else(|e| panic!("tenancy source {source:?}: router builds: {e}"));
20782
20783 let tmp = tempfile::tempdir().expect("dist parent");
20784 let dist = tmp.path().join("dist");
20785
20786 let result = crate::static_gen::render_static_routes(
20787 router,
20788 &[crate::static_gen::StaticRouteMeta {
20789 path: "/storefront",
20790 name: "tenant_page",
20791 revalidate: None,
20792 params_fn: None,
20793 seo: crate::seo::SeoRouteDefaults::EMPTY,
20794 content_type: None,
20795 }],
20796 &dist,
20797 )
20798 .await;
20799
20800 assert!(
20801 result.is_err(),
20802 "tenancy source {source:?}: a #[static_get] route reading `Tenant` must fail \
20803 the build rather than silently bake a default/missing tenant's response into \
20804 a dist/ file every tenant's requests would then share"
20805 );
20806 assert!(
20807 !dist.join("storefront/index.html").exists(),
20808 "tenancy source {source:?}: no file must be written when tenant resolution fails"
20809 );
20810 }
20811 }
20812
20813 #[tokio::test]
20837 async fn failed_rebuild_leaves_preexisting_static_file_untouched() {
20838 async fn tenant_page(tenant: crate::tenancy::Tenant) -> String {
20839 tenant.0
20840 }
20841
20842 let mut config = AutumnConfig::default();
20843 config.tenancy.enabled = true;
20844 config.tenancy.source = "header".to_owned();
20845 config.tenancy.public_paths = vec!["/storefront".to_owned()];
20846
20847 let state = AppState::for_test();
20848 state.insert_extension(config.clone());
20849
20850 let router = crate::router::try_build_router_inner(
20851 vec![Route {
20852 method: http::Method::GET,
20853 path: "/storefront",
20854 handler: axum::routing::get(tenant_page),
20855 name: "tenant_page",
20856 api_doc: crate::openapi::ApiDoc {
20857 method: "GET",
20858 path: "/storefront",
20859 operation_id: "tenant_page",
20860 success_status: 200,
20861 ..Default::default()
20862 },
20863 repository: None,
20864 idempotency: crate::route::RouteIdempotency::Direct,
20865 timeout: crate::route::RouteTimeout::Inherit,
20866 criticality: crate::admission::Criticality::Default,
20867 seo: crate::seo::SeoRouteDefaults::EMPTY,
20868 api_version: None,
20869 sunset_opt_out: false,
20870 }],
20871 &config,
20872 state,
20873 crate::router::RouterContext {
20874 exception_filters: Vec::new(),
20875 scoped_groups: Vec::new(),
20876 merge_routers: Vec::new(),
20877 nest_routers: Vec::new(),
20878 declared_routes: Vec::new(),
20879 custom_layers: Vec::new(),
20880 static_gate_layers: Vec::new(),
20881 #[cfg(feature = "maud")]
20882 error_page_renderer: None,
20883 session_store: None,
20884 #[cfg(feature = "openapi")]
20885 openapi: None,
20886 #[cfg(feature = "mcp")]
20887 mcp: None,
20888 #[cfg(feature = "webmcp")]
20889 webmcp: None,
20890 },
20891 )
20892 .expect("router builds");
20893
20894 let tmp = tempfile::tempdir().expect("dist parent");
20895 let dist = tmp.path().join("dist");
20896
20897 std::fs::create_dir_all(dist.join("storefront")).expect("mkdir storefront");
20903 let sentinel = "tenant-a-sentinel-warden-2026-09-04";
20904 std::fs::write(dist.join("storefront/index.html"), sentinel).expect("seed stale file");
20905 let mut seed_routes = std::collections::HashMap::new();
20906 seed_routes.insert(
20907 "/storefront".to_owned(),
20908 crate::static_gen::ManifestEntry::new("storefront/index.html".to_owned()),
20909 );
20910 let manifest = crate::static_gen::StaticManifest::new(seed_routes);
20911 std::fs::write(
20912 dist.join("manifest.json"),
20913 serde_json::to_string(&manifest).expect("serialize manifest"),
20914 )
20915 .expect("write manifest");
20916
20917 let result = crate::static_gen::render_static_routes(
20918 router,
20919 &[crate::static_gen::StaticRouteMeta {
20920 path: "/storefront",
20921 name: "tenant_page",
20922 revalidate: None,
20923 params_fn: None,
20924 seo: crate::seo::SeoRouteDefaults::EMPTY,
20925 content_type: None,
20926 }],
20927 &dist,
20928 )
20929 .await;
20930
20931 assert!(
20932 result.is_err(),
20933 "the rebuild must still fail closed (tenant resolution rejects the headerless \
20934 synthetic request), same as the fresh-dist case above"
20935 );
20936
20937 let surviving = std::fs::read_to_string(dist.join("storefront/index.html"))
20938 .expect("the pre-existing file must still be present after a failed rebuild");
20939 assert_eq!(
20940 surviving, sentinel,
20941 "a failed rebuild must not silently alter or remove a pre-existing static file. \
20942 This is Autumn's documented stale-while-revalidate/atomic-swap design working as \
20943 intended, but it also means a tenant-mismatched file that reached dist/ by some \
20944 other means (an operational build/serve config mismatch, not a code path in this \
20945 framework) is served indefinitely with no automatic invalidation — see \
20946 docs/security/2026-09-04-static-gen-tenancy-fails-closed/README.md"
20947 );
20948 }
20949
20950 #[test]
20951 fn app_builder_routes_adds_routes() {
20952 let builder = app();
20953 assert_eq!(builder.routes.len(), 0);
20954
20955 let builder = builder.routes(vec![test_get_route("/1", "route1")]);
20956 assert_eq!(builder.routes.len(), 1);
20957
20958 let builder = builder.routes(vec![
20959 test_get_route("/2", "route2"),
20960 test_get_route("/3", "route3"),
20961 ]);
20962 assert_eq!(builder.routes.len(), 3);
20963
20964 assert_eq!(builder.routes[0].path, "/1");
20965 assert_eq!(builder.routes[1].path, "/2");
20966 assert_eq!(builder.routes[2].path, "/3");
20967 }
20968
20969 #[test]
20970 fn app_builder_extensions_store_and_update_typed_values() {
20971 let builder = app()
20972 .with_extension::<String>("haunted".into())
20973 .update_extension::<String, _, _>(String::new, |value| value.push_str(" harvest"));
20974
20975 let value = builder
20976 .extension::<String>()
20977 .expect("string extension should be present");
20978 assert_eq!(value, "haunted harvest");
20979 }
20980
20981 #[cfg(feature = "mail")]
20982 #[tokio::test]
20983 async fn app_builder_with_mail_delivery_queue_stores_queue_for_install() {
20984 let builder = app().with_mail_delivery_queue(MailTestNoopQueue);
20985 let factory = builder
20986 .mail_delivery_queue_factory
20987 .expect("with_mail_delivery_queue should store a factory on the builder");
20988
20989 let state = AppState::for_test();
20992 let queue = factory(&state).expect("trivial factory should produce the queue");
20993 assert!(Arc::strong_count(&queue) >= 1);
20994 queue
20996 .enqueue(test_mail())
20997 .await
20998 .expect("noop queue should always succeed");
20999 }
21000
21001 #[cfg(feature = "mail")]
21002 #[test]
21003 fn app_builder_with_mail_delivery_queue_factory_runs_with_app_state() {
21004 let observed_profile: Arc<std::sync::Mutex<Option<String>>> =
21005 Arc::new(std::sync::Mutex::new(None));
21006 let captured = Arc::clone(&observed_profile);
21007 let builder = app().with_mail_delivery_queue_factory(move |state| {
21008 *captured.lock().expect("lock") = Some(state.profile().to_owned());
21009 Ok::<_, crate::AutumnError>(MailTestNoopQueue)
21010 });
21011
21012 let factory = builder
21013 .mail_delivery_queue_factory
21014 .expect("factory should be stored on the builder");
21015 let state = AppState::for_test().with_profile("dev");
21016 let _queue = factory(&state).expect("factory should succeed");
21017
21018 assert_eq!(
21019 observed_profile.lock().expect("lock").as_deref(),
21020 Some("dev"),
21021 "factory must run with the live AppState"
21022 );
21023 }
21024
21025 #[cfg(feature = "mail")]
21026 #[test]
21027 fn app_builder_with_mail_delivery_queue_factory_propagates_errors() {
21028 let builder = app().with_mail_delivery_queue_factory(|_state| {
21029 Err::<MailTestNoopQueue, _>(crate::AutumnError::service_unavailable_msg("factory boom"))
21030 });
21031
21032 let factory = builder
21033 .mail_delivery_queue_factory
21034 .expect("factory present");
21035 let state = AppState::for_test();
21036 match factory(&state) {
21037 Ok(_) => panic!("factory should have errored"),
21038 Err(err) => assert!(err.to_string().contains("factory boom")),
21039 }
21040 }
21041
21042 #[tokio::test]
21043 async fn startup_and_shutdown_hooks_run_in_expected_order() {
21044 let events = Arc::new(std::sync::Mutex::new(Vec::<&'static str>::new()));
21045 let startup_events = Arc::clone(&events);
21046 let shutdown_a = Arc::clone(&events);
21047 let shutdown_b = Arc::clone(&events);
21048 let builder = app()
21049 .on_startup(move |_state| {
21050 let startup_events = Arc::clone(&startup_events);
21051 async move {
21052 startup_events
21053 .lock()
21054 .expect("events lock poisoned")
21055 .push("start");
21056 Ok(())
21057 }
21058 })
21059 .on_shutdown(move || {
21060 let shutdown_a = Arc::clone(&shutdown_a);
21061 async move {
21062 shutdown_a
21063 .lock()
21064 .expect("events lock poisoned")
21065 .push("stop-a");
21066 }
21067 })
21068 .on_shutdown(move || {
21069 let shutdown_b = Arc::clone(&shutdown_b);
21070 async move {
21071 shutdown_b
21072 .lock()
21073 .expect("events lock poisoned")
21074 .push("stop-b");
21075 }
21076 });
21077
21078 run_startup_hooks(&builder.startup_hooks, AppState::for_test())
21079 .await
21080 .expect("startup hooks should succeed");
21081 run_shutdown_hooks(&builder.shutdown_hooks).await;
21082
21083 let recorded_events = events.lock().expect("events lock poisoned").clone();
21084 assert_eq!(recorded_events, vec!["start", "stop-b", "stop-a"]);
21085 }
21086
21087 #[derive(Default)]
21089 struct PreloadProbe {
21090 inner: crate::feature_flags::InMemoryFlagStore,
21091 preloads: std::sync::atomic::AtomicUsize,
21092 fail: bool,
21093 }
21094
21095 impl crate::feature_flags::FlagStore for PreloadProbe {
21096 fn get(
21097 &self,
21098 key: &str,
21099 ) -> Result<Option<crate::feature_flags::FlagConfig>, crate::feature_flags::FlagStoreError>
21100 {
21101 self.inner.get(key)
21102 }
21103 fn list(
21104 &self,
21105 ) -> Result<Vec<crate::feature_flags::FlagConfig>, crate::feature_flags::FlagStoreError>
21106 {
21107 self.inner.list()
21108 }
21109 fn enable(
21110 &self,
21111 key: &str,
21112 actor: Option<&str>,
21113 ) -> Result<(), crate::feature_flags::FlagStoreError> {
21114 self.inner.enable(key, actor)
21115 }
21116 fn disable(
21117 &self,
21118 key: &str,
21119 actor: Option<&str>,
21120 ) -> Result<(), crate::feature_flags::FlagStoreError> {
21121 self.inner.disable(key, actor)
21122 }
21123 fn set_rollout(
21124 &self,
21125 key: &str,
21126 pct: u8,
21127 actor: Option<&str>,
21128 ) -> Result<(), crate::feature_flags::FlagStoreError> {
21129 self.inner.set_rollout(key, pct, actor)
21130 }
21131 fn allow_actor(
21132 &self,
21133 key: &str,
21134 actor_id: &str,
21135 actor: Option<&str>,
21136 ) -> Result<(), crate::feature_flags::FlagStoreError> {
21137 self.inner.allow_actor(key, actor_id, actor)
21138 }
21139 fn add_group(
21140 &self,
21141 key: &str,
21142 group: &str,
21143 actor: Option<&str>,
21144 ) -> Result<(), crate::feature_flags::FlagStoreError> {
21145 self.inner.add_group(key, group, actor)
21146 }
21147 fn history(
21148 &self,
21149 key: &str,
21150 limit: usize,
21151 ) -> Result<Vec<crate::feature_flags::FlagChangeRecord>, crate::feature_flags::FlagStoreError>
21152 {
21153 self.inner.history(key, limit)
21154 }
21155 fn preload(&self) -> Result<(), crate::feature_flags::FlagStoreError> {
21156 self.preloads
21157 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
21158 if self.fail {
21159 return Err(crate::feature_flags::FlagStoreError::Backend(
21160 "brownout".to_owned(),
21161 ));
21162 }
21163 Ok(())
21164 }
21165 }
21166
21167 #[tokio::test]
21168 async fn with_flag_store_preloads_the_store_at_startup() {
21169 for fail in [false, true] {
21170 let store = Arc::new(PreloadProbe {
21171 fail,
21172 ..PreloadProbe::default()
21173 });
21174 let builder = app().with_flag_store(Arc::clone(&store));
21175 let state = AppState::for_test();
21176 run_state_initializers(builder.state_initializers, &state);
21177
21178 assert_eq!(
21180 store.preloads.load(std::sync::atomic::Ordering::SeqCst),
21181 1,
21182 "fail = {fail}"
21183 );
21184 }
21185 }
21186
21187 #[test]
21188 fn with_flag_service_keeps_declared_defaults() {
21189 let service = crate::feature_flags::FeatureFlagService::new(Arc::new(
21190 crate::feature_flags::InMemoryFlagStore::new(),
21191 ))
21192 .with_default("checkout_v2", true);
21193 let builder = app().with_flag_service(service);
21194 let state = AppState::for_test();
21195 run_state_initializers(builder.state_initializers, &state);
21196
21197 let installed = state
21198 .extension::<crate::feature_flags::FeatureFlagService>()
21199 .expect("service registered");
21200 assert!(installed.is_enabled("checkout_v2", None));
21201 assert!(
21202 builder.startup_hooks.is_empty(),
21203 "the state initializer preloads flags; no startup hook does"
21204 );
21205 }
21206
21207 fn startup_noop_job_handler(
21208 _state: AppState,
21209 _payload: serde_json::Value,
21210 ) -> Pin<Box<dyn Future<Output = crate::AutumnResult<()>> + Send + 'static>> {
21211 Box::pin(async move { Ok(()) })
21212 }
21213
21214 #[tokio::test]
21215 async fn startup_hooks_can_enqueue_jobs_after_runtime_init() {
21216 let _guard = crate::job::global_job_runtime_test_lock().lock().await;
21217 crate::job::clear_global_job_client();
21218
21219 let builder = app()
21220 .jobs(vec![crate::job::JobInfo {
21221 version: 1,
21222 name: "startup-seed".to_string(),
21223 max_attempts: 1,
21224 initial_backoff_ms: 1,
21225 queue: "default".to_string(),
21226 uniqueness: None,
21227 concurrency: None,
21228 handler: startup_noop_job_handler,
21229 timeout: None,
21230 }])
21231 .on_startup(|_state| async {
21232 crate::job::enqueue("startup-seed", serde_json::json!({ "kind": "warmup" })).await
21233 });
21234
21235 let state = AppState::for_test().with_profile("dev");
21236 let shutdown = tokio_util::sync::CancellationToken::new();
21237
21238 initialize_job_runtime(
21239 builder.jobs.clone(),
21240 &state,
21241 &shutdown,
21242 &crate::config::JobConfig::default(),
21243 true,
21244 )
21245 .expect("job runtime should initialize before startup hooks");
21246
21247 run_startup_hooks(&builder.startup_hooks, state.clone())
21248 .await
21249 .expect("startup hook should be able to enqueue jobs");
21250
21251 tokio::time::timeout(std::time::Duration::from_secs(1), async {
21252 loop {
21253 let snapshot = state.job_registry().snapshot();
21254 let status = snapshot
21255 .get("startup-seed")
21256 .expect("job should be registered before startup hooks run");
21257 if status.total_successes == 1 {
21258 break;
21259 }
21260 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
21261 }
21262 })
21263 .await
21264 .expect("startup-enqueued job should complete");
21265
21266 shutdown.cancel();
21267 crate::job::clear_global_job_client();
21268 }
21269
21270 #[tokio::test]
21271 async fn initialize_job_runtime_propagates_redis_init_errors() {
21272 let _guard = crate::job::global_job_runtime_test_lock().lock().await;
21273 crate::job::clear_global_job_client();
21274
21275 let state = AppState::for_test().with_profile("dev");
21276 let shutdown = tokio_util::sync::CancellationToken::new();
21277 let config = crate::config::JobConfig {
21278 backend: "redis".to_string(),
21279 ..Default::default()
21280 };
21281
21282 let error = initialize_job_runtime(
21283 vec![crate::job::JobInfo {
21284 version: 1,
21285 name: "startup-seed".to_string(),
21286 max_attempts: 1,
21287 initial_backoff_ms: 1,
21288 queue: "default".to_string(),
21289 uniqueness: None,
21290 concurrency: None,
21291 handler: startup_noop_job_handler,
21292 timeout: None,
21293 }],
21294 &state,
21295 &shutdown,
21296 &config,
21297 true,
21298 )
21299 .expect_err("redis init errors should abort startup");
21300
21301 #[cfg(feature = "redis")]
21302 assert!(
21303 error
21304 .to_string()
21305 .contains("jobs.backend=redis requires jobs.redis.url"),
21306 "unexpected error: {error}"
21307 );
21308
21309 #[cfg(not(feature = "redis"))]
21310 assert!(
21311 error
21312 .to_string()
21313 .contains("jobs.backend=redis requested but redis feature is disabled"),
21314 "unexpected error: {error}"
21315 );
21316 }
21317
21318 #[tokio::test]
21319 async fn startup_hook_errors_propagate() {
21320 let builder = app().on_startup(|_state| async {
21321 Err(crate::AutumnError::service_unavailable_msg(
21322 "startup ritual failed",
21323 ))
21324 });
21325
21326 let error = run_startup_hooks(&builder.startup_hooks, AppState::for_test())
21327 .await
21328 .expect_err("startup hook should fail");
21329 assert!(error.to_string().contains("startup ritual failed"));
21330 }
21331
21332 #[tokio::test]
21333 async fn build_router_mounts_user_routes() {
21334 let router = test_router(vec![test_get_route("/test", "test_handler")]);
21335
21336 let response = router
21337 .oneshot(Request::builder().uri("/test").body(Body::empty()).unwrap())
21338 .await
21339 .unwrap();
21340
21341 assert_eq!(response.status(), StatusCode::OK);
21342 let body = axum::body::to_bytes(response.into_body(), usize::MAX)
21343 .await
21344 .unwrap();
21345 assert_eq!(&body[..], b"ok");
21346 }
21347
21348 #[tokio::test]
21349 async fn build_router_mounts_health_check_at_default_path() {
21350 let router = test_router(vec![test_get_route("/dummy", "dummy")]);
21351
21352 let response = router
21353 .oneshot(
21354 Request::builder()
21355 .uri("/health")
21356 .body(Body::empty())
21357 .unwrap(),
21358 )
21359 .await
21360 .unwrap();
21361
21362 assert_eq!(response.status(), StatusCode::OK);
21363 let body = axum::body::to_bytes(response.into_body(), usize::MAX)
21364 .await
21365 .unwrap();
21366 let json: serde_json::Value = serde_json::from_slice(&body).unwrap();
21367 assert_eq!(json["status"], "ok");
21368 }
21369
21370 #[tokio::test]
21371 async fn build_router_mounts_health_check_at_custom_path() {
21372 let mut config = AutumnConfig::default();
21373 config.health.path = "/healthz".to_owned();
21374 let router = test_router_with_config(vec![test_get_route("/dummy", "dummy")], &config);
21375
21376 let response = router
21377 .oneshot(
21378 Request::builder()
21379 .uri("/healthz")
21380 .body(Body::empty())
21381 .unwrap(),
21382 )
21383 .await
21384 .unwrap();
21385
21386 assert_eq!(response.status(), StatusCode::OK);
21387 }
21388
21389 #[tokio::test]
21390 async fn build_router_adds_request_id_header() {
21391 let router = test_router(vec![test_get_route("/test", "test")]);
21392
21393 let response = router
21394 .oneshot(Request::builder().uri("/test").body(Body::empty()).unwrap())
21395 .await
21396 .unwrap();
21397
21398 assert!(response.headers().contains_key("x-request-id"));
21399 }
21400
21401 #[tokio::test]
21402 async fn build_router_unknown_route_returns_404() {
21403 let router = test_router(vec![test_get_route("/exists", "exists")]);
21404
21405 let response = router
21406 .oneshot(Request::builder().uri("/nope").body(Body::empty()).unwrap())
21407 .await
21408 .unwrap();
21409
21410 assert_eq!(response.status(), StatusCode::NOT_FOUND);
21411 }
21412
21413 #[tokio::test]
21414 async fn build_router_multiple_routes() {
21415 let router = test_router(vec![test_get_route("/a", "a"), test_get_route("/b", "b")]);
21416
21417 let resp_a = router
21418 .clone()
21419 .oneshot(Request::builder().uri("/a").body(Body::empty()).unwrap())
21420 .await
21421 .unwrap();
21422 assert_eq!(resp_a.status(), StatusCode::OK);
21423
21424 let resp_b = router
21425 .oneshot(Request::builder().uri("/b").body(Body::empty()).unwrap())
21426 .await
21427 .unwrap();
21428 assert_eq!(resp_b.status(), StatusCode::OK);
21429 }
21430
21431 #[tokio::test]
21432 async fn build_router_post_route() {
21433 let post_routes = vec![Route {
21434 method: http::Method::POST,
21435 path: "/submit",
21436 handler: axum::routing::post(|| async { "posted" }),
21437 name: "submit",
21438 api_doc: crate::openapi::ApiDoc {
21439 method: "POST",
21440 path: "/submit",
21441 operation_id: "submit",
21442 success_status: 200,
21443 ..Default::default()
21444 },
21445 repository: None,
21446 idempotency: crate::route::RouteIdempotency::Direct,
21447 timeout: crate::route::RouteTimeout::Inherit,
21448 criticality: crate::admission::Criticality::Default,
21449 seo: crate::seo::SeoRouteDefaults::EMPTY,
21450 api_version: None,
21451 sunset_opt_out: false,
21452 }];
21453 let router = test_router(post_routes);
21454
21455 let response = router
21456 .oneshot(
21457 Request::builder()
21458 .method("POST")
21459 .uri("/submit")
21460 .body(Body::empty())
21461 .unwrap(),
21462 )
21463 .await
21464 .unwrap();
21465
21466 assert_eq!(response.status(), StatusCode::OK);
21467 }
21468
21469 #[tokio::test]
21470 async fn build_router_merges_methods_on_same_path() {
21471 let route_list = vec![
21472 Route {
21473 method: http::Method::GET,
21474 path: "/admin",
21475 handler: axum::routing::get(|| async { "list" }),
21476 name: "admin_list",
21477 api_doc: crate::openapi::ApiDoc {
21478 method: "GET",
21479 path: "/admin",
21480 operation_id: "admin_list",
21481 success_status: 200,
21482 ..Default::default()
21483 },
21484 repository: None,
21485 idempotency: crate::route::RouteIdempotency::Direct,
21486 timeout: crate::route::RouteTimeout::Inherit,
21487 criticality: crate::admission::Criticality::Default,
21488 seo: crate::seo::SeoRouteDefaults::EMPTY,
21489 api_version: None,
21490 sunset_opt_out: false,
21491 },
21492 Route {
21493 method: http::Method::POST,
21494 path: "/admin",
21495 handler: axum::routing::post(|| async { "created" }),
21496 name: "create",
21497 api_doc: crate::openapi::ApiDoc {
21498 method: "POST",
21499 path: "/admin",
21500 operation_id: "create",
21501 success_status: 200,
21502 ..Default::default()
21503 },
21504 repository: None,
21505 idempotency: crate::route::RouteIdempotency::Direct,
21506 timeout: crate::route::RouteTimeout::Inherit,
21507 criticality: crate::admission::Criticality::Default,
21508 seo: crate::seo::SeoRouteDefaults::EMPTY,
21509 api_version: None,
21510 sunset_opt_out: false,
21511 },
21512 ];
21513 let config = AutumnConfig::default();
21514 let router = crate::router::build_router(route_list, &config, AppState::for_test());
21515
21516 let resp = router
21518 .clone()
21519 .oneshot(
21520 Request::builder()
21521 .uri("/admin")
21522 .body(Body::empty())
21523 .unwrap(),
21524 )
21525 .await
21526 .unwrap();
21527 assert_eq!(resp.status(), StatusCode::OK);
21528 let body = axum::body::to_bytes(resp.into_body(), usize::MAX)
21529 .await
21530 .unwrap();
21531 assert_eq!(&body[..], b"list");
21532
21533 let resp = router
21535 .oneshot(
21536 Request::builder()
21537 .method("POST")
21538 .uri("/admin")
21539 .body(Body::empty())
21540 .unwrap(),
21541 )
21542 .await
21543 .unwrap();
21544 assert_eq!(resp.status(), StatusCode::OK);
21545 let body = axum::body::to_bytes(resp.into_body(), usize::MAX)
21546 .await
21547 .unwrap();
21548 assert_eq!(&body[..], b"created");
21549 }
21550
21551 #[cfg(feature = "htmx")]
21556 #[tokio::test]
21557 async fn htmx_is_served_at_plain_and_fingerprinted_paths_with_matching_cache_policy() {
21558 let router = test_router(vec![test_get_route("/dummy", "dummy")]);
21559 let get = |uri: String| {
21560 let router = router.clone();
21561 async move {
21562 router
21563 .oneshot(Request::builder().uri(uri).body(Body::empty()).unwrap())
21564 .await
21565 .unwrap()
21566 }
21567 };
21568
21569 let plain = get(crate::htmx::HTMX_JS_PATH.to_owned()).await;
21570 assert_eq!(plain.status(), StatusCode::OK);
21571 let content_type = plain.headers()["content-type"].to_str().unwrap();
21572 assert!(
21573 content_type.contains("javascript"),
21574 "Expected JavaScript, got {content_type}"
21575 );
21576 let cache_control = plain.headers()["cache-control"].to_str().unwrap();
21577 assert_eq!(cache_control, "public, max-age=0, must-revalidate");
21578 assert!(plain.headers().contains_key("etag"));
21579 let body = axum::body::to_bytes(plain.into_body(), usize::MAX)
21580 .await
21581 .unwrap();
21582 assert_eq!(&body[..], crate::htmx::HTMX_JS);
21583
21584 let hashed_url = crate::assets::asset_url("js/htmx.min.js");
21585 assert_ne!(hashed_url, crate::htmx::HTMX_JS_PATH);
21586 let hashed = get(hashed_url).await;
21587 assert_eq!(hashed.status(), StatusCode::OK);
21588 assert_eq!(
21589 hashed.headers()["cache-control"].to_str().unwrap(),
21590 "public, max-age=31536000, immutable"
21591 );
21592 let body = axum::body::to_bytes(hashed.into_body(), usize::MAX)
21593 .await
21594 .unwrap();
21595 assert_eq!(&body[..], crate::htmx::HTMX_JS);
21596 }
21597
21598 #[cfg(feature = "htmx")]
21599 #[tokio::test]
21600 async fn htmx_csrf_helper_is_csp_compatible_javascript() {
21601 let router = test_router(vec![test_get_route("/dummy", "dummy")]);
21602
21603 let response = router
21604 .oneshot(
21605 Request::builder()
21606 .uri(crate::htmx::HTMX_CSRF_JS_PATH)
21607 .body(Body::empty())
21608 .unwrap(),
21609 )
21610 .await
21611 .unwrap();
21612
21613 assert_eq!(response.status(), StatusCode::OK);
21614 assert_eq!(
21615 response
21616 .headers()
21617 .get("content-type")
21618 .and_then(|value| value.to_str().ok()),
21619 Some("text/javascript; charset=utf-8")
21620 );
21621
21622 let body = axum::body::to_bytes(response.into_body(), usize::MAX)
21623 .await
21624 .unwrap();
21625 let js = std::str::from_utf8(&body).expect("csrf helper should be valid utf-8");
21626
21627 assert!(js.contains("htmx:configRequest"));
21628 assert!(js.contains("X-CSRF-Token"));
21629 assert!(!js.contains("<script"));
21630 }
21631
21632 #[cfg(feature = "htmx")]
21633 #[tokio::test]
21634 async fn build_router_serves_htmx_js() {
21635 let router = test_router(vec![test_get_route("/dummy", "dummy")]);
21636
21637 let response = router
21638 .oneshot(
21639 Request::builder()
21640 .uri(crate::htmx::HTMX_JS_PATH)
21641 .body(Body::empty())
21642 .unwrap(),
21643 )
21644 .await
21645 .unwrap();
21646
21647 assert_eq!(response.status(), StatusCode::OK);
21648 let ct = response
21649 .headers()
21650 .get("content-type")
21651 .unwrap()
21652 .to_str()
21653 .unwrap();
21654 assert!(ct.contains("javascript"));
21655 }
21656
21657 #[cfg(feature = "htmx")]
21658 #[tokio::test]
21659 async fn build_router_serves_htmx_csrf_js() {
21660 let router = test_router(vec![test_get_route("/dummy", "dummy")]);
21661
21662 let response = router
21663 .oneshot(
21664 Request::builder()
21665 .uri(crate::htmx::HTMX_CSRF_JS_PATH)
21666 .body(Body::empty())
21667 .unwrap(),
21668 )
21669 .await
21670 .unwrap();
21671
21672 assert_eq!(response.status(), StatusCode::OK);
21673 let csp = response
21674 .headers()
21675 .get("content-security-policy")
21676 .expect("framework JS should still receive security headers")
21677 .to_str()
21678 .unwrap();
21679 assert!(csp.contains("script-src 'self'"), "csp = {csp}");
21680 let body = axum::body::to_bytes(response.into_body(), usize::MAX)
21681 .await
21682 .unwrap();
21683 let js = std::str::from_utf8(&body).expect("csrf helper should be valid utf-8");
21684 assert!(js.contains("htmx:configRequest"));
21685 assert!(js.contains("X-CSRF-Token"));
21686 }
21687
21688 #[tokio::test]
21689 async fn build_router_serves_default_favicon_without_404() {
21690 let router = test_router(vec![test_get_route("/dummy", "dummy")]);
21691
21692 let response = router
21693 .oneshot(
21694 Request::builder()
21695 .uri(crate::router::DEFAULT_FAVICON_PATH)
21696 .body(Body::empty())
21697 .unwrap(),
21698 )
21699 .await
21700 .unwrap();
21701
21702 assert_eq!(response.status(), StatusCode::NO_CONTENT);
21703 assert!(
21704 response.headers().contains_key("content-security-policy"),
21705 "framework fallback responses should still receive security headers"
21706 );
21707 let body = axum::body::to_bytes(response.into_body(), usize::MAX)
21708 .await
21709 .unwrap();
21710 assert!(body.is_empty());
21711 }
21712
21713 #[tokio::test]
21714 async fn build_router_does_not_override_user_favicon_route() {
21715 let router = test_router(vec![test_get_route(
21716 crate::router::DEFAULT_FAVICON_PATH,
21717 "favicon",
21718 )]);
21719
21720 let response = router
21721 .oneshot(
21722 Request::builder()
21723 .uri(crate::router::DEFAULT_FAVICON_PATH)
21724 .body(Body::empty())
21725 .unwrap(),
21726 )
21727 .await
21728 .unwrap();
21729
21730 assert_eq!(response.status(), StatusCode::OK);
21731 let body = axum::body::to_bytes(response.into_body(), usize::MAX)
21732 .await
21733 .unwrap();
21734 assert_eq!(&body[..], b"ok");
21735 }
21736
21737 #[tokio::test]
21738 async fn build_router_serves_static_files_for_unmatched_paths() {
21739 use std::collections::HashMap;
21740
21741 let tmp = tempfile::tempdir().expect("tempdir");
21743 let dist = tmp.path().join("dist");
21744 std::fs::create_dir_all(dist.join("docs")).expect("mkdir");
21745 std::fs::write(dist.join("docs/index.html"), "<h1>Static Docs</h1>").expect("write");
21746
21747 let manifest = crate::static_gen::StaticManifest::new(HashMap::from([(
21748 "/docs".to_owned(),
21749 crate::static_gen::ManifestEntry::new("docs/index.html".to_owned()),
21750 )]))
21751 .with_generated_at("2026-03-27T00:00:00Z");
21752 let json = serde_json::to_string(&manifest).expect("serialize");
21753 std::fs::write(dist.join("manifest.json"), json).expect("write manifest");
21754
21755 let config = AutumnConfig::default();
21757 let state = AppState {
21758 health_detailed: true,
21759 ..AppState::test_default()
21760 };
21761 let router = crate::router::build_router_with_static(
21762 vec![test_get_route("/other", "other_page")],
21763 &config,
21764 state,
21765 Some(dist.as_path()),
21766 );
21767
21768 let response = router
21771 .oneshot(
21772 Request::builder()
21773 .uri("/docs/")
21774 .body(Body::empty())
21775 .unwrap(),
21776 )
21777 .await
21778 .unwrap();
21779
21780 assert_eq!(response.status(), StatusCode::OK);
21781 let csp = response
21782 .headers()
21783 .get("content-security-policy")
21784 .expect("static-first HTML should still receive security headers")
21785 .to_str()
21786 .unwrap();
21787 assert!(csp.contains("script-src 'self'"), "csp = {csp}");
21788 let body = axum::body::to_bytes(response.into_body(), usize::MAX)
21789 .await
21790 .unwrap();
21791 assert_eq!(std::str::from_utf8(&body).unwrap(), "<h1>Static Docs</h1>");
21792 }
21793
21794 #[tokio::test]
21801 async fn build_router_serves_recorded_content_type_with_nosniff() {
21802 use std::collections::HashMap;
21803
21804 let tmp = tempfile::tempdir().expect("tempdir");
21805 let dist = tmp.path().join("dist");
21806 std::fs::create_dir_all(dist.join("feed")).expect("mkdir");
21807 std::fs::write(dist.join("feed/index.html"), "<rss/>").expect("write");
21808
21809 let manifest = crate::static_gen::StaticManifest::new(HashMap::from([(
21810 "/feed".to_owned(),
21811 crate::static_gen::ManifestEntry::new("feed/index.html".to_owned())
21812 .with_content_type(Some("application/rss+xml".to_owned())),
21813 )]));
21814 std::fs::write(
21815 dist.join("manifest.json"),
21816 serde_json::to_string(&manifest).expect("serialize"),
21817 )
21818 .expect("write manifest");
21819
21820 let config = AutumnConfig::default();
21821 let router = crate::router::build_router_with_static(
21822 Vec::new(),
21823 &config,
21824 AppState::for_test(),
21825 Some(dist.as_path()),
21826 );
21827
21828 let response = router
21829 .oneshot(Request::builder().uri("/feed").body(Body::empty()).unwrap())
21830 .await
21831 .unwrap();
21832
21833 assert_eq!(response.status(), StatusCode::OK);
21834 assert_eq!(
21835 response
21836 .headers()
21837 .get(http::header::CONTENT_TYPE)
21838 .and_then(|v| v.to_str().ok()),
21839 Some("application/rss+xml"),
21840 "the recorded type must survive the full composed stack"
21841 );
21842 assert_eq!(
21843 response
21844 .headers()
21845 .get("x-content-type-options")
21846 .and_then(|v| v.to_str().ok()),
21847 Some("nosniff"),
21848 "nosniff is why the recorded type has to be correct: the browser \
21849 will not second-guess it"
21850 );
21851 }
21852
21853 #[test]
21856 fn an_empty_static_build_clears_stale_output() {
21857 let tmp = tempfile::tempdir().expect("temp dir");
21858 let dist = tmp.path().join("dist");
21859 std::fs::create_dir_all(dist.join("about")).expect("mkdir");
21860 std::fs::write(dist.join("about/index.html"), "stale").expect("write");
21861
21862 clear_stale_static_output(&dist).expect("clears");
21863 assert!(!dist.exists(), "stale dist/ must be gone");
21864
21865 clear_stale_static_output(&dist).expect("a missing dist/ is fine");
21866 }
21867
21868 #[test]
21869 fn no_static_routes_fails_the_static_build_unless_empty_is_allowed() {
21870 assert_eq!(no_static_routes_exit_code(false), 1);
21871 assert_eq!(no_static_routes_exit_code(true), 0);
21872 assert_eq!(
21873 BUILD_STATIC_ALLOW_EMPTY_ENV,
21874 "AUTUMN_BUILD_STATIC_ALLOW_EMPTY"
21875 );
21876 }
21877
21878 #[tokio::test]
21879 async fn build_mode_static_rendering_bypasses_startup_barrier() {
21880 temp_env::async_with_vars([("AUTUMN_BUILD_STATIC", Some("1"))], async {
21881 let config = AutumnConfig::default();
21882 let state = AppState::for_test().with_startup_complete(false);
21883 let router = crate::router::build_router(
21884 vec![Route {
21885 method: http::Method::GET,
21886 path: "/about",
21887 handler: axum::routing::get(|| async { "About Page Content" }),
21888 name: "about",
21889 api_doc: crate::openapi::ApiDoc {
21890 method: "GET",
21891 path: "/about",
21892 operation_id: "about",
21893 success_status: 200,
21894 ..Default::default()
21895 },
21896 repository: None,
21897 idempotency: crate::route::RouteIdempotency::Direct,
21898 timeout: crate::route::RouteTimeout::Inherit,
21899 criticality: crate::admission::Criticality::Default,
21900 seo: crate::seo::SeoRouteDefaults::EMPTY,
21901 api_version: None,
21902 sunset_opt_out: false,
21903 }],
21904 &config,
21905 state,
21906 );
21907 let tmp = tempfile::tempdir().unwrap();
21908 let dist = tmp.path().join("dist");
21909
21910 let result = crate::static_gen::render_static_routes(
21911 router,
21912 &[crate::static_gen::StaticRouteMeta {
21913 path: "/about",
21914 name: "about",
21915 revalidate: None,
21916 params_fn: None,
21917 seo: crate::seo::SeoRouteDefaults::EMPTY,
21918 content_type: None,
21919 }],
21920 &dist,
21921 )
21922 .await;
21923
21924 assert!(result.is_ok(), "build failed: {:?}", result.err());
21925 let html = std::fs::read_to_string(dist.join("about/index.html")).unwrap();
21926 assert_eq!(html, "About Page Content");
21927 })
21928 .await;
21929 }
21930
21931 #[tokio::test]
21932 async fn build_router_injects_live_reload_script_when_enabled() {
21933 let reload_file = tempfile::NamedTempFile::new().expect("reload state file");
21934 std::fs::write(reload_file.path(), r#"{"version":0,"kind":"full"}"#).expect("write");
21935 temp_env::async_with_vars(
21936 [
21937 ("AUTUMN_DEV_RELOAD", Some("1")),
21938 (
21939 "AUTUMN_DEV_RELOAD_STATE",
21940 Some(reload_file.path().to_str().expect("utf-8 path")),
21941 ),
21942 ],
21943 async {
21944 let router = test_router(vec![Route {
21945 method: http::Method::GET,
21946 path: "/page",
21947 handler: axum::routing::get(|| async {
21948 axum::response::Html("<html><body><main>ok</main></body></html>")
21949 }),
21950 name: "page",
21951 api_doc: crate::openapi::ApiDoc {
21952 method: "GET",
21953 path: "/page",
21954 operation_id: "page",
21955 success_status: 200,
21956 ..Default::default()
21957 },
21958 repository: None,
21959 idempotency: crate::route::RouteIdempotency::Direct,
21960 timeout: crate::route::RouteTimeout::Inherit,
21961 criticality: crate::admission::Criticality::Default,
21962 seo: crate::seo::SeoRouteDefaults::EMPTY,
21963 api_version: None,
21964 sunset_opt_out: false,
21965 }]);
21966
21967 let response = router
21968 .oneshot(Request::builder().uri("/page").body(Body::empty()).unwrap())
21969 .await
21970 .unwrap();
21971
21972 let body = axum::body::to_bytes(response.into_body(), usize::MAX)
21973 .await
21974 .unwrap();
21975 let html = std::str::from_utf8(&body).expect("utf-8");
21976 assert!(html.contains("/__autumn/live-reload"));
21977 },
21978 )
21979 .await;
21980 }
21981
21982 #[tokio::test]
21983 async fn build_router_mounts_dev_reload_script_endpoint_when_enabled() {
21984 let reload_file = tempfile::NamedTempFile::new().expect("reload state file");
21989 std::fs::write(reload_file.path(), r#"{"version":0,"kind":"full"}"#).expect("write");
21990 temp_env::async_with_vars(
21991 [
21992 ("AUTUMN_DEV_RELOAD", Some("1")),
21993 (
21994 "AUTUMN_DEV_RELOAD_STATE",
21995 Some(reload_file.path().to_str().expect("utf-8 path")),
21996 ),
21997 ],
21998 async {
21999 let router = test_router(vec![test_get_route("/dummy", "dummy")]);
22000
22001 let response = router
22002 .oneshot(
22003 Request::builder()
22004 .uri("/__autumn/live-reload.js")
22005 .body(Body::empty())
22006 .unwrap(),
22007 )
22008 .await
22009 .unwrap();
22010
22011 assert_eq!(response.status(), StatusCode::OK);
22012 assert_eq!(
22013 response
22014 .headers()
22015 .get("content-type")
22016 .and_then(|v| v.to_str().ok()),
22017 Some("application/javascript; charset=utf-8")
22018 );
22019 let body = axum::body::to_bytes(response.into_body(), usize::MAX)
22020 .await
22021 .unwrap();
22022 let js = std::str::from_utf8(&body).expect("utf-8");
22023 assert!(js.contains("fetch("), "js body: {js}");
22024 },
22025 )
22026 .await;
22027 }
22028
22029 #[tokio::test]
22030 async fn build_router_mounts_dev_reload_endpoint_when_enabled() {
22031 let reload_file = tempfile::NamedTempFile::new().expect("reload state file");
22032 std::fs::write(reload_file.path(), r#"{"version":7,"kind":"css"}"#).expect("write");
22033 temp_env::async_with_vars(
22034 [
22035 ("AUTUMN_DEV_RELOAD", Some("1")),
22036 (
22037 "AUTUMN_DEV_RELOAD_STATE",
22038 Some(reload_file.path().to_str().expect("utf-8 path")),
22039 ),
22040 ],
22041 async {
22042 let router = test_router(vec![test_get_route("/dummy", "dummy")]);
22043
22044 let response = router
22045 .oneshot(
22046 Request::builder()
22047 .uri("/__autumn/live-reload")
22048 .body(Body::empty())
22049 .unwrap(),
22050 )
22051 .await
22052 .unwrap();
22053
22054 assert_eq!(response.status(), StatusCode::OK);
22055 assert_eq!(
22056 response.headers().get("cache-control").unwrap(),
22057 "no-store, no-cache, must-revalidate"
22058 );
22059 let body = axum::body::to_bytes(response.into_body(), usize::MAX)
22060 .await
22061 .unwrap();
22062 assert_eq!(&body[..], br#"{"version":7,"kind":"css"}"#);
22063 },
22064 )
22065 .await;
22066 }
22067
22068 #[tokio::test]
22069 async fn build_router_disables_cache_for_static_assets_in_dev_reload_mode() {
22070 let project = tempfile::tempdir().expect("project dir");
22071 let static_dir = project.path().join("static");
22072 std::fs::create_dir_all(&static_dir).expect("mkdir");
22073 std::fs::write(static_dir.join("demo.txt"), "hello").expect("write static file");
22074 let reload_file = tempfile::NamedTempFile::new().expect("reload state file");
22075 std::fs::write(reload_file.path(), r#"{"version":0,"kind":"full"}"#).expect("write");
22076 temp_env::async_with_vars(
22077 [
22078 (
22079 "AUTUMN_MANIFEST_DIR",
22080 Some(project.path().to_str().expect("utf-8 path")),
22081 ),
22082 ("AUTUMN_DEV_RELOAD", Some("1")),
22083 (
22084 "AUTUMN_DEV_RELOAD_STATE",
22085 Some(reload_file.path().to_str().expect("utf-8 path")),
22086 ),
22087 ],
22088 async {
22089 let router = test_router(vec![test_get_route("/dummy", "dummy")]);
22090
22091 let response = router
22092 .oneshot(
22093 Request::builder()
22094 .uri("/static/demo.txt")
22095 .body(Body::empty())
22096 .unwrap(),
22097 )
22098 .await
22099 .unwrap();
22100
22101 assert_eq!(response.status(), StatusCode::OK);
22102 assert_eq!(
22103 response.headers().get("cache-control").unwrap(),
22104 "no-store, no-cache, must-revalidate"
22105 );
22106 },
22107 )
22108 .await;
22109 }
22110
22111 #[test]
22112 fn app_builder_accepts_static_routes() {
22113 use crate::static_gen::StaticRouteMeta;
22114 let metas = vec![StaticRouteMeta {
22115 path: "/about",
22116 name: "about",
22117 revalidate: None,
22118 params_fn: None,
22119 seo: crate::seo::SeoRouteDefaults::EMPTY,
22120 content_type: None,
22121 }];
22122 let builder = app().static_routes(metas);
22123 assert_eq!(builder.static_metas.len(), 1);
22124 }
22125
22126 #[test]
22127 fn project_dir_defaults_to_subdir() {
22128 let env = crate::config::MockEnv::new();
22131 let dir = super::project_dir("dist", &env);
22132 assert_eq!(dir, std::path::PathBuf::from("dist"));
22133 }
22134
22135 pub fn test_router_with_config(routes: Vec<Route>, config: &AutumnConfig) -> axum::Router {
22137 let state = AppState {
22138 health_detailed: true,
22139 ..AppState::test_default()
22140 };
22141 crate::router::build_router(routes, config, state)
22142 }
22143
22144 #[tokio::test]
22145 async fn cors_wildcard_allows_any_origin() {
22146 let mut config = AutumnConfig::default();
22147 config.cors.allowed_origins = vec!["*".to_owned()];
22148 let router = test_router_with_config(vec![test_get_route("/test", "test")], &config);
22149
22150 let response = router
22151 .oneshot(
22152 Request::builder()
22153 .uri("/test")
22154 .header("Origin", "https://example.com")
22155 .body(Body::empty())
22156 .unwrap(),
22157 )
22158 .await
22159 .unwrap();
22160
22161 assert_eq!(response.status(), StatusCode::OK);
22162 assert_eq!(
22163 response
22164 .headers()
22165 .get("access-control-allow-origin")
22166 .unwrap(),
22167 "*"
22168 );
22169 }
22170
22171 #[tokio::test]
22172 async fn cors_specific_origin_reflected() {
22173 let mut config = AutumnConfig::default();
22174 config.cors.allowed_origins = vec!["https://example.com".to_owned()];
22175 let router = test_router_with_config(vec![test_get_route("/test", "test")], &config);
22176
22177 let response = router
22178 .oneshot(
22179 Request::builder()
22180 .uri("/test")
22181 .header("Origin", "https://example.com")
22182 .body(Body::empty())
22183 .unwrap(),
22184 )
22185 .await
22186 .unwrap();
22187
22188 assert_eq!(response.status(), StatusCode::OK);
22189 assert_eq!(
22190 response
22191 .headers()
22192 .get("access-control-allow-origin")
22193 .unwrap(),
22194 "https://example.com"
22195 );
22196 }
22197
22198 #[tokio::test]
22199 async fn cors_disabled_when_no_origins() {
22200 let config = AutumnConfig::default();
22201 assert!(config.cors.allowed_origins.is_empty());
22202 let router = test_router_with_config(vec![test_get_route("/test", "test")], &config);
22203
22204 let response = router
22205 .oneshot(
22206 Request::builder()
22207 .uri("/test")
22208 .header("Origin", "https://example.com")
22209 .body(Body::empty())
22210 .unwrap(),
22211 )
22212 .await
22213 .unwrap();
22214
22215 assert_eq!(response.status(), StatusCode::OK);
22216 assert!(
22217 response
22218 .headers()
22219 .get("access-control-allow-origin")
22220 .is_none()
22221 );
22222 }
22223
22224 #[tokio::test]
22225 async fn cors_preflight_returns_204() {
22226 let mut config = AutumnConfig::default();
22227 config.cors.allowed_origins = vec!["https://example.com".to_owned()];
22228 let router = test_router_with_config(vec![test_get_route("/test", "test")], &config);
22229
22230 let response = router
22231 .oneshot(
22232 Request::builder()
22233 .method("OPTIONS")
22234 .uri("/test")
22235 .header("Origin", "https://example.com")
22236 .header("Access-Control-Request-Method", "GET")
22237 .body(Body::empty())
22238 .unwrap(),
22239 )
22240 .await
22241 .unwrap();
22242
22243 assert_eq!(response.status(), StatusCode::OK);
22244 assert!(
22245 response
22246 .headers()
22247 .contains_key("access-control-allow-methods")
22248 );
22249 }
22250
22251 #[tokio::test]
22252 async fn build_router_with_static_skips_without_manifest() {
22253 let tmp = tempfile::tempdir().expect("tempdir");
22256 let dist = tmp.path().join("dist");
22257 std::fs::create_dir_all(&dist).expect("mkdir");
22258 let config = AutumnConfig::default();
22261 let state = AppState {
22262 health_detailed: true,
22263 ..AppState::test_default()
22264 };
22265 let router = crate::router::build_router_with_static(
22266 vec![test_get_route("/test", "test")],
22267 &config,
22268 state,
22269 Some(dist.as_path()),
22270 );
22271
22272 let response = router
22273 .oneshot(Request::builder().uri("/test").body(Body::empty()).unwrap())
22274 .await
22275 .unwrap();
22276 assert_eq!(response.status(), StatusCode::OK);
22277 }
22278
22279 #[tokio::test]
22280 async fn build_router_with_static_none_dist() {
22281 let config = AutumnConfig::default();
22283 let state = AppState {
22284 health_detailed: true,
22285 ..AppState::test_default()
22286 };
22287 let router = crate::router::build_router_with_static(
22288 vec![test_get_route("/test", "test")],
22289 &config,
22290 state,
22291 None,
22292 );
22293
22294 let response = router
22295 .oneshot(Request::builder().uri("/test").body(Body::empty()).unwrap())
22296 .await
22297 .unwrap();
22298 assert_eq!(response.status(), StatusCode::OK);
22299 }
22300
22301 #[test]
22304 fn format_route_lines_lists_user_routes() {
22305 let routes = vec![
22306 test_get_route("/", "index"),
22307 test_get_route("/users/{id}", "get_user"),
22308 ];
22309 let config = AutumnConfig::default();
22310 let output = format_route_lines(&routes, &[], &config);
22311 assert!(output.contains("-> index"));
22312 assert!(output.contains("/ GET"));
22313 assert!(output.contains("/users/{id}"));
22314 assert!(output.contains("-> get_user"));
22315 }
22316
22317 #[test]
22318 fn config_runtime_drift_format_route_lines_uses_actuator_prefix() {
22319 let mut config = AutumnConfig::default();
22320 config.actuator.prefix = "/ops".to_owned();
22321 let output = format_route_lines(&[], &[], &config);
22322 assert!(output.contains("-> health"));
22323 assert!(output.contains("/ops/*"));
22324 }
22325
22326 #[test]
22327 fn format_task_lines_none_when_empty() {
22328 assert!(format_task_lines(&[]).is_none());
22329 }
22330
22331 #[test]
22332 fn format_task_lines_fixed_delay() {
22333 let tasks = vec![crate::task::TaskInfo {
22334 name: "cleanup".into(),
22335 schedule: crate::task::Schedule::FixedDelay(std::time::Duration::from_secs(300)),
22336 coordination: crate::task::TaskCoordination::Fleet,
22337 handler: |_| Box::pin(async { Ok(()) }),
22338 }];
22339 let output = format_task_lines(&tasks).unwrap();
22340 assert!(output.contains("cleanup (every 300s)"));
22341 }
22342
22343 #[test]
22344 fn format_task_lines_cron() {
22345 let tasks = vec![crate::task::TaskInfo {
22346 name: "nightly".into(),
22347 schedule: crate::task::Schedule::Cron {
22348 expression: "0 0 * * *".into(),
22349 timezone: None,
22350 },
22351 coordination: crate::task::TaskCoordination::Fleet,
22352 handler: |_| Box::pin(async { Ok(()) }),
22353 }];
22354 let output = format_task_lines(&tasks).unwrap();
22355 assert!(output.contains("nightly (cron 0 0 * * *)"));
22356 }
22357
22358 #[test]
22359 fn format_middleware_list_default() {
22360 let config = AutumnConfig::default();
22361 let output = format_middleware_list(&config);
22362 assert!(output.contains("RequestId"));
22363 assert!(output.contains("SecurityHeaders"));
22364 assert!(output.contains("Session (in-memory)"));
22365 assert!(output.contains("Metrics"));
22366 assert!(!output.contains("CORS"));
22368 assert!(!output.contains("CSRF"));
22369 }
22370
22371 #[test]
22372 fn format_middleware_list_with_cors_and_csrf() {
22373 let config = AutumnConfig {
22374 cors: crate::config::CorsConfig {
22375 allowed_origins: vec!["https://example.com".into()],
22376 ..crate::config::CorsConfig::default()
22377 },
22378 security: crate::security::config::SecurityConfig {
22379 csrf: crate::security::config::CsrfConfig {
22380 enabled: true,
22381 ..crate::security::config::CsrfConfig::default()
22382 },
22383 ..crate::security::config::SecurityConfig::default()
22384 },
22385 ..AutumnConfig::default()
22386 };
22387 let output = format_middleware_list(&config);
22388 assert!(output.contains("CORS"));
22389 assert!(output.contains("CSRF"));
22390 }
22391
22392 #[test]
22393 fn mask_database_url_with_password() {
22394 let masked = mask_database_url("postgres://user:secret@localhost:5432/mydb", 10);
22395 assert!(masked.contains("****"));
22396 assert!(!masked.contains("secret"));
22397 assert!(masked.contains("postgres://user:****@localhost:5432/mydb"));
22398 assert!(masked.contains("pool_size=10"));
22399 }
22400
22401 #[test]
22402 fn mask_database_url_without_password() {
22403 let masked = mask_database_url("postgres://localhost/mydb", 5);
22404 assert!(!masked.contains("****"));
22405 assert!(masked.contains("postgres://localhost/mydb"));
22406 assert!(masked.contains("pool_size=5"));
22407 }
22408
22409 #[test]
22410 fn mask_database_url_edge_cases() {
22411 let masked2 = mask_database_url("postgres://user:p%40ssw%3Ard%21@localhost:5432/mydb", 10);
22418 assert!(masked2.contains("****"));
22419 assert!(!masked2.contains("p%40ssw%3Ard%21"));
22420 assert!(masked2.contains("postgres://user:****@localhost:5432/mydb"));
22421
22422 let masked3 = mask_database_url("postgres://:secret@localhost:5432/mydb", 10);
22424 assert!(masked3.contains("****"));
22425 assert!(!masked3.contains("secret"));
22426 assert!(masked3.contains("postgres://:****@localhost:5432/mydb"));
22427 }
22428 #[test]
22433 fn mask_database_url_keeps_the_diagnostic_parts_of_a_target() {
22434 let sqlite = mask_database_url("sqlite:///var/lib/app.db", 1);
22435 assert!(sqlite.contains("sqlite:///var/lib/app.db"), "{sqlite}");
22436
22437 let read_only = mask_database_url("sqlite://file:app.db?mode=ro", 1);
22438 assert!(read_only.contains("mode=ro"), "{read_only}");
22439
22440 let pg = mask_database_url("postgres://app@db/app?sslmode=verify-full", 10);
22441 assert!(pg.contains("sslmode=verify-full"), "{pg}");
22442
22443 let secret = mask_database_url("postgres://app@db/app?sslpassword=hunter2", 10);
22444 assert!(!secret.contains("hunter2"), "{secret}");
22445 }
22446
22447 #[test]
22448 fn mask_database_url_invalid_url_fallback() {
22449 let masked = mask_database_url("this is completely invalid as a URL with supersecret", 10);
22450 assert!(masked.contains("****"));
22451 assert!(!masked.contains("supersecret"));
22452 assert!(masked.contains("pool_size=10"));
22453 }
22454
22455 #[test]
22456 fn format_config_summary_defaults() {
22457 let config = AutumnConfig::default();
22458 let output = format_config_summary(&config);
22459 assert!(output.contains("profile: none"));
22460 assert!(output.contains("server: 127.0.0.1:3000"));
22461 assert!(output.contains("database: not configured"));
22462 assert!(output.contains("log_level:"));
22463 assert!(output.contains("telemetry: disabled"));
22464 assert!(output.contains("health: /health"));
22465 }
22466
22467 #[test]
22468 fn format_config_summary_with_db() {
22469 let config = AutumnConfig {
22470 database: crate::config::DatabaseConfig {
22471 url: Some("postgres://user:pass@host/db".into()),
22472 pool_size: 20,
22473 ..crate::config::DatabaseConfig::default()
22474 },
22475 ..AutumnConfig::default()
22476 };
22477 let output = format_config_summary(&config);
22478 assert!(output.contains("user:****@host/db"));
22479 assert!(output.contains("pool_size=20"));
22480 assert!(!output.contains("pass"));
22481 }
22482
22483 #[test]
22484 fn format_config_summary_with_profile() {
22485 let config = AutumnConfig {
22486 profile: Some("prod".into()),
22487 ..AutumnConfig::default()
22488 };
22489 let output = format_config_summary(&config);
22490 assert!(output.contains("profile: prod"));
22491 }
22492
22493 #[test]
22494 fn format_config_summary_with_telemetry() {
22495 let config = AutumnConfig {
22496 telemetry: crate::config::TelemetryConfig {
22497 enabled: true,
22498 service_name: "orders-api".into(),
22499 otlp_endpoint: Some("http://otel-collector:4317".into()),
22500 ..crate::config::TelemetryConfig::default()
22501 },
22502 ..AutumnConfig::default()
22503 };
22504 let output = format_config_summary(&config);
22505 assert!(output.contains("telemetry: Grpc -> http://otel-collector:4317"));
22506 }
22507
22508 #[test]
22509 fn log_startup_transparency_runs_without_panic() {
22510 let routes = vec![test_get_route("/", "index")];
22514 let tasks = vec![crate::task::TaskInfo {
22515 name: "cleanup".into(),
22516 schedule: crate::task::Schedule::FixedDelay(std::time::Duration::from_secs(60)),
22517 coordination: crate::task::TaskCoordination::Fleet,
22518 handler: |_| Box::pin(async { Ok(()) }),
22519 }];
22520 let config = AutumnConfig::default();
22521 log_startup_transparency(&routes, &tasks, &[], &config);
22522 }
22523
22524 #[test]
22525 fn log_startup_transparency_no_tasks() {
22526 let routes = vec![test_get_route("/health", "check")];
22527 let config = AutumnConfig::default();
22528 log_startup_transparency(&routes, &[], &[], &config);
22529 }
22530
22531 #[cfg(feature = "ws")]
22532 #[tokio::test]
22533 async fn start_task_scheduler_broadcasts_events() {
22534 let state = AppState {
22535 health_detailed: true,
22536 ..AppState::test_default()
22537 };
22538
22539 let mut rx = state.channels().subscribe("sys:tasks");
22540
22541 let task = crate::task::TaskInfo {
22542 name: "test_broadcaster".into(),
22543 schedule: crate::task::Schedule::FixedDelay(std::time::Duration::from_millis(1)),
22545 coordination: crate::task::TaskCoordination::Fleet,
22546 handler: |_| Box::pin(async { Ok(()) }),
22547 };
22548
22549 let state_clone = state.clone();
22551 tokio::spawn(async move {
22552 super::start_task_scheduler(
22553 vec![task],
22554 &state_clone,
22555 &tokio_util::sync::CancellationToken::new(),
22556 );
22557 });
22558
22559 let msg1 = tokio::time::timeout(std::time::Duration::from_secs(1), rx.recv())
22561 .await
22562 .expect("timeout waiting for start event")
22563 .expect("channel closed");
22564 let json1: serde_json::Value = serde_json::from_str(msg1.as_str()).unwrap();
22565 assert_eq!(json1["event"], "started");
22566 assert_eq!(json1["task"], "test_broadcaster");
22567
22568 let msg2 = tokio::time::timeout(std::time::Duration::from_secs(1), rx.recv())
22570 .await
22571 .expect("timeout waiting for success event")
22572 .expect("channel closed");
22573 let json2: serde_json::Value = serde_json::from_str(msg2.as_str()).unwrap();
22574 assert_eq!(json2["event"], "success");
22575 assert_eq!(json2["task"], "test_broadcaster");
22576 assert!(json2.get("duration_ms").is_some());
22577 }
22578
22579 #[cfg(feature = "ws")]
22580 #[tokio::test]
22581 async fn start_task_scheduler_broadcasts_failure_events() {
22582 let state = AppState {
22583 health_detailed: true,
22584 ..AppState::test_default()
22585 };
22586
22587 let mut rx = state.channels().subscribe("sys:tasks");
22588
22589 let task = crate::task::TaskInfo {
22590 name: "test_failing_task".into(),
22591 schedule: crate::task::Schedule::FixedDelay(std::time::Duration::from_millis(1)),
22592 coordination: crate::task::TaskCoordination::Fleet,
22593 handler: |_| {
22594 Box::pin(async { Err(crate::AutumnError::bad_request_msg("forced error")) })
22595 },
22596 };
22597
22598 let state_clone = state.clone();
22599 tokio::spawn(async move {
22600 super::start_task_scheduler(
22601 vec![task],
22602 &state_clone,
22603 &tokio_util::sync::CancellationToken::new(),
22604 );
22605 });
22606
22607 let _ = rx.recv().await.unwrap();
22609
22610 let msg2 = tokio::time::timeout(std::time::Duration::from_secs(1), rx.recv())
22612 .await
22613 .expect("timeout waiting for failure event")
22614 .expect("channel closed");
22615 let json2: serde_json::Value = serde_json::from_str(msg2.as_str()).unwrap();
22616 assert_eq!(json2["event"], "failure");
22617 assert_eq!(json2["task"], "test_failing_task");
22618 assert_eq!(json2["error"], "forced error");
22619 }
22620
22621 #[tokio::test]
22622 async fn execute_task_result_ok_returns_duration() {
22623 let state = AppState::for_test();
22624 let handler: crate::task::TaskHandler = |_| Box::pin(async { Ok(()) });
22625 let start = state.monotonic();
22626 let result =
22627 super::execute_task_result(&state, handler, start, "test_task", "fixed_delay", false)
22628 .await;
22629 assert!(result.is_ok(), "expected Ok from successful handler");
22630 assert!(result.unwrap() < u64::MAX);
22632 }
22633
22634 #[tokio::test]
22635 async fn execute_task_result_err_returns_duration_and_message() {
22636 let state = AppState::for_test();
22637 let handler: crate::task::TaskHandler =
22638 |_| Box::pin(async { Err(crate::AutumnError::bad_request_msg("test error")) });
22639 let start = state.monotonic();
22640 let result =
22641 super::execute_task_result(&state, handler, start, "test_task", "fixed_delay", false)
22642 .await;
22643 assert!(result.is_err(), "expected Err from failing handler");
22644 let (duration_ms, msg) = result.unwrap_err();
22645 assert!(duration_ms < u64::MAX);
22646 assert!(msg.contains("test error"));
22647 }
22648
22649 fn instantly_panicking_scheduled_handler(
22650 _state: AppState,
22651 ) -> std::pin::Pin<Box<dyn std::future::Future<Output = crate::AutumnResult<()>> + Send>> {
22652 panic!("panic before scheduled future")
22653 }
22654
22655 #[tokio::test]
22656 async fn execute_task_result_reports_immediate_handler_panics() {
22657 let state = AppState::for_test();
22658 let start = state.monotonic();
22659 let result = super::execute_task_result(
22660 &state,
22661 instantly_panicking_scheduled_handler,
22662 start,
22663 "test_task",
22664 "fixed_delay",
22665 false,
22666 )
22667 .await;
22668
22669 let (duration_ms, msg) = result.expect_err("expected Err from panicking handler");
22670 assert!(duration_ms < u64::MAX);
22671 assert!(msg.contains("scheduled task handler panicked: panic before scheduled future"));
22672 }
22673
22674 #[tokio::test]
22677 async fn execute_task_result_meters_an_immediate_handler_panic() {
22678 let state = AppState::for_test();
22679 let accountant = crate::cost::CostAccountant::new(10);
22680 state.insert_extension(accountant.clone());
22681 let start = state.monotonic();
22682 let result = super::execute_task_result(
22683 &state,
22684 instantly_panicking_scheduled_handler,
22685 start,
22686 "test_task",
22687 "fixed_delay",
22688 false,
22689 )
22690 .await;
22691
22692 assert!(result.is_err());
22693 assert_eq!(accountant.snapshot().tasks.total.runs, 1);
22694 }
22695
22696 #[tokio::test]
22697 async fn execute_fixed_delay_task_does_not_timeout_in_process_runs() {
22698 let state = AppState::for_test();
22699 state.task_registry.register_scheduled(
22700 "slow_task",
22701 "every 1s",
22702 crate::task::TaskCoordination::Fleet,
22703 "in_process",
22704 "replica-a",
22705 );
22706 let handler: crate::task::TaskHandler = |_| {
22707 Box::pin(async {
22708 tokio::time::sleep(std::time::Duration::from_millis(30)).await;
22709 Ok(())
22710 })
22711 };
22712 let coordinator = std::sync::Arc::new(
22713 crate::scheduler::InProcessSchedulerCoordinator::new("replica-a"),
22714 );
22715
22716 super::execute_fixed_delay_task(
22717 "slow_task".to_owned(),
22718 state.clone(),
22719 handler,
22720 std::time::Duration::from_secs(1),
22721 crate::task::TaskCoordination::Fleet,
22722 coordinator,
22723 std::time::Duration::from_millis(10),
22724 tokio_util::sync::CancellationToken::new(),
22725 )
22726 .await;
22727
22728 let snapshot = state.task_registry.snapshot();
22729 let status = &snapshot["slow_task"];
22730 assert_eq!(status.status, "idle");
22731 assert_eq!(status.last_result.as_deref(), Some("ok"));
22732 assert_eq!(status.total_runs, 1);
22733 assert_eq!(status.total_failures, 0);
22734 assert!(status.last_error.is_none());
22735 }
22736
22737 static SKIPPED_LEASE_HANDLER_CALLS: AtomicUsize = AtomicUsize::new(0);
22738
22739 struct DenyingSchedulerCoordinator;
22740
22741 impl crate::scheduler::SchedulerCoordinator for DenyingSchedulerCoordinator {
22742 fn backend(&self) -> &'static str {
22743 "postgres"
22744 }
22745
22746 fn replica_id(&self) -> &'static str {
22747 "replica-a"
22748 }
22749
22750 fn try_acquire<'a>(
22751 &'a self,
22752 _task_name: &'a str,
22753 _tick_key: &'a str,
22754 _coordination: crate::task::TaskCoordination,
22755 ) -> crate::scheduler::SchedulerFuture<
22756 'a,
22757 crate::AutumnResult<Option<crate::scheduler::SchedulerLease>>,
22758 > {
22759 Box::pin(async { Ok(None) })
22760 }
22761 }
22762
22763 struct GrantingSchedulerCoordinator {
22764 backend: &'static str,
22765 tick_keys: std::sync::Arc<std::sync::Mutex<Vec<String>>>,
22766 release_count: Option<std::sync::Arc<AtomicUsize>>,
22767 }
22768
22769 impl crate::scheduler::SchedulerCoordinator for GrantingSchedulerCoordinator {
22770 fn backend(&self) -> &'static str {
22771 self.backend
22772 }
22773
22774 fn replica_id(&self) -> &'static str {
22775 "replica-a"
22776 }
22777
22778 fn try_acquire<'a>(
22779 &'a self,
22780 _task_name: &'a str,
22781 tick_key: &'a str,
22782 _coordination: crate::task::TaskCoordination,
22783 ) -> crate::scheduler::SchedulerFuture<
22784 'a,
22785 crate::AutumnResult<Option<crate::scheduler::SchedulerLease>>,
22786 > {
22787 Box::pin(async move {
22788 self.tick_keys.lock().unwrap().push(tick_key.to_owned());
22789 let lease = self.release_count.as_ref().map_or_else(
22790 || crate::scheduler::SchedulerLease::local(self.backend, "replica-a"),
22791 |release_count| {
22792 crate::scheduler::SchedulerLease::tracked(
22793 self.backend,
22794 "replica-a",
22795 std::sync::Arc::clone(release_count),
22796 )
22797 },
22798 );
22799 Ok(Some(lease))
22800 })
22801 }
22802 }
22803
22804 fn counted_scheduled_handler(
22805 _state: AppState,
22806 ) -> std::pin::Pin<Box<dyn std::future::Future<Output = crate::AutumnResult<()>> + Send>> {
22807 Box::pin(async {
22808 SKIPPED_LEASE_HANDLER_CALLS.fetch_add(1, Ordering::SeqCst);
22809 Ok(())
22810 })
22811 }
22812
22813 #[tokio::test]
22814 async fn execute_fixed_delay_task_skips_handler_when_lease_is_not_acquired() {
22815 SKIPPED_LEASE_HANDLER_CALLS.store(0, Ordering::SeqCst);
22816 let state = AppState::for_test();
22817 state.task_registry.register_scheduled(
22818 "claimed_elsewhere",
22819 "every 1s",
22820 crate::task::TaskCoordination::Fleet,
22821 "postgres",
22822 "replica-a",
22823 );
22824 let coordinator = std::sync::Arc::new(DenyingSchedulerCoordinator);
22825
22826 super::execute_fixed_delay_task(
22827 "claimed_elsewhere".to_owned(),
22828 state.clone(),
22829 counted_scheduled_handler,
22830 std::time::Duration::from_secs(1),
22831 crate::task::TaskCoordination::Fleet,
22832 coordinator,
22833 std::time::Duration::from_secs(1),
22834 tokio_util::sync::CancellationToken::new(),
22835 )
22836 .await;
22837
22838 let snapshot = state.task_registry.snapshot();
22839 let status = &snapshot["claimed_elsewhere"];
22840 assert_eq!(SKIPPED_LEASE_HANDLER_CALLS.load(Ordering::SeqCst), 0);
22841 assert_eq!(status.total_runs, 0);
22842 assert!(status.current_leader.is_none());
22843 assert!(status.last_tick.is_none());
22844 }
22845
22846 #[tokio::test(start_paused = true)]
22850 async fn deferred_tick_takes_an_expiring_lease_only_after_the_wait() {
22851 for (backend, acquired_while_high) in [("sqlite", 0), ("postgres", 0), ("in_process", 1)] {
22852 let name = format!("cost_lease_order_{backend}");
22853 crate::cost::mark_deferrable(crate::cost::WorkKind::Task, &name);
22854 let state = cron_test_state();
22855 let signal = crate::cost::CostSignal::new(Some(1.0));
22856 signal.set(5.0);
22857 state.insert_extension(signal.clone());
22858 let tick_keys = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
22859 let coordinator = std::sync::Arc::new(GrantingSchedulerCoordinator {
22860 backend,
22861 tick_keys: std::sync::Arc::clone(&tick_keys),
22862 release_count: None,
22863 });
22864 let handler: crate::task::TaskHandler = |_| Box::pin(async { Ok(()) });
22865 let run = tokio::spawn(super::execute_cron_task(
22866 name,
22867 state,
22868 handler,
22869 crate::task::TaskCoordination::Fleet,
22870 coordinator,
22871 std::time::Duration::from_secs(30),
22872 super::CronTick {
22873 unix_secs: 1_700_000_000,
22874 window: std::time::Duration::from_secs(60),
22875 },
22876 super::CostGate {
22877 shutdown: tokio_util::sync::CancellationToken::new(),
22878 waiting: None,
22879 waited: std::sync::atomic::AtomicBool::new(false),
22880 },
22881 ));
22882
22883 tokio::time::sleep(std::time::Duration::from_secs(90)).await;
22884 assert_eq!(
22885 tick_keys.lock().unwrap().len(),
22886 acquired_while_high,
22887 "{backend}"
22888 );
22889
22890 signal.set(0.5);
22891 run.await.expect("the tick ends");
22892 assert_eq!(tick_keys.lock().unwrap().len(), 1, "{backend}");
22893 }
22894 }
22895
22896 struct RisingSignal {
22899 signal: crate::cost::CostSignal,
22900 acquired: std::sync::Arc<AtomicUsize>,
22901 seen: std::sync::Arc<std::sync::Mutex<Vec<(String, bool)>>>,
22903 flag: std::sync::Arc<std::sync::atomic::AtomicBool>,
22904 }
22905
22906 impl crate::scheduler::SchedulerCoordinator for RisingSignal {
22907 fn backend(&self) -> &'static str {
22908 "sqlite"
22909 }
22910
22911 fn replica_id(&self) -> &'static str {
22912 "replica-a"
22913 }
22914
22915 fn try_acquire<'a>(
22916 &'a self,
22917 _task_name: &'a str,
22918 tick_key: &'a str,
22919 _coordination: crate::task::TaskCoordination,
22920 ) -> crate::scheduler::SchedulerFuture<
22921 'a,
22922 crate::AutumnResult<Option<crate::scheduler::SchedulerLease>>,
22923 > {
22924 Box::pin(async move {
22925 self.seen
22926 .lock()
22927 .unwrap()
22928 .push((tick_key.to_owned(), self.flag.load(Ordering::SeqCst)));
22929 if self.acquired.fetch_add(1, Ordering::SeqCst) == 0 {
22930 self.signal.set(5.0);
22931 }
22932 Ok(Some(crate::scheduler::SchedulerLease::local(
22933 "sqlite",
22934 "replica-a",
22935 )))
22936 })
22937 }
22938 }
22939
22940 #[tokio::test(start_paused = true)]
22944 async fn deferred_tick_gives_back_a_sqlite_lease_when_the_signal_rises() {
22945 static RAN: AtomicUsize = AtomicUsize::new(0);
22946 static SEEN_RETRY_TICK: std::sync::Mutex<Option<String>> = std::sync::Mutex::new(None);
22947 let name = "cost_sqlite_rising_signal".to_owned();
22948 crate::cost::mark_deferrable(crate::cost::WorkKind::Task, &name);
22949 let state = cron_test_state();
22950 let signal = crate::cost::CostSignal::new(Some(1.0));
22951 state.insert_extension(signal.clone());
22952 let acquired = std::sync::Arc::new(AtomicUsize::new(0));
22953 let seen = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
22954 let flag = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(true));
22956 let coordinator = std::sync::Arc::new(RisingSignal {
22957 signal: signal.clone(),
22958 acquired: std::sync::Arc::clone(&acquired),
22959 seen: std::sync::Arc::clone(&seen),
22960 flag: std::sync::Arc::clone(&flag),
22961 });
22962 let handler: crate::task::TaskHandler = |_| {
22963 Box::pin(async {
22964 RAN.fetch_add(1, Ordering::SeqCst);
22965 *SEEN_RETRY_TICK.lock().unwrap() =
22966 crate::scheduler::current_tick().map(|tick| tick.tick_key().to_owned());
22967 Ok(())
22968 })
22969 };
22970 let run = tokio::spawn(super::execute_cron_task(
22971 name,
22972 state,
22973 handler,
22974 crate::task::TaskCoordination::Fleet,
22975 coordinator,
22976 std::time::Duration::from_secs(30),
22977 super::CronTick {
22978 unix_secs: 1_700_000_000,
22979 window: std::time::Duration::from_secs(60),
22980 },
22981 super::CostGate {
22982 shutdown: tokio_util::sync::CancellationToken::new(),
22983 waiting: Some(std::sync::Arc::clone(&flag)),
22984 waited: std::sync::atomic::AtomicBool::new(false),
22985 },
22986 ));
22987
22988 tokio::time::sleep(std::time::Duration::from_secs(90)).await;
22989 assert_eq!(
22990 RAN.load(Ordering::SeqCst),
22991 0,
22992 "no run while the signal is high"
22993 );
22994
22995 signal.set(0.5);
22996 run.await.expect("the tick ends");
22997 assert_eq!(RAN.load(Ordering::SeqCst), 1);
22998 assert_eq!(
22999 acquired.load(Ordering::SeqCst),
23000 2,
23001 "the lease is taken again"
23002 );
23003 let seen = seen.lock().unwrap().clone();
23004 assert_eq!(
23005 seen[0].0, "cost_sqlite_rising_signal:1700000000",
23006 "{seen:?}"
23007 );
23008 assert_eq!(
23009 seen[1].0, seen[0].0,
23010 "every replica resumes against the one tick key: {seen:?}"
23011 );
23012 assert!(
23013 seen.iter().all(|(_, reserved)| *reserved),
23014 "later ticks stay folded while a lease attempt is in flight: {seen:?}"
23015 );
23016 assert!(
23017 !flag.load(Ordering::SeqCst),
23018 "the tick clears the reservation"
23019 );
23020 assert_eq!(
23021 SEEN_RETRY_TICK.lock().unwrap().as_deref(),
23022 Some("cost_sqlite_rising_signal:1700000000"),
23023 "the task sees the scheduled tick, not the retry key"
23024 );
23025 }
23026
23027 #[tokio::test]
23030 async fn execute_cron_task_clears_the_cost_flag_when_the_lease_is_taken() {
23031 let state = cron_test_state();
23032 let waiting = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(true));
23033 let handler: crate::task::TaskHandler = |_| Box::pin(async { Ok(()) });
23034
23035 super::execute_cron_task(
23036 "cron_lease_taken".to_owned(),
23037 state,
23038 handler,
23039 crate::task::TaskCoordination::Fleet,
23040 std::sync::Arc::new(DenyingSchedulerCoordinator),
23041 std::time::Duration::from_secs(30),
23042 super::CronTick {
23043 unix_secs: 1_700_000_000,
23044 window: std::time::Duration::from_secs(60),
23045 },
23046 super::CostGate {
23047 shutdown: tokio_util::sync::CancellationToken::new(),
23048 waiting: Some(std::sync::Arc::clone(&waiting)),
23049 waited: std::sync::atomic::AtomicBool::new(false),
23050 },
23051 )
23052 .await;
23053
23054 assert!(!waiting.load(Ordering::SeqCst), "later ticks run again");
23055 }
23056
23057 #[tokio::test]
23058 async fn execute_fixed_delay_task_records_distributed_lease_ttl_timeout() {
23059 let state = AppState::for_test();
23060 state.task_registry.register_scheduled(
23061 "slow_distributed_task",
23062 "every 1s",
23063 crate::task::TaskCoordination::Fleet,
23064 "postgres",
23065 "replica-a",
23066 );
23067 let handler: crate::task::TaskHandler = |_| {
23068 Box::pin(async {
23069 tokio::time::sleep(std::time::Duration::from_secs(5)).await;
23070 Ok(())
23071 })
23072 };
23073 let coordinator = std::sync::Arc::new(GrantingSchedulerCoordinator {
23074 backend: "postgres",
23075 tick_keys: std::sync::Arc::new(std::sync::Mutex::new(Vec::new())),
23076 release_count: None,
23077 });
23078
23079 super::execute_fixed_delay_task(
23080 "slow_distributed_task".to_owned(),
23081 state.clone(),
23082 handler,
23083 std::time::Duration::from_secs(1),
23084 crate::task::TaskCoordination::Fleet,
23085 coordinator,
23086 std::time::Duration::from_millis(10),
23087 tokio_util::sync::CancellationToken::new(),
23088 )
23089 .await;
23090
23091 let snapshot = state.task_registry.snapshot();
23092 let status = &snapshot["slow_distributed_task"];
23093 assert_eq!(status.status, "idle");
23094 assert_eq!(status.last_result.as_deref(), Some("failed"));
23095 assert_eq!(status.total_runs, 1);
23096 assert_eq!(status.total_failures, 1);
23097 assert!(
23098 status
23099 .last_error
23100 .as_deref()
23101 .is_some_and(|error| error.contains("lease TTL"))
23102 );
23103 }
23104
23105 static SEEN_TICK: std::sync::Mutex<Option<crate::scheduler::ScheduledTick>> =
23106 std::sync::Mutex::new(None);
23107
23108 struct FencingSchedulerCoordinator;
23110
23111 impl crate::scheduler::SchedulerCoordinator for FencingSchedulerCoordinator {
23112 fn backend(&self) -> &'static str {
23113 "postgres"
23114 }
23115
23116 fn replica_id(&self) -> &'static str {
23117 "replica-a"
23118 }
23119
23120 fn try_acquire<'a>(
23121 &'a self,
23122 _task_name: &'a str,
23123 _tick_key: &'a str,
23124 _coordination: crate::task::TaskCoordination,
23125 ) -> crate::scheduler::SchedulerFuture<
23126 'a,
23127 crate::AutumnResult<Option<crate::scheduler::SchedulerLease>>,
23128 > {
23129 Box::pin(async {
23130 Ok(Some(
23131 crate::scheduler::SchedulerLease::local("postgres", "replica-a")
23132 .with_fencing_token(99),
23133 ))
23134 })
23135 }
23136 }
23137
23138 struct PeriodRecordingCoordinator {
23140 period: std::sync::Mutex<Option<std::time::Duration>>,
23141 }
23142
23143 impl crate::scheduler::SchedulerCoordinator for PeriodRecordingCoordinator {
23144 fn backend(&self) -> &'static str {
23145 "postgres"
23146 }
23147
23148 fn replica_id(&self) -> &'static str {
23149 "replica-a"
23150 }
23151
23152 fn try_acquire<'a>(
23153 &'a self,
23154 _task_name: &'a str,
23155 _tick_key: &'a str,
23156 _coordination: crate::task::TaskCoordination,
23157 ) -> crate::scheduler::SchedulerFuture<
23158 'a,
23159 crate::AutumnResult<Option<crate::scheduler::SchedulerLease>>,
23160 > {
23161 Box::pin(async { Ok(None) })
23162 }
23163
23164 fn try_acquire_for_period<'a>(
23165 &'a self,
23166 _task_name: &'a str,
23167 _tick_key: &'a str,
23168 _coordination: crate::task::TaskCoordination,
23169 period: std::time::Duration,
23170 ) -> crate::scheduler::SchedulerFuture<
23171 'a,
23172 crate::AutumnResult<Option<crate::scheduler::SchedulerLease>>,
23173 > {
23174 *self.period.lock().unwrap() = Some(period);
23175 Box::pin(async { Ok(None) })
23176 }
23177 }
23178
23179 #[tokio::test]
23181 async fn fixed_delay_task_claims_its_tick_for_the_delay() {
23182 let coordinator = std::sync::Arc::new(PeriodRecordingCoordinator {
23183 period: std::sync::Mutex::new(None),
23184 });
23185 let handler: crate::task::TaskHandler = |_| Box::pin(async { Ok(()) });
23186
23187 super::execute_fixed_delay_task(
23188 "hourly_task".to_owned(),
23189 AppState::for_test(),
23190 handler,
23191 std::time::Duration::from_secs(3_600),
23192 crate::task::TaskCoordination::Fleet,
23193 std::sync::Arc::clone(&coordinator) as _,
23194 std::time::Duration::from_secs(300),
23195 tokio_util::sync::CancellationToken::new(),
23196 )
23197 .await;
23198
23199 assert_eq!(
23200 *coordinator.period.lock().unwrap(),
23201 Some(std::time::Duration::from_secs(3_600))
23202 );
23203 }
23204
23205 fn fleet_task_info(coordination: crate::task::TaskCoordination) -> crate::task::TaskInfo {
23206 crate::task::TaskInfo {
23207 name: "nightly".to_owned(),
23208 schedule: crate::task::Schedule::FixedDelay(std::time::Duration::from_secs(60)),
23209 coordination,
23210 handler: |_| Box::pin(async { Ok(()) }),
23211 }
23212 }
23213
23214 #[test]
23216 fn in_process_fleet_warning_names_the_hint_and_the_fix() {
23217 let config = AutumnConfig::default();
23218 let tasks = [fleet_task_info(crate::task::TaskCoordination::Fleet)];
23219 let env = crate::config::MockEnv::new().with("AUTUMN_REPLICAS", "3");
23220
23221 let warning = super::in_process_fleet_warning(&config, &tasks, &env)
23222 .expect("three replicas on in_process must warn");
23223 assert!(
23224 warning.contains("AUTUMN_REPLICAS is more than 1"),
23225 "{warning}"
23226 );
23227 assert!(warning.contains("scheduler.backend = \""), "{warning}");
23228
23229 let k8s = crate::config::MockEnv::new().with("KUBERNETES_SERVICE_HOST", "10.0.0.1");
23230 assert!(super::in_process_fleet_warning(&config, &tasks, &k8s).is_some());
23231 }
23232
23233 #[test]
23234 fn in_process_fleet_warning_is_silent_without_a_hint_or_a_fleet_task() {
23235 let config = AutumnConfig::default();
23236 let fleet = [fleet_task_info(crate::task::TaskCoordination::Fleet)];
23237 let local = [fleet_task_info(crate::task::TaskCoordination::PerReplica)];
23238 let crowd = crate::config::MockEnv::new().with("AUTUMN_REPLICAS", "5");
23239
23240 assert!(
23241 super::in_process_fleet_warning(&config, &fleet, &crate::config::MockEnv::new())
23242 .is_none(),
23243 "no hint"
23244 );
23245 assert!(
23246 super::in_process_fleet_warning(&config, &local, &crowd).is_none(),
23247 "per-replica tasks run on every replica by design"
23248 );
23249
23250 let mut coordinated = AutumnConfig::default();
23251 coordinated.scheduler.backend = crate::config::SchedulerBackend::Postgres;
23252 assert!(super::in_process_fleet_warning(&coordinated, &fleet, &crowd).is_none());
23253 }
23254
23255 #[tokio::test]
23257 async fn scheduled_handler_sees_its_tick_and_fencing_token() {
23258 let state = cron_test_state();
23259 state.task_registry.register_scheduled(
23260 "cron_fence_task",
23261 "cron 0 * * * * *",
23262 crate::task::TaskCoordination::Fleet,
23263 "postgres",
23264 "replica-a",
23265 );
23266 let handler: crate::task::TaskHandler = |_| {
23267 Box::pin(async {
23268 *SEEN_TICK.lock().unwrap() = crate::scheduler::current_tick();
23269 Ok(())
23270 })
23271 };
23272
23273 super::execute_cron_task(
23274 "cron_fence_task".to_owned(),
23275 state,
23276 handler,
23277 crate::task::TaskCoordination::Fleet,
23278 std::sync::Arc::new(FencingSchedulerCoordinator),
23279 std::time::Duration::from_secs(5),
23280 super::CronTick {
23281 unix_secs: 1_700_000_000,
23282 window: std::time::Duration::from_secs(60),
23283 },
23284 super::CostGate {
23285 shutdown: tokio_util::sync::CancellationToken::new(),
23286 waiting: None,
23287 waited: std::sync::atomic::AtomicBool::new(false),
23288 },
23289 )
23290 .await;
23291
23292 let seen = SEEN_TICK
23293 .lock()
23294 .unwrap()
23295 .take()
23296 .expect("handler saw a tick");
23297 assert_eq!(seen.tick_key(), "cron_fence_task:1700000000");
23298 assert_eq!(seen.fencing_token(), Some(99));
23299 }
23300
23301 #[tokio::test]
23302 async fn execute_cron_task_uses_scheduled_occurrence_for_tick_key() {
23303 let state = AppState::for_test();
23304 state.task_registry.register_scheduled(
23305 "cron_review_task",
23306 "cron */10 * * * * *",
23307 crate::task::TaskCoordination::Fleet,
23308 "postgres",
23309 "replica-a",
23310 );
23311 let tick_keys = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
23312 let coordinator = std::sync::Arc::new(GrantingSchedulerCoordinator {
23313 backend: "postgres",
23314 tick_keys: std::sync::Arc::clone(&tick_keys),
23315 release_count: None,
23316 });
23317 let handler: crate::task::TaskHandler = |_| Box::pin(async { Ok(()) });
23318 let scheduled_unix_secs = 1_700_000_000;
23319
23320 super::execute_cron_task(
23321 "cron_review_task".to_owned(),
23322 state.clone(),
23323 handler,
23324 crate::task::TaskCoordination::Fleet,
23325 coordinator,
23326 std::time::Duration::from_secs(30),
23327 super::CronTick {
23328 unix_secs: scheduled_unix_secs,
23329 window: std::time::Duration::from_secs(10),
23330 },
23331 super::CostGate {
23332 shutdown: tokio_util::sync::CancellationToken::new(),
23333 waiting: None,
23334 waited: std::sync::atomic::AtomicBool::new(false),
23335 },
23336 )
23337 .await;
23338
23339 assert_eq!(
23340 tick_keys.lock().unwrap().as_slice(),
23341 ["cron_review_task:1700000000"]
23342 );
23343 }
23344
23345 fn cron_test_state() -> AppState {
23348 AppState::for_test().with_clock(std::sync::Arc::new(crate::time::FixedClock::at(
23349 chrono::DateTime::from_timestamp(1_700_000_001, 0).expect("a valid timestamp"),
23350 )))
23351 }
23352
23353 #[tokio::test]
23357 async fn execute_cron_task_skips_an_occurrence_past_its_window() {
23358 static RUNS: AtomicUsize = AtomicUsize::new(0);
23359
23360 async fn claims(scheduled_unix_secs: u64, waited: bool) -> Vec<String> {
23361 let state = AppState::for_test();
23362 state.task_registry.register_scheduled(
23363 "late_cron_task",
23364 "cron */10 * * * * *",
23365 crate::task::TaskCoordination::Fleet,
23366 "postgres",
23367 "replica-a",
23368 );
23369 let tick_keys = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
23370 let coordinator = std::sync::Arc::new(GrantingSchedulerCoordinator {
23371 backend: "postgres",
23372 tick_keys: std::sync::Arc::clone(&tick_keys),
23373 release_count: None,
23374 });
23375 let handler: crate::task::TaskHandler = |_| {
23376 RUNS.fetch_add(1, Ordering::SeqCst);
23377 Box::pin(async { Ok(()) })
23378 };
23379 super::execute_cron_task(
23380 "late_cron_task".to_owned(),
23381 state.clone(),
23382 handler,
23383 crate::task::TaskCoordination::Fleet,
23384 coordinator,
23385 std::time::Duration::from_secs(30),
23386 super::CronTick {
23387 unix_secs: scheduled_unix_secs,
23388 window: std::time::Duration::from_secs(10),
23389 },
23390 super::CostGate {
23391 shutdown: tokio_util::sync::CancellationToken::new(),
23392 waiting: None,
23393 waited: std::sync::atomic::AtomicBool::new(waited),
23394 },
23395 )
23396 .await;
23397 tick_keys.lock().unwrap().clone()
23398 }
23399
23400 let now = crate::time::clock_unix_secs(AppState::for_test().clock());
23401 assert!(claims(1_700_000_000, true).await.is_empty());
23403 assert_eq!(claims(1_700_000_000, false).await.len(), 1);
23405 assert_eq!(
23406 RUNS.load(Ordering::SeqCst),
23407 0,
23408 "a late occurrence does not run"
23409 );
23410 assert_eq!(claims(now, true).await, [format!("late_cron_task:{now}")]);
23412 assert_eq!(RUNS.load(Ordering::SeqCst), 1);
23413 }
23414
23415 #[tokio::test]
23416 async fn execute_fixed_delay_task_releases_lease_when_handler_panics() {
23417 let state = AppState::for_test();
23418 state.task_registry.register_scheduled(
23419 "panic_task",
23420 "every 1s",
23421 crate::task::TaskCoordination::Fleet,
23422 "postgres",
23423 "replica-a",
23424 );
23425 let release_count = std::sync::Arc::new(AtomicUsize::new(0));
23426 let coordinator = std::sync::Arc::new(GrantingSchedulerCoordinator {
23427 backend: "postgres",
23428 tick_keys: std::sync::Arc::new(std::sync::Mutex::new(Vec::new())),
23429 release_count: Some(std::sync::Arc::clone(&release_count)),
23430 });
23431 let handler: crate::task::TaskHandler = |_| {
23432 Box::pin(async {
23433 panic!("forced scheduled panic");
23434 #[allow(unreachable_code)]
23435 Ok(())
23436 })
23437 };
23438
23439 super::execute_fixed_delay_task(
23440 "panic_task".to_owned(),
23441 state.clone(),
23442 handler,
23443 std::time::Duration::from_secs(1),
23444 crate::task::TaskCoordination::Fleet,
23445 coordinator,
23446 std::time::Duration::from_secs(30),
23447 tokio_util::sync::CancellationToken::new(),
23448 )
23449 .await;
23450
23451 let snapshot = state.task_registry.snapshot();
23452 let status = &snapshot["panic_task"];
23453 assert_eq!(release_count.load(Ordering::SeqCst), 1);
23454 assert_eq!(status.status, "idle");
23455 assert_eq!(status.last_result.as_deref(), Some("failed"));
23456 assert_eq!(status.total_runs, 1);
23457 assert_eq!(status.total_failures, 1);
23458 assert!(
23459 status
23460 .last_error
23461 .as_deref()
23462 .is_some_and(|error| error.contains("scheduled task handler panicked"))
23463 );
23464 }
23465
23466 #[test]
23469 fn cron_expressions_keep_accepting_shorthand_steps() {
23470 use chrono::TimeZone as _;
23471
23472 let shorthand = super::parse_cron_expression("5/5 * * * *").expect("shorthand step");
23473 let explicit = super::parse_cron_expression("5-59/5 * * * *").expect("explicit range");
23474 let now = chrono_tz::UTC
23475 .with_ymd_and_hms(2026, 5, 5, 12, 6, 0)
23476 .unwrap();
23477 let next = shorthand
23478 .find_next_occurrence(&now, false)
23479 .expect("next occurrence");
23480 assert_eq!(
23481 next,
23482 explicit
23483 .find_next_occurrence(&now, false)
23484 .expect("next occurrence")
23485 );
23486 assert_eq!(
23487 next,
23488 chrono_tz::UTC
23489 .with_ymd_and_hms(2026, 5, 5, 12, 10, 0)
23490 .unwrap()
23491 );
23492 assert!(super::parse_cron_expression("not a cron").is_err());
23493 }
23494
23495 #[test]
23496 fn next_cron_occurrence_skips_overdue_slots() {
23497 use chrono::TimeZone as _;
23498
23499 let cron = "0 * * * * *"
23500 .parse::<croner::Cron>()
23501 .expect("cron expression should parse");
23502 let stale_cursor = chrono_tz::UTC
23503 .with_ymd_and_hms(2026, 5, 5, 12, 0, 0)
23504 .unwrap();
23505 let now = chrono_tz::UTC
23506 .with_ymd_and_hms(2026, 5, 5, 12, 30, 5)
23507 .unwrap();
23508 let next = super::next_cron_occurrence_after(&cron, &stale_cursor, &now)
23509 .expect("next cron occurrence should resolve");
23510
23511 assert_eq!(
23512 next,
23513 chrono_tz::UTC
23514 .with_ymd_and_hms(2026, 5, 5, 12, 31, 0)
23515 .unwrap()
23516 );
23517 }
23518
23519 #[test]
23522 fn cron_occurrence_window_spans_to_the_next_occurrence() {
23523 use chrono::TimeZone as _;
23524
23525 let daily = "0 0 3 * * *".parse::<croner::Cron>().expect("parse");
23526 let at = chrono_tz::UTC
23527 .with_ymd_and_hms(2026, 5, 5, 3, 0, 0)
23528 .unwrap();
23529 assert_eq!(
23530 super::cron_occurrence_window(&daily, &at),
23531 std::time::Duration::from_secs(24 * 60 * 60)
23532 );
23533 let every_ten = "*/10 * * * * *".parse::<croner::Cron>().expect("parse");
23534 let at = chrono_tz::UTC
23535 .with_ymd_and_hms(2026, 5, 5, 3, 0, 10)
23536 .unwrap();
23537 assert_eq!(
23538 super::cron_occurrence_window(&every_ten, &at),
23539 std::time::Duration::from_secs(10)
23540 );
23541 }
23542
23543 #[tokio::test]
23544 async fn cron_task_claims_its_tick_for_the_occurrence_window() {
23545 let coordinator = std::sync::Arc::new(PeriodRecordingCoordinator {
23546 period: std::sync::Mutex::new(None),
23547 });
23548 let handler: crate::task::TaskHandler = |_| Box::pin(async { Ok(()) });
23549
23550 super::execute_cron_task(
23551 "daily_task".to_owned(),
23552 cron_test_state(),
23553 handler,
23554 crate::task::TaskCoordination::Fleet,
23555 std::sync::Arc::clone(&coordinator) as _,
23556 std::time::Duration::from_secs(300),
23557 super::CronTick {
23558 unix_secs: 1_700_000_000,
23559 window: std::time::Duration::from_secs(86_400),
23560 },
23561 super::CostGate {
23562 shutdown: tokio_util::sync::CancellationToken::new(),
23563 waiting: None,
23564 waited: std::sync::atomic::AtomicBool::new(false),
23565 },
23566 )
23567 .await;
23568
23569 assert_eq!(
23570 *coordinator.period.lock().unwrap(),
23571 Some(std::time::Duration::from_secs(86_400))
23572 );
23573 }
23574
23575 #[test]
23576 fn cron_occurrence_is_overdue_after_later_slot_passed() {
23577 use chrono::TimeZone as _;
23578
23579 let cron = "0 * * * * *"
23580 .parse::<croner::Cron>()
23581 .expect("cron expression should parse");
23582 let scheduled_at = chrono_tz::UTC
23583 .with_ymd_and_hms(2026, 5, 5, 12, 1, 0)
23584 .unwrap();
23585 let slightly_late = chrono_tz::UTC
23586 .with_ymd_and_hms(2026, 5, 5, 12, 1, 5)
23587 .unwrap();
23588 let after_later_slot = chrono_tz::UTC
23589 .with_ymd_and_hms(2026, 5, 5, 12, 30, 5)
23590 .unwrap();
23591
23592 assert!(
23593 !super::cron_occurrence_is_overdue(&cron, &scheduled_at, &slightly_late)
23594 .expect("overdue check should resolve")
23595 );
23596 assert!(
23597 super::cron_occurrence_is_overdue(&cron, &scheduled_at, &after_later_slot)
23598 .expect("overdue check should resolve")
23599 );
23600 }
23601
23602 #[cfg(feature = "storage")]
23603 mod storage_preflight {
23604 use super::super::{StorageBootstrap, preflight_storage};
23605 use crate::AppState;
23606 use crate::config::AutumnConfig;
23607 use crate::storage::{BlobStoreState, StorageBackend, StorageConfig, StorageLocalConfig};
23608
23609 fn config_with_storage(storage: StorageConfig) -> AutumnConfig {
23610 AutumnConfig {
23611 profile: Some("dev".into()),
23612 storage,
23613 ..AutumnConfig::default()
23614 }
23615 }
23616
23617 #[test]
23618 fn preflight_returns_none_when_disabled() {
23619 let cfg = config_with_storage(StorageConfig {
23620 backend: StorageBackend::Disabled,
23621 ..StorageConfig::default()
23622 });
23623 assert!(preflight_storage(&cfg).is_none());
23624 }
23625
23626 #[test]
23627 fn preflight_provisions_local_backend_against_tempdir() {
23628 let dir = tempfile::tempdir().unwrap();
23629 let cfg = config_with_storage(StorageConfig {
23630 backend: StorageBackend::Local,
23631 local: StorageLocalConfig {
23632 root: dir.path().to_path_buf(),
23633 ..StorageLocalConfig::default()
23634 },
23635 ..StorageConfig::default()
23636 });
23637 let bootstrap = preflight_storage(&cfg).expect("local backend should provision");
23638 assert_eq!(bootstrap.store.provider_id(), "default");
23639 assert!(bootstrap.serving.is_some(), "local backend mounts a route");
23640 }
23641
23642 #[tokio::test]
23643 async fn install_registers_blob_store_on_state() {
23644 let dir = tempfile::tempdir().unwrap();
23645 let cfg = config_with_storage(StorageConfig {
23646 backend: StorageBackend::Local,
23647 local: StorageLocalConfig {
23648 root: dir.path().to_path_buf(),
23649 ..StorageLocalConfig::default()
23650 },
23651 ..StorageConfig::default()
23652 });
23653 let bootstrap: StorageBootstrap = preflight_storage(&cfg).unwrap();
23654
23655 let state = AppState::for_test();
23656 assert!(state.extension::<BlobStoreState>().is_none());
23657 let serving = bootstrap.install(&state);
23658 assert!(serving.is_some());
23659 assert!(state.extension::<BlobStoreState>().is_some());
23660 }
23661
23662 #[test]
23663 fn with_blob_store_stores_custom_store() {
23664 use crate::storage::{
23665 Blob, BlobFuture, BlobMeta, BlobStore, BlobStoreError, ByteStream,
23666 };
23667 use bytes::Bytes;
23668 use std::time::Duration;
23669
23670 struct FakeStore;
23671 impl BlobStore for FakeStore {
23672 fn provider_id(&self) -> &'static str {
23673 "fake"
23674 }
23675 fn put<'a>(&'a self, _k: &'a str, _ct: &'a str, _b: Bytes) -> BlobFuture<'a, Blob> {
23676 Box::pin(async { Err(BlobStoreError::Unsupported("fake".into())) })
23677 }
23678 fn put_stream<'a>(
23679 &'a self,
23680 _k: &'a str,
23681 _ct: &'a str,
23682 _d: ByteStream<'a>,
23683 ) -> BlobFuture<'a, Blob> {
23684 Box::pin(async { Err(BlobStoreError::Unsupported("fake".into())) })
23685 }
23686 fn get<'a>(&'a self, _k: &'a str) -> BlobFuture<'a, Bytes> {
23687 Box::pin(async { Err(BlobStoreError::Unsupported("fake".into())) })
23688 }
23689 fn delete<'a>(&'a self, _k: &'a str) -> BlobFuture<'a, ()> {
23690 Box::pin(async { Err(BlobStoreError::Unsupported("fake".into())) })
23691 }
23692 fn head<'a>(&'a self, _k: &'a str) -> BlobFuture<'a, Option<BlobMeta>> {
23693 Box::pin(async { Err(BlobStoreError::Unsupported("fake".into())) })
23694 }
23695 fn presigned_url<'a>(
23696 &'a self,
23697 _k: &'a str,
23698 _e: Duration,
23699 ) -> BlobFuture<'a, String> {
23700 Box::pin(async { Err(BlobStoreError::Unsupported("fake".into())) })
23701 }
23702 }
23703
23704 let builder = crate::app().with_blob_store(FakeStore);
23705 assert!(builder.blob_store.is_some());
23706 }
23707
23708 #[tokio::test]
23709 async fn with_blob_store_is_installed_on_state() {
23710 use crate::storage::{
23711 Blob, BlobFuture, BlobMeta, BlobStore, BlobStoreError, ByteStream,
23712 };
23713 use bytes::Bytes;
23714 use std::time::Duration;
23715
23716 struct FakeStore;
23717 impl BlobStore for FakeStore {
23718 fn provider_id(&self) -> &'static str {
23719 "fake-installed"
23720 }
23721 fn put<'a>(&'a self, _k: &'a str, _ct: &'a str, _b: Bytes) -> BlobFuture<'a, Blob> {
23722 Box::pin(async { Err(BlobStoreError::Unsupported("fake".into())) })
23723 }
23724 fn put_stream<'a>(
23725 &'a self,
23726 _k: &'a str,
23727 _ct: &'a str,
23728 _d: ByteStream<'a>,
23729 ) -> BlobFuture<'a, Blob> {
23730 Box::pin(async { Err(BlobStoreError::Unsupported("fake".into())) })
23731 }
23732 fn get<'a>(&'a self, _k: &'a str) -> BlobFuture<'a, Bytes> {
23733 Box::pin(async { Err(BlobStoreError::Unsupported("fake".into())) })
23734 }
23735 fn delete<'a>(&'a self, _k: &'a str) -> BlobFuture<'a, ()> {
23736 Box::pin(async { Err(BlobStoreError::Unsupported("fake".into())) })
23737 }
23738 fn head<'a>(&'a self, _k: &'a str) -> BlobFuture<'a, Option<BlobMeta>> {
23739 Box::pin(async { Err(BlobStoreError::Unsupported("fake".into())) })
23740 }
23741 fn presigned_url<'a>(
23742 &'a self,
23743 _k: &'a str,
23744 _e: Duration,
23745 ) -> BlobFuture<'a, String> {
23746 Box::pin(async { Err(BlobStoreError::Unsupported("fake".into())) })
23747 }
23748 }
23749
23750 let builder = crate::app().with_blob_store(FakeStore);
23751 let bootstrap = builder.blob_store.map(|store| StorageBootstrap {
23752 store,
23753 serving: None,
23754 });
23755 let state = AppState::for_test();
23756 assert!(state.extension::<BlobStoreState>().is_none());
23757 if let Some(b) = bootstrap {
23758 b.install(&state);
23759 }
23760 let installed = state
23761 .extension::<BlobStoreState>()
23762 .expect("store should be installed");
23763 assert_eq!(installed.store().provider_id(), "fake-installed");
23764 }
23765 }
23766
23767 struct TestPlugin {
23771 name: &'static str,
23772 route: Route,
23773 }
23774
23775 impl crate::plugin::Plugin for TestPlugin {
23776 fn name(&self) -> std::borrow::Cow<'static, str> {
23777 std::borrow::Cow::Borrowed(self.name)
23778 }
23779
23780 fn build(self, app: AppBuilder) -> AppBuilder {
23781 app.routes(vec![self.route])
23782 }
23783 }
23784
23785 #[test]
23786 fn routes_registered_before_plugin_are_user_sourced() {
23787 let user_route = test_get_route("/home", "home");
23788 let builder = app().routes(vec![user_route]);
23789 assert_eq!(builder.route_sources.len(), 1);
23790 assert_eq!(
23791 builder.route_sources[0],
23792 crate::route_listing::RouteSource::User
23793 );
23794 }
23795
23796 #[test]
23797 fn routes_registered_inside_plugin_are_plugin_sourced() {
23798 let plugin_route = test_get_route("/plugin-page", "plugin_page");
23799 let plugin = TestPlugin {
23800 name: "my-plugin",
23801 route: plugin_route,
23802 };
23803 let builder = app().plugin(plugin);
23804 assert_eq!(builder.route_sources.len(), 1);
23805 assert_eq!(
23806 builder.route_sources[0],
23807 crate::route_listing::RouteSource::Plugin("my-plugin".to_owned())
23808 );
23809 }
23810
23811 #[test]
23812 fn routes_registered_after_plugin_revert_to_user_sourced() {
23813 let plugin_route = test_get_route("/plugin-page", "plugin_page");
23814 let user_route = test_get_route("/home", "home");
23815 let plugin = TestPlugin {
23816 name: "my-plugin",
23817 route: plugin_route,
23818 };
23819 let builder = app().plugin(plugin).routes(vec![user_route]);
23820 assert_eq!(builder.route_sources.len(), 2);
23821 assert_eq!(
23822 builder.route_sources[0],
23823 crate::route_listing::RouteSource::Plugin("my-plugin".to_owned())
23824 );
23825 assert_eq!(
23826 builder.route_sources[1],
23827 crate::route_listing::RouteSource::User
23828 );
23829 }
23830
23831 struct OuterPlugin;
23833
23834 impl crate::plugin::Plugin for OuterPlugin {
23835 fn name(&self) -> std::borrow::Cow<'static, str> {
23836 "outer".into()
23837 }
23838
23839 fn build(self, app: AppBuilder) -> AppBuilder {
23840 let inner = TestPlugin {
23841 name: "inner",
23842 route: test_get_route("/inner", "inner"),
23843 };
23844 app.plugin(inner)
23845 .routes(vec![test_get_route("/outer-after", "outer_after")])
23846 }
23847 }
23848
23849 #[test]
23850 fn outer_plugin_source_restored_after_nested_plugin() {
23851 let builder = app().plugin(OuterPlugin);
23852 assert_eq!(builder.route_sources.len(), 2);
23854 assert_eq!(
23855 builder.route_sources[0],
23856 crate::route_listing::RouteSource::Plugin("inner".to_owned()),
23857 "first route should be attributed to inner plugin"
23858 );
23859 assert_eq!(
23860 builder.route_sources[1],
23861 crate::route_listing::RouteSource::Plugin("outer".to_owned()),
23862 "second route should be re-attributed to outer plugin after nested build"
23863 );
23864 }
23865
23866 #[tokio::test]
23869 async fn shutdown_hooks_with_timeout_runs_all_fast_hooks() {
23870 use std::sync::atomic::{AtomicUsize, Ordering};
23871 let counter = Arc::new(AtomicUsize::new(0));
23872 let c1 = Arc::clone(&counter);
23873 let c2 = Arc::clone(&counter);
23874
23875 let hooks: Vec<ShutdownHook> = vec![
23876 Box::new(move || {
23877 let c = Arc::clone(&c1);
23878 Box::pin(async move {
23879 c.fetch_add(1, Ordering::SeqCst);
23880 })
23881 }),
23882 Box::new(move || {
23883 let c = Arc::clone(&c2);
23884 Box::pin(async move {
23885 c.fetch_add(1, Ordering::SeqCst);
23886 })
23887 }),
23888 ];
23889
23890 run_shutdown_hooks_with_timeout(
23891 &hooks,
23892 std::time::Duration::from_secs(2),
23893 std::time::Duration::from_secs(10),
23894 )
23895 .await;
23896
23897 assert_eq!(counter.load(Ordering::SeqCst), 2, "both hooks must run");
23898 }
23899
23900 #[tokio::test]
23901 async fn shutdown_hooks_with_timeout_tolerates_slow_hook_overrun() {
23902 use std::sync::atomic::{AtomicBool, Ordering};
23903 let fast_ran = Arc::new(AtomicBool::new(false));
23904 let fr = Arc::clone(&fast_ran);
23905
23906 let hooks: Vec<ShutdownHook> = vec![
23907 Box::new(move || {
23909 let fr = Arc::clone(&fr);
23910 Box::pin(async move {
23911 fr.store(true, Ordering::SeqCst);
23912 })
23913 }),
23914 Box::new(|| {
23916 Box::pin(async move {
23917 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
23918 })
23919 }),
23920 ];
23921
23922 run_shutdown_hooks_with_timeout(
23925 &hooks,
23926 std::time::Duration::from_millis(50),
23927 std::time::Duration::from_secs(1),
23928 )
23929 .await;
23930
23931 assert!(
23932 fast_ran.load(Ordering::SeqCst),
23933 "fast hook must still run even after slow hook overruns its per-hook budget"
23934 );
23935 }
23936
23937 #[cfg(feature = "http-client")]
23940 #[test]
23941 fn build_state_registers_shared_reqwest_client() {
23942 let config = AutumnConfig::default();
23943 let state = build_state(
23944 &config,
23945 #[cfg(feature = "db")]
23946 None,
23947 #[cfg(feature = "db")]
23948 None,
23949 #[cfg(feature = "ws")]
23950 None,
23951 );
23952 assert!(
23953 state
23954 .extension::<crate::http_client::SharedReqwestClient>()
23955 .is_some(),
23956 "build_state must register a SharedReqwestClient for connection-pool sharing"
23957 );
23958 }
23959
23960 #[cfg(feature = "maud")]
23965 #[test]
23966 fn with_story_gallery_installs_story_registry_extension() {
23967 let builder = crate::app().with_story_gallery(crate::stories::StoryGallery::builtin());
23968 let gallery = builder
23969 .story_gallery
23970 .expect("with_story_gallery must store the gallery on the builder");
23971 let expected_count = gallery.stories().len();
23972 assert!(expected_count > 0, "builtin gallery must not be empty");
23973
23974 let config = AutumnConfig::default();
23975 let state = build_state(
23976 &config,
23977 #[cfg(feature = "db")]
23978 None,
23979 #[cfg(feature = "db")]
23980 None,
23981 #[cfg(feature = "ws")]
23982 None,
23983 );
23984 install_story_registry(&state, Some(gallery));
23985 let registry = state
23986 .extension::<crate::stories::StoryRegistry>()
23987 .expect("install_story_registry must publish the StoryRegistry extension");
23988 assert_eq!(
23989 registry.stories().len(),
23990 expected_count,
23991 "every registered story must reach the state extension"
23992 );
23993
23994 let bare_state = build_state(
23997 &config,
23998 #[cfg(feature = "db")]
23999 None,
24000 #[cfg(feature = "db")]
24001 None,
24002 #[cfg(feature = "ws")]
24003 None,
24004 );
24005 install_story_registry(&bare_state, None);
24006 assert!(
24007 bare_state
24008 .extension::<crate::stories::StoryRegistry>()
24009 .is_none(),
24010 "no gallery registered must mean no StoryRegistry extension"
24011 );
24012 }
24013
24014 #[cfg(feature = "openapi")]
24018 #[test]
24019 fn export_preflight_refuses_a_declared_route_on_a_context_mount() {
24020 let config = crate::config::AutumnConfig::default();
24021 let openapi =
24022 crate::openapi::OpenApiConfig::new("Demo", "1.0.0").openapi_json_path("/spec.json");
24023 let declared = vec![crate::route_listing::RouteInfo {
24024 method: "GET".to_owned(),
24025 path: "/spec.json".to_owned(),
24026 handler: "sandbox:evil-plugin".to_owned(),
24027 source: crate::route_listing::RouteSource::Plugin("evil-plugin".to_owned()),
24028 ..crate::route_listing::RouteInfo::default()
24029 }];
24030 let error = export_preflight(&ExportPreflight {
24031 routes: &[test_get_route("/hello", "hello")],
24032 scoped_groups: &[],
24033 api_versions: &[],
24034 openapi_config: &openapi,
24035 merge_routers: &[],
24036 nest_routers: &[],
24037 declared_routes: &declared,
24038 config: &config,
24039 mcp_mount_path: None,
24040 })
24041 .expect_err("the router build refuses this, so the export check must too");
24042 assert!(error.contains("/spec.json"), "{error}");
24043 }
24044}
24045
24046#[cfg(all(test, unix))]
24047mod unix_socket_tests {
24048 use super::prepare_unix_socket_path;
24049
24050 #[test]
24051 fn prepare_unix_socket_path_noop_when_absent() {
24052 let dir = tempfile::tempdir().expect("tempdir");
24053 let path = dir.path().join("missing.sock");
24054 prepare_unix_socket_path(&path).expect("absent path is fine");
24055 assert!(!path.exists());
24056 }
24057
24058 #[test]
24059 fn prepare_unix_socket_path_removes_stale_socket() {
24060 let dir = tempfile::tempdir().expect("tempdir");
24061 let path = dir.path().join("stale.sock");
24062 let listener = std::os::unix::net::UnixListener::bind(&path).expect("bind socket");
24064 drop(listener);
24065 assert!(path.exists(), "socket file should exist before prepare");
24066 prepare_unix_socket_path(&path).expect("stale socket should be removed");
24067 assert!(!path.exists(), "stale socket should be unlinked");
24068 }
24069
24070 #[test]
24071 fn prepare_unix_socket_path_refuses_live_socket() {
24072 let dir = tempfile::tempdir().expect("tempdir");
24073 let path = dir.path().join("live.sock");
24074 let _listener = std::os::unix::net::UnixListener::bind(&path).expect("bind socket");
24076 let err = prepare_unix_socket_path(&path).expect_err("must refuse a live socket");
24077 assert_eq!(err.kind(), std::io::ErrorKind::AddrInUse);
24078 assert!(path.exists(), "live socket must not be removed");
24079 }
24080
24081 #[test]
24082 fn prepare_unix_socket_path_errors_on_regular_file() {
24083 let dir = tempfile::tempdir().expect("tempdir");
24084 let path = dir.path().join("not-a-socket");
24085 std::fs::write(&path, b"i am a regular file").expect("write file");
24086 let err = prepare_unix_socket_path(&path).expect_err("must refuse a non-socket file");
24087 assert_eq!(err.kind(), std::io::ErrorKind::AlreadyExists);
24088 assert!(path.exists(), "regular file must not be removed");
24089 }
24090
24091 #[cfg(feature = "acme")]
24097 #[tokio::test]
24098 async fn retention_mode_installs_the_custom_domain_pruner() {
24099 fn acme_with_custom_domains(enabled: bool) -> crate::config::AcmeConfig {
24100 let toml = format!(
24101 "domains = [\"app.example.com\"]\ncontact_email = \"ops@example.com\"\n\
24102 [custom_domains]\nenabled = {enabled}\n"
24103 );
24104 toml::from_str(&toml).expect("test ACME config parses")
24105 }
24106
24107 let state = crate::state::AppState::for_test();
24111 super::install_custom_domain_retention_pruner(
24112 Some(&acme_with_custom_domains(true)),
24113 None,
24114 &state,
24115 true,
24116 )
24117 .await;
24118 assert!(
24119 state
24120 .extension::<std::sync::Arc<dyn crate::custom_domain::CustomDomainPruner>>()
24121 .is_some(),
24122 "one-shot retention must install the custom-domain pruner"
24123 );
24124
24125 let state = crate::state::AppState::for_test();
24128 super::install_custom_domain_retention_pruner(
24129 Some(&acme_with_custom_domains(false)),
24130 None,
24131 &state,
24132 true,
24133 )
24134 .await;
24135 assert!(
24136 state
24137 .extension::<std::sync::Arc<dyn crate::custom_domain::CustomDomainPruner>>()
24138 .is_none()
24139 );
24140
24141 let state = crate::state::AppState::for_test();
24143 super::install_custom_domain_retention_pruner(None, None, &state, true).await;
24144 assert!(
24145 state
24146 .extension::<std::sync::Arc<dyn crate::custom_domain::CustomDomainPruner>>()
24147 .is_none()
24148 );
24149 }
24150}