1#![forbid(unsafe_code)]
9
10pub(crate) mod auth;
11pub(crate) mod body;
12pub mod config;
14pub mod error;
16pub(crate) mod handlers;
17pub mod hooks;
19pub(crate) mod http;
20pub mod middleware;
22pub mod s3;
24pub mod storage;
26
27use std::{collections::HashSet, convert::Infallible, future::Future, sync::Arc};
28
29use axum::{
30 Router,
31 body::Body,
32 http::{HeaderMap, Request},
33 response::IntoResponse,
34 routing::Route,
35};
36use hooks::{
37 AuthenticationProvider, AuthenticationRequest, AuthenticationResult, AuthorizationContext,
38 AuthorizationPolicy, Clock, ErrorMapper, OperationContext, OperationPolicy, RequestIdFactory,
39 RequestObserver, ResponseMapper, S3ErrorContext, S3RequestContext, S3ResponseContext,
40 SharedAuthenticationProvider, SharedAuthorizationPolicy, SharedClock, SharedErrorMapper,
41 SharedOperationPolicy, SharedRequestIdFactory, SharedRequestObserver, SharedResponseMapper,
42 SharedTargetPolicy, SharedTenantLimitsProvider, SharedUploadPolicy, SystemClock, TargetContext,
43 TargetPolicy, TenantLimitContext, TenantLimitsProvider, TenantOperationLease,
44 TenantOperationOutcome, UploadPolicy, UploadPolicyContext,
45};
46use middleware::{
47 SharedUploadProcessor, UploadProcessor, UploadProcessorAction, UploadProcessorError,
48 UploadProcessorFn, UploadProcessorHandle,
49};
50use storage::{
51 FileMultipartStore, FileObjectStore, MultipartStore, ObjectStore, SharedMultipartStore,
52 SharedObjectStore,
53};
54use thiserror::Error;
55use tokio::sync::{OwnedSemaphorePermit, Semaphore};
56use tower::{Layer, Service};
57
58#[derive(Clone)]
59pub struct AppState {
66 pub config: config::Config,
68 pub auth: Arc<config::AuthState>,
70 pub object_store: SharedObjectStore,
72 pub multipart_store: SharedMultipartStore,
74 pub(crate) io_tuning: config::IoTuning,
75 pub(crate) upload_processors: Arc<Vec<SharedUploadProcessor>>,
76 request_observer: Option<SharedRequestObserver>,
77 error_mapper: Option<SharedErrorMapper>,
78 response_mapper: Option<SharedResponseMapper>,
79 request_id_factory: SharedRequestIdFactory,
80 clock: SharedClock,
81 authentication_provider: Option<SharedAuthenticationProvider>,
82 authorization_policy: Option<SharedAuthorizationPolicy>,
83 operation_policy: Option<SharedOperationPolicy>,
84 target_policy: Option<SharedTargetPolicy>,
85 upload_policy: Option<SharedUploadPolicy>,
86 tenant_limits_provider: SharedTenantLimitsProvider,
87 disabled_operations: Arc<HashSet<config::S3Action>>,
88 admission: Arc<AdmissionControl>,
89}
90
91impl AppState {
92 pub fn builder(config: config::Config) -> AppBuilder {
94 AppBuilder {
95 config,
96 upload_processors: Vec::new(),
97 object_store: None,
98 multipart_store: None,
99 io_tuning: config::IoTuning::default(),
100 multipart_lifecycle: config::MultipartLifecycle::default(),
101 request_observer: None,
102 error_mapper: None,
103 response_mapper: None,
104 request_id_factory: Arc::new(s3::types::RequestId::new),
105 clock: Arc::new(SystemClock),
106 authentication_provider: None,
107 authorization_policy: None,
108 operation_policy: None,
109 target_policy: None,
110 upload_policy: None,
111 tenant_limits_provider: Arc::new(hooks::NoopTenantLimitsProvider),
112 disabled_operations: HashSet::new(),
113 health_path: Some("/health".to_owned()),
114 router_layers: Vec::new(),
115 }
116 }
117
118 pub async fn new(config: config::Config) -> Result<Self, AppInitError> {
125 Self::builder(config).build().await
126 }
127
128 async fn from_builder(builder: AppBuilder) -> Result<Self, AppInitError> {
129 let AppBuilder {
130 config,
131 upload_processors,
132 object_store,
133 multipart_store,
134 io_tuning,
135 multipart_lifecycle,
136 request_observer,
137 error_mapper,
138 response_mapper,
139 request_id_factory,
140 clock,
141 authentication_provider,
142 authorization_policy,
143 operation_policy,
144 target_policy,
145 upload_policy,
146 tenant_limits_provider,
147 disabled_operations,
148 health_path: _,
149 router_layers: _,
150 } = builder;
151 config.validate()?;
152 io_tuning.validate()?;
153 let upload_limits = config.upload_limits.validated()?;
154 let auth = Arc::new(config::AuthState::new(&config.auth)?);
155 let object_store = match object_store {
156 Some(store) => store,
157 None => Arc::new(FileObjectStore::new(config.storage_root.clone()).await?),
158 };
159 let multipart_store = match multipart_store {
160 Some(store) => store,
161 None => Arc::new(
162 FileMultipartStore::new_with_options(
163 config.storage_root.clone(),
164 multipart_lifecycle,
165 clock.clone(),
166 )
167 .await?,
168 ),
169 };
170 Ok(Self {
171 config,
172 auth,
173 object_store,
174 multipart_store,
175 io_tuning,
176 upload_processors: Arc::new(upload_processors),
177 request_observer,
178 error_mapper,
179 response_mapper,
180 request_id_factory,
181 clock,
182 authentication_provider,
183 authorization_policy,
184 operation_policy,
185 target_policy,
186 upload_policy,
187 tenant_limits_provider,
188 disabled_operations: Arc::new(disabled_operations),
189 admission: Arc::new(AdmissionControl::new(&upload_limits)),
190 })
191 }
192
193 pub(crate) fn upload_processors(&self) -> &[SharedUploadProcessor] {
195 &self.upload_processors
196 }
197
198 pub(crate) fn request_id(&self) -> s3::types::RequestId {
200 self.request_id_factory.request_id()
201 }
202
203 pub(crate) fn now(&self) -> chrono::DateTime<chrono::Utc> {
205 self.clock.now()
206 }
207
208 pub(crate) async fn authenticate_with_provider(
212 &self,
213 request: AuthenticationRequest,
214 ) -> Option<Result<AuthenticationResult, error::S3Error>> {
215 let provider = self.authentication_provider.as_ref()?;
216 Some(provider.authenticate(request).await)
217 }
218
219 pub(crate) async fn observe_request(&self, context: S3RequestContext) {
221 if let Some(observer) = &self.request_observer {
222 observer.observe(context).await;
223 }
224 }
225
226 pub(crate) async fn map_error(
228 &self,
229 context: S3ErrorContext,
230 error: error::S3Error,
231 ) -> error::S3Error {
232 if let Some(mapper) = &self.error_mapper {
233 mapper.map_error(context, error).await
234 } else {
235 error
236 }
237 }
238
239 pub(crate) async fn map_response(
241 &self,
242 context: S3ResponseContext,
243 response: axum::response::Response,
244 ) -> axum::response::Response {
245 if let Some(mapper) = &self.response_mapper {
246 mapper.map_response(context, response).await
247 } else {
248 response
249 }
250 }
251
252 pub(crate) fn allow_operation(&self, context: OperationContext) -> Result<(), error::S3Error> {
254 if self.disabled_operations.contains(&context.action) {
255 return Err(error::S3Error::method_not_allowed());
256 }
257 if let Some(policy) = &self.operation_policy {
258 policy.allow_operation(context)
259 } else {
260 Ok(())
261 }
262 }
263
264 pub(crate) fn allow_target(&self, context: TargetContext) -> Result<(), error::S3Error> {
266 if let Some(policy) = &self.target_policy {
267 policy.allow_target(context)
268 } else {
269 Ok(())
270 }
271 }
272
273 pub(crate) fn allow_upload(&self, context: UploadPolicyContext) -> Result<(), error::S3Error> {
275 if let Some(policy) = &self.upload_policy {
276 policy.allow_upload(context)
277 } else {
278 Ok(())
279 }
280 }
281
282 pub(crate) async fn begin_tenant_operation(
284 &self,
285 context: TenantLimitContext,
286 ) -> Result<TenantOperationLease, error::S3Error> {
287 self.tenant_limits_provider.begin_operation(context).await
288 }
289
290 pub(crate) async fn finish_tenant_operation(
292 &self,
293 context: TenantLimitContext,
294 outcome: TenantOperationOutcome,
295 ) {
296 self.tenant_limits_provider
297 .finish_operation(context, outcome)
298 .await;
299 }
300
301 pub(crate) fn authorize_with_policy(
303 &self,
304 context: AuthorizationContext,
305 ) -> Result<(), error::S3Error> {
306 if let Some(policy) = &self.authorization_policy {
307 policy.authorize(context)
308 } else {
309 Ok(())
310 }
311 }
312
313 pub(crate) fn try_acquire_s3_request(&self) -> Result<OwnedSemaphorePermit, error::S3Error> {
315 self.admission
316 .s3_requests
317 .clone()
318 .try_acquire_owned()
319 .map_err(|_| {
320 error::S3Error::slow_down("too many active S3 requests; retry the request")
321 })
322 }
323
324 pub(crate) fn try_acquire_object_writer(&self) -> Result<OwnedSemaphorePermit, error::S3Error> {
326 self.admission
327 .object_writers
328 .clone()
329 .try_acquire_owned()
330 .map_err(|_| {
331 error::S3Error::slow_down("too many active object uploads; retry the request")
332 })
333 }
334
335 pub(crate) fn try_acquire_multipart_part_writer(
337 &self,
338 ) -> Result<OwnedSemaphorePermit, error::S3Error> {
339 self.admission
340 .multipart_part_writers
341 .clone()
342 .try_acquire_owned()
343 .map_err(|_| {
344 error::S3Error::slow_down(
345 "too many active multipart part uploads; retry the request",
346 )
347 })
348 }
349
350 pub(crate) fn try_acquire_aws_chunked_decoder(
354 &self,
355 headers: &HeaderMap,
356 ) -> Result<Option<OwnedSemaphorePermit>, error::S3Error> {
357 if !body::upload::is_aws_chunked_request(headers) {
358 return Ok(None);
359 }
360 self.admission
361 .aws_chunked_decoders
362 .clone()
363 .try_acquire_owned()
364 .map(Some)
365 .map_err(|_| {
366 error::S3Error::slow_down("too many active aws-chunked uploads; retry the request")
367 })
368 }
369}
370
371pub struct AppBuilder {
373 config: config::Config,
374 upload_processors: Vec<SharedUploadProcessor>,
375 object_store: Option<SharedObjectStore>,
376 multipart_store: Option<SharedMultipartStore>,
377 io_tuning: config::IoTuning,
378 multipart_lifecycle: config::MultipartLifecycle,
379 request_observer: Option<SharedRequestObserver>,
380 error_mapper: Option<SharedErrorMapper>,
381 response_mapper: Option<SharedResponseMapper>,
382 request_id_factory: SharedRequestIdFactory,
383 clock: SharedClock,
384 authentication_provider: Option<SharedAuthenticationProvider>,
385 authorization_policy: Option<SharedAuthorizationPolicy>,
386 operation_policy: Option<SharedOperationPolicy>,
387 target_policy: Option<SharedTargetPolicy>,
388 upload_policy: Option<SharedUploadPolicy>,
389 tenant_limits_provider: SharedTenantLimitsProvider,
390 disabled_operations: HashSet<config::S3Action>,
391 health_path: Option<String>,
392 router_layers: Vec<RouterTransform>,
393}
394
395type RouterTransform = Arc<dyn Fn(Router) -> Router + Send + Sync>;
396
397impl AppBuilder {
398 pub fn object_store<S>(mut self, store: S) -> Self
400 where
401 S: ObjectStore,
402 {
403 self.object_store = Some(Arc::new(store));
404 self
405 }
406
407 pub fn multipart_store<S>(mut self, store: S) -> Self
409 where
410 S: MultipartStore,
411 {
412 self.multipart_store = Some(Arc::new(store));
413 self
414 }
415
416 pub fn io_tuning(mut self, io_tuning: config::IoTuning) -> Self {
418 self.io_tuning = io_tuning;
419 self
420 }
421
422 pub fn multipart_lifecycle(mut self, lifecycle: config::MultipartLifecycle) -> Self {
424 self.multipart_lifecycle = lifecycle;
425 self
426 }
427
428 pub fn upload_processor<P>(mut self, processor: P) -> Self
433 where
434 P: UploadProcessor,
435 {
436 self.upload_processors.push(Arc::new(processor));
437 self
438 }
439
440 pub fn upload_processor_fn<F, Fut>(mut self, name: impl Into<String>, handler: F) -> Self
445 where
446 F: Fn(UploadProcessorHandle) -> Fut + Send + Sync + 'static,
447 Fut: Future<Output = Result<UploadProcessorAction, UploadProcessorError>> + Send + 'static,
448 {
449 self.upload_processors
450 .push(Arc::new(UploadProcessorFn::new(name, handler)));
451 self
452 }
453
454 pub fn health_path(mut self, path: impl Into<String>) -> Self {
456 self.health_path = Some(path.into());
457 self
458 }
459
460 pub fn disable_health_route(mut self) -> Self {
462 self.health_path = None;
463 self
464 }
465
466 pub fn router_layer<L>(mut self, layer: L) -> Self
468 where
469 L: Layer<Route> + Clone + Send + Sync + 'static,
470 L::Service: Service<Request<Body>> + Clone + Send + Sync + 'static,
471 <L::Service as Service<Request<Body>>>::Response: IntoResponse + 'static,
472 <L::Service as Service<Request<Body>>>::Error: Into<Infallible> + 'static,
473 <L::Service as Service<Request<Body>>>::Future: Send + 'static,
474 {
475 self.router_layers
476 .push(Arc::new(move |router| router.layer(layer.clone())));
477 self
478 }
479
480 pub fn on_request<O>(mut self, observer: O) -> Self
482 where
483 O: RequestObserver,
484 {
485 self.request_observer = Some(Arc::new(observer));
486 self
487 }
488
489 pub fn on_error<M>(mut self, mapper: M) -> Self
491 where
492 M: ErrorMapper,
493 {
494 self.error_mapper = Some(Arc::new(mapper));
495 self
496 }
497
498 pub fn on_response<M>(mut self, mapper: M) -> Self
500 where
501 M: ResponseMapper,
502 {
503 self.response_mapper = Some(Arc::new(mapper));
504 self
505 }
506
507 pub fn request_id_factory<F>(mut self, factory: F) -> Self
509 where
510 F: RequestIdFactory,
511 {
512 self.request_id_factory = Arc::new(factory);
513 self
514 }
515
516 pub fn clock<C>(mut self, clock: C) -> Self
518 where
519 C: Clock,
520 {
521 self.clock = Arc::new(clock);
522 self
523 }
524
525 pub fn authentication_provider<P>(mut self, provider: P) -> Self
532 where
533 P: AuthenticationProvider,
534 {
535 self.authentication_provider = Some(Arc::new(provider));
536 self
537 }
538
539 pub fn authorization_policy<P>(mut self, policy: P) -> Self
541 where
542 P: AuthorizationPolicy,
543 {
544 self.authorization_policy = Some(Arc::new(policy));
545 self
546 }
547
548 pub fn operation_policy<P>(mut self, policy: P) -> Self
550 where
551 P: OperationPolicy,
552 {
553 self.operation_policy = Some(Arc::new(policy));
554 self
555 }
556
557 pub fn disable_operation(mut self, action: config::S3Action) -> Self {
559 self.disabled_operations.insert(action);
560 self
561 }
562
563 pub fn target_policy<P>(mut self, policy: P) -> Self
565 where
566 P: TargetPolicy,
567 {
568 self.target_policy = Some(Arc::new(policy));
569 self
570 }
571
572 pub fn upload_policy<P>(mut self, policy: P) -> Self
574 where
575 P: UploadPolicy,
576 {
577 self.upload_policy = Some(Arc::new(policy));
578 self
579 }
580
581 pub fn tenant_limits_provider<P>(mut self, provider: P) -> Self
583 where
584 P: TenantLimitsProvider,
585 {
586 self.tenant_limits_provider = Arc::new(provider);
587 self
588 }
589
590 pub async fn build(self) -> Result<AppState, AppInitError> {
592 AppState::from_builder(self).await
593 }
594
595 pub async fn build_router(self) -> Result<Router, AppInitError> {
597 let health_path = self.health_path.clone();
598 let router_layers = self.router_layers.clone();
599 validate_health_path(health_path.as_deref())?;
600 let state = AppState::from_builder(self).await?;
601 let mut router = http::router_with_options(state, http::RouterOptions { health_path });
602 for layer in router_layers {
603 router = layer(router);
604 }
605 Ok(router)
606 }
607}
608
609fn validate_health_path(path: Option<&str>) -> Result<(), AppInitError> {
610 let Some(path) = path else {
611 return Ok(());
612 };
613 if path.starts_with('/') && path.len() > 1 && !path.contains('*') {
614 Ok(())
615 } else {
616 Err(AppInitError::RouterConfig(
617 "health path must start with '/' and must not contain wildcards".to_owned(),
618 ))
619 }
620}
621
622struct AdmissionControl {
623 s3_requests: Arc<Semaphore>,
624 object_writers: Arc<Semaphore>,
625 multipart_part_writers: Arc<Semaphore>,
626 aws_chunked_decoders: Arc<Semaphore>,
627}
628
629impl AdmissionControl {
630 fn new(limits: &config::ValidatedUploadLimits) -> Self {
631 Self {
632 s3_requests: Arc::new(Semaphore::new(limits.max_concurrent_s3_requests.get())),
633 object_writers: Arc::new(Semaphore::new(limits.max_active_object_writers.get())),
634 multipart_part_writers: Arc::new(Semaphore::new(
635 limits.max_active_multipart_part_writers.get(),
636 )),
637 aws_chunked_decoders: Arc::new(Semaphore::new(
638 limits.max_active_aws_chunked_decoders.get(),
639 )),
640 }
641 }
642}
643
644#[derive(Debug, Error)]
645pub enum AppInitError {
647 #[error("invalid configuration")]
649 Config(#[from] config::ConfigError),
650
651 #[error("failed to initialize storage")]
653 Store(#[from] storage::StoreError),
654
655 #[error("invalid router configuration: {0}")]
657 RouterConfig(String),
658}
659
660pub fn router(state: AppState) -> Router {
665 http::router(state)
666}
667
668#[cfg(test)]
669mod tests {
670 use super::*;
671 use axum::{
672 body::{Body, Bytes, to_bytes},
673 http::{HeaderMap, HeaderValue, Request, StatusCode, header},
674 };
675 use futures_util::{future::BoxFuture, stream};
676 use std::{convert::Infallible, path::Path, sync::Mutex, time::Duration};
677 use tower::ServiceExt;
678
679 #[tokio::test]
680 async fn app_state_rejects_invalid_upload_limits() {
681 let temp_dir = tempfile::tempdir().expect("create temp dir");
682 let result = AppState::new(config::Config {
683 storage_root: temp_dir.path().to_path_buf(),
684 auth: Default::default(),
685 virtual_host_base_domain: None,
686 upload_limits: config::UploadLimits {
687 max_object_size: 1,
688 max_part_size: 0,
689 min_non_final_part_size: 1,
690 ..Default::default()
691 },
692 })
693 .await;
694
695 let Err(error) = result else {
696 panic!("invalid config should fail");
697 };
698
699 assert!(matches!(
700 error,
701 AppInitError::Config(config::ConfigError::InvalidUploadLimit(
702 "max_part_size must be greater than 0"
703 ))
704 ));
705 }
706
707 #[tokio::test]
708 async fn app_state_builder_registers_upload_processors_in_order() {
709 let temp_dir = tempfile::tempdir().expect("create temp dir");
710 let state = AppState::builder(config::Config {
711 storage_root: temp_dir.path().to_path_buf(),
712 auth: Default::default(),
713 virtual_host_base_domain: None,
714 upload_limits: Default::default(),
715 })
716 .upload_processor(NamedProcessor("first"))
717 .upload_processor(NamedProcessor("second"))
718 .build()
719 .await
720 .expect("create app state");
721
722 assert_eq!(state.upload_processors.len(), 2);
723 }
724
725 #[tokio::test]
726 async fn app_state_builder_registers_upload_processor_fn() {
727 let temp_dir = tempfile::tempdir().expect("create temp dir");
728 let state = AppState::builder(config::Config::new(temp_dir.path().to_path_buf()))
729 .upload_processor_fn("keep", |upload| async move { Ok(upload.keep()) })
730 .build()
731 .await
732 .expect("create app state");
733
734 assert_eq!(state.upload_processors.len(), 1);
735 }
736
737 #[tokio::test]
738 async fn build_router_serves_custom_health_path_and_s3_routes() {
739 let temp_dir = tempfile::tempdir().expect("create temp dir");
740 let app = AppState::builder(
741 config::Config::builder(temp_dir.path())
742 .allow_anonymous(true)
743 .build(),
744 )
745 .health_path("/ready")
746 .build_router()
747 .await
748 .expect("router");
749
750 let health = app
751 .clone()
752 .oneshot(
753 Request::builder()
754 .method("GET")
755 .uri("/ready")
756 .body(Body::empty())
757 .expect("request"),
758 )
759 .await
760 .expect("health response");
761 assert_eq!(health.status(), StatusCode::NO_CONTENT);
762
763 let head_bucket = app
764 .oneshot(
765 Request::builder()
766 .method("HEAD")
767 .uri("/test-bucket")
768 .body(Body::empty())
769 .expect("request"),
770 )
771 .await
772 .expect("head bucket response");
773 assert_eq!(head_bucket.status(), StatusCode::OK);
774 }
775
776 #[tokio::test]
777 async fn build_router_can_disable_health_route() {
778 let temp_dir = tempfile::tempdir().expect("create temp dir");
779 let app = AppState::builder(
780 config::Config::builder(temp_dir.path())
781 .allow_anonymous(true)
782 .build(),
783 )
784 .disable_health_route()
785 .build_router()
786 .await
787 .expect("router");
788
789 let response = app
790 .oneshot(
791 Request::builder()
792 .method("GET")
793 .uri("/health")
794 .body(Body::empty())
795 .expect("request"),
796 )
797 .await
798 .expect("response");
799
800 assert_ne!(response.status(), StatusCode::NO_CONTENT);
801 }
802
803 #[tokio::test]
804 async fn authorization_policy_can_deny_allowed_request() {
805 let temp_dir = tempfile::tempdir().expect("create temp dir");
806 let app = AppState::builder(
807 config::Config::builder(temp_dir.path())
808 .allow_anonymous(true)
809 .build(),
810 )
811 .authorization_policy(|context: hooks::AuthorizationContext| {
812 assert_eq!(context.bucket.as_str(), "test-bucket");
813 assert_eq!(
814 context.key.as_ref().map(|key| key.as_str()),
815 Some("blocked.txt")
816 );
817 Err(error::S3Error::access_denied("blocked by policy"))
818 })
819 .build_router()
820 .await
821 .expect("router");
822
823 let response = app
824 .oneshot(
825 Request::builder()
826 .method("PUT")
827 .uri("/test-bucket/blocked.txt")
828 .header(header::CONTENT_LENGTH, "2")
829 .body(Body::from("hi"))
830 .expect("request"),
831 )
832 .await
833 .expect("response");
834
835 assert_eq!(response.status(), StatusCode::FORBIDDEN);
836 }
837
838 #[tokio::test]
839 async fn authentication_provider_can_replace_static_auth() {
840 let temp_dir = tempfile::tempdir().expect("create temp dir");
841 let app = AppState::builder(config::Config::new(temp_dir.path().to_path_buf()))
842 .authentication_provider(|request: hooks::AuthenticationRequest| async move {
843 if request
844 .headers
845 .get("x-api-key")
846 .and_then(|value| value.to_str().ok())
847 == Some("let-me-in")
848 {
849 Ok(hooks::AuthenticationResult::custom("tenant-a"))
850 } else {
851 Err(error::S3Error::access_denied("invalid api key"))
852 }
853 })
854 .build_router()
855 .await
856 .expect("router");
857
858 let allowed = app
859 .clone()
860 .oneshot(
861 Request::builder()
862 .method("HEAD")
863 .uri("/test-bucket")
864 .header("x-api-key", "let-me-in")
865 .body(Body::empty())
866 .expect("request"),
867 )
868 .await
869 .expect("allowed response");
870 assert_eq!(allowed.status(), StatusCode::OK);
871
872 let denied = app
873 .oneshot(
874 Request::builder()
875 .method("HEAD")
876 .uri("/test-bucket")
877 .body(Body::empty())
878 .expect("request"),
879 )
880 .await
881 .expect("denied response");
882 assert_eq!(denied.status(), StatusCode::FORBIDDEN);
883 }
884
885 #[tokio::test]
886 async fn authorization_policy_receives_custom_auth_principal() {
887 let temp_dir = tempfile::tempdir().expect("create temp dir");
888 let app = AppState::builder(config::Config::new(temp_dir.path().to_path_buf()))
889 .authentication_provider(|_request: hooks::AuthenticationRequest| async {
890 Ok(hooks::AuthenticationResult::custom("tenant-a"))
891 })
892 .authorization_policy(|context: hooks::AuthorizationContext| {
893 assert_eq!(
894 context.principal,
895 hooks::RequestPrincipal::Custom {
896 id: "tenant-a".to_owned()
897 }
898 );
899 Ok(())
900 })
901 .build_router()
902 .await
903 .expect("router");
904
905 let response = app
906 .oneshot(
907 Request::builder()
908 .method("HEAD")
909 .uri("/test-bucket")
910 .body(Body::empty())
911 .expect("request"),
912 )
913 .await
914 .expect("response");
915
916 assert_eq!(response.status(), StatusCode::OK);
917 }
918
919 #[tokio::test]
920 async fn error_mapper_can_rewrite_error_and_preserve_request_id() {
921 let temp_dir = tempfile::tempdir().expect("create temp dir");
922 let app = AppState::builder(
923 config::Config::builder(temp_dir.path())
924 .allow_anonymous(true)
925 .build(),
926 )
927 .request_id_factory(|| s3::types::RequestId::parse("fixed-request-id").expect("request id"))
928 .on_error(|context: hooks::S3ErrorContext, _error| async move {
929 assert_eq!(context.request_id.as_str(), "fixed-request-id");
930 error::S3Error::access_denied("mapped error")
931 })
932 .build_router()
933 .await
934 .expect("router");
935
936 let response = app
937 .oneshot(
938 Request::builder()
939 .method("GET")
940 .uri("/test-bucket/missing.txt")
941 .body(Body::empty())
942 .expect("request"),
943 )
944 .await
945 .expect("response");
946
947 assert_eq!(response.status(), StatusCode::FORBIDDEN);
948 assert_eq!(
949 response
950 .headers()
951 .get("x-amz-request-id")
952 .and_then(|value| value.to_str().ok()),
953 Some("fixed-request-id")
954 );
955 }
956
957 #[tokio::test]
958 async fn request_observer_receives_parsed_request_context() {
959 let temp_dir = tempfile::tempdir().expect("create temp dir");
960 let observed = Arc::new(Mutex::new(Vec::new()));
961 let observed_clone = Arc::clone(&observed);
962 let app = AppState::builder(
963 config::Config::builder(temp_dir.path())
964 .allow_anonymous(true)
965 .build(),
966 )
967 .request_id_factory(|| s3::types::RequestId::parse("observed-request").expect("request id"))
968 .on_request(move |context: hooks::S3RequestContext| {
969 let observed = Arc::clone(&observed_clone);
970 async move {
971 observed.lock().expect("observed lock").push(context);
972 }
973 })
974 .build_router()
975 .await
976 .expect("router");
977
978 let response = app
979 .oneshot(
980 Request::builder()
981 .method("HEAD")
982 .uri("/test-bucket")
983 .body(Body::empty())
984 .expect("request"),
985 )
986 .await
987 .expect("response");
988 assert_eq!(response.status(), StatusCode::OK);
989
990 let observed = observed.lock().expect("observed lock");
991 assert_eq!(observed.len(), 1);
992 assert_eq!(observed[0].request_id.as_str(), "observed-request");
993 assert_eq!(observed[0].operation, "HeadBucket");
994 assert_eq!(
995 observed[0].bucket.as_ref().map(|bucket| bucket.as_str()),
996 Some("test-bucket")
997 );
998 }
999
1000 #[tokio::test]
1001 async fn upload_processor_fn_can_transform_stored_object() {
1002 let temp_dir = tempfile::tempdir().expect("create temp dir");
1003 let app = AppState::builder(
1004 config::Config::builder(temp_dir.path())
1005 .allow_anonymous(true)
1006 .build(),
1007 )
1008 .upload_processor_fn("uppercase", |upload| async move {
1009 let bytes = upload.read_current().await?;
1010 let upper = bytes
1011 .into_iter()
1012 .map(|byte| byte.to_ascii_uppercase())
1013 .collect::<Vec<_>>();
1014 upload.write_replacement(upper).await?;
1015 Ok(upload.replace())
1016 })
1017 .build_router()
1018 .await
1019 .expect("router");
1020
1021 let put = app
1022 .clone()
1023 .oneshot(
1024 Request::builder()
1025 .method("PUT")
1026 .uri("/test-bucket/greeting.txt")
1027 .header(header::CONTENT_LENGTH, "5")
1028 .body(Body::from("hello"))
1029 .expect("request"),
1030 )
1031 .await
1032 .expect("put response");
1033 assert_eq!(put.status(), StatusCode::OK);
1034
1035 let get = app
1036 .oneshot(
1037 Request::builder()
1038 .method("GET")
1039 .uri("/test-bucket/greeting.txt")
1040 .body(Body::empty())
1041 .expect("request"),
1042 )
1043 .await
1044 .expect("get response");
1045 assert_eq!(get.status(), StatusCode::OK);
1046 let body = to_bytes(get.into_body(), 1024).await.expect("body");
1047 assert_eq!(&body[..], b"HELLO");
1048 }
1049
1050 #[tokio::test]
1051 async fn app_state_new_registers_no_upload_processors() {
1052 let temp_dir = tempfile::tempdir().expect("create temp dir");
1053 let state = AppState::new(config::Config {
1054 storage_root: temp_dir.path().to_path_buf(),
1055 auth: Default::default(),
1056 virtual_host_base_domain: None,
1057 upload_limits: Default::default(),
1058 })
1059 .await
1060 .expect("create app state");
1061
1062 assert!(state.upload_processors.is_empty());
1063 }
1064
1065 #[tokio::test]
1066 async fn app_builder_rejects_invalid_io_tuning() {
1067 let temp_dir = tempfile::tempdir().expect("create temp dir");
1068 let result = AppState::builder(config::Config::new(temp_dir.path().to_path_buf()))
1069 .io_tuning(
1070 config::IoTuning::builder()
1071 .object_stream_buffer_size(0)
1072 .build(),
1073 )
1074 .build()
1075 .await;
1076
1077 assert!(matches!(
1078 result,
1079 Err(AppInitError::Config(config::ConfigError::InvalidIoTuning(
1080 "object_stream_buffer_size must be greater than 0"
1081 )))
1082 ));
1083 }
1084
1085 #[tokio::test]
1086 async fn disabled_operation_rejects_before_handler_dispatch() {
1087 let temp_dir = tempfile::tempdir().expect("create temp dir");
1088 let app = AppState::builder(
1089 config::Config::builder(temp_dir.path())
1090 .allow_anonymous(true)
1091 .build(),
1092 )
1093 .disable_operation(config::S3Action::GetObject)
1094 .build_router()
1095 .await
1096 .expect("router");
1097
1098 let response = app
1099 .oneshot(
1100 Request::builder()
1101 .method("GET")
1102 .uri("/test-bucket/file.txt")
1103 .body(Body::empty())
1104 .expect("request"),
1105 )
1106 .await
1107 .expect("response");
1108
1109 assert_eq!(response.status(), StatusCode::METHOD_NOT_ALLOWED);
1110 }
1111
1112 #[tokio::test]
1113 async fn operation_policy_can_deny_specific_operations() {
1114 let temp_dir = tempfile::tempdir().expect("create temp dir");
1115 let app = AppState::builder(
1116 config::Config::builder(temp_dir.path())
1117 .allow_anonymous(true)
1118 .build(),
1119 )
1120 .operation_policy(|context: hooks::OperationContext| {
1121 assert_eq!(context.action, config::S3Action::HeadBucket);
1122 Err(error::S3Error::access_denied("operation blocked"))
1123 })
1124 .build_router()
1125 .await
1126 .expect("router");
1127
1128 let response = app
1129 .oneshot(
1130 Request::builder()
1131 .method("HEAD")
1132 .uri("/test-bucket")
1133 .body(Body::empty())
1134 .expect("request"),
1135 )
1136 .await
1137 .expect("response");
1138
1139 assert_eq!(response.status(), StatusCode::FORBIDDEN);
1140 }
1141
1142 #[tokio::test]
1143 async fn response_mapper_can_add_headers() {
1144 let temp_dir = tempfile::tempdir().expect("create temp dir");
1145 let app = AppState::builder(
1146 config::Config::builder(temp_dir.path())
1147 .allow_anonymous(true)
1148 .build(),
1149 )
1150 .on_response(
1151 |context: hooks::S3ResponseContext, mut response: axum::response::Response| async move {
1152 assert_eq!(context.operation, "HeadBucket");
1153 response
1154 .headers_mut()
1155 .insert("x-app-trace", HeaderValue::from_static("trace-1"));
1156 response
1157 },
1158 )
1159 .build_router()
1160 .await
1161 .expect("router");
1162
1163 let response = app
1164 .oneshot(
1165 Request::builder()
1166 .method("HEAD")
1167 .uri("/test-bucket")
1168 .body(Body::empty())
1169 .expect("request"),
1170 )
1171 .await
1172 .expect("response");
1173
1174 assert_eq!(response.status(), StatusCode::OK);
1175 assert_eq!(
1176 response
1177 .headers()
1178 .get("x-app-trace")
1179 .and_then(|value| value.to_str().ok()),
1180 Some("trace-1")
1181 );
1182 }
1183
1184 #[tokio::test]
1185 async fn target_policy_can_deny_key_prefixes() {
1186 let temp_dir = tempfile::tempdir().expect("create temp dir");
1187 let app = AppState::builder(
1188 config::Config::builder(temp_dir.path())
1189 .allow_anonymous(true)
1190 .build(),
1191 )
1192 .target_policy(|context: hooks::TargetContext| {
1193 if context
1194 .key
1195 .as_ref()
1196 .is_some_and(|key| key.as_str().starts_with("private/"))
1197 {
1198 return Err(error::S3Error::access_denied("private prefix"));
1199 }
1200 Ok(())
1201 })
1202 .build_router()
1203 .await
1204 .expect("router");
1205
1206 let response = app
1207 .oneshot(
1208 Request::builder()
1209 .method("GET")
1210 .uri("/test-bucket/private/file.txt")
1211 .body(Body::empty())
1212 .expect("request"),
1213 )
1214 .await
1215 .expect("response");
1216
1217 assert_eq!(response.status(), StatusCode::FORBIDDEN);
1218 }
1219
1220 #[tokio::test]
1221 async fn upload_policy_can_deny_uploads_before_body_is_stored() {
1222 let temp_dir = tempfile::tempdir().expect("create temp dir");
1223 let app = AppState::builder(
1224 config::Config::builder(temp_dir.path())
1225 .allow_anonymous(true)
1226 .build(),
1227 )
1228 .upload_policy(|context: hooks::UploadPolicyContext| {
1229 assert_eq!(context.action, config::S3Action::PutObject);
1230 Err(error::S3Error::access_denied("uploads disabled"))
1231 })
1232 .build_router()
1233 .await
1234 .expect("router");
1235
1236 let response = app
1237 .oneshot(
1238 Request::builder()
1239 .method("PUT")
1240 .uri("/test-bucket/file.txt")
1241 .header(header::CONTENT_LENGTH, "4")
1242 .body(Body::from("body"))
1243 .expect("request"),
1244 )
1245 .await
1246 .expect("response");
1247
1248 assert_eq!(response.status(), StatusCode::FORBIDDEN);
1249 }
1250
1251 #[tokio::test]
1252 async fn clock_controls_object_last_modified_metadata() {
1253 let temp_dir = tempfile::tempdir().expect("create temp dir");
1254 let fixed_now = chrono::DateTime::parse_from_rfc3339("2026-06-17T12:34:56Z")
1255 .expect("timestamp")
1256 .with_timezone(&chrono::Utc);
1257 let state = AppState::builder(
1258 config::Config::builder(temp_dir.path())
1259 .allow_anonymous(true)
1260 .build(),
1261 )
1262 .clock(move || fixed_now)
1263 .build()
1264 .await
1265 .expect("state");
1266 let app = router(state.clone());
1267
1268 let response = app
1269 .oneshot(
1270 Request::builder()
1271 .method("PUT")
1272 .uri("/test-bucket/file.txt")
1273 .header(header::CONTENT_LENGTH, "4")
1274 .body(Body::from("body"))
1275 .expect("request"),
1276 )
1277 .await
1278 .expect("response");
1279 assert_eq!(response.status(), StatusCode::OK);
1280
1281 let metadata = state
1282 .object_store
1283 .head_object(
1284 &s3::types::BucketName::parse("test-bucket").expect("bucket"),
1285 &s3::types::ObjectKey::parse("file.txt").expect("key"),
1286 )
1287 .await
1288 .expect("metadata read")
1289 .expect("metadata");
1290 assert_eq!(metadata.last_modified, fixed_now);
1291 }
1292
1293 #[tokio::test]
1294 async fn builder_accepts_custom_store_trait_objects() {
1295 let temp_dir = tempfile::tempdir().expect("create temp dir");
1296 let object_store = storage::FileObjectStore::new(temp_dir.path().to_path_buf())
1297 .await
1298 .expect("object store");
1299 let multipart_store = storage::FileMultipartStore::new(temp_dir.path().to_path_buf())
1300 .await
1301 .expect("multipart store");
1302
1303 let state = AppState::builder(config::Config::new(temp_dir.path().to_path_buf()))
1304 .object_store(object_store)
1305 .multipart_store(multipart_store)
1306 .build()
1307 .await
1308 .expect("state");
1309
1310 assert!(state.upload_processors.is_empty());
1311 }
1312
1313 #[tokio::test]
1314 async fn tenant_limits_provider_can_deny_before_handler_execution() {
1315 let temp_dir = tempfile::tempdir().expect("create temp dir");
1316 let app = AppState::builder(
1317 config::Config::builder(temp_dir.path())
1318 .allow_anonymous(true)
1319 .build(),
1320 )
1321 .tenant_limits_provider(DenyTenantLimitsProvider)
1322 .build_router()
1323 .await
1324 .expect("router");
1325
1326 let response = app
1327 .oneshot(
1328 Request::builder()
1329 .method("HEAD")
1330 .uri("/test-bucket")
1331 .body(Body::empty())
1332 .expect("request"),
1333 )
1334 .await
1335 .expect("response");
1336
1337 assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
1338 }
1339
1340 #[tokio::test]
1341 async fn tenant_limits_timeout_returns_slow_down_and_preserves_request_id() {
1342 let temp_dir = tempfile::tempdir().expect("create temp dir");
1343 let finishes = Arc::new(Mutex::new(Vec::new()));
1344 let app = AppState::builder(
1345 config::Config::builder(temp_dir.path())
1346 .allow_anonymous(true)
1347 .build(),
1348 )
1349 .request_id_factory(|| s3::types::RequestId::parse("timeout-request").expect("request id"))
1350 .tenant_limits_provider(RecordingTenantLimitsProvider {
1351 begins: Arc::new(Mutex::new(Vec::new())),
1352 finishes: Arc::clone(&finishes),
1353 timeout: Some(std::time::Duration::from_millis(1)),
1354 permit_drops: None,
1355 })
1356 .upload_processor_fn("slow", |_upload| async move {
1357 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1358 Ok(middleware::UploadProcessorAction::Keep)
1359 })
1360 .build_router()
1361 .await
1362 .expect("router");
1363
1364 let response = app
1365 .oneshot(
1366 Request::builder()
1367 .method("PUT")
1368 .uri("/test-bucket/slow.txt")
1369 .header(header::CONTENT_LENGTH, "4")
1370 .body(Body::from("slow"))
1371 .expect("request"),
1372 )
1373 .await
1374 .expect("response");
1375
1376 assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
1377 assert_eq!(
1378 response
1379 .headers()
1380 .get("x-amz-request-id")
1381 .and_then(|value| value.to_str().ok()),
1382 Some("timeout-request")
1383 );
1384 let finishes = finishes.lock().expect("finishes lock");
1385 assert_eq!(finishes.len(), 1);
1386 assert!(finishes[0].1.timed_out);
1387 assert_eq!(finishes[0].1.error_code.as_deref(), Some("SlowDown"));
1388 }
1389
1390 #[tokio::test]
1391 async fn custom_auth_with_invalid_principal_id_is_rejected() {
1392 let temp_dir = tempfile::tempdir().expect("create temp dir");
1393 let app = AppState::builder(config::Config::new(temp_dir.path().to_path_buf()))
1394 .request_id_factory(|| {
1395 s3::types::RequestId::parse("invalid-principal").expect("request id")
1396 })
1397 .authentication_provider(|_request: hooks::AuthenticationRequest| async {
1398 Ok(hooks::AuthenticationResult::custom("tenant\nid"))
1399 })
1400 .build_router()
1401 .await
1402 .expect("router");
1403
1404 let response = app
1405 .oneshot(
1406 Request::builder()
1407 .method("HEAD")
1408 .uri("/test-bucket")
1409 .body(Body::empty())
1410 .expect("request"),
1411 )
1412 .await
1413 .expect("response");
1414
1415 assert_eq!(response.status(), StatusCode::FORBIDDEN);
1416 assert_eq!(
1417 response
1418 .headers()
1419 .get("x-amz-request-id")
1420 .and_then(|value| value.to_str().ok()),
1421 Some("invalid-principal")
1422 );
1423 }
1424
1425 #[tokio::test]
1426 async fn custom_auth_success_still_runs_downstream_policies() {
1427 let temp_dir = tempfile::tempdir().expect("create temp dir");
1428 let app = AppState::builder(config::Config::new(temp_dir.path().to_path_buf()))
1429 .authentication_provider(|_request: hooks::AuthenticationRequest| async {
1430 hooks::AuthenticationResult::try_custom("tenant-a")
1431 .map_err(|err| error::S3Error::access_denied(err.to_string()))
1432 })
1433 .authorization_policy(|_context: hooks::AuthorizationContext| {
1434 Err(error::S3Error::access_denied("policy denied"))
1435 })
1436 .build_router()
1437 .await
1438 .expect("router");
1439
1440 let response = app
1441 .oneshot(
1442 Request::builder()
1443 .method("HEAD")
1444 .uri("/test-bucket")
1445 .body(Body::empty())
1446 .expect("request"),
1447 )
1448 .await
1449 .expect("response");
1450
1451 assert_eq!(response.status(), StatusCode::FORBIDDEN);
1452 }
1453
1454 #[tokio::test]
1455 async fn timed_out_put_object_discards_temporary_object_file() {
1456 let temp_dir = tempfile::tempdir().expect("create temp dir");
1457 let app = AppState::builder(
1458 config::Config::builder(temp_dir.path())
1459 .allow_anonymous(true)
1460 .build(),
1461 )
1462 .tenant_limits_provider(RecordingTenantLimitsProvider {
1463 begins: Arc::new(Mutex::new(Vec::new())),
1464 finishes: Arc::new(Mutex::new(Vec::new())),
1465 timeout: Some(Duration::from_millis(1)),
1466 permit_drops: None,
1467 })
1468 .upload_processor_fn("slow", |_upload| async move {
1469 tokio::time::sleep(Duration::from_millis(50)).await;
1470 Ok(middleware::UploadProcessorAction::Keep)
1471 })
1472 .build_router()
1473 .await
1474 .expect("router");
1475
1476 let response = app
1477 .oneshot(
1478 Request::builder()
1479 .method("PUT")
1480 .uri("/test-bucket/slow.txt")
1481 .header(header::CONTENT_LENGTH, "4")
1482 .body(Body::from("slow"))
1483 .expect("request"),
1484 )
1485 .await
1486 .expect("response");
1487
1488 assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
1489 assert_dir_empty_eventually(&temp_dir.path().join("tmp")).await;
1490 }
1491
1492 #[tokio::test]
1493 async fn timed_out_upload_part_discards_temporary_part_file() {
1494 let temp_dir = tempfile::tempdir().expect("create temp dir");
1495 let app = AppState::builder(
1496 config::Config::builder(temp_dir.path())
1497 .allow_anonymous(true)
1498 .build(),
1499 )
1500 .tenant_limits_provider(RecordingTenantLimitsProvider {
1501 begins: Arc::new(Mutex::new(Vec::new())),
1502 finishes: Arc::new(Mutex::new(Vec::new())),
1503 timeout: Some(Duration::from_millis(1)),
1504 permit_drops: None,
1505 })
1506 .build_router()
1507 .await
1508 .expect("router");
1509
1510 let create = app
1511 .clone()
1512 .oneshot(
1513 Request::builder()
1514 .method("POST")
1515 .uri("/test-bucket/parts.txt?uploads")
1516 .body(Body::empty())
1517 .expect("request"),
1518 )
1519 .await
1520 .expect("create response");
1521 assert_eq!(create.status(), StatusCode::OK);
1522 let body = to_bytes(create.into_body(), usize::MAX)
1523 .await
1524 .expect("create body");
1525 let upload_id = upload_id_from_xml(std::str::from_utf8(&body).expect("xml utf8"));
1526
1527 let delayed = stream::once(async {
1528 tokio::time::sleep(Duration::from_millis(50)).await;
1529 Ok::<_, Infallible>(Bytes::from_static(b"part"))
1530 });
1531 let response = app
1532 .oneshot(
1533 Request::builder()
1534 .method("PUT")
1535 .uri(format!(
1536 "/test-bucket/parts.txt?partNumber=1&uploadId={}",
1537 upload_id
1538 ))
1539 .header(header::CONTENT_LENGTH, "4")
1540 .body(Body::from_stream(delayed))
1541 .expect("request"),
1542 )
1543 .await
1544 .expect("response");
1545
1546 assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
1547 assert_no_multipart_tmp_files_eventually(temp_dir.path()).await;
1548 }
1549
1550 #[tokio::test]
1551 async fn tenant_limits_provider_permit_is_dropped_after_completion() {
1552 let temp_dir = tempfile::tempdir().expect("create temp dir");
1553 let drops = Arc::new(Mutex::new(0_usize));
1554 let app = AppState::builder(
1555 config::Config::builder(temp_dir.path())
1556 .allow_anonymous(true)
1557 .build(),
1558 )
1559 .tenant_limits_provider(RecordingTenantLimitsProvider {
1560 begins: Arc::new(Mutex::new(Vec::new())),
1561 finishes: Arc::new(Mutex::new(Vec::new())),
1562 timeout: None,
1563 permit_drops: Some(Arc::clone(&drops)),
1564 })
1565 .build_router()
1566 .await
1567 .expect("router");
1568
1569 let response = app
1570 .oneshot(
1571 Request::builder()
1572 .method("HEAD")
1573 .uri("/test-bucket")
1574 .body(Body::empty())
1575 .expect("request"),
1576 )
1577 .await
1578 .expect("response");
1579
1580 assert_eq!(response.status(), StatusCode::OK);
1581 assert_eq!(*drops.lock().expect("drop lock"), 1);
1582 }
1583
1584 #[tokio::test]
1585 async fn tenant_limits_finish_receives_operation_outcomes() {
1586 let temp_dir = tempfile::tempdir().expect("create temp dir");
1587 let begins = Arc::new(Mutex::new(Vec::new()));
1588 let finishes = Arc::new(Mutex::new(Vec::new()));
1589 let app = AppState::builder(
1590 config::Config::builder(temp_dir.path())
1591 .allow_anonymous(true)
1592 .build(),
1593 )
1594 .tenant_limits_provider(RecordingTenantLimitsProvider {
1595 begins: Arc::clone(&begins),
1596 finishes: Arc::clone(&finishes),
1597 timeout: None,
1598 permit_drops: None,
1599 })
1600 .build_router()
1601 .await
1602 .expect("router");
1603
1604 let head = app
1605 .clone()
1606 .oneshot(
1607 Request::builder()
1608 .method("HEAD")
1609 .uri("/test-bucket")
1610 .body(Body::empty())
1611 .expect("request"),
1612 )
1613 .await
1614 .expect("head response");
1615 assert_eq!(head.status(), StatusCode::OK);
1616
1617 let missing = app
1618 .clone()
1619 .oneshot(
1620 Request::builder()
1621 .method("GET")
1622 .uri("/test-bucket/missing.txt")
1623 .body(Body::empty())
1624 .expect("request"),
1625 )
1626 .await
1627 .expect("missing response");
1628 assert_eq!(missing.status(), StatusCode::NOT_FOUND);
1629
1630 let put = app
1631 .oneshot(
1632 Request::builder()
1633 .method("PUT")
1634 .uri("/test-bucket/body.txt")
1635 .header(header::CONTENT_LENGTH, "5")
1636 .body(Body::from("hello"))
1637 .expect("request"),
1638 )
1639 .await
1640 .expect("put response");
1641 assert_eq!(put.status(), StatusCode::OK);
1642
1643 let begins = begins.lock().expect("begins lock");
1644 assert_eq!(begins.len(), 3);
1645 assert!(
1646 begins
1647 .iter()
1648 .all(|context| context.tenant.as_str() == "anonymous")
1649 );
1650 assert_eq!(begins[0].action, config::S3Action::HeadBucket);
1651 assert_eq!(begins[1].action, config::S3Action::GetObject);
1652 assert_eq!(begins[2].action, config::S3Action::PutObject);
1653
1654 let finishes = finishes.lock().expect("finishes lock");
1655 assert_eq!(finishes.len(), 3);
1656 assert_eq!(finishes[0].1.status, StatusCode::OK);
1657 assert_eq!(finishes[1].1.status, StatusCode::NOT_FOUND);
1658 assert_eq!(finishes[1].1.error_code.as_deref(), Some("NoSuchKey"));
1659 assert_eq!(finishes[2].1.status, StatusCode::OK);
1660 assert_eq!(finishes[2].1.decoded_bytes, Some(5));
1661 }
1662
1663 #[tokio::test]
1664 async fn tenant_limits_use_custom_auth_principal_and_authenticate_once() {
1665 let temp_dir = tempfile::tempdir().expect("create temp dir");
1666 let auth_calls = Arc::new(Mutex::new(0_usize));
1667 let begins = Arc::new(Mutex::new(Vec::new()));
1668 let auth_calls_clone = Arc::clone(&auth_calls);
1669 let app = AppState::builder(config::Config::new(temp_dir.path().to_path_buf()))
1670 .authentication_provider(move |_request: hooks::AuthenticationRequest| {
1671 let auth_calls = Arc::clone(&auth_calls_clone);
1672 async move {
1673 *auth_calls.lock().expect("auth calls lock") += 1;
1674 Ok(hooks::AuthenticationResult::custom("tenant-a"))
1675 }
1676 })
1677 .tenant_limits_provider(RecordingTenantLimitsProvider {
1678 begins: Arc::clone(&begins),
1679 finishes: Arc::new(Mutex::new(Vec::new())),
1680 timeout: None,
1681 permit_drops: None,
1682 })
1683 .build_router()
1684 .await
1685 .expect("router");
1686
1687 let response = app
1688 .oneshot(
1689 Request::builder()
1690 .method("HEAD")
1691 .uri("/test-bucket")
1692 .body(Body::empty())
1693 .expect("request"),
1694 )
1695 .await
1696 .expect("response");
1697
1698 assert_eq!(response.status(), StatusCode::OK);
1699 assert_eq!(*auth_calls.lock().expect("auth calls lock"), 1);
1700 let begins = begins.lock().expect("begins lock");
1701 assert_eq!(begins.len(), 1);
1702 assert_eq!(begins[0].tenant.as_str(), "tenant-a");
1703 assert_eq!(
1704 begins[0].principal,
1705 hooks::RequestPrincipal::Custom {
1706 id: "tenant-a".to_owned()
1707 }
1708 );
1709 }
1710
1711 struct DenyTenantLimitsProvider;
1712
1713 impl hooks::TenantLimitsProvider for DenyTenantLimitsProvider {
1714 fn begin_operation<'a>(
1715 &'a self,
1716 _context: hooks::TenantLimitContext,
1717 ) -> BoxFuture<'a, Result<hooks::TenantOperationLease, error::S3Error>> {
1718 Box::pin(async { Err(error::S3Error::slow_down("tenant quota exceeded")) })
1719 }
1720
1721 fn finish_operation<'a>(
1722 &'a self,
1723 _context: hooks::TenantLimitContext,
1724 _outcome: hooks::TenantOperationOutcome,
1725 ) -> BoxFuture<'a, ()> {
1726 Box::pin(async {})
1727 }
1728 }
1729
1730 struct RecordingTenantLimitsProvider {
1731 begins: Arc<Mutex<Vec<hooks::TenantLimitContext>>>,
1732 finishes: Arc<Mutex<Vec<(hooks::TenantLimitContext, hooks::TenantOperationOutcome)>>>,
1733 timeout: Option<std::time::Duration>,
1734 permit_drops: Option<Arc<Mutex<usize>>>,
1735 }
1736
1737 impl hooks::TenantLimitsProvider for RecordingTenantLimitsProvider {
1738 fn begin_operation<'a>(
1739 &'a self,
1740 context: hooks::TenantLimitContext,
1741 ) -> BoxFuture<'a, Result<hooks::TenantOperationLease, error::S3Error>> {
1742 let timeout = self.timeout;
1743 let permit_drops = self.permit_drops.clone();
1744 self.begins.lock().expect("begins lock").push(context);
1745 Box::pin(async move {
1746 let mut lease = hooks::TenantOperationLease::new();
1747 if let Some(timeout) = timeout {
1748 lease = lease.timeout(timeout);
1749 }
1750 if let Some(permit_drops) = permit_drops {
1751 lease = lease.permit(DropCounter(permit_drops));
1752 }
1753 Ok(lease)
1754 })
1755 }
1756
1757 fn finish_operation<'a>(
1758 &'a self,
1759 context: hooks::TenantLimitContext,
1760 outcome: hooks::TenantOperationOutcome,
1761 ) -> BoxFuture<'a, ()> {
1762 self.finishes
1763 .lock()
1764 .expect("finishes lock")
1765 .push((context, outcome));
1766 Box::pin(async {})
1767 }
1768 }
1769
1770 struct DropCounter(Arc<Mutex<usize>>);
1771
1772 impl Drop for DropCounter {
1773 fn drop(&mut self) {
1774 *self.0.lock().expect("drop counter lock") += 1;
1775 }
1776 }
1777
1778 async fn assert_dir_empty_eventually(path: &Path) {
1779 for _ in 0..20 {
1780 if dir_entries(path).is_empty() {
1781 return;
1782 }
1783 tokio::time::sleep(Duration::from_millis(10)).await;
1784 }
1785 panic!("directory still contains entries: {:?}", dir_entries(path));
1786 }
1787
1788 async fn assert_no_multipart_tmp_files_eventually(root: &Path) {
1789 for _ in 0..20 {
1790 let tmp_files = multipart_tmp_files(root);
1791 if tmp_files.is_empty() {
1792 return;
1793 }
1794 tokio::time::sleep(Duration::from_millis(10)).await;
1795 }
1796 panic!(
1797 "multipart storage still contains tmp files: {:?}",
1798 multipart_tmp_files(root)
1799 );
1800 }
1801
1802 fn dir_entries(path: &Path) -> Vec<std::path::PathBuf> {
1803 std::fs::read_dir(path)
1804 .map(|entries| {
1805 entries
1806 .map(|entry| entry.expect("dir entry").path())
1807 .collect::<Vec<_>>()
1808 })
1809 .unwrap_or_default()
1810 }
1811
1812 fn multipart_tmp_files(root: &Path) -> Vec<std::path::PathBuf> {
1813 let multipart = root.join("multipart");
1814 let mut tmp_files = Vec::new();
1815 for upload in dir_entries(&multipart) {
1816 for entry in dir_entries(&upload) {
1817 if entry
1818 .file_name()
1819 .and_then(|name| name.to_str())
1820 .is_some_and(|name| name.ends_with(".tmp"))
1821 {
1822 tmp_files.push(entry);
1823 }
1824 }
1825 }
1826 tmp_files
1827 }
1828
1829 fn upload_id_from_xml(xml: &str) -> String {
1830 let start = xml.find("<UploadId>").expect("upload id start") + "<UploadId>".len();
1831 let end = xml[start..].find("</UploadId>").expect("upload id end") + start;
1832 xml[start..end].to_owned()
1833 }
1834
1835 struct NamedProcessor(&'static str);
1836
1837 impl middleware::UploadProcessor for NamedProcessor {
1838 fn process<'a>(
1839 &'a self,
1840 _request: middleware::UploadProcessorRequest<'a>,
1841 ) -> BoxFuture<
1842 'a,
1843 Result<middleware::UploadProcessorAction, middleware::UploadProcessorError>,
1844 > {
1845 let _name = self.0;
1846 Box::pin(async { Ok(middleware::UploadProcessorAction::Keep) })
1847 }
1848 }
1849
1850 #[tokio::test]
1851 async fn app_state_rejects_invalid_auth_config() {
1852 let temp_dir = tempfile::tempdir().expect("create temp dir");
1853 let result = AppState::new(config::Config {
1854 storage_root: temp_dir.path().to_path_buf(),
1855 auth: config::AuthConfig {
1856 region: "US-EAST-1".to_owned(),
1857 ..Default::default()
1858 },
1859 virtual_host_base_domain: None,
1860 upload_limits: Default::default(),
1861 })
1862 .await;
1863
1864 let Err(error) = result else {
1865 panic!("invalid auth config should fail");
1866 };
1867
1868 assert!(matches!(
1869 error,
1870 AppInitError::Config(config::ConfigError::InvalidAuthConfig(
1871 "region must be a non-empty lowercase AWS region identifier"
1872 ))
1873 ));
1874 }
1875
1876 #[tokio::test]
1877 async fn s3_request_admission_rejects_when_limit_is_exhausted() {
1878 let temp_dir = tempfile::tempdir().expect("create temp dir");
1879 let state = AppState::new(config::Config::new(temp_dir.path().to_path_buf()))
1880 .await
1881 .expect("create app state");
1882 let permits = (0..state.config.upload_limits.max_concurrent_s3_requests)
1883 .map(|_| state.try_acquire_s3_request().expect("permit"))
1884 .collect::<Vec<_>>();
1885
1886 let error = state
1887 .try_acquire_s3_request()
1888 .expect_err("S3 request limit should reject");
1889 let response = error.into_response_with_request_id(&s3::types::RequestId::new());
1890 assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
1891
1892 drop(permits);
1893 let _permit = state
1894 .try_acquire_s3_request()
1895 .expect("permit released after drop");
1896 }
1897
1898 #[tokio::test]
1899 async fn router_returns_slow_down_when_s3_request_admission_is_exhausted() {
1900 let temp_dir = tempfile::tempdir().expect("create temp dir");
1901 let state = AppState::new(config::Config::new(temp_dir.path().to_path_buf()))
1902 .await
1903 .expect("create app state");
1904 let _permits = (0..state.config.upload_limits.max_concurrent_s3_requests)
1905 .map(|_| state.try_acquire_s3_request().expect("permit"))
1906 .collect::<Vec<_>>();
1907
1908 let response = router(state)
1909 .oneshot(
1910 Request::builder()
1911 .method("PUT")
1912 .uri("/test-bucket/admission.txt")
1913 .body(Body::empty())
1914 .expect("build request"),
1915 )
1916 .await
1917 .expect("send request");
1918
1919 assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
1920 let body = axum::body::to_bytes(response.into_body(), usize::MAX)
1921 .await
1922 .expect("read response body");
1923 let body = std::str::from_utf8(&body).expect("utf8 body");
1924 assert!(body.contains("<Code>SlowDown</Code>"));
1925 assert!(body.contains("too many active S3 requests"));
1926 }
1927
1928 #[tokio::test]
1929 async fn object_writer_admission_rejects_when_limit_is_exhausted() {
1930 let temp_dir = tempfile::tempdir().expect("create temp dir");
1931 let state = AppState::new(config::Config::new(temp_dir.path().to_path_buf()))
1932 .await
1933 .expect("create app state");
1934 let permits = (0..state.config.upload_limits.max_active_object_writers)
1935 .map(|_| state.try_acquire_object_writer().expect("permit"))
1936 .collect::<Vec<_>>();
1937
1938 let error = state
1939 .try_acquire_object_writer()
1940 .expect_err("object writer limit should reject");
1941 let response = error.into_response_with_request_id(&s3::types::RequestId::new());
1942 assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
1943
1944 drop(permits);
1945 let _permit = state
1946 .try_acquire_object_writer()
1947 .expect("permit released after drop");
1948 }
1949
1950 #[tokio::test]
1951 async fn admission_limits_are_configurable() {
1952 let temp_dir = tempfile::tempdir().expect("create temp dir");
1953 let state = AppState::new(config::Config {
1954 storage_root: temp_dir.path().to_path_buf(),
1955 auth: Default::default(),
1956 virtual_host_base_domain: None,
1957 upload_limits: config::UploadLimits {
1958 max_concurrent_s3_requests: 1,
1959 max_active_object_writers: 1,
1960 max_active_multipart_part_writers: 1,
1961 max_active_aws_chunked_decoders: 1,
1962 ..Default::default()
1963 },
1964 })
1965 .await
1966 .expect("create app state");
1967
1968 let _request = state.try_acquire_s3_request().expect("request permit");
1969 assert!(state.try_acquire_s3_request().is_err());
1970
1971 let _object = state.try_acquire_object_writer().expect("object permit");
1972 assert!(state.try_acquire_object_writer().is_err());
1973
1974 let _part = state
1975 .try_acquire_multipart_part_writer()
1976 .expect("part permit");
1977 assert!(state.try_acquire_multipart_part_writer().is_err());
1978
1979 let mut headers = HeaderMap::new();
1980 headers.insert("content-encoding", HeaderValue::from_static("aws-chunked"));
1981 let _decoder = state
1982 .try_acquire_aws_chunked_decoder(&headers)
1983 .expect("decoder permit")
1984 .expect("aws-chunked permit");
1985 assert!(state.try_acquire_aws_chunked_decoder(&headers).is_err());
1986 }
1987
1988 #[tokio::test]
1989 async fn multipart_part_writer_admission_rejects_when_limit_is_exhausted() {
1990 let temp_dir = tempfile::tempdir().expect("create temp dir");
1991 let state = AppState::new(config::Config::new(temp_dir.path().to_path_buf()))
1992 .await
1993 .expect("create app state");
1994 let permits = (0..state.config.upload_limits.max_active_multipart_part_writers)
1995 .map(|_| state.try_acquire_multipart_part_writer().expect("permit"))
1996 .collect::<Vec<_>>();
1997
1998 let error = state
1999 .try_acquire_multipart_part_writer()
2000 .expect_err("multipart part writer limit should reject");
2001 let response = error.into_response_with_request_id(&s3::types::RequestId::new());
2002 assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
2003
2004 drop(permits);
2005 let _permit = state
2006 .try_acquire_multipart_part_writer()
2007 .expect("permit released after drop");
2008 }
2009
2010 #[tokio::test]
2011 async fn aws_chunked_decoder_admission_only_applies_to_aws_chunked_requests() {
2012 let temp_dir = tempfile::tempdir().expect("create temp dir");
2013 let state = AppState::new(config::Config::new(temp_dir.path().to_path_buf()))
2014 .await
2015 .expect("create app state");
2016 let permits = (0..state.config.upload_limits.max_active_aws_chunked_decoders)
2017 .map(|_| {
2018 state
2019 .admission
2020 .aws_chunked_decoders
2021 .clone()
2022 .try_acquire_owned()
2023 .expect("permit")
2024 })
2025 .collect::<Vec<_>>();
2026
2027 let plain_headers = HeaderMap::new();
2028 assert!(
2029 state
2030 .try_acquire_aws_chunked_decoder(&plain_headers)
2031 .expect("plain request")
2032 .is_none()
2033 );
2034
2035 let mut aws_chunked_headers = HeaderMap::new();
2036 aws_chunked_headers.insert("content-encoding", HeaderValue::from_static("aws-chunked"));
2037 let error = state
2038 .try_acquire_aws_chunked_decoder(&aws_chunked_headers)
2039 .expect_err("aws-chunked decoder limit should reject");
2040 let response = error.into_response_with_request_id(&s3::types::RequestId::new());
2041 assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
2042
2043 drop(permits);
2044 assert!(
2045 state
2046 .try_acquire_aws_chunked_decoder(&aws_chunked_headers)
2047 .expect("permit released after drop")
2048 .is_some()
2049 );
2050 }
2051}