Skip to main content

s3_endpoint/
lib.rs

1//! S3-compatible upload endpoint implementation.
2//!
3//! The crate exposes an Axum router plus typed configuration, authentication,
4//! storage, and S3-domain modules for a focused S3-compatible subset:
5//! single-object upload, multipart upload, object reads/metadata, deletion,
6//! SigV4 header authentication, presigned URLs, and aws-chunked bodies.
7
8#![forbid(unsafe_code)]
9
10pub(crate) mod auth;
11pub(crate) mod body;
12/// Runtime configuration, credentials, authorization policy, and upload limits.
13pub mod config;
14/// S3 XML error construction and HTTP response conversion.
15pub mod error;
16pub(crate) mod handlers;
17/// Developer tuning hooks for embedded applications.
18pub mod hooks;
19pub(crate) mod http;
20/// S3-aware upload middleware extension points.
21pub mod middleware;
22/// S3 request target parsing and validated domain value types.
23pub mod s3;
24/// Filesystem-backed object and multipart storage types.
25pub mod storage;
26
27use std::{collections::HashSet, convert::Infallible, future::Future, sync::Arc};
28
29use axum::{
30    Router,
31    body::Body,
32    http::{HeaderMap, Request},
33    response::IntoResponse,
34    routing::Route,
35};
36use hooks::{
37    AuthenticationProvider, AuthenticationRequest, AuthenticationResult, AuthorizationContext,
38    AuthorizationPolicy, Clock, ErrorMapper, OperationContext, OperationPolicy, RequestIdFactory,
39    RequestObserver, ResponseMapper, S3ErrorContext, S3RequestContext, S3ResponseContext,
40    SharedAuthenticationProvider, SharedAuthorizationPolicy, SharedClock, SharedErrorMapper,
41    SharedOperationPolicy, SharedRequestIdFactory, SharedRequestObserver, SharedResponseMapper,
42    SharedTargetPolicy, SharedTenantLimitsProvider, SharedUploadPolicy, SystemClock, TargetContext,
43    TargetPolicy, TenantLimitContext, TenantLimitsProvider, TenantOperationLease,
44    TenantOperationOutcome, UploadPolicy, UploadPolicyContext,
45};
46use middleware::{
47    SharedUploadProcessor, UploadProcessor, UploadProcessorAction, UploadProcessorError,
48    UploadProcessorFn, UploadProcessorHandle,
49};
50use storage::{
51    FileMultipartStore, FileObjectStore, MultipartStore, ObjectStore, SharedMultipartStore,
52    SharedObjectStore,
53};
54use thiserror::Error;
55use tokio::sync::{OwnedSemaphorePermit, Semaphore};
56use tower::{Layer, Service};
57
58#[derive(Clone)]
59/// Shared application state for the S3 endpoint.
60///
61/// `AppState` owns validated configuration, credential state, filesystem-backed
62/// object and multipart stores, and bounded admission controls. Construct it
63/// with [`AppState::new`] or [`AppState::builder`] and pass it to [`router`] to
64/// build the HTTP service.
65pub struct AppState {
66    /// Validated runtime configuration.
67    pub config: config::Config,
68    /// Credential lookup and authorization state.
69    pub auth: Arc<config::AuthState>,
70    /// Filesystem-backed committed object store.
71    pub object_store: SharedObjectStore,
72    /// Filesystem-backed multipart upload store.
73    pub multipart_store: SharedMultipartStore,
74    pub(crate) io_tuning: config::IoTuning,
75    pub(crate) upload_processors: Arc<Vec<SharedUploadProcessor>>,
76    request_observer: Option<SharedRequestObserver>,
77    error_mapper: Option<SharedErrorMapper>,
78    response_mapper: Option<SharedResponseMapper>,
79    request_id_factory: SharedRequestIdFactory,
80    clock: SharedClock,
81    authentication_provider: Option<SharedAuthenticationProvider>,
82    authorization_policy: Option<SharedAuthorizationPolicy>,
83    operation_policy: Option<SharedOperationPolicy>,
84    target_policy: Option<SharedTargetPolicy>,
85    upload_policy: Option<SharedUploadPolicy>,
86    tenant_limits_provider: SharedTenantLimitsProvider,
87    disabled_operations: Arc<HashSet<config::S3Action>>,
88    admission: Arc<AdmissionControl>,
89}
90
91impl AppState {
92    /// Creates a builder for application state with optional upload processors.
93    pub fn builder(config: config::Config) -> AppBuilder {
94        AppBuilder {
95            config,
96            upload_processors: Vec::new(),
97            object_store: None,
98            multipart_store: None,
99            io_tuning: config::IoTuning::default(),
100            multipart_lifecycle: config::MultipartLifecycle::default(),
101            request_observer: None,
102            error_mapper: None,
103            response_mapper: None,
104            request_id_factory: Arc::new(s3::types::RequestId::new),
105            clock: Arc::new(SystemClock),
106            authentication_provider: None,
107            authorization_policy: None,
108            operation_policy: None,
109            target_policy: None,
110            upload_policy: None,
111            tenant_limits_provider: Arc::new(hooks::NoopTenantLimitsProvider),
112            disabled_operations: HashSet::new(),
113            health_path: Some("/health".to_owned()),
114            router_layers: Vec::new(),
115        }
116    }
117
118    /// Builds application state from configuration.
119    ///
120    /// This validates configuration and credentials, initializes storage
121    /// directories, removes stale temporary/orphaned files, and creates bounded
122    /// admission controls. Returns an error when configuration is invalid or
123    /// storage cannot be initialized.
124    pub async fn new(config: config::Config) -> Result<Self, AppInitError> {
125        Self::builder(config).build().await
126    }
127
128    async fn from_builder(builder: AppBuilder) -> Result<Self, AppInitError> {
129        let AppBuilder {
130            config,
131            upload_processors,
132            object_store,
133            multipart_store,
134            io_tuning,
135            multipart_lifecycle,
136            request_observer,
137            error_mapper,
138            response_mapper,
139            request_id_factory,
140            clock,
141            authentication_provider,
142            authorization_policy,
143            operation_policy,
144            target_policy,
145            upload_policy,
146            tenant_limits_provider,
147            disabled_operations,
148            health_path: _,
149            router_layers: _,
150        } = builder;
151        config.validate()?;
152        io_tuning.validate()?;
153        let upload_limits = config.upload_limits.validated()?;
154        let auth = Arc::new(config::AuthState::new(&config.auth)?);
155        let object_store = match object_store {
156            Some(store) => store,
157            None => Arc::new(FileObjectStore::new(config.storage_root.clone()).await?),
158        };
159        let multipart_store = match multipart_store {
160            Some(store) => store,
161            None => Arc::new(
162                FileMultipartStore::new_with_options(
163                    config.storage_root.clone(),
164                    multipart_lifecycle,
165                    clock.clone(),
166                )
167                .await?,
168            ),
169        };
170        Ok(Self {
171            config,
172            auth,
173            object_store,
174            multipart_store,
175            io_tuning,
176            upload_processors: Arc::new(upload_processors),
177            request_observer,
178            error_mapper,
179            response_mapper,
180            request_id_factory,
181            clock,
182            authentication_provider,
183            authorization_policy,
184            operation_policy,
185            target_policy,
186            upload_policy,
187            tenant_limits_provider,
188            disabled_operations: Arc::new(disabled_operations),
189            admission: Arc::new(AdmissionControl::new(&upload_limits)),
190        })
191    }
192
193    /// Returns the registered upload processors in execution order.
194    pub(crate) fn upload_processors(&self) -> &[SharedUploadProcessor] {
195        &self.upload_processors
196    }
197
198    /// Creates the next S3 request ID using the configured factory.
199    pub(crate) fn request_id(&self) -> s3::types::RequestId {
200        self.request_id_factory.request_id()
201    }
202
203    /// Returns the current application time from the configured clock.
204    pub(crate) fn now(&self) -> chrono::DateTime<chrono::Utc> {
205        self.clock.now()
206    }
207
208    /// Attempts custom authentication when an authentication provider is configured.
209    ///
210    /// Returns `None` when the built-in SigV4/anonymous authenticator should be used.
211    pub(crate) async fn authenticate_with_provider(
212        &self,
213        request: AuthenticationRequest,
214    ) -> Option<Result<AuthenticationResult, error::S3Error>> {
215        let provider = self.authentication_provider.as_ref()?;
216        Some(provider.authenticate(request).await)
217    }
218
219    /// Notifies the optional request observer without affecting request handling.
220    pub(crate) async fn observe_request(&self, context: S3RequestContext) {
221        if let Some(observer) = &self.request_observer {
222            observer.observe(context).await;
223        }
224    }
225
226    /// Applies the optional error mapper before an S3 error is serialized.
227    pub(crate) async fn map_error(
228        &self,
229        context: S3ErrorContext,
230        error: error::S3Error,
231    ) -> error::S3Error {
232        if let Some(mapper) = &self.error_mapper {
233            mapper.map_error(context, error).await
234        } else {
235            error
236        }
237    }
238
239    /// Applies the optional response mapper to a completed S3 response.
240    pub(crate) async fn map_response(
241        &self,
242        context: S3ResponseContext,
243        response: axum::response::Response,
244    ) -> axum::response::Response {
245        if let Some(mapper) = &self.response_mapper {
246            mapper.map_response(context, response).await
247        } else {
248            response
249        }
250    }
251
252    /// Checks whether an operation is enabled and allowed by the operation policy.
253    pub(crate) fn allow_operation(&self, context: OperationContext) -> Result<(), error::S3Error> {
254        if self.disabled_operations.contains(&context.action) {
255            return Err(error::S3Error::method_not_allowed());
256        }
257        if let Some(policy) = &self.operation_policy {
258            policy.allow_operation(context)
259        } else {
260            Ok(())
261        }
262    }
263
264    /// Checks whether a bucket/key target is allowed by the target policy.
265    pub(crate) fn allow_target(&self, context: TargetContext) -> Result<(), error::S3Error> {
266        if let Some(policy) = &self.target_policy {
267            policy.allow_target(context)
268        } else {
269            Ok(())
270        }
271    }
272
273    /// Checks whether an upload operation is allowed before body bytes are stored.
274    pub(crate) fn allow_upload(&self, context: UploadPolicyContext) -> Result<(), error::S3Error> {
275        if let Some(policy) = &self.upload_policy {
276            policy.allow_upload(context)
277        } else {
278            Ok(())
279        }
280    }
281
282    /// Begins a tenant-limited operation and returns its lease.
283    pub(crate) async fn begin_tenant_operation(
284        &self,
285        context: TenantLimitContext,
286    ) -> Result<TenantOperationLease, error::S3Error> {
287        self.tenant_limits_provider.begin_operation(context).await
288    }
289
290    /// Reports a tenant-limited operation outcome to the configured provider.
291    pub(crate) async fn finish_tenant_operation(
292        &self,
293        context: TenantLimitContext,
294        outcome: TenantOperationOutcome,
295    ) {
296        self.tenant_limits_provider
297            .finish_operation(context, outcome)
298            .await;
299    }
300
301    /// Checks whether the authenticated principal may access the requested target.
302    pub(crate) fn authorize_with_policy(
303        &self,
304        context: AuthorizationContext,
305    ) -> Result<(), error::S3Error> {
306        if let Some(policy) = &self.authorization_policy {
307            policy.authorize(context)
308        } else {
309            Ok(())
310        }
311    }
312
313    /// Acquires one global S3 request admission permit.
314    pub(crate) fn try_acquire_s3_request(&self) -> Result<OwnedSemaphorePermit, error::S3Error> {
315        self.admission
316            .s3_requests
317            .clone()
318            .try_acquire_owned()
319            .map_err(|_| {
320                error::S3Error::slow_down("too many active S3 requests; retry the request")
321            })
322    }
323
324    /// Acquires one single-object upload writer permit.
325    pub(crate) fn try_acquire_object_writer(&self) -> Result<OwnedSemaphorePermit, error::S3Error> {
326        self.admission
327            .object_writers
328            .clone()
329            .try_acquire_owned()
330            .map_err(|_| {
331                error::S3Error::slow_down("too many active object uploads; retry the request")
332            })
333    }
334
335    /// Acquires one multipart part writer permit.
336    pub(crate) fn try_acquire_multipart_part_writer(
337        &self,
338    ) -> Result<OwnedSemaphorePermit, error::S3Error> {
339        self.admission
340            .multipart_part_writers
341            .clone()
342            .try_acquire_owned()
343            .map_err(|_| {
344                error::S3Error::slow_down(
345                    "too many active multipart part uploads; retry the request",
346                )
347            })
348    }
349
350    /// Acquires an aws-chunked decoder permit when the request uses aws-chunked encoding.
351    ///
352    /// Returns `Ok(None)` for ordinary non-aws-chunked request bodies.
353    pub(crate) fn try_acquire_aws_chunked_decoder(
354        &self,
355        headers: &HeaderMap,
356    ) -> Result<Option<OwnedSemaphorePermit>, error::S3Error> {
357        if !body::upload::is_aws_chunked_request(headers) {
358            return Ok(None);
359        }
360        self.admission
361            .aws_chunked_decoders
362            .clone()
363            .try_acquire_owned()
364            .map(Some)
365            .map_err(|_| {
366                error::S3Error::slow_down("too many active aws-chunked uploads; retry the request")
367            })
368    }
369}
370
371/// Builder for [`AppState`].
372pub struct AppBuilder {
373    config: config::Config,
374    upload_processors: Vec<SharedUploadProcessor>,
375    object_store: Option<SharedObjectStore>,
376    multipart_store: Option<SharedMultipartStore>,
377    io_tuning: config::IoTuning,
378    multipart_lifecycle: config::MultipartLifecycle,
379    request_observer: Option<SharedRequestObserver>,
380    error_mapper: Option<SharedErrorMapper>,
381    response_mapper: Option<SharedResponseMapper>,
382    request_id_factory: SharedRequestIdFactory,
383    clock: SharedClock,
384    authentication_provider: Option<SharedAuthenticationProvider>,
385    authorization_policy: Option<SharedAuthorizationPolicy>,
386    operation_policy: Option<SharedOperationPolicy>,
387    target_policy: Option<SharedTargetPolicy>,
388    upload_policy: Option<SharedUploadPolicy>,
389    tenant_limits_provider: SharedTenantLimitsProvider,
390    disabled_operations: HashSet<config::S3Action>,
391    health_path: Option<String>,
392    router_layers: Vec<RouterTransform>,
393}
394
395type RouterTransform = Arc<dyn Fn(Router) -> Router + Send + Sync>;
396
397impl AppBuilder {
398    /// Replaces the default filesystem object store.
399    pub fn object_store<S>(mut self, store: S) -> Self
400    where
401        S: ObjectStore,
402    {
403        self.object_store = Some(Arc::new(store));
404        self
405    }
406
407    /// Replaces the default filesystem multipart store.
408    pub fn multipart_store<S>(mut self, store: S) -> Self
409    where
410        S: MultipartStore,
411    {
412        self.multipart_store = Some(Arc::new(store));
413        self
414    }
415
416    /// Replaces low-level I/O buffer tuning.
417    pub fn io_tuning(mut self, io_tuning: config::IoTuning) -> Self {
418        self.io_tuning = io_tuning;
419        self
420    }
421
422    /// Replaces multipart lifecycle and startup cleanup tuning.
423    pub fn multipart_lifecycle(mut self, lifecycle: config::MultipartLifecycle) -> Self {
424        self.multipart_lifecycle = lifecycle;
425        self
426    }
427
428    /// Registers an upload processor.
429    ///
430    /// Processors run sequentially in registration order after decoded upload
431    /// bytes have been validated and before the final object is committed.
432    pub fn upload_processor<P>(mut self, processor: P) -> Self
433    where
434        P: UploadProcessor,
435    {
436        self.upload_processors.push(Arc::new(processor));
437        self
438    }
439
440    /// Registers a closure-backed upload processor.
441    ///
442    /// The closure receives an owned [`UploadProcessorHandle`] so common async
443    /// closures do not need to name [`futures_util::future::BoxFuture`].
444    pub fn upload_processor_fn<F, Fut>(mut self, name: impl Into<String>, handler: F) -> Self
445    where
446        F: Fn(UploadProcessorHandle) -> Fut + Send + Sync + 'static,
447        Fut: Future<Output = Result<UploadProcessorAction, UploadProcessorError>> + Send + 'static,
448    {
449        self.upload_processors
450            .push(Arc::new(UploadProcessorFn::new(name, handler)));
451        self
452    }
453
454    /// Sets the health-check route path used by [`Self::build_router`].
455    pub fn health_path(mut self, path: impl Into<String>) -> Self {
456        self.health_path = Some(path.into());
457        self
458    }
459
460    /// Disables the health-check route used by [`Self::build_router`].
461    pub fn disable_health_route(mut self) -> Self {
462        self.health_path = None;
463        self
464    }
465
466    /// Applies a Tower layer to routers built by [`Self::build_router`].
467    pub fn router_layer<L>(mut self, layer: L) -> Self
468    where
469        L: Layer<Route> + Clone + Send + Sync + 'static,
470        L::Service: Service<Request<Body>> + Clone + Send + Sync + 'static,
471        <L::Service as Service<Request<Body>>>::Response: IntoResponse + 'static,
472        <L::Service as Service<Request<Body>>>::Error: Into<Infallible> + 'static,
473        <L::Service as Service<Request<Body>>>::Future: Send + 'static,
474    {
475        self.router_layers
476            .push(Arc::new(move |router| router.layer(layer.clone())));
477        self
478    }
479
480    /// Registers a read-only request observer.
481    pub fn on_request<O>(mut self, observer: O) -> Self
482    where
483        O: RequestObserver,
484    {
485        self.request_observer = Some(Arc::new(observer));
486        self
487    }
488
489    /// Registers an S3 error mapper.
490    pub fn on_error<M>(mut self, mapper: M) -> Self
491    where
492        M: ErrorMapper,
493    {
494        self.error_mapper = Some(Arc::new(mapper));
495        self
496    }
497
498    /// Registers an HTTP response mapper.
499    pub fn on_response<M>(mut self, mapper: M) -> Self
500    where
501        M: ResponseMapper,
502    {
503        self.response_mapper = Some(Arc::new(mapper));
504        self
505    }
506
507    /// Sets the request ID factory used for S3 responses and hooks.
508    pub fn request_id_factory<F>(mut self, factory: F) -> Self
509    where
510        F: RequestIdFactory,
511    {
512        self.request_id_factory = Arc::new(factory);
513        self
514    }
515
516    /// Sets the clock used for generated object and multipart timestamps.
517    pub fn clock<C>(mut self, clock: C) -> Self
518    where
519        C: Clock,
520    {
521        self.clock = Arc::new(clock);
522        self
523    }
524
525    /// Registers a custom authentication provider.
526    ///
527    /// When configured, this provider replaces the built-in static SigV4 and
528    /// anonymous authentication path. Built-in SigV4 remains the default when
529    /// no custom provider is registered. Target, upload, authorization, and
530    /// tenant-limit hooks still run after custom authentication succeeds.
531    pub fn authentication_provider<P>(mut self, provider: P) -> Self
532    where
533        P: AuthenticationProvider,
534    {
535        self.authentication_provider = Some(Arc::new(provider));
536        self
537    }
538
539    /// Registers an additional authorization policy.
540    pub fn authorization_policy<P>(mut self, policy: P) -> Self
541    where
542        P: AuthorizationPolicy,
543    {
544        self.authorization_policy = Some(Arc::new(policy));
545        self
546    }
547
548    /// Registers an additional operation policy.
549    pub fn operation_policy<P>(mut self, policy: P) -> Self
550    where
551        P: OperationPolicy,
552    {
553        self.operation_policy = Some(Arc::new(policy));
554        self
555    }
556
557    /// Disables one implemented S3 operation by action.
558    pub fn disable_operation(mut self, action: config::S3Action) -> Self {
559        self.disabled_operations.insert(action);
560        self
561    }
562
563    /// Registers an additional bucket/key target policy.
564    pub fn target_policy<P>(mut self, policy: P) -> Self
565    where
566        P: TargetPolicy,
567    {
568        self.target_policy = Some(Arc::new(policy));
569        self
570    }
571
572    /// Registers an additional upload policy.
573    pub fn upload_policy<P>(mut self, policy: P) -> Self
574    where
575        P: UploadPolicy,
576    {
577        self.upload_policy = Some(Arc::new(policy));
578        self
579    }
580
581    /// Registers a per-tenant operation limits provider.
582    pub fn tenant_limits_provider<P>(mut self, provider: P) -> Self
583    where
584        P: TenantLimitsProvider,
585    {
586        self.tenant_limits_provider = Arc::new(provider);
587        self
588    }
589
590    /// Builds application state.
591    pub async fn build(self) -> Result<AppState, AppInitError> {
592        AppState::from_builder(self).await
593    }
594
595    /// Builds application state and returns an Axum router.
596    pub async fn build_router(self) -> Result<Router, AppInitError> {
597        let health_path = self.health_path.clone();
598        let router_layers = self.router_layers.clone();
599        validate_health_path(health_path.as_deref())?;
600        let state = AppState::from_builder(self).await?;
601        let mut router = http::router_with_options(state, http::RouterOptions { health_path });
602        for layer in router_layers {
603            router = layer(router);
604        }
605        Ok(router)
606    }
607}
608
609fn validate_health_path(path: Option<&str>) -> Result<(), AppInitError> {
610    let Some(path) = path else {
611        return Ok(());
612    };
613    if path.starts_with('/') && path.len() > 1 && !path.contains('*') {
614        Ok(())
615    } else {
616        Err(AppInitError::RouterConfig(
617            "health path must start with '/' and must not contain wildcards".to_owned(),
618        ))
619    }
620}
621
622struct AdmissionControl {
623    s3_requests: Arc<Semaphore>,
624    object_writers: Arc<Semaphore>,
625    multipart_part_writers: Arc<Semaphore>,
626    aws_chunked_decoders: Arc<Semaphore>,
627}
628
629impl AdmissionControl {
630    fn new(limits: &config::ValidatedUploadLimits) -> Self {
631        Self {
632            s3_requests: Arc::new(Semaphore::new(limits.max_concurrent_s3_requests.get())),
633            object_writers: Arc::new(Semaphore::new(limits.max_active_object_writers.get())),
634            multipart_part_writers: Arc::new(Semaphore::new(
635                limits.max_active_multipart_part_writers.get(),
636            )),
637            aws_chunked_decoders: Arc::new(Semaphore::new(
638                limits.max_active_aws_chunked_decoders.get(),
639            )),
640        }
641    }
642}
643
644#[derive(Debug, Error)]
645/// Errors returned while initializing [`AppState`].
646pub enum AppInitError {
647    /// Configuration or credential validation failed.
648    #[error("invalid configuration")]
649    Config(#[from] config::ConfigError),
650
651    /// Storage initialization or cleanup failed.
652    #[error("failed to initialize storage")]
653    Store(#[from] storage::StoreError),
654
655    /// Router tuning configuration is invalid.
656    #[error("invalid router configuration: {0}")]
657    RouterConfig(String),
658}
659
660/// Builds the Axum router for the S3 endpoint.
661///
662/// The returned router contains the health route and S3 operation dispatcher.
663/// It does not bind sockets; callers choose how to serve it.
664pub fn router(state: AppState) -> Router {
665    http::router(state)
666}
667
668#[cfg(test)]
669mod tests {
670    use super::*;
671    use axum::{
672        body::{Body, Bytes, to_bytes},
673        http::{HeaderMap, HeaderValue, Request, StatusCode, header},
674    };
675    use futures_util::{future::BoxFuture, stream};
676    use std::{convert::Infallible, path::Path, sync::Mutex, time::Duration};
677    use tower::ServiceExt;
678
679    #[tokio::test]
680    async fn app_state_rejects_invalid_upload_limits() {
681        let temp_dir = tempfile::tempdir().expect("create temp dir");
682        let result = AppState::new(config::Config {
683            storage_root: temp_dir.path().to_path_buf(),
684            auth: Default::default(),
685            virtual_host_base_domain: None,
686            upload_limits: config::UploadLimits {
687                max_object_size: 1,
688                max_part_size: 0,
689                min_non_final_part_size: 1,
690                ..Default::default()
691            },
692        })
693        .await;
694
695        let Err(error) = result else {
696            panic!("invalid config should fail");
697        };
698
699        assert!(matches!(
700            error,
701            AppInitError::Config(config::ConfigError::InvalidUploadLimit(
702                "max_part_size must be greater than 0"
703            ))
704        ));
705    }
706
707    #[tokio::test]
708    async fn app_state_builder_registers_upload_processors_in_order() {
709        let temp_dir = tempfile::tempdir().expect("create temp dir");
710        let state = AppState::builder(config::Config {
711            storage_root: temp_dir.path().to_path_buf(),
712            auth: Default::default(),
713            virtual_host_base_domain: None,
714            upload_limits: Default::default(),
715        })
716        .upload_processor(NamedProcessor("first"))
717        .upload_processor(NamedProcessor("second"))
718        .build()
719        .await
720        .expect("create app state");
721
722        assert_eq!(state.upload_processors.len(), 2);
723    }
724
725    #[tokio::test]
726    async fn app_state_builder_registers_upload_processor_fn() {
727        let temp_dir = tempfile::tempdir().expect("create temp dir");
728        let state = AppState::builder(config::Config::new(temp_dir.path().to_path_buf()))
729            .upload_processor_fn("keep", |upload| async move { Ok(upload.keep()) })
730            .build()
731            .await
732            .expect("create app state");
733
734        assert_eq!(state.upload_processors.len(), 1);
735    }
736
737    #[tokio::test]
738    async fn build_router_serves_custom_health_path_and_s3_routes() {
739        let temp_dir = tempfile::tempdir().expect("create temp dir");
740        let app = AppState::builder(
741            config::Config::builder(temp_dir.path())
742                .allow_anonymous(true)
743                .build(),
744        )
745        .health_path("/ready")
746        .build_router()
747        .await
748        .expect("router");
749
750        let health = app
751            .clone()
752            .oneshot(
753                Request::builder()
754                    .method("GET")
755                    .uri("/ready")
756                    .body(Body::empty())
757                    .expect("request"),
758            )
759            .await
760            .expect("health response");
761        assert_eq!(health.status(), StatusCode::NO_CONTENT);
762
763        let head_bucket = app
764            .oneshot(
765                Request::builder()
766                    .method("HEAD")
767                    .uri("/test-bucket")
768                    .body(Body::empty())
769                    .expect("request"),
770            )
771            .await
772            .expect("head bucket response");
773        assert_eq!(head_bucket.status(), StatusCode::OK);
774    }
775
776    #[tokio::test]
777    async fn build_router_can_disable_health_route() {
778        let temp_dir = tempfile::tempdir().expect("create temp dir");
779        let app = AppState::builder(
780            config::Config::builder(temp_dir.path())
781                .allow_anonymous(true)
782                .build(),
783        )
784        .disable_health_route()
785        .build_router()
786        .await
787        .expect("router");
788
789        let response = app
790            .oneshot(
791                Request::builder()
792                    .method("GET")
793                    .uri("/health")
794                    .body(Body::empty())
795                    .expect("request"),
796            )
797            .await
798            .expect("response");
799
800        assert_ne!(response.status(), StatusCode::NO_CONTENT);
801    }
802
803    #[tokio::test]
804    async fn authorization_policy_can_deny_allowed_request() {
805        let temp_dir = tempfile::tempdir().expect("create temp dir");
806        let app = AppState::builder(
807            config::Config::builder(temp_dir.path())
808                .allow_anonymous(true)
809                .build(),
810        )
811        .authorization_policy(|context: hooks::AuthorizationContext| {
812            assert_eq!(context.bucket.as_str(), "test-bucket");
813            assert_eq!(
814                context.key.as_ref().map(|key| key.as_str()),
815                Some("blocked.txt")
816            );
817            Err(error::S3Error::access_denied("blocked by policy"))
818        })
819        .build_router()
820        .await
821        .expect("router");
822
823        let response = app
824            .oneshot(
825                Request::builder()
826                    .method("PUT")
827                    .uri("/test-bucket/blocked.txt")
828                    .header(header::CONTENT_LENGTH, "2")
829                    .body(Body::from("hi"))
830                    .expect("request"),
831            )
832            .await
833            .expect("response");
834
835        assert_eq!(response.status(), StatusCode::FORBIDDEN);
836    }
837
838    #[tokio::test]
839    async fn authentication_provider_can_replace_static_auth() {
840        let temp_dir = tempfile::tempdir().expect("create temp dir");
841        let app = AppState::builder(config::Config::new(temp_dir.path().to_path_buf()))
842            .authentication_provider(|request: hooks::AuthenticationRequest| async move {
843                if request
844                    .headers
845                    .get("x-api-key")
846                    .and_then(|value| value.to_str().ok())
847                    == Some("let-me-in")
848                {
849                    Ok(hooks::AuthenticationResult::custom("tenant-a"))
850                } else {
851                    Err(error::S3Error::access_denied("invalid api key"))
852                }
853            })
854            .build_router()
855            .await
856            .expect("router");
857
858        let allowed = app
859            .clone()
860            .oneshot(
861                Request::builder()
862                    .method("HEAD")
863                    .uri("/test-bucket")
864                    .header("x-api-key", "let-me-in")
865                    .body(Body::empty())
866                    .expect("request"),
867            )
868            .await
869            .expect("allowed response");
870        assert_eq!(allowed.status(), StatusCode::OK);
871
872        let denied = app
873            .oneshot(
874                Request::builder()
875                    .method("HEAD")
876                    .uri("/test-bucket")
877                    .body(Body::empty())
878                    .expect("request"),
879            )
880            .await
881            .expect("denied response");
882        assert_eq!(denied.status(), StatusCode::FORBIDDEN);
883    }
884
885    #[tokio::test]
886    async fn authorization_policy_receives_custom_auth_principal() {
887        let temp_dir = tempfile::tempdir().expect("create temp dir");
888        let app = AppState::builder(config::Config::new(temp_dir.path().to_path_buf()))
889            .authentication_provider(|_request: hooks::AuthenticationRequest| async {
890                Ok(hooks::AuthenticationResult::custom("tenant-a"))
891            })
892            .authorization_policy(|context: hooks::AuthorizationContext| {
893                assert_eq!(
894                    context.principal,
895                    hooks::RequestPrincipal::Custom {
896                        id: "tenant-a".to_owned()
897                    }
898                );
899                Ok(())
900            })
901            .build_router()
902            .await
903            .expect("router");
904
905        let response = app
906            .oneshot(
907                Request::builder()
908                    .method("HEAD")
909                    .uri("/test-bucket")
910                    .body(Body::empty())
911                    .expect("request"),
912            )
913            .await
914            .expect("response");
915
916        assert_eq!(response.status(), StatusCode::OK);
917    }
918
919    #[tokio::test]
920    async fn error_mapper_can_rewrite_error_and_preserve_request_id() {
921        let temp_dir = tempfile::tempdir().expect("create temp dir");
922        let app = AppState::builder(
923            config::Config::builder(temp_dir.path())
924                .allow_anonymous(true)
925                .build(),
926        )
927        .request_id_factory(|| s3::types::RequestId::parse("fixed-request-id").expect("request id"))
928        .on_error(|context: hooks::S3ErrorContext, _error| async move {
929            assert_eq!(context.request_id.as_str(), "fixed-request-id");
930            error::S3Error::access_denied("mapped error")
931        })
932        .build_router()
933        .await
934        .expect("router");
935
936        let response = app
937            .oneshot(
938                Request::builder()
939                    .method("GET")
940                    .uri("/test-bucket/missing.txt")
941                    .body(Body::empty())
942                    .expect("request"),
943            )
944            .await
945            .expect("response");
946
947        assert_eq!(response.status(), StatusCode::FORBIDDEN);
948        assert_eq!(
949            response
950                .headers()
951                .get("x-amz-request-id")
952                .and_then(|value| value.to_str().ok()),
953            Some("fixed-request-id")
954        );
955    }
956
957    #[tokio::test]
958    async fn request_observer_receives_parsed_request_context() {
959        let temp_dir = tempfile::tempdir().expect("create temp dir");
960        let observed = Arc::new(Mutex::new(Vec::new()));
961        let observed_clone = Arc::clone(&observed);
962        let app = AppState::builder(
963            config::Config::builder(temp_dir.path())
964                .allow_anonymous(true)
965                .build(),
966        )
967        .request_id_factory(|| s3::types::RequestId::parse("observed-request").expect("request id"))
968        .on_request(move |context: hooks::S3RequestContext| {
969            let observed = Arc::clone(&observed_clone);
970            async move {
971                observed.lock().expect("observed lock").push(context);
972            }
973        })
974        .build_router()
975        .await
976        .expect("router");
977
978        let response = app
979            .oneshot(
980                Request::builder()
981                    .method("HEAD")
982                    .uri("/test-bucket")
983                    .body(Body::empty())
984                    .expect("request"),
985            )
986            .await
987            .expect("response");
988        assert_eq!(response.status(), StatusCode::OK);
989
990        let observed = observed.lock().expect("observed lock");
991        assert_eq!(observed.len(), 1);
992        assert_eq!(observed[0].request_id.as_str(), "observed-request");
993        assert_eq!(observed[0].operation, "HeadBucket");
994        assert_eq!(
995            observed[0].bucket.as_ref().map(|bucket| bucket.as_str()),
996            Some("test-bucket")
997        );
998    }
999
1000    #[tokio::test]
1001    async fn upload_processor_fn_can_transform_stored_object() {
1002        let temp_dir = tempfile::tempdir().expect("create temp dir");
1003        let app = AppState::builder(
1004            config::Config::builder(temp_dir.path())
1005                .allow_anonymous(true)
1006                .build(),
1007        )
1008        .upload_processor_fn("uppercase", |upload| async move {
1009            let bytes = upload.read_current().await?;
1010            let upper = bytes
1011                .into_iter()
1012                .map(|byte| byte.to_ascii_uppercase())
1013                .collect::<Vec<_>>();
1014            upload.write_replacement(upper).await?;
1015            Ok(upload.replace())
1016        })
1017        .build_router()
1018        .await
1019        .expect("router");
1020
1021        let put = app
1022            .clone()
1023            .oneshot(
1024                Request::builder()
1025                    .method("PUT")
1026                    .uri("/test-bucket/greeting.txt")
1027                    .header(header::CONTENT_LENGTH, "5")
1028                    .body(Body::from("hello"))
1029                    .expect("request"),
1030            )
1031            .await
1032            .expect("put response");
1033        assert_eq!(put.status(), StatusCode::OK);
1034
1035        let get = app
1036            .oneshot(
1037                Request::builder()
1038                    .method("GET")
1039                    .uri("/test-bucket/greeting.txt")
1040                    .body(Body::empty())
1041                    .expect("request"),
1042            )
1043            .await
1044            .expect("get response");
1045        assert_eq!(get.status(), StatusCode::OK);
1046        let body = to_bytes(get.into_body(), 1024).await.expect("body");
1047        assert_eq!(&body[..], b"HELLO");
1048    }
1049
1050    #[tokio::test]
1051    async fn app_state_new_registers_no_upload_processors() {
1052        let temp_dir = tempfile::tempdir().expect("create temp dir");
1053        let state = AppState::new(config::Config {
1054            storage_root: temp_dir.path().to_path_buf(),
1055            auth: Default::default(),
1056            virtual_host_base_domain: None,
1057            upload_limits: Default::default(),
1058        })
1059        .await
1060        .expect("create app state");
1061
1062        assert!(state.upload_processors.is_empty());
1063    }
1064
1065    #[tokio::test]
1066    async fn app_builder_rejects_invalid_io_tuning() {
1067        let temp_dir = tempfile::tempdir().expect("create temp dir");
1068        let result = AppState::builder(config::Config::new(temp_dir.path().to_path_buf()))
1069            .io_tuning(
1070                config::IoTuning::builder()
1071                    .object_stream_buffer_size(0)
1072                    .build(),
1073            )
1074            .build()
1075            .await;
1076
1077        assert!(matches!(
1078            result,
1079            Err(AppInitError::Config(config::ConfigError::InvalidIoTuning(
1080                "object_stream_buffer_size must be greater than 0"
1081            )))
1082        ));
1083    }
1084
1085    #[tokio::test]
1086    async fn disabled_operation_rejects_before_handler_dispatch() {
1087        let temp_dir = tempfile::tempdir().expect("create temp dir");
1088        let app = AppState::builder(
1089            config::Config::builder(temp_dir.path())
1090                .allow_anonymous(true)
1091                .build(),
1092        )
1093        .disable_operation(config::S3Action::GetObject)
1094        .build_router()
1095        .await
1096        .expect("router");
1097
1098        let response = app
1099            .oneshot(
1100                Request::builder()
1101                    .method("GET")
1102                    .uri("/test-bucket/file.txt")
1103                    .body(Body::empty())
1104                    .expect("request"),
1105            )
1106            .await
1107            .expect("response");
1108
1109        assert_eq!(response.status(), StatusCode::METHOD_NOT_ALLOWED);
1110    }
1111
1112    #[tokio::test]
1113    async fn operation_policy_can_deny_specific_operations() {
1114        let temp_dir = tempfile::tempdir().expect("create temp dir");
1115        let app = AppState::builder(
1116            config::Config::builder(temp_dir.path())
1117                .allow_anonymous(true)
1118                .build(),
1119        )
1120        .operation_policy(|context: hooks::OperationContext| {
1121            assert_eq!(context.action, config::S3Action::HeadBucket);
1122            Err(error::S3Error::access_denied("operation blocked"))
1123        })
1124        .build_router()
1125        .await
1126        .expect("router");
1127
1128        let response = app
1129            .oneshot(
1130                Request::builder()
1131                    .method("HEAD")
1132                    .uri("/test-bucket")
1133                    .body(Body::empty())
1134                    .expect("request"),
1135            )
1136            .await
1137            .expect("response");
1138
1139        assert_eq!(response.status(), StatusCode::FORBIDDEN);
1140    }
1141
1142    #[tokio::test]
1143    async fn response_mapper_can_add_headers() {
1144        let temp_dir = tempfile::tempdir().expect("create temp dir");
1145        let app = AppState::builder(
1146            config::Config::builder(temp_dir.path())
1147                .allow_anonymous(true)
1148                .build(),
1149        )
1150        .on_response(
1151            |context: hooks::S3ResponseContext, mut response: axum::response::Response| async move {
1152                assert_eq!(context.operation, "HeadBucket");
1153                response
1154                    .headers_mut()
1155                    .insert("x-app-trace", HeaderValue::from_static("trace-1"));
1156                response
1157            },
1158        )
1159        .build_router()
1160        .await
1161        .expect("router");
1162
1163        let response = app
1164            .oneshot(
1165                Request::builder()
1166                    .method("HEAD")
1167                    .uri("/test-bucket")
1168                    .body(Body::empty())
1169                    .expect("request"),
1170            )
1171            .await
1172            .expect("response");
1173
1174        assert_eq!(response.status(), StatusCode::OK);
1175        assert_eq!(
1176            response
1177                .headers()
1178                .get("x-app-trace")
1179                .and_then(|value| value.to_str().ok()),
1180            Some("trace-1")
1181        );
1182    }
1183
1184    #[tokio::test]
1185    async fn target_policy_can_deny_key_prefixes() {
1186        let temp_dir = tempfile::tempdir().expect("create temp dir");
1187        let app = AppState::builder(
1188            config::Config::builder(temp_dir.path())
1189                .allow_anonymous(true)
1190                .build(),
1191        )
1192        .target_policy(|context: hooks::TargetContext| {
1193            if context
1194                .key
1195                .as_ref()
1196                .is_some_and(|key| key.as_str().starts_with("private/"))
1197            {
1198                return Err(error::S3Error::access_denied("private prefix"));
1199            }
1200            Ok(())
1201        })
1202        .build_router()
1203        .await
1204        .expect("router");
1205
1206        let response = app
1207            .oneshot(
1208                Request::builder()
1209                    .method("GET")
1210                    .uri("/test-bucket/private/file.txt")
1211                    .body(Body::empty())
1212                    .expect("request"),
1213            )
1214            .await
1215            .expect("response");
1216
1217        assert_eq!(response.status(), StatusCode::FORBIDDEN);
1218    }
1219
1220    #[tokio::test]
1221    async fn upload_policy_can_deny_uploads_before_body_is_stored() {
1222        let temp_dir = tempfile::tempdir().expect("create temp dir");
1223        let app = AppState::builder(
1224            config::Config::builder(temp_dir.path())
1225                .allow_anonymous(true)
1226                .build(),
1227        )
1228        .upload_policy(|context: hooks::UploadPolicyContext| {
1229            assert_eq!(context.action, config::S3Action::PutObject);
1230            Err(error::S3Error::access_denied("uploads disabled"))
1231        })
1232        .build_router()
1233        .await
1234        .expect("router");
1235
1236        let response = app
1237            .oneshot(
1238                Request::builder()
1239                    .method("PUT")
1240                    .uri("/test-bucket/file.txt")
1241                    .header(header::CONTENT_LENGTH, "4")
1242                    .body(Body::from("body"))
1243                    .expect("request"),
1244            )
1245            .await
1246            .expect("response");
1247
1248        assert_eq!(response.status(), StatusCode::FORBIDDEN);
1249    }
1250
1251    #[tokio::test]
1252    async fn clock_controls_object_last_modified_metadata() {
1253        let temp_dir = tempfile::tempdir().expect("create temp dir");
1254        let fixed_now = chrono::DateTime::parse_from_rfc3339("2026-06-17T12:34:56Z")
1255            .expect("timestamp")
1256            .with_timezone(&chrono::Utc);
1257        let state = AppState::builder(
1258            config::Config::builder(temp_dir.path())
1259                .allow_anonymous(true)
1260                .build(),
1261        )
1262        .clock(move || fixed_now)
1263        .build()
1264        .await
1265        .expect("state");
1266        let app = router(state.clone());
1267
1268        let response = app
1269            .oneshot(
1270                Request::builder()
1271                    .method("PUT")
1272                    .uri("/test-bucket/file.txt")
1273                    .header(header::CONTENT_LENGTH, "4")
1274                    .body(Body::from("body"))
1275                    .expect("request"),
1276            )
1277            .await
1278            .expect("response");
1279        assert_eq!(response.status(), StatusCode::OK);
1280
1281        let metadata = state
1282            .object_store
1283            .head_object(
1284                &s3::types::BucketName::parse("test-bucket").expect("bucket"),
1285                &s3::types::ObjectKey::parse("file.txt").expect("key"),
1286            )
1287            .await
1288            .expect("metadata read")
1289            .expect("metadata");
1290        assert_eq!(metadata.last_modified, fixed_now);
1291    }
1292
1293    #[tokio::test]
1294    async fn builder_accepts_custom_store_trait_objects() {
1295        let temp_dir = tempfile::tempdir().expect("create temp dir");
1296        let object_store = storage::FileObjectStore::new(temp_dir.path().to_path_buf())
1297            .await
1298            .expect("object store");
1299        let multipart_store = storage::FileMultipartStore::new(temp_dir.path().to_path_buf())
1300            .await
1301            .expect("multipart store");
1302
1303        let state = AppState::builder(config::Config::new(temp_dir.path().to_path_buf()))
1304            .object_store(object_store)
1305            .multipart_store(multipart_store)
1306            .build()
1307            .await
1308            .expect("state");
1309
1310        assert!(state.upload_processors.is_empty());
1311    }
1312
1313    #[tokio::test]
1314    async fn tenant_limits_provider_can_deny_before_handler_execution() {
1315        let temp_dir = tempfile::tempdir().expect("create temp dir");
1316        let app = AppState::builder(
1317            config::Config::builder(temp_dir.path())
1318                .allow_anonymous(true)
1319                .build(),
1320        )
1321        .tenant_limits_provider(DenyTenantLimitsProvider)
1322        .build_router()
1323        .await
1324        .expect("router");
1325
1326        let response = app
1327            .oneshot(
1328                Request::builder()
1329                    .method("HEAD")
1330                    .uri("/test-bucket")
1331                    .body(Body::empty())
1332                    .expect("request"),
1333            )
1334            .await
1335            .expect("response");
1336
1337        assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
1338    }
1339
1340    #[tokio::test]
1341    async fn tenant_limits_timeout_returns_slow_down_and_preserves_request_id() {
1342        let temp_dir = tempfile::tempdir().expect("create temp dir");
1343        let finishes = Arc::new(Mutex::new(Vec::new()));
1344        let app = AppState::builder(
1345            config::Config::builder(temp_dir.path())
1346                .allow_anonymous(true)
1347                .build(),
1348        )
1349        .request_id_factory(|| s3::types::RequestId::parse("timeout-request").expect("request id"))
1350        .tenant_limits_provider(RecordingTenantLimitsProvider {
1351            begins: Arc::new(Mutex::new(Vec::new())),
1352            finishes: Arc::clone(&finishes),
1353            timeout: Some(std::time::Duration::from_millis(1)),
1354            permit_drops: None,
1355        })
1356        .upload_processor_fn("slow", |_upload| async move {
1357            tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1358            Ok(middleware::UploadProcessorAction::Keep)
1359        })
1360        .build_router()
1361        .await
1362        .expect("router");
1363
1364        let response = app
1365            .oneshot(
1366                Request::builder()
1367                    .method("PUT")
1368                    .uri("/test-bucket/slow.txt")
1369                    .header(header::CONTENT_LENGTH, "4")
1370                    .body(Body::from("slow"))
1371                    .expect("request"),
1372            )
1373            .await
1374            .expect("response");
1375
1376        assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
1377        assert_eq!(
1378            response
1379                .headers()
1380                .get("x-amz-request-id")
1381                .and_then(|value| value.to_str().ok()),
1382            Some("timeout-request")
1383        );
1384        let finishes = finishes.lock().expect("finishes lock");
1385        assert_eq!(finishes.len(), 1);
1386        assert!(finishes[0].1.timed_out);
1387        assert_eq!(finishes[0].1.error_code.as_deref(), Some("SlowDown"));
1388    }
1389
1390    #[tokio::test]
1391    async fn custom_auth_with_invalid_principal_id_is_rejected() {
1392        let temp_dir = tempfile::tempdir().expect("create temp dir");
1393        let app = AppState::builder(config::Config::new(temp_dir.path().to_path_buf()))
1394            .request_id_factory(|| {
1395                s3::types::RequestId::parse("invalid-principal").expect("request id")
1396            })
1397            .authentication_provider(|_request: hooks::AuthenticationRequest| async {
1398                Ok(hooks::AuthenticationResult::custom("tenant\nid"))
1399            })
1400            .build_router()
1401            .await
1402            .expect("router");
1403
1404        let response = app
1405            .oneshot(
1406                Request::builder()
1407                    .method("HEAD")
1408                    .uri("/test-bucket")
1409                    .body(Body::empty())
1410                    .expect("request"),
1411            )
1412            .await
1413            .expect("response");
1414
1415        assert_eq!(response.status(), StatusCode::FORBIDDEN);
1416        assert_eq!(
1417            response
1418                .headers()
1419                .get("x-amz-request-id")
1420                .and_then(|value| value.to_str().ok()),
1421            Some("invalid-principal")
1422        );
1423    }
1424
1425    #[tokio::test]
1426    async fn custom_auth_success_still_runs_downstream_policies() {
1427        let temp_dir = tempfile::tempdir().expect("create temp dir");
1428        let app = AppState::builder(config::Config::new(temp_dir.path().to_path_buf()))
1429            .authentication_provider(|_request: hooks::AuthenticationRequest| async {
1430                hooks::AuthenticationResult::try_custom("tenant-a")
1431                    .map_err(|err| error::S3Error::access_denied(err.to_string()))
1432            })
1433            .authorization_policy(|_context: hooks::AuthorizationContext| {
1434                Err(error::S3Error::access_denied("policy denied"))
1435            })
1436            .build_router()
1437            .await
1438            .expect("router");
1439
1440        let response = app
1441            .oneshot(
1442                Request::builder()
1443                    .method("HEAD")
1444                    .uri("/test-bucket")
1445                    .body(Body::empty())
1446                    .expect("request"),
1447            )
1448            .await
1449            .expect("response");
1450
1451        assert_eq!(response.status(), StatusCode::FORBIDDEN);
1452    }
1453
1454    #[tokio::test]
1455    async fn timed_out_put_object_discards_temporary_object_file() {
1456        let temp_dir = tempfile::tempdir().expect("create temp dir");
1457        let app = AppState::builder(
1458            config::Config::builder(temp_dir.path())
1459                .allow_anonymous(true)
1460                .build(),
1461        )
1462        .tenant_limits_provider(RecordingTenantLimitsProvider {
1463            begins: Arc::new(Mutex::new(Vec::new())),
1464            finishes: Arc::new(Mutex::new(Vec::new())),
1465            timeout: Some(Duration::from_millis(1)),
1466            permit_drops: None,
1467        })
1468        .upload_processor_fn("slow", |_upload| async move {
1469            tokio::time::sleep(Duration::from_millis(50)).await;
1470            Ok(middleware::UploadProcessorAction::Keep)
1471        })
1472        .build_router()
1473        .await
1474        .expect("router");
1475
1476        let response = app
1477            .oneshot(
1478                Request::builder()
1479                    .method("PUT")
1480                    .uri("/test-bucket/slow.txt")
1481                    .header(header::CONTENT_LENGTH, "4")
1482                    .body(Body::from("slow"))
1483                    .expect("request"),
1484            )
1485            .await
1486            .expect("response");
1487
1488        assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
1489        assert_dir_empty_eventually(&temp_dir.path().join("tmp")).await;
1490    }
1491
1492    #[tokio::test]
1493    async fn timed_out_upload_part_discards_temporary_part_file() {
1494        let temp_dir = tempfile::tempdir().expect("create temp dir");
1495        let app = AppState::builder(
1496            config::Config::builder(temp_dir.path())
1497                .allow_anonymous(true)
1498                .build(),
1499        )
1500        .tenant_limits_provider(RecordingTenantLimitsProvider {
1501            begins: Arc::new(Mutex::new(Vec::new())),
1502            finishes: Arc::new(Mutex::new(Vec::new())),
1503            timeout: Some(Duration::from_millis(1)),
1504            permit_drops: None,
1505        })
1506        .build_router()
1507        .await
1508        .expect("router");
1509
1510        let create = app
1511            .clone()
1512            .oneshot(
1513                Request::builder()
1514                    .method("POST")
1515                    .uri("/test-bucket/parts.txt?uploads")
1516                    .body(Body::empty())
1517                    .expect("request"),
1518            )
1519            .await
1520            .expect("create response");
1521        assert_eq!(create.status(), StatusCode::OK);
1522        let body = to_bytes(create.into_body(), usize::MAX)
1523            .await
1524            .expect("create body");
1525        let upload_id = upload_id_from_xml(std::str::from_utf8(&body).expect("xml utf8"));
1526
1527        let delayed = stream::once(async {
1528            tokio::time::sleep(Duration::from_millis(50)).await;
1529            Ok::<_, Infallible>(Bytes::from_static(b"part"))
1530        });
1531        let response = app
1532            .oneshot(
1533                Request::builder()
1534                    .method("PUT")
1535                    .uri(format!(
1536                        "/test-bucket/parts.txt?partNumber=1&uploadId={}",
1537                        upload_id
1538                    ))
1539                    .header(header::CONTENT_LENGTH, "4")
1540                    .body(Body::from_stream(delayed))
1541                    .expect("request"),
1542            )
1543            .await
1544            .expect("response");
1545
1546        assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
1547        assert_no_multipart_tmp_files_eventually(temp_dir.path()).await;
1548    }
1549
1550    #[tokio::test]
1551    async fn tenant_limits_provider_permit_is_dropped_after_completion() {
1552        let temp_dir = tempfile::tempdir().expect("create temp dir");
1553        let drops = Arc::new(Mutex::new(0_usize));
1554        let app = AppState::builder(
1555            config::Config::builder(temp_dir.path())
1556                .allow_anonymous(true)
1557                .build(),
1558        )
1559        .tenant_limits_provider(RecordingTenantLimitsProvider {
1560            begins: Arc::new(Mutex::new(Vec::new())),
1561            finishes: Arc::new(Mutex::new(Vec::new())),
1562            timeout: None,
1563            permit_drops: Some(Arc::clone(&drops)),
1564        })
1565        .build_router()
1566        .await
1567        .expect("router");
1568
1569        let response = app
1570            .oneshot(
1571                Request::builder()
1572                    .method("HEAD")
1573                    .uri("/test-bucket")
1574                    .body(Body::empty())
1575                    .expect("request"),
1576            )
1577            .await
1578            .expect("response");
1579
1580        assert_eq!(response.status(), StatusCode::OK);
1581        assert_eq!(*drops.lock().expect("drop lock"), 1);
1582    }
1583
1584    #[tokio::test]
1585    async fn tenant_limits_finish_receives_operation_outcomes() {
1586        let temp_dir = tempfile::tempdir().expect("create temp dir");
1587        let begins = Arc::new(Mutex::new(Vec::new()));
1588        let finishes = Arc::new(Mutex::new(Vec::new()));
1589        let app = AppState::builder(
1590            config::Config::builder(temp_dir.path())
1591                .allow_anonymous(true)
1592                .build(),
1593        )
1594        .tenant_limits_provider(RecordingTenantLimitsProvider {
1595            begins: Arc::clone(&begins),
1596            finishes: Arc::clone(&finishes),
1597            timeout: None,
1598            permit_drops: None,
1599        })
1600        .build_router()
1601        .await
1602        .expect("router");
1603
1604        let head = app
1605            .clone()
1606            .oneshot(
1607                Request::builder()
1608                    .method("HEAD")
1609                    .uri("/test-bucket")
1610                    .body(Body::empty())
1611                    .expect("request"),
1612            )
1613            .await
1614            .expect("head response");
1615        assert_eq!(head.status(), StatusCode::OK);
1616
1617        let missing = app
1618            .clone()
1619            .oneshot(
1620                Request::builder()
1621                    .method("GET")
1622                    .uri("/test-bucket/missing.txt")
1623                    .body(Body::empty())
1624                    .expect("request"),
1625            )
1626            .await
1627            .expect("missing response");
1628        assert_eq!(missing.status(), StatusCode::NOT_FOUND);
1629
1630        let put = app
1631            .oneshot(
1632                Request::builder()
1633                    .method("PUT")
1634                    .uri("/test-bucket/body.txt")
1635                    .header(header::CONTENT_LENGTH, "5")
1636                    .body(Body::from("hello"))
1637                    .expect("request"),
1638            )
1639            .await
1640            .expect("put response");
1641        assert_eq!(put.status(), StatusCode::OK);
1642
1643        let begins = begins.lock().expect("begins lock");
1644        assert_eq!(begins.len(), 3);
1645        assert!(
1646            begins
1647                .iter()
1648                .all(|context| context.tenant.as_str() == "anonymous")
1649        );
1650        assert_eq!(begins[0].action, config::S3Action::HeadBucket);
1651        assert_eq!(begins[1].action, config::S3Action::GetObject);
1652        assert_eq!(begins[2].action, config::S3Action::PutObject);
1653
1654        let finishes = finishes.lock().expect("finishes lock");
1655        assert_eq!(finishes.len(), 3);
1656        assert_eq!(finishes[0].1.status, StatusCode::OK);
1657        assert_eq!(finishes[1].1.status, StatusCode::NOT_FOUND);
1658        assert_eq!(finishes[1].1.error_code.as_deref(), Some("NoSuchKey"));
1659        assert_eq!(finishes[2].1.status, StatusCode::OK);
1660        assert_eq!(finishes[2].1.decoded_bytes, Some(5));
1661    }
1662
1663    #[tokio::test]
1664    async fn tenant_limits_use_custom_auth_principal_and_authenticate_once() {
1665        let temp_dir = tempfile::tempdir().expect("create temp dir");
1666        let auth_calls = Arc::new(Mutex::new(0_usize));
1667        let begins = Arc::new(Mutex::new(Vec::new()));
1668        let auth_calls_clone = Arc::clone(&auth_calls);
1669        let app = AppState::builder(config::Config::new(temp_dir.path().to_path_buf()))
1670            .authentication_provider(move |_request: hooks::AuthenticationRequest| {
1671                let auth_calls = Arc::clone(&auth_calls_clone);
1672                async move {
1673                    *auth_calls.lock().expect("auth calls lock") += 1;
1674                    Ok(hooks::AuthenticationResult::custom("tenant-a"))
1675                }
1676            })
1677            .tenant_limits_provider(RecordingTenantLimitsProvider {
1678                begins: Arc::clone(&begins),
1679                finishes: Arc::new(Mutex::new(Vec::new())),
1680                timeout: None,
1681                permit_drops: None,
1682            })
1683            .build_router()
1684            .await
1685            .expect("router");
1686
1687        let response = app
1688            .oneshot(
1689                Request::builder()
1690                    .method("HEAD")
1691                    .uri("/test-bucket")
1692                    .body(Body::empty())
1693                    .expect("request"),
1694            )
1695            .await
1696            .expect("response");
1697
1698        assert_eq!(response.status(), StatusCode::OK);
1699        assert_eq!(*auth_calls.lock().expect("auth calls lock"), 1);
1700        let begins = begins.lock().expect("begins lock");
1701        assert_eq!(begins.len(), 1);
1702        assert_eq!(begins[0].tenant.as_str(), "tenant-a");
1703        assert_eq!(
1704            begins[0].principal,
1705            hooks::RequestPrincipal::Custom {
1706                id: "tenant-a".to_owned()
1707            }
1708        );
1709    }
1710
1711    struct DenyTenantLimitsProvider;
1712
1713    impl hooks::TenantLimitsProvider for DenyTenantLimitsProvider {
1714        fn begin_operation<'a>(
1715            &'a self,
1716            _context: hooks::TenantLimitContext,
1717        ) -> BoxFuture<'a, Result<hooks::TenantOperationLease, error::S3Error>> {
1718            Box::pin(async { Err(error::S3Error::slow_down("tenant quota exceeded")) })
1719        }
1720
1721        fn finish_operation<'a>(
1722            &'a self,
1723            _context: hooks::TenantLimitContext,
1724            _outcome: hooks::TenantOperationOutcome,
1725        ) -> BoxFuture<'a, ()> {
1726            Box::pin(async {})
1727        }
1728    }
1729
1730    struct RecordingTenantLimitsProvider {
1731        begins: Arc<Mutex<Vec<hooks::TenantLimitContext>>>,
1732        finishes: Arc<Mutex<Vec<(hooks::TenantLimitContext, hooks::TenantOperationOutcome)>>>,
1733        timeout: Option<std::time::Duration>,
1734        permit_drops: Option<Arc<Mutex<usize>>>,
1735    }
1736
1737    impl hooks::TenantLimitsProvider for RecordingTenantLimitsProvider {
1738        fn begin_operation<'a>(
1739            &'a self,
1740            context: hooks::TenantLimitContext,
1741        ) -> BoxFuture<'a, Result<hooks::TenantOperationLease, error::S3Error>> {
1742            let timeout = self.timeout;
1743            let permit_drops = self.permit_drops.clone();
1744            self.begins.lock().expect("begins lock").push(context);
1745            Box::pin(async move {
1746                let mut lease = hooks::TenantOperationLease::new();
1747                if let Some(timeout) = timeout {
1748                    lease = lease.timeout(timeout);
1749                }
1750                if let Some(permit_drops) = permit_drops {
1751                    lease = lease.permit(DropCounter(permit_drops));
1752                }
1753                Ok(lease)
1754            })
1755        }
1756
1757        fn finish_operation<'a>(
1758            &'a self,
1759            context: hooks::TenantLimitContext,
1760            outcome: hooks::TenantOperationOutcome,
1761        ) -> BoxFuture<'a, ()> {
1762            self.finishes
1763                .lock()
1764                .expect("finishes lock")
1765                .push((context, outcome));
1766            Box::pin(async {})
1767        }
1768    }
1769
1770    struct DropCounter(Arc<Mutex<usize>>);
1771
1772    impl Drop for DropCounter {
1773        fn drop(&mut self) {
1774            *self.0.lock().expect("drop counter lock") += 1;
1775        }
1776    }
1777
1778    async fn assert_dir_empty_eventually(path: &Path) {
1779        for _ in 0..20 {
1780            if dir_entries(path).is_empty() {
1781                return;
1782            }
1783            tokio::time::sleep(Duration::from_millis(10)).await;
1784        }
1785        panic!("directory still contains entries: {:?}", dir_entries(path));
1786    }
1787
1788    async fn assert_no_multipart_tmp_files_eventually(root: &Path) {
1789        for _ in 0..20 {
1790            let tmp_files = multipart_tmp_files(root);
1791            if tmp_files.is_empty() {
1792                return;
1793            }
1794            tokio::time::sleep(Duration::from_millis(10)).await;
1795        }
1796        panic!(
1797            "multipart storage still contains tmp files: {:?}",
1798            multipart_tmp_files(root)
1799        );
1800    }
1801
1802    fn dir_entries(path: &Path) -> Vec<std::path::PathBuf> {
1803        std::fs::read_dir(path)
1804            .map(|entries| {
1805                entries
1806                    .map(|entry| entry.expect("dir entry").path())
1807                    .collect::<Vec<_>>()
1808            })
1809            .unwrap_or_default()
1810    }
1811
1812    fn multipart_tmp_files(root: &Path) -> Vec<std::path::PathBuf> {
1813        let multipart = root.join("multipart");
1814        let mut tmp_files = Vec::new();
1815        for upload in dir_entries(&multipart) {
1816            for entry in dir_entries(&upload) {
1817                if entry
1818                    .file_name()
1819                    .and_then(|name| name.to_str())
1820                    .is_some_and(|name| name.ends_with(".tmp"))
1821                {
1822                    tmp_files.push(entry);
1823                }
1824            }
1825        }
1826        tmp_files
1827    }
1828
1829    fn upload_id_from_xml(xml: &str) -> String {
1830        let start = xml.find("<UploadId>").expect("upload id start") + "<UploadId>".len();
1831        let end = xml[start..].find("</UploadId>").expect("upload id end") + start;
1832        xml[start..end].to_owned()
1833    }
1834
1835    struct NamedProcessor(&'static str);
1836
1837    impl middleware::UploadProcessor for NamedProcessor {
1838        fn process<'a>(
1839            &'a self,
1840            _request: middleware::UploadProcessorRequest<'a>,
1841        ) -> BoxFuture<
1842            'a,
1843            Result<middleware::UploadProcessorAction, middleware::UploadProcessorError>,
1844        > {
1845            let _name = self.0;
1846            Box::pin(async { Ok(middleware::UploadProcessorAction::Keep) })
1847        }
1848    }
1849
1850    #[tokio::test]
1851    async fn app_state_rejects_invalid_auth_config() {
1852        let temp_dir = tempfile::tempdir().expect("create temp dir");
1853        let result = AppState::new(config::Config {
1854            storage_root: temp_dir.path().to_path_buf(),
1855            auth: config::AuthConfig {
1856                region: "US-EAST-1".to_owned(),
1857                ..Default::default()
1858            },
1859            virtual_host_base_domain: None,
1860            upload_limits: Default::default(),
1861        })
1862        .await;
1863
1864        let Err(error) = result else {
1865            panic!("invalid auth config should fail");
1866        };
1867
1868        assert!(matches!(
1869            error,
1870            AppInitError::Config(config::ConfigError::InvalidAuthConfig(
1871                "region must be a non-empty lowercase AWS region identifier"
1872            ))
1873        ));
1874    }
1875
1876    #[tokio::test]
1877    async fn s3_request_admission_rejects_when_limit_is_exhausted() {
1878        let temp_dir = tempfile::tempdir().expect("create temp dir");
1879        let state = AppState::new(config::Config::new(temp_dir.path().to_path_buf()))
1880            .await
1881            .expect("create app state");
1882        let permits = (0..state.config.upload_limits.max_concurrent_s3_requests)
1883            .map(|_| state.try_acquire_s3_request().expect("permit"))
1884            .collect::<Vec<_>>();
1885
1886        let error = state
1887            .try_acquire_s3_request()
1888            .expect_err("S3 request limit should reject");
1889        let response = error.into_response_with_request_id(&s3::types::RequestId::new());
1890        assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
1891
1892        drop(permits);
1893        let _permit = state
1894            .try_acquire_s3_request()
1895            .expect("permit released after drop");
1896    }
1897
1898    #[tokio::test]
1899    async fn router_returns_slow_down_when_s3_request_admission_is_exhausted() {
1900        let temp_dir = tempfile::tempdir().expect("create temp dir");
1901        let state = AppState::new(config::Config::new(temp_dir.path().to_path_buf()))
1902            .await
1903            .expect("create app state");
1904        let _permits = (0..state.config.upload_limits.max_concurrent_s3_requests)
1905            .map(|_| state.try_acquire_s3_request().expect("permit"))
1906            .collect::<Vec<_>>();
1907
1908        let response = router(state)
1909            .oneshot(
1910                Request::builder()
1911                    .method("PUT")
1912                    .uri("/test-bucket/admission.txt")
1913                    .body(Body::empty())
1914                    .expect("build request"),
1915            )
1916            .await
1917            .expect("send request");
1918
1919        assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
1920        let body = axum::body::to_bytes(response.into_body(), usize::MAX)
1921            .await
1922            .expect("read response body");
1923        let body = std::str::from_utf8(&body).expect("utf8 body");
1924        assert!(body.contains("<Code>SlowDown</Code>"));
1925        assert!(body.contains("too many active S3 requests"));
1926    }
1927
1928    #[tokio::test]
1929    async fn object_writer_admission_rejects_when_limit_is_exhausted() {
1930        let temp_dir = tempfile::tempdir().expect("create temp dir");
1931        let state = AppState::new(config::Config::new(temp_dir.path().to_path_buf()))
1932            .await
1933            .expect("create app state");
1934        let permits = (0..state.config.upload_limits.max_active_object_writers)
1935            .map(|_| state.try_acquire_object_writer().expect("permit"))
1936            .collect::<Vec<_>>();
1937
1938        let error = state
1939            .try_acquire_object_writer()
1940            .expect_err("object writer limit should reject");
1941        let response = error.into_response_with_request_id(&s3::types::RequestId::new());
1942        assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
1943
1944        drop(permits);
1945        let _permit = state
1946            .try_acquire_object_writer()
1947            .expect("permit released after drop");
1948    }
1949
1950    #[tokio::test]
1951    async fn admission_limits_are_configurable() {
1952        let temp_dir = tempfile::tempdir().expect("create temp dir");
1953        let state = AppState::new(config::Config {
1954            storage_root: temp_dir.path().to_path_buf(),
1955            auth: Default::default(),
1956            virtual_host_base_domain: None,
1957            upload_limits: config::UploadLimits {
1958                max_concurrent_s3_requests: 1,
1959                max_active_object_writers: 1,
1960                max_active_multipart_part_writers: 1,
1961                max_active_aws_chunked_decoders: 1,
1962                ..Default::default()
1963            },
1964        })
1965        .await
1966        .expect("create app state");
1967
1968        let _request = state.try_acquire_s3_request().expect("request permit");
1969        assert!(state.try_acquire_s3_request().is_err());
1970
1971        let _object = state.try_acquire_object_writer().expect("object permit");
1972        assert!(state.try_acquire_object_writer().is_err());
1973
1974        let _part = state
1975            .try_acquire_multipart_part_writer()
1976            .expect("part permit");
1977        assert!(state.try_acquire_multipart_part_writer().is_err());
1978
1979        let mut headers = HeaderMap::new();
1980        headers.insert("content-encoding", HeaderValue::from_static("aws-chunked"));
1981        let _decoder = state
1982            .try_acquire_aws_chunked_decoder(&headers)
1983            .expect("decoder permit")
1984            .expect("aws-chunked permit");
1985        assert!(state.try_acquire_aws_chunked_decoder(&headers).is_err());
1986    }
1987
1988    #[tokio::test]
1989    async fn multipart_part_writer_admission_rejects_when_limit_is_exhausted() {
1990        let temp_dir = tempfile::tempdir().expect("create temp dir");
1991        let state = AppState::new(config::Config::new(temp_dir.path().to_path_buf()))
1992            .await
1993            .expect("create app state");
1994        let permits = (0..state.config.upload_limits.max_active_multipart_part_writers)
1995            .map(|_| state.try_acquire_multipart_part_writer().expect("permit"))
1996            .collect::<Vec<_>>();
1997
1998        let error = state
1999            .try_acquire_multipart_part_writer()
2000            .expect_err("multipart part writer limit should reject");
2001        let response = error.into_response_with_request_id(&s3::types::RequestId::new());
2002        assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
2003
2004        drop(permits);
2005        let _permit = state
2006            .try_acquire_multipart_part_writer()
2007            .expect("permit released after drop");
2008    }
2009
2010    #[tokio::test]
2011    async fn aws_chunked_decoder_admission_only_applies_to_aws_chunked_requests() {
2012        let temp_dir = tempfile::tempdir().expect("create temp dir");
2013        let state = AppState::new(config::Config::new(temp_dir.path().to_path_buf()))
2014            .await
2015            .expect("create app state");
2016        let permits = (0..state.config.upload_limits.max_active_aws_chunked_decoders)
2017            .map(|_| {
2018                state
2019                    .admission
2020                    .aws_chunked_decoders
2021                    .clone()
2022                    .try_acquire_owned()
2023                    .expect("permit")
2024            })
2025            .collect::<Vec<_>>();
2026
2027        let plain_headers = HeaderMap::new();
2028        assert!(
2029            state
2030                .try_acquire_aws_chunked_decoder(&plain_headers)
2031                .expect("plain request")
2032                .is_none()
2033        );
2034
2035        let mut aws_chunked_headers = HeaderMap::new();
2036        aws_chunked_headers.insert("content-encoding", HeaderValue::from_static("aws-chunked"));
2037        let error = state
2038            .try_acquire_aws_chunked_decoder(&aws_chunked_headers)
2039            .expect_err("aws-chunked decoder limit should reject");
2040        let response = error.into_response_with_request_id(&s3::types::RequestId::new());
2041        assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
2042
2043        drop(permits);
2044        assert!(
2045            state
2046                .try_acquire_aws_chunked_decoder(&aws_chunked_headers)
2047                .expect("permit released after drop")
2048                .is_some()
2049        );
2050    }
2051}