s3-endpoint 0.1.1

S3-compatible endpoint for upload and object-access workflows.
Documentation
#![forbid(unsafe_code)]

use std::{collections::BTreeSet, net::SocketAddr, path::PathBuf};

use clap::Parser;
use s3_endpoint::{
    AppState,
    config::{AccessKeyConfig, Config, S3Action, UploadLimits},
    router,
};
use serde::Deserialize;
use tokio::net::TcpListener;

#[derive(Parser)]
#[command(name = "s3-endpoint")]
struct Cli {
    #[arg(long, env = "S3_ENDPOINT_ADDR", default_value = "127.0.0.1:9000")]
    addr: SocketAddr,

    #[arg(long, env = "S3_ENDPOINT_STORAGE_ROOT", default_value = "./data")]
    storage_root: std::path::PathBuf,

    #[arg(long, env = "S3_ENDPOINT_ALLOW_ANONYMOUS", default_value_t = false)]
    allow_anonymous: bool,

    #[arg(long, env = "S3_ENDPOINT_ACCESS_KEY_ID", default_value = "test")]
    access_key_id: String,

    #[arg(long, env = "S3_ENDPOINT_SECRET_KEY", default_value = "testsecret")]
    secret_key: String,

    #[arg(long, env = "S3_ENDPOINT_SESSION_TOKEN")]
    session_token: Option<String>,

    #[arg(
        long,
        env = "S3_ENDPOINT_CREDENTIALS_FILE",
        help = "JSON file containing additional credential entries"
    )]
    credentials_file: Option<PathBuf>,

    #[arg(long, env = "S3_ENDPOINT_REGION", default_value = "us-east-1")]
    region: String,

    #[arg(long, env = "S3_ENDPOINT_MAX_SKEW_SECONDS", default_value_t = 900)]
    max_skew_seconds: i64,

    #[arg(
        long,
        env = "S3_ENDPOINT_ALLOWED_BUCKETS",
        value_delimiter = ',',
        help = "Comma-separated bucket allow-list for authenticated requests; empty permits all buckets"
    )]
    allowed_buckets: Vec<String>,

    #[arg(
        long,
        env = "S3_ENDPOINT_ALLOWED_ACTIONS",
        value_delimiter = ',',
        value_parser = parse_s3_action,
        help = "Comma-separated action allow-list such as s3:PutObject,upload-part; empty permits all actions"
    )]
    allowed_actions: Vec<S3Action>,

    #[arg(long, env = "S3_ENDPOINT_VIRTUAL_HOST_BASE_DOMAIN")]
    virtual_host_base_domain: Option<String>,

    #[arg(long, env = "S3_ENDPOINT_MAX_OBJECT_SIZE")]
    max_object_size: Option<u64>,

    #[arg(long, env = "S3_ENDPOINT_MAX_PART_SIZE")]
    max_part_size: Option<u64>,

    #[arg(long, env = "S3_ENDPOINT_MIN_NON_FINAL_PART_SIZE")]
    min_non_final_part_size: Option<u64>,

    #[arg(long, env = "S3_ENDPOINT_MAX_CONCURRENT_S3_REQUESTS")]
    max_concurrent_s3_requests: Option<usize>,

    #[arg(long, env = "S3_ENDPOINT_MAX_ACTIVE_OBJECT_WRITERS")]
    max_active_object_writers: Option<usize>,

    #[arg(long, env = "S3_ENDPOINT_MAX_ACTIVE_MULTIPART_PART_WRITERS")]
    max_active_multipart_part_writers: Option<usize>,

    #[arg(long, env = "S3_ENDPOINT_MAX_ACTIVE_AWS_CHUNKED_DECODERS")]
    max_active_aws_chunked_decoders: Option<usize>,
}

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
    let subscriber = tracing_subscriber::fmt()
        .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
        .finish();
    tracing::subscriber::set_global_default(subscriber)?;

    let cli = Cli::parse();
    let default_limits = UploadLimits::default();
    let credentials = load_credentials_file(cli.credentials_file.as_deref()).await?;
    let storage_root = cli.storage_root.clone();
    let region = cli.region.clone();
    let upload_limits = UploadLimits {
        max_object_size: cli
            .max_object_size
            .unwrap_or(default_limits.max_object_size),
        max_part_size: cli.max_part_size.unwrap_or(default_limits.max_part_size),
        min_non_final_part_size: cli
            .min_non_final_part_size
            .unwrap_or(default_limits.min_non_final_part_size),
        max_concurrent_s3_requests: cli
            .max_concurrent_s3_requests
            .unwrap_or(default_limits.max_concurrent_s3_requests),
        max_active_object_writers: cli
            .max_active_object_writers
            .unwrap_or(default_limits.max_active_object_writers),
        max_active_multipart_part_writers: cli
            .max_active_multipart_part_writers
            .unwrap_or(default_limits.max_active_multipart_part_writers),
        max_active_aws_chunked_decoders: cli
            .max_active_aws_chunked_decoders
            .unwrap_or(default_limits.max_active_aws_chunked_decoders),
    };
    log::info!(
        "starting s3-endpoint addr={} storage_root={} allow_anonymous={} region={} max_object_size={} max_part_size={} min_non_final_part_size={} max_concurrent_s3_requests={} max_active_object_writers={} max_active_multipart_part_writers={} max_active_aws_chunked_decoders={} extra_credentials={}",
        cli.addr,
        storage_root.display(),
        cli.allow_anonymous,
        region,
        upload_limits.max_object_size,
        upload_limits.max_part_size,
        upload_limits.min_non_final_part_size,
        upload_limits.max_concurrent_s3_requests,
        upload_limits.max_active_object_writers,
        upload_limits.max_active_multipart_part_writers,
        upload_limits.max_active_aws_chunked_decoders,
        credentials.len()
    );
    let state = AppState::new(Config {
        storage_root: cli.storage_root,
        auth: s3_endpoint::config::AuthConfig {
            allow_anonymous: cli.allow_anonymous,
            access_key_id: cli.access_key_id,
            secret_key: cli.secret_key,
            session_token: cli.session_token,
            region: cli.region,
            max_skew_seconds: cli.max_skew_seconds,
            allowed_buckets: cli.allowed_buckets.into_iter().collect(),
            allowed_actions: cli.allowed_actions.into_iter().collect(),
            credentials,
        },
        virtual_host_base_domain: cli.virtual_host_base_domain,
        upload_limits,
    })
    .await?;

    let listener = TcpListener::bind(cli.addr).await?;
    log::info!("s3-endpoint listening addr={}", cli.addr);
    tracing::info!(addr = %cli.addr, "listening");
    axum::serve(listener, router(state)).await?;
    Ok(())
}

fn parse_s3_action(value: &str) -> Result<S3Action, String> {
    value.parse().map_err(|err| format!("{err}"))
}

async fn load_credentials_file(
    path: Option<&std::path::Path>,
) -> Result<Vec<AccessKeyConfig>, Box<dyn std::error::Error>> {
    let Some(path) = path else {
        return Ok(Vec::new());
    };

    log::info!("loading additional credentials path={}", path.display());
    let json = match tokio::fs::read_to_string(path).await {
        Ok(json) => json,
        Err(error) => {
            log::warn!(
                "failed to read additional credentials path={} error={}",
                path.display(),
                error
            );
            return Err(error.into());
        }
    };
    let credentials = match parse_credentials_json(&json) {
        Ok(credentials) => credentials,
        Err(error) => {
            log::warn!(
                "failed to parse additional credentials path={} error={}",
                path.display(),
                error
            );
            return Err(error.into());
        }
    };
    log::info!(
        "loaded additional credentials path={} count={}",
        path.display(),
        credentials.len()
    );
    Ok(credentials)
}

fn parse_credentials_json(json: &str) -> Result<Vec<AccessKeyConfig>, String> {
    let entries = serde_json::from_str::<Vec<CredentialFileEntry>>(json)
        .map_err(|err| format!("failed to parse credentials file: {err}"))?;

    entries
        .into_iter()
        .map(CredentialFileEntry::try_into_config)
        .collect()
}

#[derive(Debug, Deserialize)]
struct CredentialFileEntry {
    access_key_id: String,
    secret_key: String,
    #[serde(default)]
    session_token: Option<String>,
    #[serde(default = "default_active")]
    active: bool,
    #[serde(default)]
    allowed_buckets: Vec<String>,
    #[serde(default)]
    allowed_actions: Vec<String>,
}

impl CredentialFileEntry {
    fn try_into_config(self) -> Result<AccessKeyConfig, String> {
        Ok(AccessKeyConfig {
            access_key_id: self.access_key_id,
            secret_key: self.secret_key,
            session_token: self.session_token,
            active: self.active,
            allowed_buckets: self.allowed_buckets.into_iter().collect(),
            allowed_actions: self
                .allowed_actions
                .into_iter()
                .map(|action| action.parse::<S3Action>().map_err(|err| format!("{err}")))
                .collect::<Result<BTreeSet<_>, _>>()?,
        })
    }
}

fn default_active() -> bool {
    true
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn credentials_json_parses_additional_credentials() {
        let credentials = parse_credentials_json(
            r#"[
              {
                "access_key_id": "client-a",
                "secret_key": "client-secret",
                "session_token": "client-session",
                "allowed_buckets": ["bucket-a", "bucket-b"],
                "allowed_actions": ["s3:PutObject", "upload-part"]
              }
            ]"#,
        )
        .expect("parse credentials");

        assert_eq!(credentials.len(), 1);
        let credential = &credentials[0];
        assert_eq!(credential.access_key_id, "client-a");
        assert_eq!(credential.secret_key, "client-secret");
        assert_eq!(credential.session_token.as_deref(), Some("client-session"));
        assert!(credential.active);
        assert!(credential.allowed_buckets.contains("bucket-a"));
        assert!(credential.allowed_buckets.contains("bucket-b"));
        assert!(credential.allowed_actions.contains(&S3Action::PutObject));
        assert!(credential.allowed_actions.contains(&S3Action::UploadPart));
    }

    #[test]
    fn credentials_json_allows_inactive_credentials() {
        let credentials = parse_credentials_json(
            r#"[
              {
                "access_key_id": "inactive-client",
                "secret_key": "inactive-secret",
                "active": false
              }
            ]"#,
        )
        .expect("parse credentials");

        assert_eq!(credentials.len(), 1);
        assert!(!credentials[0].active);
        assert!(credentials[0].allowed_buckets.is_empty());
        assert!(credentials[0].allowed_actions.is_empty());
    }

    #[test]
    fn credentials_json_rejects_unsupported_actions() {
        let error = parse_credentials_json(
            r#"[
              {
                "access_key_id": "bad-client",
                "secret_key": "bad-secret",
                "allowed_actions": ["s3:ListBucket"]
              }
            ]"#,
        )
        .expect_err("invalid action should fail");

        assert!(error.contains("allowed_actions must contain supported S3 action names"));
    }
}