icechunk_storage/
s3_config.rs

1//! Shared S3 configuration types used by both the native AWS SDK backend and the
2//! `object_store`-based S3 backend.
3//!
4//! These types are always compiled (no feature gate) so that [`ObjectStoreConfig::S3`],
5//! [`ObjectStoreConfig::S3Compatible`], [`ObjectStoreConfig::Tigris`], and
6//! [`Credentials::S3`] are available regardless of which S3 implementation is enabled.
7
8use core::fmt;
9use std::sync::Arc;
10
11use async_trait::async_trait;
12use chrono::{DateTime, Utc};
13use serde::{Deserialize, Serialize};
14
15/// Checksum algorithm to use for S3 write requests.
16///
17/// When unset, the AWS Rust SDK picks its own default for the `x-amz-checksum-*`
18/// header on `PutObject`, `UploadPart`, and `DeleteObjects`. Some S3-compatible
19/// services only accept a subset of algorithms; set this option to override the
20/// SDK's choice.
21#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)]
22#[serde(rename_all = "snake_case")]
23pub enum S3ChecksumAlgorithm {
24    Crc32,
25    Crc32c,
26    Crc64Nvme,
27    Sha1,
28    Sha256,
29}
30
31#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
32#[non_exhaustive]
33pub struct S3Options {
34    pub region: Option<String>,
35    pub endpoint_url: Option<String>,
36    pub anonymous: bool,
37    pub allow_http: bool,
38    // field was added in v0.2.6
39    #[serde(default = "default_force_path_style")]
40    pub force_path_style: bool,
41    pub network_stream_timeout_seconds: Option<u32>,
42    #[serde(default)]
43    pub requester_pays: bool,
44    #[serde(default)]
45    pub checksum_algorithm: Option<S3ChecksumAlgorithm>,
46}
47
48fn default_force_path_style() -> bool {
49    false
50}
51
52impl Default for S3Options {
53    fn default() -> Self {
54        Self {
55            region: None,
56            endpoint_url: None,
57            anonymous: false,
58            allow_http: false,
59            force_path_style: default_force_path_style(),
60            network_stream_timeout_seconds: None,
61            requester_pays: false,
62            checksum_algorithm: None,
63        }
64    }
65}
66
67impl S3Options {
68    pub fn with_region(mut self, region: impl Into<String>) -> Self {
69        self.region = Some(region.into());
70        self
71    }
72
73    pub fn with_endpoint_url(mut self, endpoint_url: impl Into<String>) -> Self {
74        self.endpoint_url = Some(endpoint_url.into());
75        self
76    }
77
78    pub fn with_anonymous(mut self, anonymous: bool) -> Self {
79        self.anonymous = anonymous;
80        self
81    }
82
83    pub fn with_allow_http(mut self, allow_http: bool) -> Self {
84        self.allow_http = allow_http;
85        self
86    }
87
88    pub fn with_force_path_style(mut self, force_path_style: bool) -> Self {
89        self.force_path_style = force_path_style;
90        self
91    }
92
93    pub fn with_network_stream_timeout_seconds(mut self, seconds: u32) -> Self {
94        self.network_stream_timeout_seconds = Some(seconds);
95        self
96    }
97
98    pub fn with_requester_pays(mut self, requester_pays: bool) -> Self {
99        self.requester_pays = requester_pays;
100        self
101    }
102
103    pub fn with_checksum_algorithm(mut self, algorithm: S3ChecksumAlgorithm) -> Self {
104        self.checksum_algorithm = Some(algorithm);
105        self
106    }
107
108    /// Return key-value pairs of non-default configuration for display purposes.
109    pub fn info_fields(&self) -> Vec<(&'static str, String)> {
110        let mut fields = Vec::new();
111        if let Some(region) = &self.region {
112            fields.push(("region", region.clone()));
113        }
114        if let Some(endpoint) = &self.endpoint_url {
115            fields.push(("endpoint_url", endpoint.clone()));
116        }
117        if self.anonymous {
118            fields.push(("anonymous", "True".to_string()));
119        }
120        if self.allow_http {
121            fields.push(("allow_http", "True".to_string()));
122        }
123        if self.force_path_style {
124            fields.push(("force_path_style", "True".to_string()));
125        }
126        if self.requester_pays {
127            fields.push(("requester_pays", "True".to_string()));
128        }
129        if let Some(algo) = self.checksum_algorithm {
130            fields.push(("checksum_algorithm", format!("{algo:?}")));
131        }
132        fields
133    }
134}
135
136impl fmt::Display for S3ChecksumAlgorithm {
137    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
138        let s = match self {
139            S3ChecksumAlgorithm::Crc32 => "crc32",
140            S3ChecksumAlgorithm::Crc32c => "crc32c",
141            S3ChecksumAlgorithm::Crc64Nvme => "crc64nvme",
142            S3ChecksumAlgorithm::Sha1 => "sha1",
143            S3ChecksumAlgorithm::Sha256 => "sha256",
144        };
145        f.write_str(s)
146    }
147}
148
149impl fmt::Display for S3Options {
150    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
151        write!(
152            f,
153            "S3Options(region={}, endpoint_url={}, anonymous={}, allow_http={}, force_path_style={}, network_stream_timeout_seconds={}, requester_pays={}, checksum_algorithm={})",
154            self.region.as_deref().unwrap_or("None"),
155            self.endpoint_url.as_deref().unwrap_or("None"),
156            self.anonymous,
157            self.allow_http,
158            self.force_path_style,
159            self.network_stream_timeout_seconds
160                .map(|n| n.to_string())
161                .unwrap_or("None".to_string()),
162            self.requester_pays,
163            self.checksum_algorithm.map(|a| a.to_string()).unwrap_or("None".to_string()),
164        )
165    }
166}
167
168#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
169pub struct S3StaticCredentials {
170    pub access_key_id: String,
171    pub secret_access_key: String,
172    pub session_token: Option<String>,
173    pub expires_after: Option<DateTime<Utc>>,
174}
175
176#[async_trait]
177#[typetag::serde(tag = "s3_credentials_fetcher_type")]
178pub trait S3CredentialsFetcher: fmt::Debug + Sync + Send {
179    async fn get(&self) -> Result<S3StaticCredentials, String>;
180}
181
182/// S3 authentication credentials.
183#[derive(Clone, Debug, Deserialize, Serialize, Default)]
184#[serde(tag = "s3_credential_type")]
185#[serde(rename_all = "snake_case")]
186pub enum S3Credentials {
187    #[default]
188    FromEnv,
189    Anonymous,
190    Static(S3StaticCredentials),
191    Refreshable(Arc<dyn S3CredentialsFetcher>),
192}
icechunk_storage/s3_config.rs

icechunk_storage/
s3_config.rs
