1use crate::credentials::{CredentialType, Profile};
2use crate::error::{AppError, Result};
3use aws_config::Region;
4use aws_sdk_s3::config::{RequestChecksumCalculation, ResponseChecksumValidation};
5use aws_sdk_s3::Client;
6use serde::{Deserialize, Serialize};
7
8use std::collections::{HashMap, VecDeque};
9
10type SortedFolderCacheKey = (String, String, String, String, String);
11
12const MAX_SORTED_CACHE_ENTRIES: usize = 32;
13const MAX_SORTED_CACHE_ITEMS: usize = 100_000;
14
15pub(crate) fn normalize_endpoint_url(url: &str) -> String {
19 let trimmed = url.trim();
20 if trimmed.starts_with("http://") || trimmed.starts_with("https://") {
21 trimmed.to_string()
22 } else {
23 format!("https://{}", trimmed)
24 }
25}
26
27fn build_s3_config(sdk_config: &aws_config::SdkConfig, profile: &Profile) -> aws_sdk_s3::Config {
28 let mut builder = aws_sdk_s3::config::Builder::from(sdk_config);
29
30 if let CredentialType::CustomEndpoint { endpoint_url, .. } = &profile.credential_type {
31 builder = builder.endpoint_url(normalize_endpoint_url(endpoint_url));
32 }
33
34 if matches!(
39 profile.credential_type,
40 CredentialType::CustomEndpoint { .. }
41 ) || sdk_config.endpoint_url().is_some()
42 {
43 builder = builder
44 .force_path_style(true)
45 .response_checksum_validation(ResponseChecksumValidation::WhenRequired);
46 }
47
48 builder = builder.request_checksum_calculation(RequestChecksumCalculation::WhenRequired);
49
50 builder.build()
51}
52
53pub(crate) async fn load_sdk_config(
56 profile: &Profile,
57 override_region: Option<String>,
58) -> aws_config::SdkConfig {
59 let region_str = override_region
60 .or_else(|| profile.region.clone())
61 .unwrap_or_else(|| "us-east-1".to_string());
62 let region = Region::new(region_str);
63
64 match &profile.credential_type {
65 CredentialType::Environment => {
66 aws_config::defaults(aws_config::BehaviorVersion::latest())
67 .region(region)
68 .load()
69 .await
70 }
71 CredentialType::SharedConfig { profile_name } => {
72 aws_config::defaults(aws_config::BehaviorVersion::latest())
73 .region(region)
74 .profile_name(profile_name.as_deref().unwrap_or("default"))
75 .load()
76 .await
77 }
78 CredentialType::Manual {
79 access_key_id,
80 secret_access_key,
81 } => {
82 let creds = aws_credential_types::Credentials::new(
83 access_key_id,
84 secret_access_key,
85 None,
86 None,
87 "manual",
88 );
89 aws_config::defaults(aws_config::BehaviorVersion::latest())
90 .region(region)
91 .credentials_provider(creds)
92 .load()
93 .await
94 }
95 CredentialType::CustomEndpoint {
96 access_key_id,
97 secret_access_key,
98 ..
99 } => {
100 let creds = aws_credential_types::Credentials::new(
101 access_key_id,
102 secret_access_key,
103 None,
104 None,
105 "custom_endpoint",
106 );
107 aws_config::defaults(aws_config::BehaviorVersion::latest())
108 .region(region)
109 .credentials_provider(creds)
110 .load()
111 .await
112 }
113 }
114}
115
116pub(crate) fn client_from_sdk_config(
117 sdk_config: &aws_config::SdkConfig,
118 profile: &Profile,
119) -> Client {
120 Client::from_conf(build_s3_config(sdk_config, profile))
121}
122
123pub(crate) async fn build_s3_client(
124 profile: &Profile,
125 override_region: Option<String>,
126) -> Result<Client> {
127 let sdk_config = load_sdk_config(profile, override_region).await;
128 Ok(client_from_sdk_config(&sdk_config, profile))
129}
130
131#[derive(Debug, Clone, Serialize, Deserialize)]
132pub struct S3Object {
133 pub key: String,
134 pub last_modified: Option<String>,
135 pub size: i64,
136 pub storage_class: Option<String>,
137}
138
139#[derive(Debug, Clone, Serialize, Deserialize, Default)]
140pub struct FolderContent {
141 pub generation: String,
142 pub objects: Vec<S3Object>,
143 pub common_prefixes: Vec<String>,
144}
145
146pub struct S3ClientManager {
148 clients: HashMap<(String, String), Client>,
149 sorted_folder_cache: HashMap<SortedFolderCacheKey, FolderContent>,
150 sorted_folder_cache_order: VecDeque<SortedFolderCacheKey>,
151 sorted_folder_cache_items: usize,
152 bucket_regions: HashMap<(String, String), String>, }
154
155impl S3ClientManager {
156 pub fn new() -> Self {
157 Self {
158 clients: HashMap::new(),
159 sorted_folder_cache: HashMap::new(),
160 sorted_folder_cache_order: VecDeque::new(),
161 sorted_folder_cache_items: 0,
162 bucket_regions: HashMap::new(),
163 }
164 }
165
166 pub async fn get_client(&mut self, profile: &Profile) -> Result<&Client> {
168 let region = profile
169 .region
170 .clone()
171 .unwrap_or_else(|| "us-east-1".to_string());
172 self.get_client_for_region(profile, ®ion).await
173 }
174
175 pub async fn get_client_for_region(
177 &mut self,
178 profile: &Profile,
179 region: &str,
180 ) -> Result<&Client> {
181 let key = (profile.cache_identity(), region.to_string());
182
183 if !self.clients.contains_key(&key) {
184 let client = self.build_client(profile, Some(region.to_string())).await?;
185 self.clients.insert(key.clone(), client);
186 }
187
188 Ok(self.clients.get(&key).unwrap())
189 }
190
191 async fn build_client(
193 &self,
194 profile: &Profile,
195 override_region: Option<String>,
196 ) -> Result<Client> {
197 build_s3_client(profile, override_region).await
198 }
199
200 pub fn clear_cache(&mut self) {
202 self.clients.clear();
203 self.sorted_folder_cache.clear();
204 self.sorted_folder_cache_order.clear();
205 self.sorted_folder_cache_items = 0;
206 self.bucket_regions.clear();
207 }
208
209 pub fn get_bucket_region(&self, profile: &Profile, bucket_name: &str) -> Option<String> {
211 self.bucket_regions
212 .get(&(profile.cache_identity(), bucket_name.to_string()))
213 .cloned()
214 }
215
216 pub fn set_bucket_region(&mut self, profile: &Profile, bucket_name: &str, region: String) {
218 self.bucket_regions
219 .insert((profile.cache_identity(), bucket_name.to_string()), region);
220 }
221
222 pub fn set_bucket_regions<I>(&mut self, profile: &Profile, bucket_names: I, region: &str)
224 where
225 I: IntoIterator,
226 I::Item: AsRef<str>,
227 {
228 for bucket_name in bucket_names {
229 self.set_bucket_region(profile, bucket_name.as_ref(), region.to_string());
230 }
231 }
232
233 pub fn get_sorted_folder_content(
234 &self,
235 profile_id: &str,
236 bucket_name: &str,
237 prefix: &str,
238 sort_field: &str,
239 sort_direction: &str,
240 ) -> Option<&FolderContent> {
241 self.sorted_folder_cache.get(&(
242 profile_id.to_string(),
243 bucket_name.to_string(),
244 prefix.to_string(),
245 sort_field.to_string(),
246 sort_direction.to_string(),
247 ))
248 }
249
250 pub fn set_sorted_folder_content(
251 &mut self,
252 profile_id: &str,
253 bucket_name: &str,
254 prefix: &str,
255 sort_field: &str,
256 sort_direction: &str,
257 content: FolderContent,
258 ) {
259 let key = (
260 profile_id.to_string(),
261 bucket_name.to_string(),
262 prefix.to_string(),
263 sort_field.to_string(),
264 sort_direction.to_string(),
265 );
266 let item_count = content
267 .objects
268 .len()
269 .saturating_add(content.common_prefixes.len());
270
271 if let Some(previous) = self.sorted_folder_cache.remove(&key) {
272 self.sorted_folder_cache_items = self.sorted_folder_cache_items.saturating_sub(
273 previous
274 .objects
275 .len()
276 .saturating_add(previous.common_prefixes.len()),
277 );
278 self.sorted_folder_cache_order
279 .retain(|cached_key| cached_key != &key);
280 }
281
282 if item_count > MAX_SORTED_CACHE_ITEMS {
283 return;
284 }
285
286 while self.sorted_folder_cache.len() >= MAX_SORTED_CACHE_ENTRIES
287 || self.sorted_folder_cache_items.saturating_add(item_count) > MAX_SORTED_CACHE_ITEMS
288 {
289 let Some(oldest_key) = self.sorted_folder_cache_order.pop_front() else {
290 break;
291 };
292 if let Some(evicted) = self.sorted_folder_cache.remove(&oldest_key) {
293 self.sorted_folder_cache_items = self.sorted_folder_cache_items.saturating_sub(
294 evicted
295 .objects
296 .len()
297 .saturating_add(evicted.common_prefixes.len()),
298 );
299 }
300 }
301
302 self.sorted_folder_cache_items = self.sorted_folder_cache_items.saturating_add(item_count);
303 self.sorted_folder_cache_order.push_back(key.clone());
304 self.sorted_folder_cache.insert(key, content);
305 }
306
307 pub fn remove_bucket_cache(&mut self, profile_id: &str, bucket_name: &str) {
309 let pid = profile_id.to_string();
310 let revision_prefix = format!("{profile_id}:");
311 let bname = bucket_name.to_string();
312
313 self.sorted_folder_cache.retain(|(p, b, _, _, _), _| {
314 (p != &pid && !p.starts_with(&revision_prefix)) || b != &bname
315 });
316 self.sorted_folder_cache_order.retain(|(p, b, _, _, _)| {
317 (p != &pid && !p.starts_with(&revision_prefix)) || b != &bname
318 });
319 self.sorted_folder_cache_items = self
320 .sorted_folder_cache
321 .values()
322 .map(|content| {
323 content
324 .objects
325 .len()
326 .saturating_add(content.common_prefixes.len())
327 })
328 .sum();
329 }
330}
331
332impl Default for S3ClientManager {
333 fn default() -> Self {
334 Self::new()
335 }
336}
337
338#[derive(Debug, Clone, Serialize, Deserialize)]
340pub struct BucketInfo {
341 pub name: String,
342 pub region: Option<String>,
343 pub creation_date: Option<String>,
344 pub object_count: Option<u64>,
345 pub total_size: Option<u64>,
346 pub total_size_formatted: Option<String>,
347}
348
349pub async fn list_buckets(client: &Client) -> Result<Vec<BucketInfo>> {
351 let response = client
352 .list_buckets()
353 .send()
354 .await
355 .map_err(|e| AppError::S3Error(e.to_string()))?;
356
357 let buckets = response
358 .buckets()
359 .iter()
360 .map(|b| BucketInfo {
361 name: b.name().unwrap_or_default().to_string(),
362 region: None, creation_date: b.creation_date().map(|d| d.to_string()),
364 object_count: None,
365 total_size: None,
366 total_size_formatted: None,
367 })
368 .collect();
369
370 Ok(buckets)
371}
372
373pub async fn get_bucket_region(client: &Client, bucket_name: &str) -> Result<String> {
378 let result = tokio::time::timeout(
380 std::time::Duration::from_secs(10),
381 client.get_bucket_location().bucket(bucket_name).send(),
382 )
383 .await;
384
385 let response = match result {
386 Ok(Ok(resp)) => resp,
387 Ok(Err(e)) => {
388 return Err(AppError::S3Error(format!(
389 "GetBucketLocation failed for '{}': {}",
390 bucket_name, e
391 )));
392 }
393 Err(_) => {
394 return Err(AppError::S3Error(format!(
395 "GetBucketLocation timed out for '{}'",
396 bucket_name
397 )));
398 }
399 };
400
401 let region = response
403 .location_constraint()
404 .map(|l| l.as_str().to_string())
405 .unwrap_or_else(|| "us-east-1".to_string());
406
407 let region = if region.is_empty() {
408 "us-east-1".to_string()
409 } else {
410 region
411 };
412
413 Ok(region)
414}
415
416pub fn format_size(bytes: u64) -> String {
418 const KB: u64 = 1024;
419 const MB: u64 = KB * 1024;
420 const GB: u64 = MB * 1024;
421 const TB: u64 = GB * 1024;
422
423 if bytes >= TB {
424 format!("{:.2} TB", bytes as f64 / TB as f64)
425 } else if bytes >= GB {
426 format!("{:.2} GB", bytes as f64 / GB as f64)
427 } else if bytes >= MB {
428 format!("{:.2} MB", bytes as f64 / MB as f64)
429 } else if bytes >= KB {
430 format!("{:.2} KB", bytes as f64 / KB as f64)
431 } else {
432 format!("{} B", bytes)
433 }
434}
435
436#[cfg(test)]
437mod tests {
438 use super::{
439 build_s3_config, normalize_endpoint_url, FolderContent, S3ClientManager, S3Object,
440 MAX_SORTED_CACHE_ENTRIES,
441 };
442 use crate::credentials::{CredentialType, Profile};
443 use aws_config::{Region, SdkConfig};
444 use aws_sdk_s3::presigning::PresigningConfig;
445 use std::time::Duration;
446
447 #[test]
448 fn bucket_regions_are_isolated_by_profile_and_revision() {
449 let mut manager = S3ClientManager::new();
450 let a = Profile::new("A".into(), CredentialType::Environment, None);
451 let b = Profile::new("B".into(), CredentialType::Environment, None);
452 manager.set_bucket_region(&a, "shared", "region-a".into());
453 manager.set_bucket_region(&b, "shared", "region-b".into());
454 assert_eq!(
455 manager.get_bucket_region(&a, "shared").as_deref(),
456 Some("region-a")
457 );
458 assert_eq!(
459 manager.get_bucket_region(&b, "shared").as_deref(),
460 Some("region-b")
461 );
462 let mut edited = a.clone();
463 edited.updated_at = a.updated_at.map(|date| date + chrono::Duration::seconds(1));
464 assert!(manager.get_bucket_region(&edited, "shared").is_none());
465 }
466
467 #[test]
468 fn normalize_endpoint_url_preserves_existing_scheme() {
469 assert_eq!(
470 normalize_endpoint_url("https://us-east-1.linodeobjects.com"),
471 "https://us-east-1.linodeobjects.com"
472 );
473 assert_eq!(
474 normalize_endpoint_url("http://localhost:9000"),
475 "http://localhost:9000"
476 );
477 }
478
479 #[test]
480 fn normalize_endpoint_url_defaults_to_https() {
481 assert_eq!(
482 normalize_endpoint_url("us-east-1.linodeobjects.com"),
483 "https://us-east-1.linodeobjects.com"
484 );
485 }
486
487 #[test]
488 fn normalize_endpoint_url_trims_whitespace() {
489 assert_eq!(
490 normalize_endpoint_url(" us-east-1.linodeobjects.com "),
491 "https://us-east-1.linodeobjects.com"
492 );
493 }
494
495 #[tokio::test]
496 async fn shared_config_endpoint_uses_path_style_addressing() {
497 let sdk_config = SdkConfig::builder()
498 .behavior_version(aws_config::BehaviorVersion::latest())
499 .region(Region::new("us-east-1"))
500 .endpoint_url("http://localhost:9100")
501 .credentials_provider(
502 aws_credential_types::provider::SharedCredentialsProvider::new(
503 aws_credential_types::Credentials::new(
504 "access-key",
505 "secret-key",
506 None,
507 None,
508 "test",
509 ),
510 ),
511 )
512 .build();
513 let profile = Profile::new(
514 "MinIO shared profile".to_string(),
515 CredentialType::SharedConfig {
516 profile_name: Some("minio-local".to_string()),
517 },
518 Some("us-east-1".to_string()),
519 );
520 let client = aws_sdk_s3::Client::from_conf(build_s3_config(&sdk_config, &profile));
521
522 let request = client
523 .get_object()
524 .bucket("test-bucket")
525 .key("folder/file.txt")
526 .presigned(PresigningConfig::expires_in(Duration::from_secs(60)).unwrap())
527 .await
528 .unwrap();
529
530 assert!(
531 request
532 .uri()
533 .starts_with("http://localhost:9100/test-bucket/folder/file.txt?"),
534 "unexpected presigned URI: {}",
535 request.uri()
536 );
537 }
538
539 #[test]
540 fn aws_upload_compatibility_preserves_download_checksum_validation() {
541 let config = SdkConfig::builder()
542 .behavior_version(aws_config::BehaviorVersion::latest())
543 .region(Region::new("us-east-1"))
544 .response_checksum_validation(
545 aws_sdk_s3::config::ResponseChecksumValidation::WhenSupported,
546 )
547 .build();
548 let profile = Profile::new("AWS".into(), CredentialType::Environment, None);
549 let result = build_s3_config(&config, &profile);
550 assert_eq!(
551 result.request_checksum_calculation(),
552 Some(&aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired)
553 );
554 assert_eq!(
555 result.response_checksum_validation(),
556 Some(&aws_sdk_s3::config::ResponseChecksumValidation::WhenSupported)
557 );
558 }
559
560 #[test]
561 fn sorted_folder_cache_evicts_the_oldest_entry_at_its_limit() {
562 let mut manager = S3ClientManager::new();
563
564 for index in 0..=MAX_SORTED_CACHE_ENTRIES {
565 manager.set_sorted_folder_content(
566 "profile",
567 "bucket",
568 &format!("prefix-{index}/"),
569 "size",
570 "desc",
571 FolderContent {
572 objects: vec![S3Object {
573 key: format!("object-{index}"),
574 last_modified: None,
575 size: index as i64,
576 storage_class: None,
577 }],
578 common_prefixes: Vec::new(),
579 generation: index.to_string(),
580 },
581 );
582 }
583
584 assert!(manager
585 .get_sorted_folder_content("profile", "bucket", "prefix-0/", "size", "desc")
586 .is_none());
587 assert!(manager
588 .get_sorted_folder_content(
589 "profile",
590 "bucket",
591 &format!("prefix-{MAX_SORTED_CACHE_ENTRIES}/"),
592 "size",
593 "desc"
594 )
595 .is_some());
596 assert_eq!(manager.sorted_folder_cache.len(), MAX_SORTED_CACHE_ENTRIES);
597 }
598}