Skip to main content

brows3_lib/s3/
client.rs

1use crate::credentials::{CredentialType, Profile};
2use crate::error::{AppError, Result};
3use aws_config::Region;
4use aws_sdk_s3::config::{RequestChecksumCalculation, ResponseChecksumValidation};
5use aws_sdk_s3::Client;
6use serde::{Deserialize, Serialize};
7
8use std::collections::{HashMap, VecDeque};
9
10type SortedFolderCacheKey = (String, String, String, String, String);
11
12const MAX_SORTED_CACHE_ENTRIES: usize = 32;
13const MAX_SORTED_CACHE_ITEMS: usize = 100_000;
14
15/// Normalize an endpoint URL to ensure it has a scheme.
16/// Many S3-compatible providers (Linode, DigitalOcean, etc.) may be configured
17/// without a scheme, causing the AWS SDK to fail with "dispatch failure".
18pub(crate) fn normalize_endpoint_url(url: &str) -> String {
19    let trimmed = url.trim();
20    if trimmed.starts_with("http://") || trimmed.starts_with("https://") {
21        trimmed.to_string()
22    } else {
23        format!("https://{}", trimmed)
24    }
25}
26
27fn build_s3_config(sdk_config: &aws_config::SdkConfig, profile: &Profile) -> aws_sdk_s3::Config {
28    let mut builder = aws_sdk_s3::config::Builder::from(sdk_config);
29
30    if let CredentialType::CustomEndpoint { endpoint_url, .. } = &profile.credential_type {
31        builder = builder.endpoint_url(normalize_endpoint_url(endpoint_url));
32    }
33
34    // A global endpoint_url loaded from an environment variable or shared
35    // profile is a custom S3 endpoint just as much as one entered in the UI.
36    // Path-style addressing is required by common providers such as MinIO
37    // unless they have explicitly configured wildcard bucket DNS.
38    if matches!(
39        profile.credential_type,
40        CredentialType::CustomEndpoint { .. }
41    ) || sdk_config.endpoint_url().is_some()
42    {
43        builder = builder
44            .force_path_style(true)
45            .response_checksum_validation(ResponseChecksumValidation::WhenRequired);
46    }
47
48    builder = builder.request_checksum_calculation(RequestChecksumCalculation::WhenRequired);
49
50    builder.build()
51}
52
53/// Build an S3 client for both production operations and connection tests so
54/// profile, endpoint, and compatibility behavior cannot drift between them.
55pub(crate) async fn load_sdk_config(
56    profile: &Profile,
57    override_region: Option<String>,
58) -> aws_config::SdkConfig {
59    let region_str = override_region
60        .or_else(|| profile.region.clone())
61        .unwrap_or_else(|| "us-east-1".to_string());
62    let region = Region::new(region_str);
63
64    match &profile.credential_type {
65        CredentialType::Environment => {
66            aws_config::defaults(aws_config::BehaviorVersion::latest())
67                .region(region)
68                .load()
69                .await
70        }
71        CredentialType::SharedConfig { profile_name } => {
72            aws_config::defaults(aws_config::BehaviorVersion::latest())
73                .region(region)
74                .profile_name(profile_name.as_deref().unwrap_or("default"))
75                .load()
76                .await
77        }
78        CredentialType::Manual {
79            access_key_id,
80            secret_access_key,
81        } => {
82            let creds = aws_credential_types::Credentials::new(
83                access_key_id,
84                secret_access_key,
85                None,
86                None,
87                "manual",
88            );
89            aws_config::defaults(aws_config::BehaviorVersion::latest())
90                .region(region)
91                .credentials_provider(creds)
92                .load()
93                .await
94        }
95        CredentialType::CustomEndpoint {
96            access_key_id,
97            secret_access_key,
98            ..
99        } => {
100            let creds = aws_credential_types::Credentials::new(
101                access_key_id,
102                secret_access_key,
103                None,
104                None,
105                "custom_endpoint",
106            );
107            aws_config::defaults(aws_config::BehaviorVersion::latest())
108                .region(region)
109                .credentials_provider(creds)
110                .load()
111                .await
112        }
113    }
114}
115
116pub(crate) fn client_from_sdk_config(
117    sdk_config: &aws_config::SdkConfig,
118    profile: &Profile,
119) -> Client {
120    Client::from_conf(build_s3_config(sdk_config, profile))
121}
122
123pub(crate) async fn build_s3_client(
124    profile: &Profile,
125    override_region: Option<String>,
126) -> Result<Client> {
127    let sdk_config = load_sdk_config(profile, override_region).await;
128    Ok(client_from_sdk_config(&sdk_config, profile))
129}
130
131#[derive(Debug, Clone, Serialize, Deserialize)]
132pub struct S3Object {
133    pub key: String,
134    pub last_modified: Option<String>,
135    pub size: i64,
136    pub storage_class: Option<String>,
137}
138
139#[derive(Debug, Clone, Serialize, Deserialize, Default)]
140pub struct FolderContent {
141    pub generation: String,
142    pub objects: Vec<S3Object>,
143    pub common_prefixes: Vec<String>,
144}
145
146/// S3 Client Manager - creates and caches S3 clients per profile and region
147pub struct S3ClientManager {
148    clients: HashMap<(String, String), Client>,
149    sorted_folder_cache: HashMap<SortedFolderCacheKey, FolderContent>,
150    sorted_folder_cache_order: VecDeque<SortedFolderCacheKey>,
151    sorted_folder_cache_items: usize,
152    bucket_regions: HashMap<(String, String), String>, // profile revision + bucket
153}
154
155impl S3ClientManager {
156    pub fn new() -> Self {
157        Self {
158            clients: HashMap::new(),
159            sorted_folder_cache: HashMap::new(),
160            sorted_folder_cache_order: VecDeque::new(),
161            sorted_folder_cache_items: 0,
162            bucket_regions: HashMap::new(),
163        }
164    }
165
166    /// Get or create an S3 client for the given profile's default region
167    pub async fn get_client(&mut self, profile: &Profile) -> Result<&Client> {
168        let region = profile
169            .region
170            .clone()
171            .unwrap_or_else(|| "us-east-1".to_string());
172        self.get_client_for_region(profile, &region).await
173    }
174
175    /// Get or create an S3 client for the given profile and specific region
176    pub async fn get_client_for_region(
177        &mut self,
178        profile: &Profile,
179        region: &str,
180    ) -> Result<&Client> {
181        let key = (profile.cache_identity(), region.to_string());
182
183        if !self.clients.contains_key(&key) {
184            let client = self.build_client(profile, Some(region.to_string())).await?;
185            self.clients.insert(key.clone(), client);
186        }
187
188        Ok(self.clients.get(&key).unwrap())
189    }
190
191    /// Build a new S3 client for the given profile
192    async fn build_client(
193        &self,
194        profile: &Profile,
195        override_region: Option<String>,
196    ) -> Result<Client> {
197        build_s3_client(profile, override_region).await
198    }
199
200    /// Clear cached clients, sorted folder results, and discovered regions.
201    pub fn clear_cache(&mut self) {
202        self.clients.clear();
203        self.sorted_folder_cache.clear();
204        self.sorted_folder_cache_order.clear();
205        self.sorted_folder_cache_items = 0;
206        self.bucket_regions.clear();
207    }
208
209    /// Get cached region for a bucket
210    pub fn get_bucket_region(&self, profile: &Profile, bucket_name: &str) -> Option<String> {
211        self.bucket_regions
212            .get(&(profile.cache_identity(), bucket_name.to_string()))
213            .cloned()
214    }
215
216    /// Cache the region for a bucket
217    pub fn set_bucket_region(&mut self, profile: &Profile, bucket_name: &str, region: String) {
218        self.bucket_regions
219            .insert((profile.cache_identity(), bucket_name.to_string()), region);
220    }
221
222    /// Cache the same region for a set of buckets.
223    pub fn set_bucket_regions<I>(&mut self, profile: &Profile, bucket_names: I, region: &str)
224    where
225        I: IntoIterator,
226        I::Item: AsRef<str>,
227    {
228        for bucket_name in bucket_names {
229            self.set_bucket_region(profile, bucket_name.as_ref(), region.to_string());
230        }
231    }
232
233    pub fn get_sorted_folder_content(
234        &self,
235        profile_id: &str,
236        bucket_name: &str,
237        prefix: &str,
238        sort_field: &str,
239        sort_direction: &str,
240    ) -> Option<&FolderContent> {
241        self.sorted_folder_cache.get(&(
242            profile_id.to_string(),
243            bucket_name.to_string(),
244            prefix.to_string(),
245            sort_field.to_string(),
246            sort_direction.to_string(),
247        ))
248    }
249
250    pub fn set_sorted_folder_content(
251        &mut self,
252        profile_id: &str,
253        bucket_name: &str,
254        prefix: &str,
255        sort_field: &str,
256        sort_direction: &str,
257        content: FolderContent,
258    ) {
259        let key = (
260            profile_id.to_string(),
261            bucket_name.to_string(),
262            prefix.to_string(),
263            sort_field.to_string(),
264            sort_direction.to_string(),
265        );
266        let item_count = content
267            .objects
268            .len()
269            .saturating_add(content.common_prefixes.len());
270
271        if let Some(previous) = self.sorted_folder_cache.remove(&key) {
272            self.sorted_folder_cache_items = self.sorted_folder_cache_items.saturating_sub(
273                previous
274                    .objects
275                    .len()
276                    .saturating_add(previous.common_prefixes.len()),
277            );
278            self.sorted_folder_cache_order
279                .retain(|cached_key| cached_key != &key);
280        }
281
282        if item_count > MAX_SORTED_CACHE_ITEMS {
283            return;
284        }
285
286        while self.sorted_folder_cache.len() >= MAX_SORTED_CACHE_ENTRIES
287            || self.sorted_folder_cache_items.saturating_add(item_count) > MAX_SORTED_CACHE_ITEMS
288        {
289            let Some(oldest_key) = self.sorted_folder_cache_order.pop_front() else {
290                break;
291            };
292            if let Some(evicted) = self.sorted_folder_cache.remove(&oldest_key) {
293                self.sorted_folder_cache_items = self.sorted_folder_cache_items.saturating_sub(
294                    evicted
295                        .objects
296                        .len()
297                        .saturating_add(evicted.common_prefixes.len()),
298                );
299            }
300        }
301
302        self.sorted_folder_cache_items = self.sorted_folder_cache_items.saturating_add(item_count);
303        self.sorted_folder_cache_order.push_back(key.clone());
304        self.sorted_folder_cache.insert(key, content);
305    }
306
307    /// Remove cached sorted results for a specific profile and bucket.
308    pub fn remove_bucket_cache(&mut self, profile_id: &str, bucket_name: &str) {
309        let pid = profile_id.to_string();
310        let revision_prefix = format!("{profile_id}:");
311        let bname = bucket_name.to_string();
312
313        self.sorted_folder_cache.retain(|(p, b, _, _, _), _| {
314            (p != &pid && !p.starts_with(&revision_prefix)) || b != &bname
315        });
316        self.sorted_folder_cache_order.retain(|(p, b, _, _, _)| {
317            (p != &pid && !p.starts_with(&revision_prefix)) || b != &bname
318        });
319        self.sorted_folder_cache_items = self
320            .sorted_folder_cache
321            .values()
322            .map(|content| {
323                content
324                    .objects
325                    .len()
326                    .saturating_add(content.common_prefixes.len())
327            })
328            .sum();
329    }
330}
331
332impl Default for S3ClientManager {
333    fn default() -> Self {
334        Self::new()
335    }
336}
337
338/// Bucket information returned from S3
339#[derive(Debug, Clone, Serialize, Deserialize)]
340pub struct BucketInfo {
341    pub name: String,
342    pub region: Option<String>,
343    pub creation_date: Option<String>,
344    pub object_count: Option<u64>,
345    pub total_size: Option<u64>,
346    pub total_size_formatted: Option<String>,
347}
348
349/// List all buckets accessible by the current credentials
350pub async fn list_buckets(client: &Client) -> Result<Vec<BucketInfo>> {
351    let response = client
352        .list_buckets()
353        .send()
354        .await
355        .map_err(|e| AppError::S3Error(e.to_string()))?;
356
357    let buckets = response
358        .buckets()
359        .iter()
360        .map(|b| BucketInfo {
361            name: b.name().unwrap_or_default().to_string(),
362            region: None, // Will be fetched separately if needed
363            creation_date: b.creation_date().map(|d| d.to_string()),
364            object_count: None,
365            total_size: None,
366            total_size_formatted: None,
367        })
368        .collect();
369
370    Ok(buckets)
371}
372
373/// Get the region for a specific bucket.
374/// Note: GetBucketLocation is not supported by all S3-compatible providers
375/// (e.g., Linode Object Storage, DigitalOcean Spaces). Callers should handle
376/// errors gracefully and fall back to the profile's configured region.
377pub async fn get_bucket_region(client: &Client, bucket_name: &str) -> Result<String> {
378    // Use a timeout to prevent hanging on providers that don't support this API
379    let result = tokio::time::timeout(
380        std::time::Duration::from_secs(10),
381        client.get_bucket_location().bucket(bucket_name).send(),
382    )
383    .await;
384
385    let response = match result {
386        Ok(Ok(resp)) => resp,
387        Ok(Err(e)) => {
388            return Err(AppError::S3Error(format!(
389                "GetBucketLocation failed for '{}': {}",
390                bucket_name, e
391            )));
392        }
393        Err(_) => {
394            return Err(AppError::S3Error(format!(
395                "GetBucketLocation timed out for '{}'",
396                bucket_name
397            )));
398        }
399    };
400
401    // Empty string means us-east-1
402    let region = response
403        .location_constraint()
404        .map(|l| l.as_str().to_string())
405        .unwrap_or_else(|| "us-east-1".to_string());
406
407    let region = if region.is_empty() {
408        "us-east-1".to_string()
409    } else {
410        region
411    };
412
413    Ok(region)
414}
415
416/// Format bytes to human-readable size
417pub fn format_size(bytes: u64) -> String {
418    const KB: u64 = 1024;
419    const MB: u64 = KB * 1024;
420    const GB: u64 = MB * 1024;
421    const TB: u64 = GB * 1024;
422
423    if bytes >= TB {
424        format!("{:.2} TB", bytes as f64 / TB as f64)
425    } else if bytes >= GB {
426        format!("{:.2} GB", bytes as f64 / GB as f64)
427    } else if bytes >= MB {
428        format!("{:.2} MB", bytes as f64 / MB as f64)
429    } else if bytes >= KB {
430        format!("{:.2} KB", bytes as f64 / KB as f64)
431    } else {
432        format!("{} B", bytes)
433    }
434}
435
436#[cfg(test)]
437mod tests {
438    use super::{
439        build_s3_config, normalize_endpoint_url, FolderContent, S3ClientManager, S3Object,
440        MAX_SORTED_CACHE_ENTRIES,
441    };
442    use crate::credentials::{CredentialType, Profile};
443    use aws_config::{Region, SdkConfig};
444    use aws_sdk_s3::presigning::PresigningConfig;
445    use std::time::Duration;
446
447    #[test]
448    fn bucket_regions_are_isolated_by_profile_and_revision() {
449        let mut manager = S3ClientManager::new();
450        let a = Profile::new("A".into(), CredentialType::Environment, None);
451        let b = Profile::new("B".into(), CredentialType::Environment, None);
452        manager.set_bucket_region(&a, "shared", "region-a".into());
453        manager.set_bucket_region(&b, "shared", "region-b".into());
454        assert_eq!(
455            manager.get_bucket_region(&a, "shared").as_deref(),
456            Some("region-a")
457        );
458        assert_eq!(
459            manager.get_bucket_region(&b, "shared").as_deref(),
460            Some("region-b")
461        );
462        let mut edited = a.clone();
463        edited.updated_at = a.updated_at.map(|date| date + chrono::Duration::seconds(1));
464        assert!(manager.get_bucket_region(&edited, "shared").is_none());
465    }
466
467    #[test]
468    fn normalize_endpoint_url_preserves_existing_scheme() {
469        assert_eq!(
470            normalize_endpoint_url("https://us-east-1.linodeobjects.com"),
471            "https://us-east-1.linodeobjects.com"
472        );
473        assert_eq!(
474            normalize_endpoint_url("http://localhost:9000"),
475            "http://localhost:9000"
476        );
477    }
478
479    #[test]
480    fn normalize_endpoint_url_defaults_to_https() {
481        assert_eq!(
482            normalize_endpoint_url("us-east-1.linodeobjects.com"),
483            "https://us-east-1.linodeobjects.com"
484        );
485    }
486
487    #[test]
488    fn normalize_endpoint_url_trims_whitespace() {
489        assert_eq!(
490            normalize_endpoint_url("  us-east-1.linodeobjects.com  "),
491            "https://us-east-1.linodeobjects.com"
492        );
493    }
494
495    #[tokio::test]
496    async fn shared_config_endpoint_uses_path_style_addressing() {
497        let sdk_config = SdkConfig::builder()
498            .behavior_version(aws_config::BehaviorVersion::latest())
499            .region(Region::new("us-east-1"))
500            .endpoint_url("http://localhost:9100")
501            .credentials_provider(
502                aws_credential_types::provider::SharedCredentialsProvider::new(
503                    aws_credential_types::Credentials::new(
504                        "access-key",
505                        "secret-key",
506                        None,
507                        None,
508                        "test",
509                    ),
510                ),
511            )
512            .build();
513        let profile = Profile::new(
514            "MinIO shared profile".to_string(),
515            CredentialType::SharedConfig {
516                profile_name: Some("minio-local".to_string()),
517            },
518            Some("us-east-1".to_string()),
519        );
520        let client = aws_sdk_s3::Client::from_conf(build_s3_config(&sdk_config, &profile));
521
522        let request = client
523            .get_object()
524            .bucket("test-bucket")
525            .key("folder/file.txt")
526            .presigned(PresigningConfig::expires_in(Duration::from_secs(60)).unwrap())
527            .await
528            .unwrap();
529
530        assert!(
531            request
532                .uri()
533                .starts_with("http://localhost:9100/test-bucket/folder/file.txt?"),
534            "unexpected presigned URI: {}",
535            request.uri()
536        );
537    }
538
539    #[test]
540    fn aws_upload_compatibility_preserves_download_checksum_validation() {
541        let config = SdkConfig::builder()
542            .behavior_version(aws_config::BehaviorVersion::latest())
543            .region(Region::new("us-east-1"))
544            .response_checksum_validation(
545                aws_sdk_s3::config::ResponseChecksumValidation::WhenSupported,
546            )
547            .build();
548        let profile = Profile::new("AWS".into(), CredentialType::Environment, None);
549        let result = build_s3_config(&config, &profile);
550        assert_eq!(
551            result.request_checksum_calculation(),
552            Some(&aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired)
553        );
554        assert_eq!(
555            result.response_checksum_validation(),
556            Some(&aws_sdk_s3::config::ResponseChecksumValidation::WhenSupported)
557        );
558    }
559
560    #[test]
561    fn sorted_folder_cache_evicts_the_oldest_entry_at_its_limit() {
562        let mut manager = S3ClientManager::new();
563
564        for index in 0..=MAX_SORTED_CACHE_ENTRIES {
565            manager.set_sorted_folder_content(
566                "profile",
567                "bucket",
568                &format!("prefix-{index}/"),
569                "size",
570                "desc",
571                FolderContent {
572                    objects: vec![S3Object {
573                        key: format!("object-{index}"),
574                        last_modified: None,
575                        size: index as i64,
576                        storage_class: None,
577                    }],
578                    common_prefixes: Vec::new(),
579                    generation: index.to_string(),
580                },
581            );
582        }
583
584        assert!(manager
585            .get_sorted_folder_content("profile", "bucket", "prefix-0/", "size", "desc")
586            .is_none());
587        assert!(manager
588            .get_sorted_folder_content(
589                "profile",
590                "bucket",
591                &format!("prefix-{MAX_SORTED_CACHE_ENTRIES}/"),
592                "size",
593                "desc"
594            )
595            .is_some());
596        assert_eq!(manager.sorted_folder_cache.len(), MAX_SORTED_CACHE_ENTRIES);
597    }
598}