Skip to main content

oxigeo_cloud/
lib.rs

1//! Advanced cloud storage backends for OxiGeo
2//!
3//! This crate provides comprehensive cloud storage integration for OxiGeo, including:
4//!
5//! - **Cloud Providers**: S3, Azure Blob Storage, Google Cloud Storage
6//! - **Authentication**: OAuth 2.0, service accounts, API keys, SAS tokens, IAM roles
7//! - **Advanced Caching**: Multi-level cache with memory and disk tiers, compression, LRU+LFU eviction
8//! - **Intelligent Prefetching**: Predictive prefetch, access pattern analysis, bandwidth management
9//! - **Retry Logic**: Exponential backoff, jitter, circuit breaker, retry budgets
10//! - **HTTP Backend**: Enhanced HTTP/HTTPS with authentication and retry support
11//!
12//! # Features
13//!
14//! - `s3` - AWS S3 support
15//! - `azure-blob` - Azure Blob Storage support
16//! - `gcs` - Google Cloud Storage support
17//! - `http` - HTTP/HTTPS backend
18//! - `cache` - Advanced caching layer
19//! - `prefetch` - Intelligent prefetching
20//! - `retry` - Retry logic with backoff
21//!
22//! # Examples
23//!
24//! ## AWS S3
25//!
26//! ```rust,no_run
27//! # #[cfg(feature = "s3")]
28//! # async fn example() -> oxigeo_cloud::Result<()> {
29//! use oxigeo_cloud::backends::S3Backend;
30//! use oxigeo_cloud::backends::CloudStorageBackend;
31//!
32//! let backend = S3Backend::new("my-bucket", "data/zarr")
33//!     .with_region("us-west-2");
34//!
35//! // Get object
36//! let data = backend.get("file.tif").await?;
37//!
38//! // Put object
39//! backend.put("output.tif", &data).await?;
40//!
41//! # Ok(())
42//! # }
43//! ```
44//!
45//! ## Multi-cloud Abstraction
46//!
47//! ```rust,no_run
48//! # #[cfg(feature = "s3")]
49//! # async fn example() -> oxigeo_cloud::Result<()> {
50//! use oxigeo_cloud::CloudBackend;
51//!
52//! // Parse URL and create appropriate backend
53//! let backend = CloudBackend::from_url("s3://bucket/file.tif")?;
54//! let data = backend.get().await?;
55//!
56//! # Ok(())
57//! # }
58//! ```
59//!
60//! ## Advanced Caching
61//!
62//! ```rust,no_run
63//! # #[cfg(feature = "cache")]
64//! # async fn example() -> oxigeo_cloud::Result<()> {
65//! use oxigeo_cloud::cache::{CacheConfig, MultiLevelCache};
66//! use bytes::Bytes;
67//!
68//! let config = CacheConfig::new()
69//!     .with_max_memory_size(100 * 1024 * 1024) // 100 MB
70//!     .with_cache_dir("/tmp/oxigeo-cache");
71//!
72//! let cache = MultiLevelCache::new(config)?;
73//!
74//! // Cache data
75//! cache.put("key".to_string(), Bytes::from("data")).await?;
76//!
77//! // Retrieve from cache
78//! let data = cache.get(&"key".to_string()).await?;
79//!
80//! # Ok(())
81//! # }
82//! ```
83
84#![cfg_attr(not(feature = "std"), no_std)]
85// Allow partial documentation during development
86#![allow(missing_docs)]
87// Allow dead code for backend features
88#![allow(dead_code)]
89// Allow matches! suggestions - explicit patterns preferred for cloud errors
90#![allow(clippy::match_like_matches_macro)]
91// Allow expect() for internal cloud state invariants
92#![allow(clippy::expect_used)]
93// Allow complex types in cloud interfaces
94#![allow(clippy::type_complexity)]
95// Allow manual div_ceil for bandwidth calculations
96#![allow(clippy::manual_div_ceil)]
97// Allow unused variables in platform-specific code
98#![allow(unused_variables)]
99// Allow collapsible matches for explicit error handling
100#![allow(clippy::collapsible_match)]
101// Allow async fn in traits for cloud operations
102#![allow(async_fn_in_trait)]
103// Allow stripping prefix manually for URL path handling
104#![allow(clippy::manual_strip)]
105// Allow first element access with get(0)
106#![allow(clippy::get_first)]
107// Allow field assignment outside initializer
108#![allow(clippy::field_reassign_with_default)]
109// Allow unused imports in feature-gated modules
110#![allow(unused_imports)]
111// Allow method names that may conflict with std traits
112#![allow(clippy::should_implement_trait)]
113
114#[cfg(feature = "alloc")]
115extern crate alloc;
116
117pub mod auth;
118pub mod backends;
119#[cfg(feature = "cache")]
120pub mod cache;
121pub mod error;
122#[cfg(feature = "async")]
123pub mod multicloud;
124#[cfg(feature = "prefetch")]
125pub mod prefetch;
126#[cfg(feature = "retry")]
127pub mod retry;
128
129pub use error::{CloudError, Result};
130
131#[cfg(feature = "s3")]
132pub use backends::s3::S3Backend;
133
134#[cfg(feature = "azure-blob")]
135pub use backends::azure::AzureBlobBackend;
136
137#[cfg(feature = "gcs")]
138pub use backends::gcs::GcsBackend;
139
140#[cfg(feature = "http")]
141pub use backends::http::HttpBackend;
142
143#[cfg(feature = "async")]
144pub use multicloud::{
145    CloudProvider, CloudProviderConfig, CloudRegion, CrossCloudTransferConfig,
146    CrossCloudTransferResult, MultiCloudManager, MultiCloudManagerBuilder, ProviderHealth,
147    RoutingStrategy, TransferCostEstimate,
148};
149
150use url::Url;
151
152/// Multi-cloud storage backend abstraction
153#[derive(Debug)]
154pub enum CloudBackend {
155    /// AWS S3 backend
156    #[cfg(feature = "s3")]
157    S3 {
158        /// S3 backend instance
159        backend: S3Backend,
160        /// Object key
161        key: String,
162    },
163
164    /// Azure Blob Storage backend
165    #[cfg(feature = "azure-blob")]
166    Azure {
167        /// Azure backend instance
168        backend: AzureBlobBackend,
169        /// Blob name
170        blob: String,
171    },
172
173    /// Google Cloud Storage backend
174    #[cfg(feature = "gcs")]
175    Gcs {
176        /// GCS backend instance
177        backend: GcsBackend,
178        /// Object name
179        object: String,
180    },
181
182    /// HTTP/HTTPS backend
183    #[cfg(feature = "http")]
184    Http {
185        /// HTTP backend instance
186        backend: HttpBackend,
187        /// Resource path
188        path: String,
189    },
190}
191
192impl CloudBackend {
193    /// Creates a cloud backend from a URL
194    ///
195    /// Supported URL formats:
196    /// - `s3://bucket/key` - AWS S3
197    /// - `az://container/blob` - Azure Blob Storage
198    /// - `gs://bucket/object` - Google Cloud Storage
199    /// - `<http://example.com/path>` or `<https://example.com/path>` - HTTP/HTTPS
200    ///
201    /// # Examples
202    ///
203    /// ```rust,no_run
204    /// # fn example() -> oxigeo_cloud::Result<()> {
205    /// use oxigeo_cloud::CloudBackend;
206    ///
207    /// let backend = CloudBackend::from_url("s3://my-bucket/data/file.tif")?;
208    /// # Ok(())
209    /// # }
210    /// ```
211    pub fn from_url(url: &str) -> Result<Self> {
212        let parsed = Url::parse(url)?;
213
214        match parsed.scheme() {
215            #[cfg(feature = "s3")]
216            "s3" => {
217                let bucket = parsed.host_str().ok_or_else(|| CloudError::InvalidUrl {
218                    url: url.to_string(),
219                })?;
220
221                let key = parsed.path().trim_start_matches('/').to_string();
222
223                Ok(Self::S3 {
224                    backend: S3Backend::new(bucket, ""),
225                    key,
226                })
227            }
228
229            #[cfg(feature = "azure-blob")]
230            "az" | "azure" => {
231                let container = parsed.host_str().ok_or_else(|| CloudError::InvalidUrl {
232                    url: url.to_string(),
233                })?;
234
235                // Account name should be in the username part of the URL
236                let account = parsed.username();
237                if account.is_empty() {
238                    return Err(CloudError::InvalidUrl {
239                        url: url.to_string(),
240                    });
241                }
242
243                let blob = parsed.path().trim_start_matches('/').to_string();
244
245                Ok(Self::Azure {
246                    backend: AzureBlobBackend::new(account, container),
247                    blob,
248                })
249            }
250
251            #[cfg(feature = "gcs")]
252            "gs" | "gcs" => {
253                let bucket = parsed.host_str().ok_or_else(|| CloudError::InvalidUrl {
254                    url: url.to_string(),
255                })?;
256
257                let object = parsed.path().trim_start_matches('/').to_string();
258
259                Ok(Self::Gcs {
260                    backend: GcsBackend::new(bucket),
261                    object,
262                })
263            }
264
265            #[cfg(feature = "http")]
266            "http" | "https" => {
267                // Reconstruct base URL without the path
268                let base_url = format!(
269                    "{}://{}",
270                    parsed.scheme(),
271                    parsed.host_str().ok_or_else(|| CloudError::InvalidUrl {
272                        url: url.to_string(),
273                    })?
274                );
275
276                let path = parsed.path().trim_start_matches('/').to_string();
277
278                Ok(Self::Http {
279                    backend: HttpBackend::new(base_url),
280                    path,
281                })
282            }
283
284            scheme => Err(CloudError::UnsupportedProtocol {
285                protocol: scheme.to_string(),
286            }),
287        }
288    }
289
290    /// Gets data from the cloud backend
291    #[cfg(all(
292        feature = "async",
293        any(
294            feature = "s3",
295            feature = "azure-blob",
296            feature = "gcs",
297            feature = "http"
298        )
299    ))]
300    pub async fn get(&self) -> Result<bytes::Bytes> {
301        use backends::CloudStorageBackend;
302
303        match self {
304            #[cfg(feature = "s3")]
305            Self::S3 { backend, key } => backend.get(key).await,
306
307            #[cfg(feature = "azure-blob")]
308            Self::Azure { backend, blob } => backend.get(blob).await,
309
310            #[cfg(feature = "gcs")]
311            Self::Gcs { backend, object } => backend.get(object).await,
312
313            #[cfg(feature = "http")]
314            Self::Http { backend, path } => backend.get(path).await,
315        }
316    }
317
318    /// Puts data to the cloud backend
319    #[cfg(all(
320        feature = "async",
321        any(
322            feature = "s3",
323            feature = "azure-blob",
324            feature = "gcs",
325            feature = "http"
326        )
327    ))]
328    pub async fn put(&self, data: &[u8]) -> Result<()> {
329        use backends::CloudStorageBackend;
330
331        match self {
332            #[cfg(feature = "s3")]
333            Self::S3 { backend, key } => backend.put(key, data).await,
334
335            #[cfg(feature = "azure-blob")]
336            Self::Azure { backend, blob } => backend.put(blob, data).await,
337
338            #[cfg(feature = "gcs")]
339            Self::Gcs { backend, object } => backend.put(object, data).await,
340
341            #[cfg(feature = "http")]
342            Self::Http { .. } => Err(CloudError::NotSupported {
343                operation: "HTTP backend is read-only".to_string(),
344            }),
345        }
346    }
347
348    /// Checks if the object exists
349    #[cfg(all(
350        feature = "async",
351        any(
352            feature = "s3",
353            feature = "azure-blob",
354            feature = "gcs",
355            feature = "http"
356        )
357    ))]
358    pub async fn exists(&self) -> Result<bool> {
359        use backends::CloudStorageBackend;
360
361        match self {
362            #[cfg(feature = "s3")]
363            Self::S3 { backend, key } => backend.exists(key).await,
364
365            #[cfg(feature = "azure-blob")]
366            Self::Azure { backend, blob } => backend.exists(blob).await,
367
368            #[cfg(feature = "gcs")]
369            Self::Gcs { backend, object } => backend.exists(object).await,
370
371            #[cfg(feature = "http")]
372            Self::Http { backend, path } => backend.exists(path).await,
373        }
374    }
375}
376
377#[cfg(test)]
378#[allow(clippy::panic)]
379mod tests {
380    use super::*;
381
382    #[test]
383    #[cfg(feature = "s3")]
384    fn test_cloud_backend_from_url_s3() {
385        let backend = CloudBackend::from_url("s3://my-bucket/path/to/file.tif");
386        assert!(backend.is_ok());
387
388        if let Ok(CloudBackend::S3 { backend, key }) = backend {
389            assert_eq!(backend.bucket, "my-bucket");
390            assert_eq!(key, "path/to/file.tif");
391        } else {
392            panic!("Expected S3 backend");
393        }
394    }
395
396    #[test]
397    #[cfg(feature = "gcs")]
398    fn test_cloud_backend_from_url_gcs() {
399        let backend = CloudBackend::from_url("gs://my-bucket/path/to/file.tif");
400        assert!(backend.is_ok());
401
402        if let Ok(CloudBackend::Gcs { backend, object }) = backend {
403            assert_eq!(backend.bucket, "my-bucket");
404            assert_eq!(object, "path/to/file.tif");
405        } else {
406            panic!("Expected GCS backend");
407        }
408    }
409
410    #[test]
411    #[cfg(feature = "http")]
412    fn test_cloud_backend_from_url_http() {
413        let backend = CloudBackend::from_url("https://example.com/path/to/file.tif");
414        assert!(backend.is_ok());
415
416        if let Ok(CloudBackend::Http { backend, path }) = backend {
417            assert!(backend.base_url.contains("example.com"));
418            assert_eq!(path, "path/to/file.tif");
419        } else {
420            panic!("Expected HTTP backend");
421        }
422    }
423
424    #[test]
425    fn test_cloud_backend_from_url_invalid() {
426        let backend = CloudBackend::from_url("invalid://url");
427        assert!(backend.is_err());
428    }
429}