converge_storage/
config.rs

1// Copyright 2024-2026 Reflective Labs
2// SPDX-License-Identifier: MIT
3
4use serde::{Deserialize, Serialize};
5
6use crate::StorageUri;
7
8/// Storage configuration, typically loaded from TOML:
9///
10/// ```toml
11/// [storage]
12/// uri = "./data/parquet"        # local
13/// # uri = "s3://my-bucket"      # S3/MinIO/RustFS bucket
14/// # uri = "gs://my-bucket"      # GCS bucket
15/// prefix = "datasets/"
16/// ```
17#[derive(Debug, Clone, Serialize, Deserialize)]
18pub struct StorageConfig {
19    pub uri: StorageUri,
20
21    /// Object key prefix applied to all operations.
22    /// This is stored separately from `uri`.
23    ///
24    /// e.g., `"datasets/"` means `get("file.parquet")` resolves to `datasets/file.parquet`.
25    #[serde(default)]
26    pub prefix: Option<String>,
27
28    /// Skip request signing for public buckets.
29    /// Only used when `uri` is `s3://` or `gs://`.
30    #[serde(default)]
31    pub public: bool,
32
33    /// S3-compatible endpoint override (`MinIO`, `RustFS`, etc.).
34    /// Only used when `uri` is `s3://`.
35    #[serde(default)]
36    pub endpoint: Option<String>,
37
38    /// AWS region. Only used when `uri` is `s3://`.
39    #[serde(default)]
40    pub region: Option<String>,
41}
42
43impl StorageConfig {
44    /// Resolve a key relative to the configured prefix.
45    #[must_use]
46    pub fn resolve_key(&self, key: &str) -> String {
47        let key = key.trim_start_matches('/');
48
49        match self
50            .prefix
51            .as_deref()
52            .map(|prefix| prefix.trim_matches('/'))
53            .filter(|prefix| !prefix.is_empty())
54        {
55            Some(prefix) if key.is_empty() => prefix.to_string(),
56            Some(prefix) => format!("{prefix}/{key}"),
57            None => key.to_string(),
58        }
59    }
60}
61
62#[cfg(test)]
63mod tests {
64    use super::*;
65
66    #[test]
67    fn parse_local_config() {
68        let toml_str = r#"
69            uri = "./data/parquet"
70            prefix = "datasets/"
71        "#;
72        let config: StorageConfig = toml::from_str(toml_str).unwrap();
73        assert_eq!(config.resolve_key("file.parquet"), "datasets/file.parquet");
74    }
75
76    #[test]
77    fn parse_s3_config() {
78        let toml_str = r#"
79            uri = "s3://my-bucket"
80            prefix = "v1/"
81            endpoint = "http://localhost:9000"
82            region = "us-east-1"
83        "#;
84        let config: StorageConfig = toml::from_str(toml_str).unwrap();
85        assert_eq!(config.uri.scheme(), "s3");
86        assert_eq!(config.endpoint.as_deref(), Some("http://localhost:9000"));
87    }
88
89    #[test]
90    fn no_prefix() {
91        let toml_str = r#"uri = "gs://my-bucket""#;
92        let config: StorageConfig = toml::from_str(toml_str).unwrap();
93        assert_eq!(config.resolve_key("file.parquet"), "file.parquet");
94    }
95
96    #[test]
97    fn resolve_key_normalizes_slashes() {
98        let config = StorageConfig {
99            uri: StorageUri::Local("./data".into()),
100            prefix: Some("/datasets/".to_string()),
101            public: false,
102            endpoint: None,
103            region: None,
104        };
105
106        assert_eq!(
107            config.resolve_key("/nested/file.parquet"),
108            "datasets/nested/file.parquet"
109        );
110    }
111}
converge_storage/config.rs

converge_storage/
config.rs
