datui_lib/
s3_tools.rs

1//! S3-compatible servers other tools already know about: the MinIO client's aliases,
2//! `MC_HOST_<alias>`, and s3cmd's config.
3//!
4//! Anyone running MinIO has `mc`, and anyone using Ceph, Wasabi or DigitalOcean Spaces
5//! from a terminal often has s3cmd. Both keep an endpoint and its keys in a file datui
6//! can read, so those servers appear without being described again in datui's config.
7
8use crate::cloud_browse::Environment;
9use std::path::PathBuf;
10
11/// One S3-compatible server another tool describes.
12#[derive(Debug, Clone, PartialEq, Eq, Default)]
13pub struct ToolServer {
14    /// The alias, or `s3cmd`.
15    pub name: String,
16    /// `https://host:port`, or `None` for AWS itself.
17    pub endpoint: Option<String>,
18    pub access_key_id: String,
19    pub secret_access_key: String,
20    pub session_token: Option<String>,
21    pub region: Option<String>,
22    /// `Some(false)` for path-style, `Some(true)` for virtual-hosted, `None` for the
23    /// endpoint's usual style.
24    pub virtual_hosted: Option<bool>,
25    /// Where it was found, for the row's note: `mc alias`, `MC_HOST_lab`, `s3cmd`.
26    pub origin: String,
27}
28
29/// The placeholder keys `mc` writes into the aliases it creates for you.
30const MC_PLACEHOLDER_KEYS: [&str; 2] = ["YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE"];
31
32/// Where `mc` keeps `config.json`: `MC_CONFIG_DIR`, else one in the home directory
33/// named after the binary: `.mc` (or `.mcli`, as some distributions package
34/// it), and without the dot on Windows.
35pub fn mc_config_paths(env: &Environment<'_>) -> Vec<PathBuf> {
36    if let Some(dir) = (env.var)("MC_CONFIG_DIR").filter(|v| !v.trim().is_empty()) {
37        return vec![PathBuf::from(dir.trim()).join("config.json")];
38    }
39    let Some(home) = &env.home else {
40        return Vec::new();
41    };
42    ["mc", "mcli"]
43        .iter()
44        .map(|name| {
45            let directory = if env.windows {
46                name.to_string()
47            } else {
48                format!(".{name}")
49            };
50            home.join(directory).join("config.json")
51        })
52        .collect()
53}
54
55/// Aliases from an `mc` `config.json` (format version 10), leaving out the ones with no
56/// keys or `mc`'s placeholders, and the public `play` server `mc` adds by default.
57pub fn parse_mc_config(text: &str) -> Vec<ToolServer> {
58    let Ok(value) = serde_json::from_str::<serde_json::Value>(text) else {
59        return Vec::new();
60    };
61    let Some(aliases) = value.get("aliases").and_then(|a| a.as_object()) else {
62        return Vec::new();
63    };
64    let mut servers: Vec<ToolServer> = aliases
65        .iter()
66        .filter(|(name, _)| name.as_str() != "play")
67        .filter_map(|(name, alias)| {
68            let field = |key: &str| {
69                alias
70                    .get(key)
71                    .and_then(|v| v.as_str())
72                    .map(str::to_string)
73                    .filter(|v| !v.is_empty())
74            };
75            let url = field("url")?;
76            let access_key_id = field("accessKey")?;
77            let secret_access_key = field("secretKey")?;
78            if MC_PLACEHOLDER_KEYS.contains(&access_key_id.as_str()) {
79                return None;
80            }
81            Some(ToolServer {
82                name: name.clone(),
83                endpoint: server_endpoint(&url),
84                access_key_id,
85                secret_access_key,
86                session_token: field("sessionToken"),
87                region: None,
88                virtual_hosted: match field("path").as_deref() {
89                    Some("on") => Some(false),
90                    Some("off") => Some(true),
91                    _ => None,
92                },
93                origin: "mc alias".to_string(),
94            })
95        })
96        .collect();
97    servers.sort_by(|a, b| a.name.cmp(&b.name));
98    servers
99}
100
101/// An alias from `MC_HOST_<alias>`: `https://ACCESS:SECRET@host:port`, or
102/// `https://ACCESS:SECRET:TOKEN@host:port`, as `mc` reads it.
103pub fn parse_mc_host(alias: &str, value: &str) -> Option<ToolServer> {
104    let value = value.trim();
105    let (scheme, rest) = value.split_once("://")?;
106    if !matches!(scheme, "http" | "https") {
107        return None;
108    }
109    let (credentials, host) = rest.rsplit_once('@')?;
110    let host = host.trim_end_matches('/');
111    let mut parts = credentials.splitn(3, ':');
112    let access_key_id = parts.next()?.to_string();
113    let second = parts.next()?;
114    let (secret_access_key, session_token) = match parts.next() {
115        Some(token) => (second.to_string(), Some(token.to_string())),
116        None => (second.to_string(), None),
117    };
118    if access_key_id.is_empty() || secret_access_key.is_empty() || host.is_empty() {
119        return None;
120    }
121    Some(ToolServer {
122        name: alias.to_string(),
123        endpoint: server_endpoint(&format!("{scheme}://{host}")),
124        access_key_id,
125        secret_access_key,
126        session_token: session_token.filter(|t| !t.is_empty()),
127        region: None,
128        virtual_hosted: None,
129        origin: format!("MC_HOST_{alias}"),
130    })
131}
132
133/// Every `MC_HOST_<alias>` in `vars`.
134pub fn mc_hosts(vars: &[(String, String)]) -> Vec<ToolServer> {
135    let mut servers: Vec<ToolServer> = vars
136        .iter()
137        .filter_map(|(key, value)| {
138            let alias = key.strip_prefix("MC_HOST_")?;
139            (!alias.is_empty()).then(|| parse_mc_host(alias, value))?
140        })
141        .collect();
142    servers.sort_by(|a, b| a.name.cmp(&b.name));
143    servers
144}
145
146/// Where s3cmd keeps its config: `S3CMD_CONFIG`, else `%APPDATA%\s3cmd.ini` on Windows
147/// and `~/.s3cfg` elsewhere.
148pub fn s3cfg_path(env: &Environment<'_>) -> Option<PathBuf> {
149    if let Some(path) = (env.var)("S3CMD_CONFIG").filter(|v| !v.trim().is_empty()) {
150        return Some(PathBuf::from(path.trim()));
151    }
152    if env.windows {
153        return (env.var)("APPDATA").map(|dir| PathBuf::from(dir).join("s3cmd.ini"));
154    }
155    env.home.as_ref().map(|home| home.join(".s3cfg"))
156}
157
158/// The server in the `[default]` section of an s3cmd config, when it has keys.
159pub fn parse_s3cfg(text: &str) -> Option<ToolServer> {
160    let mut in_default = false;
161    let mut values = std::collections::HashMap::new();
162    for line in text.lines() {
163        let line = line.trim();
164        if line.starts_with('[') {
165            in_default = line == "[default]";
166            continue;
167        }
168        if !in_default || line.starts_with('#') || line.starts_with(';') {
169            continue;
170        }
171        if let Some((key, value)) = line.split_once('=') {
172            values.insert(key.trim().to_string(), value.trim().to_string());
173        }
174    }
175    let get = |key: &str| values.get(key).filter(|v| !v.is_empty()).cloned();
176    let access_key_id = get("access_key")?;
177    let secret_access_key = get("secret_key")?;
178    let host_base = get("host_base").unwrap_or_else(|| "s3.amazonaws.com".to_string());
179    let https = get("use_https").is_none_or(|v| v.eq_ignore_ascii_case("true"));
180    let aws = host_base.eq_ignore_ascii_case("s3.amazonaws.com");
181    let region = get("bucket_location").map(|location| {
182        // s3cmd's "US" is AWS's us-east-1.
183        if location.eq_ignore_ascii_case("us") {
184            "us-east-1".to_string()
185        } else {
186            location
187        }
188    });
189    Some(ToolServer {
190        name: "s3cmd".to_string(),
191        endpoint: (!aws).then(|| format!("{}://{host_base}", if https { "https" } else { "http" })),
192        access_key_id,
193        secret_access_key,
194        session_token: get("access_token"),
195        region,
196        // `%(bucket)s` in the host template means the bucket goes in the host name.
197        virtual_hosted: get("host_bucket").map(|template| template.contains("%(bucket)s")),
198        origin: "s3cmd".to_string(),
199    })
200}
201
202/// The endpoint of an alias URL, or `None` when it is AWS itself.
203fn server_endpoint(url: &str) -> Option<String> {
204    let url = url.trim().trim_end_matches('/');
205    let host = url.split_once("://").map(|(_, h)| h).unwrap_or(url);
206    if host.eq_ignore_ascii_case("s3.amazonaws.com") {
207        return None;
208    }
209    Some(url.to_string())
210}
211
212#[cfg(test)]
213mod tests {
214    use super::*;
215
216    const MC_CONFIG: &str = r#"{
217        "version": "10",
218        "aliases": {
219            "gcs": {"url": "https://storage.googleapis.com", "accessKey": "YOUR-ACCESS-KEY-HERE", "secretKey": "YOUR-SECRET-KEY-HERE", "api": "S3v2", "path": "dns"},
220            "local": {"url": "http://localhost:9000", "accessKey": "", "secretKey": "", "api": "S3v4", "path": "auto"},
221            "play": {"url": "https://play.min.io", "accessKey": "Q3AM3UQ867SPQQA43P2F", "secretKey": "zuf+tfteSlswRu7BJ86wtrueekitnifILbZam1KYY3TG", "api": "S3v4", "path": "auto"},
222            "lab": {"url": "http://127.0.0.1:9000/", "accessKey": "minioadmin", "secretKey": "minioadmin", "api": "S3v4", "path": "on"},
223            "corp": {"url": "https://minio.corp.example", "accessKey": "k", "secretKey": "s", "sessionToken": "t", "api": "s3v4", "path": "off"}
224        }
225    }"#;
226
227    #[test]
228    fn mc_aliases_with_keys_become_servers() {
229        let servers = parse_mc_config(MC_CONFIG);
230        let names: Vec<&str> = servers.iter().map(|s| s.name.as_str()).collect();
231        assert_eq!(
232            names,
233            ["corp", "lab"],
234            "no placeholders, no empty keys, no play"
235        );
236        let lab = &servers[1];
237        assert_eq!(lab.endpoint.as_deref(), Some("http://127.0.0.1:9000"));
238        assert_eq!(lab.virtual_hosted, Some(false));
239        assert_eq!(servers[0].virtual_hosted, Some(true));
240        assert_eq!(servers[0].session_token.as_deref(), Some("t"));
241        assert!(parse_mc_config("not json").is_empty());
242    }
243
244    #[test]
245    fn mc_host_variables_read_like_mc_reads_them() {
246        let plain = parse_mc_host("lab", "http://minioadmin:minio/admin@127.0.0.1:9000").unwrap();
247        assert_eq!(plain.access_key_id, "minioadmin");
248        assert_eq!(plain.secret_access_key, "minio/admin");
249        assert_eq!(plain.session_token, None);
250        assert_eq!(plain.endpoint.as_deref(), Some("http://127.0.0.1:9000"));
251        assert_eq!(plain.origin, "MC_HOST_lab");
252
253        let token = parse_mc_host("corp", "https://AK:SK:TOKEN@minio.corp.example").unwrap();
254        assert_eq!(token.secret_access_key, "SK");
255        assert_eq!(token.session_token.as_deref(), Some("TOKEN"));
256
257        assert!(parse_mc_host("x", "https://minio.corp.example").is_none());
258        assert!(parse_mc_host("x", "ftp://a:b@host").is_none());
259
260        let found = mc_hosts(&[
261            ("MC_HOST_b".to_string(), "http://k:s@b:9000".to_string()),
262            ("PATH".to_string(), "/usr/bin".to_string()),
263            ("MC_HOST_a".to_string(), "http://k:s@a:9000".to_string()),
264        ]);
265        let names: Vec<&str> = found.iter().map(|s| s.name.as_str()).collect();
266        assert_eq!(names, ["a", "b"]);
267    }
268
269    #[test]
270    fn s3cmd_config_for_ceph_and_for_aws() {
271        let ceph = parse_s3cfg(
272            "[default]\naccess_key = CEPHKEY\nsecret_key = cephsecret\nhost_base = ceph.example:7480\n\
273             host_bucket = ceph.example:7480\nuse_https = False\nbucket_location = US\n",
274        )
275        .unwrap();
276        assert_eq!(ceph.endpoint.as_deref(), Some("http://ceph.example:7480"));
277        assert_eq!(ceph.virtual_hosted, Some(false));
278        assert_eq!(ceph.region.as_deref(), Some("us-east-1"));
279
280        let spaces = parse_s3cfg(
281            "[default]\naccess_key = DO\nsecret_key = s\nhost_base = nyc3.digitaloceanspaces.com\n\
282             host_bucket = %(bucket)s.nyc3.digitaloceanspaces.com\n",
283        )
284        .unwrap();
285        assert_eq!(
286            spaces.endpoint.as_deref(),
287            Some("https://nyc3.digitaloceanspaces.com")
288        );
289        assert_eq!(spaces.virtual_hosted, Some(true));
290
291        let aws = parse_s3cfg("[default]\naccess_key = AKIA\nsecret_key = s\n").unwrap();
292        assert_eq!(aws.endpoint, None, "no host_base is AWS");
293
294        assert!(
295            parse_s3cfg("[default]\nhost_base = x\n").is_none(),
296            "no keys"
297        );
298        assert!(parse_s3cfg("[other]\naccess_key = a\nsecret_key = b\n").is_none());
299    }
300
301    #[test]
302    fn where_each_tool_keeps_its_config() {
303        let run =
304            |_: &str, _: &[&str]| Err(crate::cloud_command::CommandError::Missing("x".to_string()));
305        let env_with = |vars: &'static [(&'static str, &'static str)], windows: bool| {
306            move |f: &dyn Fn(&Environment<'_>)| {
307                let var = |key: &str| {
308                    vars.iter()
309                        .find(|(k, _)| *k == key)
310                        .map(|(_, v)| v.to_string())
311                };
312                let env = Environment {
313                    var: &var,
314                    exists: &|_| false,
315                    read: &|_| None,
316                    home: Some(PathBuf::from(if windows {
317                        r"C:\Users\u"
318                    } else {
319                        "/home/u"
320                    })),
321                    windows,
322                    run: &run,
323                    all_vars: &|| Vec::new(),
324                    list: &|_| Vec::new(),
325                };
326                f(&env);
327            }
328        };
329        env_with(&[], false)(&|env| {
330            assert_eq!(
331                mc_config_paths(env),
332                [
333                    PathBuf::from("/home/u/.mc/config.json"),
334                    PathBuf::from("/home/u/.mcli/config.json")
335                ]
336            );
337            assert_eq!(s3cfg_path(env), Some(PathBuf::from("/home/u/.s3cfg")));
338        });
339        env_with(&[("APPDATA", r"C:\Users\u\AppData\Roaming")], true)(&|env| {
340            assert_eq!(
341                mc_config_paths(env)[0],
342                PathBuf::from(r"C:\Users\u").join("mc").join("config.json")
343            );
344            assert_eq!(
345                s3cfg_path(env),
346                Some(PathBuf::from(r"C:\Users\u\AppData\Roaming").join("s3cmd.ini"))
347            );
348        });
349        env_with(
350            &[("MC_CONFIG_DIR", "/etc/mc"), ("S3CMD_CONFIG", "/etc/s3cfg")],
351            false,
352        )(&|env| {
353            assert_eq!(mc_config_paths(env), [PathBuf::from("/etc/mc/config.json")]);
354            assert_eq!(s3cfg_path(env), Some(PathBuf::from("/etc/s3cfg")));
355        });
356    }
357}
datui_lib/s3_tools.rs

datui_lib/
s3_tools.rs
