datui_lib/cloud/
source.rs

1//! Input source detection for local paths vs remote URLs (S3, GCS, Azure, HTTP/HTTPS).
2
3use std::path::{Path, PathBuf};
4
5/// Which API a provider speaks. Not which company runs it: MinIO, Ceph, R2 and AWS
6/// itself are all [`ProviderKind::S3`], and are told apart by their endpoint.
7#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
8pub enum ProviderKind {
9    Gcs,
10    #[default]
11    S3,
12    Azure,
13}
14
15impl ProviderKind {
16    /// The URL scheme datui opens this provider's objects with.
17    pub fn scheme(self) -> &'static str {
18        match self {
19            ProviderKind::Gcs => "gs",
20            ProviderKind::S3 => "s3",
21            ProviderKind::Azure => "abfss",
22        }
23    }
24
25    /// The word for the API, as the config and the home screen write it.
26    pub fn name(self) -> &'static str {
27        match self {
28            ProviderKind::Gcs => "gcs",
29            ProviderKind::S3 => "s3",
30            ProviderKind::Azure => "azure",
31        }
32    }
33}
34
35#[derive(Clone, Debug, Eq, PartialEq)]
36pub enum InputSource {
37    Local(PathBuf),
38    S3(String),
39    Gcs(String),
40    /// Azure Blob Storage, always as `abfss://container@account.dfs.core.windows.net/path`
41    /// whichever of its forms it was written in.
42    Azure(String),
43    Http(String),
44}
45
46/// Classifies the path as local, S3, GCS, or HTTP/HTTPS using string parsing only (no filesystem calls).
47pub fn input_source(path: &Path) -> InputSource {
48    let s = path.as_os_str().to_string_lossy();
49    if let Some(after_scheme) = s.find("://") {
50        let prefix = s[..after_scheme].to_lowercase();
51        let rest = s[after_scheme + 3..].to_string();
52        if prefix == "s3" || prefix == "s3a" {
53            return InputSource::S3(rest);
54        }
55        if prefix == "gs" || prefix == "gcs" {
56            return InputSource::Gcs(rest);
57        }
58        if let Some((account, container, key)) = azure_parts(&s) {
59            return InputSource::Azure(azure_url(&account, &container, &key));
60        }
61        if prefix == "http" || prefix == "https" {
62            return InputSource::Http(s.to_string());
63        }
64    }
65    InputSource::Local(path.to_path_buf())
66}
67
68/// Whether `path` is a URL datui reads rather than a path on this machine: whatever
69/// [`input_source`] places remotely, and the Azure forms that name no account, which
70/// are expanded when opened.
71pub fn is_remote_url(path: &Path) -> bool {
72    !matches!(input_source(path), InputSource::Local(_))
73        || path
74            .to_string_lossy()
75            .split_once("://")
76            .is_some_and(|(scheme, _)| is_azure_short_scheme(scheme))
77}
78
79/// Whether this build can open `path`: object stores need the `cloud` feature and web
80/// URLs the `http` feature; a local path always opens.
81pub fn opens_in_this_build(path: &Path) -> bool {
82    match input_source(path) {
83        InputSource::Local(_) if is_remote_url(path) => cfg!(feature = "cloud"),
84        InputSource::Local(_) => true,
85        InputSource::S3(_) | InputSource::Gcs(_) | InputSource::Azure(_) => {
86            cfg!(feature = "cloud")
87        }
88        InputSource::Http(_) => cfg!(feature = "http"),
89    }
90}
91
92/// `az`, `adl` and `azure`: Azure schemes that name a container but no account.
93pub(crate) fn is_azure_short_scheme(scheme: &str) -> bool {
94    matches!(scheme.to_ascii_lowercase().as_str(), "az" | "adl" | "azure")
95}
96
97/// The source an `s3://<id>@bucket/key` URL names, and the URL without it. Only S3
98/// (S3-compatible servers repeat bucket names); bucket names cannot contain `@`, so one
99/// in the first segment is always a source. Other URLs come back unchanged.
100pub fn split_source_id(url: &str) -> (Option<&str>, std::borrow::Cow<'_, str>) {
101    let Some((scheme, rest)) = url.split_once("://") else {
102        return (None, url.into());
103    };
104    if !matches!(scheme.to_ascii_lowercase().as_str(), "s3" | "s3a") {
105        return (None, url.into());
106    }
107    let first = rest.split('/').next().unwrap_or(rest);
108    match first.split_once('@') {
109        Some((id, _)) if !id.is_empty() => {
110            let plain = format!("{scheme}://{}", &rest[id.len() + 1..]);
111            (Some(id), plain.into())
112        }
113        _ => (None, url.into()),
114    }
115}
116
117/// The account, container and path of an Azure Blob URL: `abfss://`/`abfs://`
118/// (`container@account.dfs.core.windows.net/path`) or `https://` blob/dfs endpoints
119/// (`account.blob.core.windows.net/container/path`); not `az://`, which names no
120/// account. The path has no leading slash and keeps a trailing one (a directory).
121pub fn azure_parts(url: &str) -> Option<(String, String, String)> {
122    let (scheme, rest) = url.split_once("://")?;
123    let scheme = scheme.to_ascii_lowercase();
124    let (host_part, path) = match rest.split_once('/') {
125        Some((host, path)) => (host, path),
126        None => (rest, ""),
127    };
128    let account_of = |host: &str| {
129        let host = host.to_ascii_lowercase();
130        [".dfs.core.windows.net", ".blob.core.windows.net"]
131            .iter()
132            .find_map(|suffix| host.strip_suffix(suffix).map(str::to_string))
133            .filter(|account| !account.is_empty() && !account.contains('.'))
134    };
135    match scheme.as_str() {
136        "abfss" | "abfs" => {
137            let (container, host) = host_part.split_once('@')?;
138            let account = account_of(host)?;
139            (!container.is_empty()).then(|| (account, container.to_string(), path.to_string()))
140        }
141        "https" | "http" => {
142            let account = account_of(host_part)?;
143            let (container, path) = match path.split_once('/') {
144                Some((container, path)) => (container, path),
145                None => (path, ""),
146            };
147            (!container.is_empty()).then(|| (account, container.to_string(), path.to_string()))
148        }
149        _ => None,
150    }
151}
152
153/// The canonical URL for a place in Azure Blob Storage.
154pub fn azure_url(account: &str, container: &str, path: &str) -> String {
155    format!(
156        "abfss://{container}@{account}.dfs.core.windows.net/{}",
157        path.trim_start_matches('/')
158    )
159}
160
161/// A cloud place in the form used for equality comparisons. Equivalent Azure URL
162/// forms become `abfss://`, and a trailing slash does not distinguish a place.
163pub(crate) fn canonical_cloud_place(url: &str) -> String {
164    let canonical = match azure_parts(url) {
165        Some((account, container, path)) => azure_url(&account, &container, &path),
166        None => url.to_string(),
167    };
168    canonical.trim_end_matches('/').to_string()
169}
170
171/// A cloud location whose shape is a prefix or a glob rather than one object.
172pub(crate) fn is_prefix_or_glob(url: &str) -> bool {
173    url.ends_with('/') || url.contains('*')
174}
175
176/// The characters Polars expands a path on (`polars_io::path_utils::has_glob`).
177pub(crate) fn has_glob_chars(path: &Path) -> bool {
178    path.as_os_str().to_string_lossy().contains(['*', '?', '['])
179}
180
181/// Whether a path is a pattern to expand: it has glob characters and no file has that
182/// name (an existing `d[1].csv` is that file). The `glob` flag for every local Polars
183/// scan.
184pub(crate) fn expands_as_glob(path: &Path) -> bool {
185    has_glob_chars(path) && std::fs::symlink_metadata(path).is_err()
186}
187
188/// A path for a Polars reader that always expands globs (its NDJSON scan has no
189/// `glob` flag): an existing name with a glob character comes back escaped, so the
190/// pattern matches that file alone.
191pub(crate) fn polars_literal_path(
192    path: &Path,
193) -> polars::prelude::PolarsResult<polars::prelude::PlRefPath> {
194    if !has_glob_chars(path) || expands_as_glob(path) {
195        return polars::prelude::PlRefPath::try_from_path(path);
196    }
197    // Escaped as Polars will read it: on Windows that text has `/` separators and no
198    // `\\?\` prefix, so neither a separator nor the prefix's `?` is touched.
199    let text = polars::prelude::PlRefPath::try_from_path(path)?;
200    Ok(polars::prelude::PlRefPath::new(
201        escape_glob(text.as_str()).as_str(),
202    ))
203}
204
205/// `text` as a glob that matches only itself: each glob character in brackets.
206pub(crate) fn escape_glob(text: &str) -> String {
207    let mut escaped = String::with_capacity(text.len() + 8);
208    for c in text.chars() {
209        if matches!(c, '*' | '?' | '[' | ']') {
210            escaped.extend(['[', c, ']']);
211        } else {
212            escaped.push(c);
213        }
214    }
215    escaped
216}
217
218/// Whether a path is an object-store location scanned in place with range requests
219/// rather than downloaded: Parquet, or a prefix or glob of it. Keeps a downloaded object
220/// (reaching the schema phase under its URL) from being treated as remote.
221pub(crate) fn scans_in_place(path: &Path) -> bool {
222    if !matches!(
223        input_source(path),
224        InputSource::S3(_) | InputSource::Gcs(_) | InputSource::Azure(_)
225    ) {
226        return false;
227    }
228    let url = path.to_string_lossy();
229    let (_, ext) = url_path_extension(&url);
230    !cloud_path_should_download(ext.as_deref(), is_prefix_or_glob(&url))
231}
232
233/// Returns the path segment and file extension for URL format inference.
234/// For S3, path part is everything after `://` (bucket/key). For HTTP/HTTPS, path part is the URL path only (host stripped).
235pub(crate) fn url_path_extension(url: &str) -> (String, Option<String>) {
236    let path_part = if let Some(i) = url.find("://") {
237        let scheme = url[..i].to_lowercase();
238        let after = &url[i + 3..];
239        if scheme == "http" || scheme == "https" {
240            after
241                .find('/')
242                .map(|j| after[j + 1..].to_string())
243                .unwrap_or_default()
244        } else {
245            after.to_string()
246        }
247    } else {
248        String::new()
249    };
250    let last_segment = path_part.rsplit('/').next().unwrap_or(&path_part);
251    let ext = std::path::Path::new(last_segment)
252        .extension()
253        .and_then(|e| e.to_str())
254        .map(String::from);
255    (path_part, ext)
256}
257
258/// The extension a downloaded copy of `url` should keep, so it opens as what it is:
259/// `csv.gz` rather than `gz` for a compressed file, since a temporary `.gz` does not
260/// say what is inside it.
261#[cfg(any(feature = "http", feature = "cloud"))]
262pub(crate) fn download_suffix(url: &str) -> Option<String> {
263    let (path_part, ext) = url_path_extension(url);
264    let ext = ext?;
265    const COMPRESSION: [&str; 6] = ["gz", "zst", "bz2", "xz", "lz4", "zip"];
266    if !COMPRESSION.iter().any(|c| ext.eq_ignore_ascii_case(c)) {
267        return Some(ext);
268    }
269    let name = path_part.rsplit('/').next().unwrap_or(&path_part);
270    let stem = &name[..name.len() - ext.len() - 1];
271    match Path::new(stem).extension().and_then(|e| e.to_str()) {
272        Some(inner) => Some(format!("{inner}.{ext}")),
273        None => Some(ext),
274    }
275}
276
277/// For S3/GCS, whether a path must be downloaded first: Polars scans only Parquet
278/// directly, so only Parquet or directory-like paths (no extension, trailing slash,
279/// glob) pass through; `.csv`, `.json`, `.gz` and the like download.
280pub(crate) fn cloud_path_should_download(ext: Option<&str>, is_glob: bool) -> bool {
281    if is_glob {
282        return false;
283    }
284    match ext {
285        None => false,
286        Some(e) => !e.eq_ignore_ascii_case("parquet"),
287    }
288}
289
290#[cfg(test)]
291mod tests;
datui_lib/cloud/source.rs

datui_lib/cloud/
source.rs
