ursa_engine/
paths.rs

1//! Turning a name in `FROM` into a file location.
2//!
3//! `SELECT * FROM 'edges.parquet'` names a file where a table would go. A name
4//! that reads as a location (it has a scheme, a path separator, an extension or
5//! a glob) is opened as a listing table; anything else is left to the catalog,
6//! so a misspelt table name still reports "table not found".
7//!
8//! DataFusion's [`ListingTableUrl::parse`] expands a glob only in a scheme-less
9//! local path. [`table_url`] also expands one in a URL (`s3://bucket/dir/*.parquet`,
10//! `file:///data/*.csv`): the URL is split at the path segment where the glob
11//! begins, the part before it becomes the listing prefix, and the rest filters
12//! the listing.
13
14use datafusion::datasource::listing::ListingTableUrl;
15use datafusion::error::{DataFusionError, Result};
16use glob::Pattern;
17use url::Url;
18
19/// The characters that start a glob, as DataFusion's local-path parser has them.
20const GLOB_START_CHARS: [char; 3] = ['*', '?', '['];
21
22/// Whether a table name reads as a file location rather than a table name.
23pub fn looks_like_location(name: &str) -> bool {
24    name.contains(['/', '\\', '.']) || name.contains(GLOB_START_CHARS) || name.starts_with('~')
25}
26
27/// `~` or `~/…` with the home directory put in for the `~`.
28pub fn expand_tilde(location: &str) -> String {
29    if location == "~" || location.starts_with("~/") {
30        if let Some(home) = std::env::var_os("HOME").and_then(|h| h.into_string().ok()) {
31            if !home.is_empty() {
32                return format!("{home}{}", &location[1..]);
33            }
34        }
35    }
36    location.to_string()
37}
38
39/// The listing table URL for a location: a local path (with globs, as DataFusion
40/// expands them), or a URL whose path may contain a glob.
41pub fn table_url(location: &str) -> Result<ListingTableUrl> {
42    let location = expand_tilde(location);
43    #[cfg(target_arch = "wasm32")]
44    let location = browser::resolve(&location)?;
45    if let Some((base, pattern)) = split_url_glob(&location) {
46        let url = Url::parse(base)
47            .map_err(|e| DataFusionError::Plan(format!("'{location}' is not a valid URL: {e}")))?;
48        let glob = Pattern::new(pattern).map_err(|e| {
49            DataFusionError::Plan(format!(
50                "'{pattern}' in '{location}' is not a valid glob: {e}"
51            ))
52        })?;
53        return ListingTableUrl::try_new(url, Some(glob));
54    }
55    ListingTableUrl::parse(&location)
56        .map_err(|e| DataFusionError::Plan(format!("'{location}' is not a valid path or URL: {e}")))
57}
58
59/// For a URL with a scheme, split it at the start of the path segment that holds
60/// the first glob character: `s3://b/data/*.parquet` gives `("s3://b/data/",
61/// "*.parquet")`. `None` for a local path (DataFusion expands its globs itself),
62/// a URL without a glob, and `http(s)` URLs, where `?` starts a query and a
63/// server cannot be listed anyway.
64pub fn split_url_glob(location: &str) -> Option<(&str, &str)> {
65    let scheme_end = location.find("://")?;
66    let scheme = &location[..scheme_end];
67    // A one-letter "scheme" is a Windows drive letter.
68    if scheme.len() < 2
69        || !scheme
70            .chars()
71            .all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.'))
72        || matches!(scheme, "http" | "https")
73    {
74        return None;
75    }
76    let authority_start = scheme_end + 3;
77    let path_start = authority_start + location[authority_start..].find('/')?;
78    let path = &location[path_start..];
79    let glob_at = path.find(GLOB_START_CHARS)?;
80    // `path` starts with '/', so there is always a separator before the glob.
81    let split = path_start + path[..glob_at].rfind('/')? + 1;
82    Some((&location[..split], &location[split..]))
83}
84
85/// Relative paths in the browser (wasm32).
86///
87/// The browser has no current directory, so a relative path such as
88/// `'karate_edges.csv'` resolves against a base URL the page sets, usually the
89/// directory its data is served from. The Getting Started SQL then runs there
90/// as written (#278).
91#[cfg(target_arch = "wasm32")]
92pub mod browser {
93    use std::sync::RwLock;
94
95    use datafusion::error::{DataFusionError, Result};
96    use url::Url;
97
98    static BASE_URL: RwLock<Option<Url>> = RwLock::new(None);
99
100    /// Set (or clear) the URL relative paths resolve against. A base that
101    /// names a directory should end in `/`, as in a browser.
102    pub fn set_base_url(base: Option<Url>) {
103        *BASE_URL.write().unwrap_or_else(|e| e.into_inner()) = base;
104    }
105
106    /// The URL relative paths resolve against, if one is set.
107    pub fn base_url() -> Option<Url> {
108        BASE_URL.read().unwrap_or_else(|e| e.into_inner()).clone()
109    }
110
111    /// `location` as a URL: unchanged if it has a scheme, else joined to the
112    /// base URL.
113    pub(crate) fn resolve(location: &str) -> Result<String> {
114        if matches!(Url::parse(location), Ok(url) if url.scheme().len() > 1) {
115            return Ok(location.to_string());
116        }
117        let base = base_url().ok_or_else(|| {
118            DataFusionError::Plan(format!(
119                "'{location}' is a relative path, and the browser has no current directory; \
120                 use an http:// or https:// URL"
121            ))
122        })?;
123        base.join(location).map(String::from).map_err(|e| {
124            DataFusionError::Plan(format!("'{location}' is not a valid path or URL: {e}"))
125        })
126    }
127}
128
129#[cfg(test)]
130mod tests {
131    use super::*;
132
133    #[test]
134    fn locations_and_table_names() {
135        assert!(looks_like_location("edges.csv"));
136        assert!(looks_like_location("data/edges"));
137        assert!(looks_like_location("s3://bucket/x"));
138        assert!(looks_like_location("*.parquet"));
139        assert!(looks_like_location("~/edges"));
140        assert!(!looks_like_location("edges"));
141        assert!(!looks_like_location("my_table"));
142    }
143
144    #[test]
145    fn splits_url_globs() {
146        assert_eq!(
147            split_url_glob("s3://bucket/data/*.parquet"),
148            Some(("s3://bucket/data/", "*.parquet"))
149        );
150        assert_eq!(
151            split_url_glob("gs://bucket/a/b/part-?.csv"),
152            Some(("gs://bucket/a/b/", "part-?.csv"))
153        );
154        assert_eq!(
155            split_url_glob("s3://bucket/year=*/x.parquet"),
156            Some(("s3://bucket/", "year=*/x.parquet"))
157        );
158        assert_eq!(
159            split_url_glob("file:///tmp/x/[ab].csv"),
160            Some(("file:///tmp/x/", "[ab].csv"))
161        );
162        assert_eq!(split_url_glob("s3://bucket/data/x.parquet"), None);
163        assert_eq!(split_url_glob("https://host/x.csv?y=1"), None);
164        assert_eq!(split_url_glob("data/*.csv"), None);
165        assert_eq!(split_url_glob("C://data/*.csv"), None);
166    }
167
168    #[test]
169    fn remote_glob_url_lists_its_prefix() {
170        let url = table_url("s3://bucket/data/*.parquet").unwrap();
171        assert_eq!(url.scheme(), "s3");
172        assert_eq!(url.prefix().as_ref(), "data");
173        assert!(url.is_collection());
174        assert_eq!(url.get_glob().as_ref().unwrap().as_str(), "*.parquet");
175    }
176}
ursa_engine/paths.rs

ursa_engine/
paths.rs
