lance_io/object_store/providers/
local.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright The Lance Authors
3
4use std::{collections::HashMap, sync::Arc};
5
6use crate::object_store::{
7    ObjectStore, ObjectStoreParams, ObjectStoreProvider, StorageOptions, DEFAULT_LOCAL_BLOCK_SIZE,
8    DEFAULT_LOCAL_IO_PARALLELISM, DEFAULT_MAX_IOP_SIZE,
9};
10use lance_core::error::Result;
11use lance_core::Error;
12use object_store::{local::LocalFileSystem, path::Path};
13use snafu::location;
14use url::Url;
15
16#[derive(Default, Debug)]
17pub struct FileStoreProvider;
18
19#[async_trait::async_trait]
20impl ObjectStoreProvider for FileStoreProvider {
21    async fn new_store(&self, base_path: Url, params: &ObjectStoreParams) -> Result<ObjectStore> {
22        let block_size = params.block_size.unwrap_or(DEFAULT_LOCAL_BLOCK_SIZE);
23        let storage_options = StorageOptions(params.storage_options.clone().unwrap_or_default());
24        let download_retry_count = storage_options.download_retry_count();
25        Ok(ObjectStore {
26            inner: Arc::new(LocalFileSystem::new()),
27            scheme: base_path.scheme().to_owned(),
28            block_size,
29            max_iop_size: *DEFAULT_MAX_IOP_SIZE,
30            use_constant_size_upload_parts: false,
31            list_is_lexically_ordered: false,
32            io_parallelism: DEFAULT_LOCAL_IO_PARALLELISM,
33            download_retry_count,
34            io_tracker: Default::default(),
35        })
36    }
37
38    fn extract_path(&self, url: &Url) -> Result<Path> {
39        if let Ok(file_path) = url.to_file_path() {
40            if let Ok(path) = Path::from_absolute_path(&file_path) {
41                return Ok(path);
42            }
43        }
44
45        Path::parse(url.path()).map_err(|e| {
46            Error::invalid_input(
47                format!("Failed to parse path '{}': {}", url.path(), e),
48                location!(),
49            )
50        })
51    }
52
53    fn calculate_object_store_prefix(
54        &self,
55        url: &Url,
56        _storage_options: Option<&HashMap<String, String>>,
57    ) -> Result<String> {
58        Ok(url.scheme().to_string())
59    }
60}
61
62#[cfg(test)]
63mod tests {
64    use crate::object_store::uri_to_url;
65
66    use super::*;
67
68    #[test]
69    fn test_file_store_path() {
70        let provider = FileStoreProvider;
71
72        let cases = [
73            ("file:///", ""),
74            ("file:///usr/local/bin", "usr/local/bin"),
75            ("file-object-store:///path/to/file", "path/to/file"),
76            ("file:///path/to/foo/../bar", "path/to/bar"),
77        ];
78
79        for (uri, expected_path) in cases {
80            let url = uri_to_url(uri).unwrap();
81            let path = provider.extract_path(&url).unwrap();
82            assert_eq!(path.as_ref(), expected_path, "uri: '{}'", uri);
83        }
84    }
85
86    #[test]
87    fn test_calculate_object_store_prefix() {
88        let provider = FileStoreProvider;
89        assert_eq!(
90            "file",
91            provider
92                .calculate_object_store_prefix(&Url::parse("file:///etc").unwrap(), None)
93                .unwrap()
94        );
95    }
96
97    #[test]
98    fn test_calculate_object_store_prefix_for_file_object_store() {
99        let provider = FileStoreProvider;
100        assert_eq!(
101            "file-object-store",
102            provider
103                .calculate_object_store_prefix(
104                    &Url::parse("file-object-store:///etc").unwrap(),
105                    None
106                )
107                .unwrap()
108        );
109    }
110
111    #[test]
112    #[cfg(windows)]
113    fn test_file_store_path_windows() {
114        let provider = FileStoreProvider;
115
116        let cases = [
117            (
118                "C:\\Users\\ADMINI~1\\AppData\\Local\\",
119                "C:/Users/ADMINI~1/AppData/Local",
120            ),
121            (
122                "C:\\Users\\ADMINI~1\\AppData\\Local\\..\\",
123                "C:/Users/ADMINI~1/AppData",
124            ),
125        ];
126
127        for (uri, expected_path) in cases {
128            let url = uri_to_url(uri).unwrap();
129            let path = provider.extract_path(&url).unwrap();
130            assert_eq!(path.as_ref(), expected_path);
131        }
132    }
133}
lance_io/object_store/providers/local.rs

lance_io/object_store/providers/
local.rs
