htsget_storage/
local.rs

1//! Module providing an implementation for the [StorageTrait] trait using the local file system.
2//!
3
4use std::fmt::Debug;
5use std::io::{ErrorKind, SeekFrom};
6use std::path::{Path, PathBuf};
7
8use super::{GetOptions, RangeUrlOptions, Result, StorageError};
9use crate::{HeadOptions, StorageMiddleware, StorageTrait, UrlFormatter};
10use crate::{Streamable, Url as HtsGetUrl};
11use async_trait::async_trait;
12use htsget_config::error;
13use http::HeaderMap;
14use tokio::fs;
15use tokio::fs::File;
16use tokio::io::{AsyncReadExt, AsyncSeekExt};
17use tracing::debug;
18use tracing::instrument;
19use url::Url;
20
21/// Implementation for the [StorageTrait] trait using the local file system. [T] is the type of the
22/// server struct, which is used for formatting urls.
23#[derive(Debug, Clone)]
24pub struct FileStorage<T> {
25  base_path: PathBuf,
26  url_formatter: T,
27  ticket_headers: Vec<String>,
28}
29
30impl<T: UrlFormatter + Send + Sync> FileStorage<T> {
31  pub fn new<P: AsRef<Path>>(
32    base_path: P,
33    url_formatter: T,
34    ticket_headers: Vec<String>,
35  ) -> Result<Self> {
36    base_path
37      .as_ref()
38      .to_path_buf()
39      .canonicalize()
40      .map_err(|_| StorageError::KeyNotFound(base_path.as_ref().to_string_lossy().to_string()))
41      .map(|canonicalized_base_path| Self {
42        base_path: canonicalized_base_path,
43        url_formatter,
44        ticket_headers,
45      })
46  }
47
48  pub fn base_path(&self) -> &Path {
49    self.base_path.as_path()
50  }
51
52  pub(crate) fn get_path_from_key<K: AsRef<str>>(&self, key: K) -> Result<PathBuf> {
53    let key: &str = key.as_ref();
54
55    self
56      .base_path
57      .join(key)
58      .canonicalize()
59      .map_err(|err| {
60        if let ErrorKind::NotFound = err.kind() {
61          StorageError::KeyNotFound(key.to_string())
62        } else {
63          StorageError::InvalidKey(key.to_string())
64        }
65      })
66      .and_then(|path| {
67        path
68          .starts_with(&self.base_path)
69          .then_some(path)
70          .ok_or_else(|| StorageError::InvalidKey(key.to_string()))
71      })
72      .and_then(|path| {
73        path
74          .is_file()
75          .then_some(path)
76          .ok_or_else(|| StorageError::KeyNotFound(key.to_string()))
77      })
78  }
79
80  pub async fn get<K: AsRef<str>>(&self, key: K) -> Result<File> {
81    let path = self.get_path_from_key(&key)?;
82    File::open(path)
83      .await
84      .map_err(|_| StorageError::KeyNotFound(key.as_ref().to_string()))
85  }
86}
87
88#[async_trait]
89impl<T: UrlFormatter + Send + Sync + Debug> StorageMiddleware for FileStorage<T> {}
90
91#[async_trait]
92impl<T: UrlFormatter + Send + Sync + Debug + Clone + 'static> StorageTrait for FileStorage<T> {
93  /// Get the file at the location of the key.
94  #[instrument(level = "debug", skip(self))]
95  async fn get(&self, key: &str, options: GetOptions<'_>) -> Result<Streamable> {
96    debug!(calling_from = ?self, key = key, "getting file with key {:?}", key);
97
98    // Need to ensure range options are considered for local files.
99    let mut file = self.get(key).await?;
100    let start = options.range().get_start().unwrap_or(0);
101    let seek = file.seek(SeekFrom::Start(start)).await?;
102
103    if let Some(end) = options.range().get_end() {
104      let file = file.take(end.checked_sub(seek).ok_or_else(|| {
105        StorageError::InternalError("subtraction overflow in local storage get".to_string())
106      })?);
107      Ok(Streamable::from_async_read(file))
108    } else {
109      Ok(Streamable::from_async_read(file))
110    }
111  }
112
113  /// Get a url for the file at key.
114  #[instrument(level = "debug", skip(self))]
115  async fn range_url(&self, key: &str, options: RangeUrlOptions<'_>) -> Result<HtsGetUrl> {
116    let path = self.get_path_from_key(key)?;
117
118    let base_url = Url::from_file_path(&self.base_path)
119      .map_err(|_| StorageError::UrlParseError("failed to parse base path as url".to_string()))?;
120    let path_url = Url::from_file_path(path)
121      .map_err(|_| StorageError::UrlParseError("failed to parse key path as url".to_string()))?;
122
123    // Get the difference between the two URLs and strip and leading slashes.
124    let path = path_url
125      .path()
126      .strip_prefix(base_url.path())
127      .ok_or_else(|| {
128        StorageError::UrlParseError("failed parse relative component of key path url".to_string())
129      })?;
130    let path = path.trim_start_matches('/');
131
132    let mut url = HtsGetUrl::new(self.url_formatter.format_url(path)?);
133    if !self.ticket_headers.is_empty() {
134      let headers = &HeaderMap::from_iter(options.response_headers().iter().filter_map(
135        |(name, value)| {
136          if self.ticket_headers.contains(&name.to_string()) {
137            Some((name.clone(), value.clone()))
138          } else {
139            None
140          }
141        },
142      ));
143      url = url.with_headers(
144        headers
145          .try_into()
146          .map_err(|err: error::Error| StorageError::InvalidInput(err.to_string()))?,
147      );
148    }
149    let url = options.apply(url)?;
150
151    debug!(calling_from = ?self, key = key, ?url, "getting url with key {:?}", key);
152
153    Ok(url)
154  }
155
156  /// Get the size of the file.
157  #[instrument(level = "debug", skip(self))]
158  async fn head(&self, key: &str, _options: HeadOptions<'_>) -> Result<u64> {
159    let path = self.get_path_from_key(key)?;
160    let len = fs::metadata(path)
161      .await
162      .map_err(|err| StorageError::KeyNotFound(err.to_string()))?
163      .len();
164
165    debug!(calling_from = ?self, key = key, len, "size of key {:?} is {}", key, len);
166    Ok(len)
167  }
168}
169
170#[cfg(test)]
171pub(crate) mod tests {
172  use std::future::Future;
173  use std::matches;
174
175  use htsget_config::storage;
176  use htsget_config::types::Scheme;
177  use http::uri::Authority;
178  use tempfile::TempDir;
179  use tokio::fs::{create_dir, write};
180
181  use super::*;
182  use crate::types::BytesPosition;
183  use crate::{GetOptions, RangeUrlOptions, StorageError};
184  use crate::{Headers, Url};
185
186  #[tokio::test]
187  async fn get_non_existing_key() {
188    with_local_storage(|storage, _| async move {
189      let result = storage.get("non-existing-key").await;
190      assert!(matches!(result, Err(StorageError::KeyNotFound(msg)) if msg == "non-existing-key"));
191    })
192    .await;
193  }
194
195  #[tokio::test]
196  async fn get_folder() {
197    with_local_storage(|storage, _| async move {
198      let result = StorageTrait::get(
199        &storage,
200        "folder",
201        GetOptions::new_with_default_range(&Default::default()),
202      )
203      .await;
204      assert!(matches!(result, Err(StorageError::KeyNotFound(msg)) if msg == "folder"));
205    })
206    .await;
207  }
208
209  #[tokio::test]
210  async fn get_forbidden_path() {
211    with_local_storage(|storage, _| async move {
212      let result = StorageTrait::get(
213        &storage,
214        "folder/../../passwords",
215        GetOptions::new_with_default_range(&Default::default()),
216      )
217      .await;
218      assert!(
219        matches!(result, Err(StorageError::KeyNotFound(msg)) if msg == "folder/../../passwords")
220      );
221    })
222    .await;
223  }
224
225  #[tokio::test]
226  async fn get_existing_key() {
227    with_local_storage(|storage, _| async move {
228      let result = StorageTrait::get(
229        &storage,
230        "folder/../key1",
231        GetOptions::new_with_default_range(&Default::default()),
232      )
233      .await;
234      assert!(result.is_ok());
235    })
236    .await;
237  }
238
239  #[tokio::test]
240  async fn url_of_non_existing_key() {
241    with_local_storage(|storage, _| async move {
242      let result = StorageTrait::range_url(
243        &storage,
244        "non-existing-key",
245        RangeUrlOptions::new_with_default_range(&Default::default()),
246      )
247      .await;
248      assert!(matches!(result, Err(StorageError::KeyNotFound(msg)) if msg == "non-existing-key"));
249    })
250    .await;
251  }
252
253  #[tokio::test]
254  async fn url_of_folder() {
255    with_local_storage(|storage, _| async move {
256      let result = StorageTrait::range_url(
257        &storage,
258        "folder",
259        RangeUrlOptions::new_with_default_range(&Default::default()),
260      )
261      .await;
262      assert!(matches!(result, Err(StorageError::KeyNotFound(msg)) if msg == "folder"));
263    })
264    .await;
265  }
266
267  #[tokio::test]
268  async fn url_with_forbidden_path() {
269    with_local_storage(|storage, _| async move {
270      let result = StorageTrait::range_url(
271        &storage,
272        "folder/../../passwords",
273        RangeUrlOptions::new_with_default_range(&Default::default()),
274      )
275      .await;
276      assert!(
277        matches!(result, Err(StorageError::KeyNotFound(msg)) if msg == "folder/../../passwords")
278      );
279    })
280    .await;
281  }
282
283  #[tokio::test]
284  async fn url_of_existing_key() {
285    with_local_storage(|storage, _| async move {
286      let result = StorageTrait::range_url(
287        &storage,
288        "folder/../key1",
289        RangeUrlOptions::new_with_default_range(&Default::default()),
290      )
291      .await;
292      let expected = Url::new("http://127.0.0.1:8081/key1");
293      assert!(matches!(result, Ok(url) if url == expected));
294    })
295    .await;
296  }
297
298  #[tokio::test]
299  async fn url_of_existing_key_with_specified_range() {
300    with_local_storage(|storage, _| async move {
301      let result = StorageTrait::range_url(
302        &storage,
303        "folder/../key1",
304        RangeUrlOptions::new(
305          BytesPosition::builder()
306            .with_start(7)
307            .with_end(10)
308            .build()
309            .unwrap(),
310          &Default::default(),
311        ),
312      )
313      .await;
314      let expected = Url::new("http://127.0.0.1:8081/key1")
315        .with_headers(Headers::default().with_header("Range", "bytes=7-9"));
316      assert!(matches!(result, Ok(url) if url == expected));
317    })
318    .await;
319  }
320
321  #[tokio::test]
322  async fn url_of_existing_key_with_specified_open_ended_range() {
323    with_local_storage(|storage, _| async move {
324      let result = StorageTrait::range_url(
325        &storage,
326        "folder/../key1",
327        RangeUrlOptions::new(
328          BytesPosition::builder().with_start(7).build().unwrap(),
329          &Default::default(),
330        ),
331      )
332      .await;
333      let expected = Url::new("http://127.0.0.1:8081/key1")
334        .with_headers(Headers::default().with_header("Range", "bytes=7-"));
335      assert!(matches!(result, Ok(url) if url == expected));
336    })
337    .await;
338  }
339
340  #[tokio::test]
341  async fn file_size() {
342    with_local_storage(|storage, _| async move {
343      let result = StorageTrait::head(
344        &storage,
345        "folder/../key1",
346        HeadOptions::new(&Default::default()),
347      )
348      .await;
349      let expected: u64 = 6;
350      assert!(matches!(result, Ok(size) if size == expected));
351    })
352    .await;
353  }
354
355  pub(crate) async fn create_local_test_files() -> (String, TempDir) {
356    let base_path = TempDir::new().unwrap();
357
358    let folder_name = "folder";
359    let key1 = "key1";
360    let value1 = b"value1";
361    let key2 = "key2";
362    let value2 = b"value2";
363    write(base_path.path().join(key1), value1).await.unwrap();
364    create_dir(base_path.path().join(folder_name))
365      .await
366      .unwrap();
367    write(base_path.path().join(folder_name).join(key2), value2)
368      .await
369      .unwrap();
370
371    (folder_name.to_string(), base_path)
372  }
373
374  pub(crate) fn test_local_storage(base_path: &Path) -> FileStorage<storage::file::File> {
375    FileStorage::new(
376      base_path,
377      storage::file::File::new(
378        Scheme::Http,
379        Authority::from_static("127.0.0.1:8081"),
380        "data".to_string(),
381      ),
382      vec![],
383    )
384    .unwrap()
385  }
386
387  pub(crate) async fn with_local_storage<F, Fut>(test: F)
388  where
389    F: FnOnce(FileStorage<storage::file::File>, PathBuf) -> Fut,
390    Fut: Future<Output = ()>,
391  {
392    let (_, base_path) = create_local_test_files().await;
393    test(
394      test_local_storage(base_path.path()),
395      base_path.path().to_path_buf(),
396    )
397    .await
398  }
399}
htsget_storage/local.rs

htsget_storage/
local.rs
