vortex_io/filesystem/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4//! A filesystem abstraction for discovering and opening Vortex files.
5//!
6//! [`FileSystem`] provides a storage-agnostic interface for listing files under a prefix
7//! and opening them for reading. Implementations can target local filesystems, object stores,
8//! or any other storage backend.
9
10mod glob;
11mod prefix;
12
13use std::fmt::Debug;
14use std::sync::Arc;
15
16use async_trait::async_trait;
17use futures::stream::BoxStream;
18use vortex_error::VortexResult;
19
20use crate::VortexReadAt;
21
22/// A file discovered during listing, with its path and optional size in bytes.
23#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
24pub struct FileListing {
25    /// The file path (relative to the filesystem root).
26    pub path: String,
27    /// The file size in bytes, if known from the listing metadata.
28    pub size: Option<u64>,
29}
30
31/// A reference-counted handle to a file system.
32pub type FileSystemRef = Arc<dyn FileSystem>;
33
34/// A storage-agnostic filesystem interface for discovering and reading Vortex files.
35///
36/// Implementations handle the details of a particular storage backend (local disk, S3, GCS, etc.)
37/// while consumers work through this uniform interface.
38///
39/// # Paths
40///
41/// Path strings are *literal* object keys / file paths: the characters are used verbatim, with no
42/// shell-style `~` expansion (`~` is a literal tilde, not the home directory) and no
43/// percent-encoding or -decoding applied by this layer (`%20` is the three characters `%`, `2`,
44/// `0`, not a space). A path produced by [`list`](FileSystem::list) or [`head`](FileSystem::head)
45/// is the object's actual key, so it can be passed straight back to
46/// [`open_read`](FileSystem::open_read) — including when it contains characters such as `~`, `%`,
47/// `[`, `]`, or `#`.
48///
49/// # Future Work
50///
51/// An `open_write` method will be added once [`VortexWrite`](crate::VortexWrite) is
52/// object-safe (it currently uses `impl Future` return types which prevent trait-object usage).
53#[async_trait]
54pub trait FileSystem: Debug + Send + Sync {
55    /// Recursively list files whose paths start with `prefix`.
56    ///
57    /// When `prefix` is empty, all files are listed. Implementations must recurse into
58    /// subdirectories so that the returned stream contains every file reachable under the prefix.
59    ///
60    /// Returns a stream of [`FileListing`] entries. The stream may yield entries in any order;
61    /// callers should sort if deterministic ordering is required.
62    fn list(&self, prefix: &str) -> BoxStream<'_, VortexResult<FileListing>>;
63
64    /// Fetch metadata for the file at the exact `path`, if it exists.
65    ///
66    /// Unlike [`list`](FileSystem::list), which enumerates files *under* a prefix on a
67    /// path-segment basis and never yields the prefix itself, `head` looks up the object at
68    /// exactly `path`. It is the correct primitive for confirming that a single known file
69    /// exists and reading its size.
70    ///
71    /// Returns `Ok(Some(_))` with the file's [`FileListing`] when it exists, `Ok(None)` when no
72    /// file exists at `path`, and `Err(_)` for any other failure (I/O or permission errors, etc.).
73    async fn head(&self, path: &str) -> VortexResult<Option<FileListing>>;
74
75    /// Open a file for reading at the given path.
76    async fn open_read(&self, path: &str) -> VortexResult<Arc<dyn VortexReadAt>>;
77
78    async fn delete(&self, path: &str) -> VortexResult<()>;
79}
vortex_io/filesystem/mod.rs

vortex_io/filesystem/
mod.rs
