vortex_io/filesystem/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4//! A filesystem abstraction for discovering and opening Vortex files.
5//!
6//! [`FileSystem`] provides a storage-agnostic interface for listing files under a prefix
7//! and opening them for reading. Implementations can target local filesystems, object stores,
8//! or any other storage backend.
9
10mod glob;
11mod prefix;
12
13use std::fmt::Debug;
14use std::sync::Arc;
15
16use async_trait::async_trait;
17use futures::stream::BoxStream;
18use vortex_error::VortexResult;
19
20use crate::VortexReadAt;
21
22/// A file discovered during listing, with its path and optional size in bytes.
23#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
24pub struct FileListing {
25    /// The file path (relative to the filesystem root).
26    pub path: String,
27    /// The file size in bytes, if known from the listing metadata.
28    pub size: Option<u64>,
29}
30
31/// A reference-counted handle to a file system.
32pub type FileSystemRef = Arc<dyn FileSystem>;
33
34/// A storage-agnostic filesystem interface for discovering and reading Vortex files.
35///
36/// Implementations handle the details of a particular storage backend (local disk, S3, GCS, etc.)
37/// while consumers work through this uniform interface.
38///
39/// # Future Work
40///
41/// An `open_write` method will be added once [`VortexWrite`](crate::VortexWrite) is
42/// object-safe (it currently uses `impl Future` return types which prevent trait-object usage).
43#[async_trait]
44pub trait FileSystem: Debug + Send + Sync {
45    /// Recursively list files whose paths start with `prefix`.
46    ///
47    /// When `prefix` is empty, all files are listed. Implementations must recurse into
48    /// subdirectories so that the returned stream contains every file reachable under the prefix.
49    ///
50    /// Returns a stream of [`FileListing`] entries. The stream may yield entries in any order;
51    /// callers should sort if deterministic ordering is required.
52    fn list(&self, prefix: &str) -> BoxStream<'_, VortexResult<FileListing>>;
53
54    /// Open a file for reading at the given path.
55    async fn open_read(&self, path: &str) -> VortexResult<Arc<dyn VortexReadAt>>;
56}
vortex_io/filesystem/mod.rs

vortex_io/filesystem/
mod.rs
