use std::io;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::SystemTime;
pub mod memory;
pub use memory::InMemoryFs;
pub trait ReadStorage {
fn read(&self, path: &Path) -> impl Future<Output = io::Result<Vec<u8>>>;
fn read_to_string(&self, path: &Path) -> impl Future<Output = io::Result<String>>;
fn read_dir(&self, path: &Path) -> impl Future<Output = io::Result<Vec<DirEntry>>>;
fn metadata(&self, path: &Path) -> impl Future<Output = io::Result<Metadata>>;
fn try_exists(&self, path: &Path) -> impl Future<Output = io::Result<bool>> {
async move {
match self.metadata(path).await {
Ok(_) => Ok(true),
Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false),
Err(e) => Err(e),
}
}
}
fn executable(&self, path: &Path) -> impl Future<Output = io::Result<Option<bool>>> {
async move {
let _ = path;
Ok(None)
}
}
fn read_link(&self, path: &Path) -> impl Future<Output = io::Result<Option<PathBuf>>> {
async move {
let _ = path;
Ok(None)
}
}
}
impl<S: ReadStorage + ?Sized> ReadStorage for &S {
async fn read(&self, path: &Path) -> io::Result<Vec<u8>> {
(**self).read(path).await
}
async fn read_to_string(&self, path: &Path) -> io::Result<String> {
(**self).read_to_string(path).await
}
async fn read_dir(&self, path: &Path) -> io::Result<Vec<DirEntry>> {
(**self).read_dir(path).await
}
async fn metadata(&self, path: &Path) -> io::Result<Metadata> {
(**self).metadata(path).await
}
async fn try_exists(&self, path: &Path) -> io::Result<bool> {
(**self).try_exists(path).await
}
async fn executable(&self, path: &Path) -> io::Result<Option<bool>> {
(**self).executable(path).await
}
async fn read_link(&self, path: &Path) -> io::Result<Option<PathBuf>> {
(**self).read_link(path).await
}
}
impl<S: ReadStorage + ?Sized> ReadStorage for Arc<S> {
async fn read(&self, path: &Path) -> io::Result<Vec<u8>> {
(**self).read(path).await
}
async fn read_to_string(&self, path: &Path) -> io::Result<String> {
(**self).read_to_string(path).await
}
async fn read_dir(&self, path: &Path) -> io::Result<Vec<DirEntry>> {
(**self).read_dir(path).await
}
async fn metadata(&self, path: &Path) -> io::Result<Metadata> {
(**self).metadata(path).await
}
async fn try_exists(&self, path: &Path) -> io::Result<bool> {
(**self).try_exists(path).await
}
async fn executable(&self, path: &Path) -> io::Result<Option<bool>> {
(**self).executable(path).await
}
async fn read_link(&self, path: &Path) -> io::Result<Option<PathBuf>> {
(**self).read_link(path).await
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DirEntry {
path: PathBuf,
file_type: FileType,
}
impl DirEntry {
pub fn new(path: impl Into<PathBuf>, file_type: FileType) -> Self {
Self {
path: path.into(),
file_type,
}
}
pub fn path(&self) -> &Path {
&self.path
}
pub fn file_name(&self) -> Option<&std::ffi::OsStr> {
self.path.file_name()
}
pub fn file_type(&self) -> FileType {
self.file_type
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Metadata {
file_type: FileType,
len: u64,
modified: Option<SystemTime>,
}
impl Metadata {
pub fn new(file_type: FileType, len: u64, modified: Option<SystemTime>) -> Self {
Self {
file_type,
len,
modified,
}
}
pub fn file_type(&self) -> FileType {
self.file_type
}
pub fn is_file(&self) -> bool {
self.file_type.is_file()
}
pub fn is_dir(&self) -> bool {
self.file_type.is_dir()
}
pub fn len(&self) -> u64 {
self.len
}
pub fn is_empty(&self) -> bool {
self.len == 0
}
pub fn modified(&self) -> io::Result<SystemTime> {
self.modified
.ok_or_else(|| io::Error::new(io::ErrorKind::Unsupported, "modified time unavailable"))
}
}
#[derive(Debug, Clone, Copy, Default)]
pub struct StdFs;
impl ReadStorage for StdFs {
async fn read(&self, path: &Path) -> io::Result<Vec<u8>> {
std::fs::read(path)
}
async fn read_to_string(&self, path: &Path) -> io::Result<String> {
std::fs::read_to_string(path)
}
async fn read_dir(&self, path: &Path) -> io::Result<Vec<DirEntry>> {
std::fs::read_dir(path)?
.map(|entry| {
let entry = entry?;
Ok(DirEntry::new(
entry.path(),
convert_file_type(entry.file_type()?),
))
})
.collect()
}
async fn metadata(&self, path: &Path) -> io::Result<Metadata> {
let md = std::fs::metadata(path)?;
Ok(Metadata::new(
convert_file_type(md.file_type()),
md.len(),
md.modified().ok(),
))
}
#[cfg(unix)]
async fn executable(&self, path: &Path) -> io::Result<Option<bool>> {
use std::os::unix::fs::PermissionsExt as _;
let md = std::fs::metadata(path)?;
Ok(Some(md.permissions().mode() & 0o111 != 0))
}
async fn read_link(&self, path: &Path) -> io::Result<Option<PathBuf>> {
std::fs::read_link(path).map(Some)
}
}
fn convert_file_type(ft: std::fs::FileType) -> FileType {
if ft.is_dir() {
FileType::DIR
} else if ft.is_file() {
FileType::FILE
} else {
FileType::SYMLINK
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct FileType {
is_dir: bool,
is_file: bool,
is_symlink: bool,
}
impl FileType {
pub const FILE: FileType = FileType {
is_dir: false,
is_file: true,
is_symlink: false,
};
pub const DIR: FileType = FileType {
is_dir: true,
is_file: false,
is_symlink: false,
};
pub const SYMLINK: FileType = FileType {
is_dir: false,
is_file: false,
is_symlink: true,
};
pub fn is_file(&self) -> bool {
self.is_file
}
pub fn is_dir(&self) -> bool {
self.is_dir
}
pub fn is_symlink(&self) -> bool {
self.is_symlink
}
}
pub trait Storage: ReadStorage {
fn write(&self, path: &Path, contents: &[u8]) -> impl Future<Output = io::Result<()>>;
fn create_new(&self, path: &Path, contents: &[u8]) -> impl Future<Output = io::Result<()>> {
async move {
let _ = (path, contents);
Err(io::Error::new(
io::ErrorKind::Unsupported,
"this backend does not support exclusive create",
))
}
}
fn create_dir_all(&self, path: &Path) -> impl Future<Output = io::Result<()>>;
fn remove_file(&self, path: &Path) -> impl Future<Output = io::Result<()>>;
fn remove_dir_all(&self, path: &Path) -> impl Future<Output = io::Result<()>>;
fn rename(&self, from: &Path, to: &Path) -> impl Future<Output = io::Result<()>>;
fn copy_permissions(&self, from: &Path, to: &Path) -> impl Future<Output = io::Result<()>> {
async move {
let _ = (from, to);
Ok(())
}
}
fn set_executable(
&self,
path: &Path,
executable: bool,
) -> impl Future<Output = io::Result<()>> {
async move {
let _ = (path, executable);
Ok(())
}
}
fn set_link(&self, path: &Path, target: &Path) -> impl Future<Output = io::Result<()>> {
async move {
let _ = (path, target);
Err(io::Error::new(
io::ErrorKind::Unsupported,
"this backend does not model symbolic links",
))
}
}
fn capabilities(&self) -> Capabilities {
Capabilities::NONE
}
fn sync(&self, path: &Path, need: Durability) -> impl Future<Output = io::Result<()>> {
async move {
let _ = (path, need);
Ok(())
}
}
fn replace(&self, path: &Path, contents: &[u8]) -> impl Future<Output = io::Result<()>> {
async move {
if !self.capabilities().atomic_replace {
return self.write(path, contents).await;
}
let tmp = temp_sibling(path);
let staged = async {
self.write(&tmp, contents).await?;
self.sync(&tmp, Durability::Ordered).await?;
self.copy_permissions(path, &tmp).await?;
self.rename(&tmp, path).await
}
.await;
match staged {
Ok(()) => Ok(()),
Err(e) => {
let _ = self.remove_file(&tmp).await;
Err(e)
}
}
}
}
fn write_atomic(&self, path: &Path, contents: &[u8]) -> impl Future<Output = io::Result<()>> {
async move {
self.replace(path, contents).await?;
if !self.capabilities().atomic_replace {
self.sync(path, Durability::Durable).await?;
}
match parent_dir(path) {
Some(dir) => self.sync(dir, Durability::Durable).await,
None => Ok(()),
}
}
}
}
impl<S: Storage + ?Sized> Storage for &S {
async fn write(&self, path: &Path, contents: &[u8]) -> io::Result<()> {
(**self).write(path, contents).await
}
async fn create_new(&self, path: &Path, contents: &[u8]) -> io::Result<()> {
(**self).create_new(path, contents).await
}
async fn create_dir_all(&self, path: &Path) -> io::Result<()> {
(**self).create_dir_all(path).await
}
async fn remove_file(&self, path: &Path) -> io::Result<()> {
(**self).remove_file(path).await
}
async fn remove_dir_all(&self, path: &Path) -> io::Result<()> {
(**self).remove_dir_all(path).await
}
async fn rename(&self, from: &Path, to: &Path) -> io::Result<()> {
(**self).rename(from, to).await
}
async fn copy_permissions(&self, from: &Path, to: &Path) -> io::Result<()> {
(**self).copy_permissions(from, to).await
}
async fn set_executable(&self, path: &Path, executable: bool) -> io::Result<()> {
(**self).set_executable(path, executable).await
}
async fn set_link(&self, path: &Path, target: &Path) -> io::Result<()> {
(**self).set_link(path, target).await
}
fn capabilities(&self) -> Capabilities {
(**self).capabilities()
}
async fn sync(&self, path: &Path, need: Durability) -> io::Result<()> {
(**self).sync(path, need).await
}
async fn replace(&self, path: &Path, contents: &[u8]) -> io::Result<()> {
(**self).replace(path, contents).await
}
async fn write_atomic(&self, path: &Path, contents: &[u8]) -> io::Result<()> {
(**self).write_atomic(path, contents).await
}
}
impl<S: Storage + ?Sized> Storage for Arc<S> {
async fn write(&self, path: &Path, contents: &[u8]) -> io::Result<()> {
(**self).write(path, contents).await
}
async fn create_new(&self, path: &Path, contents: &[u8]) -> io::Result<()> {
(**self).create_new(path, contents).await
}
async fn create_dir_all(&self, path: &Path) -> io::Result<()> {
(**self).create_dir_all(path).await
}
async fn remove_file(&self, path: &Path) -> io::Result<()> {
(**self).remove_file(path).await
}
async fn remove_dir_all(&self, path: &Path) -> io::Result<()> {
(**self).remove_dir_all(path).await
}
async fn rename(&self, from: &Path, to: &Path) -> io::Result<()> {
(**self).rename(from, to).await
}
async fn copy_permissions(&self, from: &Path, to: &Path) -> io::Result<()> {
(**self).copy_permissions(from, to).await
}
async fn set_executable(&self, path: &Path, executable: bool) -> io::Result<()> {
(**self).set_executable(path, executable).await
}
async fn set_link(&self, path: &Path, target: &Path) -> io::Result<()> {
(**self).set_link(path, target).await
}
fn capabilities(&self) -> Capabilities {
(**self).capabilities()
}
async fn sync(&self, path: &Path, need: Durability) -> io::Result<()> {
(**self).sync(path, need).await
}
async fn replace(&self, path: &Path, contents: &[u8]) -> io::Result<()> {
(**self).replace(path, contents).await
}
async fn write_atomic(&self, path: &Path, contents: &[u8]) -> io::Result<()> {
(**self).write_atomic(path, contents).await
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Capabilities {
pub atomic_replace: bool,
pub exclusive_create: bool,
pub sync_guarantee: SyncGuarantee,
pub native_transactions: bool,
}
impl Capabilities {
pub const NONE: Self = Self {
atomic_replace: false,
exclusive_create: false,
sync_guarantee: SyncGuarantee::None,
native_transactions: false,
};
pub const LOCAL_FS: Self = Self {
atomic_replace: true,
exclusive_create: true,
sync_guarantee: SyncGuarantee::Durable,
native_transactions: false,
};
pub const IN_MEMORY: Self = Self {
atomic_replace: true,
exclusive_create: true,
sync_guarantee: SyncGuarantee::None,
native_transactions: false,
};
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum Durability {
Ordered,
Durable,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum SyncGuarantee {
None,
Ordered,
Durable,
}
impl SyncGuarantee {
pub const fn satisfies(self, need: Durability) -> bool {
match need {
Durability::Ordered => !matches!(self, SyncGuarantee::None),
Durability::Durable => matches!(self, SyncGuarantee::Durable),
}
}
}
pub(crate) async fn create_dir_all_traced<FS: Storage>(
fs: &FS,
dir: &Path,
) -> io::Result<Vec<PathBuf>> {
let mut changed = Vec::new();
let mut cur = Some(dir);
while let Some(d) = cur {
if fs.try_exists(d).await? {
if !changed.is_empty() {
changed.push(d.to_path_buf());
}
break;
}
changed.push(d.to_path_buf());
cur = parent_dir(d);
}
if changed.is_empty() {
return Ok(changed);
}
fs.create_dir_all(dir).await?;
Ok(changed)
}
pub(crate) async fn flush_all_durable<FS: Storage>(
fs: &FS,
paths: impl IntoIterator<Item = PathBuf>,
anchor: &Path,
) -> io::Result<()> {
let mut owed = false;
for path in paths {
owed = true;
if path != anchor {
fs.sync(&path, Durability::Ordered).await?;
}
}
if owed {
fs.sync(anchor, Durability::Durable).await?;
}
Ok(())
}
pub(crate) fn parent_dir(path: &Path) -> Option<&Path> {
path.parent().filter(|p| !p.as_os_str().is_empty())
}
pub(crate) fn temp_sibling(path: &Path) -> PathBuf {
let name = path.file_name().and_then(|n| n.to_str()).unwrap_or("file");
path.with_file_name(format!(".{name}.fstx-tmp"))
}
impl Storage for StdFs {
async fn write(&self, path: &Path, contents: &[u8]) -> io::Result<()> {
std::fs::write(path, contents)
}
async fn create_new(&self, path: &Path, contents: &[u8]) -> io::Result<()> {
use std::io::Write as _;
let mut file = std::fs::File::create_new(path)?;
file.write_all(contents)
}
async fn create_dir_all(&self, path: &Path) -> io::Result<()> {
std::fs::create_dir_all(path)
}
async fn remove_file(&self, path: &Path) -> io::Result<()> {
std::fs::remove_file(path)
}
async fn remove_dir_all(&self, path: &Path) -> io::Result<()> {
std::fs::remove_dir_all(path)
}
async fn rename(&self, from: &Path, to: &Path) -> io::Result<()> {
std::fs::rename(from, to)
}
#[cfg(unix)]
async fn set_executable(&self, path: &Path, executable: bool) -> io::Result<()> {
use std::os::unix::fs::PermissionsExt as _;
let mut permissions = std::fs::metadata(path)?.permissions();
let held = permissions.mode();
let mode = if executable {
held | ((held & 0o444) >> 2)
} else {
held & !0o111
};
if mode == held {
return Ok(());
}
permissions.set_mode(mode);
std::fs::set_permissions(path, permissions)
}
#[cfg(unix)]
async fn set_link(&self, path: &Path, target: &Path) -> io::Result<()> {
let tmp = temp_sibling(path);
let _ = std::fs::remove_file(&tmp);
std::os::unix::fs::symlink(target, &tmp)?;
match std::fs::rename(&tmp, path) {
Ok(()) => Ok(()),
Err(e) => {
let _ = std::fs::remove_file(&tmp);
Err(e)
}
}
}
async fn copy_permissions(&self, from: &Path, to: &Path) -> io::Result<()> {
let perms = match std::fs::metadata(from) {
Ok(meta) => meta.permissions(),
Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()),
Err(e) => return Err(e),
};
let _ = std::fs::set_permissions(to, perms);
Ok(())
}
fn capabilities(&self) -> Capabilities {
Capabilities::LOCAL_FS
}
async fn sync(&self, path: &Path, need: Durability) -> io::Result<()> {
sync_path(path, need)
}
}
fn sync_path(path: &Path, need: Durability) -> io::Result<()> {
#[cfg(not(unix))]
if path.is_dir() {
return Ok(());
}
match std::fs::File::open(path) {
Ok(file) => sync_file(&file, need)?,
Err(e) if e.kind() == io::ErrorKind::NotFound => {}
Err(e) => return Err(e),
}
Ok(())
}
#[cfg(all(feature = "barrier-fsync", target_vendor = "apple"))]
fn sync_file(file: &std::fs::File, need: Durability) -> io::Result<()> {
use std::os::fd::AsRawFd as _;
match need {
Durability::Ordered => {
if unsafe { libc::fcntl(file.as_raw_fd(), libc::F_BARRIERFSYNC) } != -1 {
return Ok(());
}
if unsafe { libc::fsync(file.as_raw_fd()) } != -1 {
return Ok(());
}
Err(io::Error::last_os_error())
}
Durability::Durable => file.sync_all(),
}
}
#[cfg(not(all(feature = "barrier-fsync", target_vendor = "apple")))]
fn sync_file(file: &std::fs::File, need: Durability) -> io::Result<()> {
let _ = need;
file.sync_all()
}
#[cfg(test)]
mod tests {
use crate::exec::block_on;
use super::*;
fn tmp(name: &str) -> PathBuf {
let dir = std::env::temp_dir().join(format!("fstx-fs-{name}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
dir
}
#[test]
fn stdfs_declares_the_local_filesystem_guarantees() {
assert_eq!(StdFs.capabilities(), Capabilities::LOCAL_FS);
assert!(StdFs.capabilities().atomic_replace);
assert!(StdFs.capabilities().exclusive_create);
assert_eq!(StdFs.capabilities().sync_guarantee, SyncGuarantee::Durable);
assert!(!StdFs.capabilities().native_transactions);
}
#[test]
fn a_guarantee_answers_only_the_requests_it_can_keep() {
assert!(!SyncGuarantee::None.satisfies(Durability::Ordered));
assert!(!SyncGuarantee::None.satisfies(Durability::Durable));
assert!(SyncGuarantee::Ordered.satisfies(Durability::Ordered));
assert!(!SyncGuarantee::Ordered.satisfies(Durability::Durable));
assert!(SyncGuarantee::Durable.satisfies(Durability::Ordered));
assert!(SyncGuarantee::Durable.satisfies(Durability::Durable));
}
#[test]
fn replace_stages_and_barriers_but_never_drains() {
let root = tmp("replace-protocol");
std::fs::write(root.join("doc.md"), "old").unwrap();
let fs = crate::fs_faults::RecordingFs::local();
block_on(fs.replace(&root.join("doc.md"), b"new")).unwrap();
use crate::fs_faults::FsEvent;
let tmp_name = temp_sibling(&root.join("doc.md"));
assert_eq!(
fs.events(),
vec![
FsEvent::Write(tmp_name.clone()),
FsEvent::Sync(tmp_name.clone(), Durability::Ordered),
FsEvent::Rename(tmp_name, root.join("doc.md")),
]
);
assert_eq!(std::fs::read_to_string(root.join("doc.md")).unwrap(), "new");
}
#[test]
fn write_atomic_is_replace_plus_the_flushes_it_left_behind() {
let root = tmp("write-atomic-composed");
let fs = crate::fs_faults::RecordingFs::local();
block_on(fs.write_atomic(&root.join("doc.md"), b"bytes")).unwrap();
use crate::fs_faults::FsEvent;
let tmp_name = temp_sibling(&root.join("doc.md"));
assert_eq!(
fs.events(),
vec![
FsEvent::Write(tmp_name.clone()),
FsEvent::Sync(tmp_name.clone(), Durability::Ordered),
FsEvent::Rename(tmp_name, root.join("doc.md")),
FsEvent::Sync(root.clone(), Durability::Durable),
]
);
}
#[test]
fn create_new_writes_a_fresh_file() {
let root = tmp("create-new");
let path = root.join("once.md");
block_on(StdFs.create_new(&path, b"first")).unwrap();
assert_eq!(std::fs::read_to_string(&path).unwrap(), "first");
}
#[test]
fn create_new_refuses_an_occupied_path_with_already_exists() {
let root = tmp("create-new-taken");
let path = root.join("once.md");
block_on(StdFs.create_new(&path, b"first")).unwrap();
let err = block_on(StdFs.create_new(&path, b"second")).unwrap_err();
assert_eq!(err.kind(), io::ErrorKind::AlreadyExists);
assert_eq!(std::fs::read_to_string(&path).unwrap(), "first");
}
#[test]
fn the_default_create_new_declines_rather_than_emulating() {
struct Bare;
impl ReadStorage for Bare {
async fn read(&self, _: &Path) -> io::Result<Vec<u8>> {
unreachable!()
}
async fn read_to_string(&self, _: &Path) -> io::Result<String> {
unreachable!()
}
async fn read_dir(&self, _: &Path) -> io::Result<Vec<DirEntry>> {
unreachable!()
}
async fn metadata(&self, _: &Path) -> io::Result<Metadata> {
unreachable!()
}
}
impl Storage for Bare {
async fn write(&self, _: &Path, _: &[u8]) -> io::Result<()> {
unreachable!()
}
async fn create_dir_all(&self, _: &Path) -> io::Result<()> {
unreachable!()
}
async fn remove_file(&self, _: &Path) -> io::Result<()> {
unreachable!()
}
async fn remove_dir_all(&self, _: &Path) -> io::Result<()> {
unreachable!()
}
async fn rename(&self, _: &Path, _: &Path) -> io::Result<()> {
unreachable!()
}
}
assert!(!Bare.capabilities().exclusive_create);
let err = block_on(Bare.create_new(Path::new("x"), b"")).unwrap_err();
assert_eq!(err.kind(), io::ErrorKind::Unsupported);
}
#[test]
fn sync_of_a_missing_path_is_not_an_error() {
let root = tmp("sync-missing");
block_on(StdFs.sync(&root.join("never-created.md"), Durability::Durable)).unwrap();
}
#[test]
fn sync_flushes_a_directory_as_readily_as_a_file() {
let root = tmp("sync-dir");
block_on(StdFs.sync(&root, Durability::Durable)).unwrap();
}
#[test]
fn sync_answers_both_strengths_on_files_and_directories() {
let root = tmp("sync-strengths");
let file = root.join("doc.md");
std::fs::write(&file, "bytes").unwrap();
for need in [Durability::Ordered, Durability::Durable] {
block_on(StdFs.sync(&file, need)).unwrap();
block_on(StdFs.sync(&root, need)).unwrap();
}
}
}