coordinode-lsm-tree 7.3.0

// SPDX-License-Identifier: Apache-2.0
// Copyright (c) 2024-present, fjall-rs
// Copyright (c) 2026-present, Dmitry Prudnikov

use super::{FileHint, Fs, FsDirEntry, FsFile, FsMetadata, FsOpenOptions, SyncMode};
use crate::io;
use crate::path::{Path, PathBuf};
#[cfg(not(feature = "std"))]
use alloc::{boxed::Box, string::String, vec::Vec};
use std::fs::{File, OpenOptions};

/// Delays between the attempts of a rename or delete that another process is
/// briefly holding the file against, before the last error is returned: about
/// half a second in all, longer than an antivirus or indexer scan of a fresh
/// file usually takes.
#[cfg(windows)]
const TRANSIENT_HOLD_BACKOFF_MS: [u64; 9] = [1, 2, 4, 8, 16, 32, 64, 128, 256];

/// Runs `op`, retrying it while it fails because another process holds the
/// file open without `FILE_SHARE_DELETE` (an antivirus, a search indexer, a
/// backup agent). Windows refuses a rename or delete of such a file with
/// `ERROR_ACCESS_DENIED` (5), `ERROR_SHARING_VIOLATION` (32),
/// `ERROR_LOCK_VIOLATION` (33) or `ERROR_USER_MAPPED_FILE` (1224) for as long
/// as the hold lasts, usually milliseconds. POSIX renames and unlinks ignore
/// open handles, so elsewhere `op` runs once.
#[cfg(windows)]
fn retry_transient_hold(mut op: impl FnMut() -> std::io::Result<()>) -> std::io::Result<()> {
    const ERROR_ACCESS_DENIED: i32 = 5;
    const ERROR_SHARING_VIOLATION: i32 = 32;
    const ERROR_LOCK_VIOLATION: i32 = 33;
    const ERROR_USER_MAPPED_FILE: i32 = 1224;

    for delay in TRANSIENT_HOLD_BACKOFF_MS {
        match op() {
            Err(e)
                if matches!(
                    e.raw_os_error(),
                    Some(
                        ERROR_ACCESS_DENIED
                            | ERROR_SHARING_VIOLATION
                            | ERROR_LOCK_VIOLATION
                            | ERROR_USER_MAPPED_FILE
                    )
                ) =>
            {
                std::thread::sleep(std::time::Duration::from_millis(delay));
            }
            result => return result,
        }
    }
    op()
}

/// See the Windows variant: elsewhere no open handle blocks a rename or delete.
#[cfg(not(windows))]
fn retry_transient_hold(mut op: impl FnMut() -> std::io::Result<()>) -> std::io::Result<()> {
    op()
}

/// Plain `fsync` (no `F_FULLFSYNC`) for [`SyncMode::Normal`].
///
/// On macOS, `File::sync_all` issues `fcntl(F_FULLFSYNC)` (a full hardware
/// barrier, ~50× slower than `fsync`). `Normal` mode wants the cheaper plain
/// `fsync` - which std does not expose on macOS - so call it directly via
/// `libc`. On every other platform `File::sync_all` IS plain `fsync`, so just
/// delegate.
#[cfg(target_os = "macos")]
fn normal_fsync(file: &File) -> io::Result<()> {
    use std::os::unix::io::AsRawFd;
    // SAFETY: `fd` is a valid open descriptor for the lifetime of `file`.
    let rc = unsafe { libc::fsync(file.as_raw_fd()) };
    if rc == 0 {
        Ok(())
    } else {
        Err(std::io::Error::last_os_error().into())
    }
}

#[cfg(not(target_os = "macos"))]
fn normal_fsync(file: &File) -> io::Result<()> {
    // No `F_FULLFSYNC` distinction off macOS - `sync_all` is plain `fsync`.
    File::sync_all(file).map_err(io::Error::from)
}

/// `fcntl(F_BARRIERFSYNC)` for [`SyncMode::Barrier`]: an `fsync` that also
/// keeps the drive from reordering these writes past later ones.
#[cfg(target_os = "macos")]
fn barrier_fsync(file: &File) -> io::Result<()> {
    use std::os::unix::io::AsRawFd;
    // SAFETY: `fd` is a valid open descriptor for the lifetime of `file`.
    let rc = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_BARRIERFSYNC) };
    if rc == -1 {
        Err(std::io::Error::last_os_error().into())
    } else {
        Ok(())
    }
}

#[cfg(not(target_os = "macos"))]
fn barrier_fsync(file: &File) -> io::Result<()> {
    // Off macOS `fsync` already reaches the medium in order.
    normal_fsync(file)
}

/// The sync `mode` asks of `file`.
fn sync_file_with(file: &File, mode: SyncMode) -> io::Result<()> {
    match mode {
        // `File::sync_all` is `fcntl(F_FULLFSYNC)` on macOS.
        SyncMode::Full => File::sync_all(file).map_err(io::Error::from),
        SyncMode::Normal => normal_fsync(file),
        SyncMode::Barrier => barrier_fsync(file),
    }
}

/// Default [`Fs`] implementation backed by [`std::fs`].
///
/// This is a zero-sized type - when used as a monomorphized generic
/// parameter it adds no runtime overhead.
#[derive(Clone, Copy, Debug, Default)]
pub struct StdFs;

// ---------------------------------------------------------------------------
// FsFile for std::fs::File
// ---------------------------------------------------------------------------
// Self:: calls delegate to File's inherent methods (clippy::use_self preference).

impl FsFile for File {
    fn sync_all(&self) -> io::Result<()> {
        Self::sync_all(self).map_err(io::Error::from)
    }

    fn sync_data(&self) -> io::Result<()> {
        Self::sync_data(self).map_err(io::Error::from)
    }

    fn sync_all_with(&self, mode: SyncMode) -> io::Result<()> {
        sync_file_with(self, mode)
    }

    fn sync_data_with(&self, mode: SyncMode) -> io::Result<()> {
        // Off macOS every mode is the same `fdatasync`, which spares the
        // metadata a read does not need (timestamps).
        #[cfg(not(target_os = "macos"))]
        {
            let _ = mode;
            Self::sync_data(self).map_err(io::Error::from)
        }
        #[cfg(target_os = "macos")]
        match mode {
            SyncMode::Full => Self::sync_data(self).map_err(io::Error::from),
            // On macOS `sync_data` is `F_FULLFSYNC` too, and a plain `fsync`
            // or a barrier already covers the data, so there is no cheaper
            // data-only sync to issue.
            SyncMode::Normal | SyncMode::Barrier => sync_file_with(self, mode),
        }
    }

    fn metadata(&self) -> io::Result<FsMetadata> {
        let m = Self::metadata(self)?;
        Ok(FsMetadata {
            len: m.len(),
            is_dir: m.is_dir(),
            is_file: m.is_file(),
        })
    }

    #[cfg(unix)]
    fn hard_link_count(&self) -> io::Result<u64> {
        use std::os::unix::fs::MetadataExt as _;
        Ok(Self::metadata(self)?.nlink())
    }

    // `std` exposes the NTFS link count only behind the unstable
    // `windows_by_handle` feature, so query the Win32 API directly (same
    // dependency-free pattern as `available_space_sys` below).
    #[cfg(windows)]
    fn hard_link_count(&self) -> io::Result<u64> {
        hard_link_count_sys::hard_link_count(self).map_err(io::Error::from)
    }

    fn set_len(&self, size: u64) -> io::Result<()> {
        Self::set_len(self, size).map_err(io::Error::from)
    }

    fn read_at(&self, buf: &mut [u8], offset: u64) -> io::Result<usize> {
        // Fill-or-EOF loop: retry on short reads and EINTR so that callers
        // see either a full buffer or a short read that signals EOF.
        let mut filled = 0usize;

        while filled < buf.len() {
            // SAFETY: loop guard `filled < buf.len()` ensures this is in-bounds.
            #[expect(clippy::expect_used, reason = "filled < buf.len() by loop guard")]
            let remaining = buf.get_mut(filled..).expect("filled < buf.len()");
            // `offset` is caller-supplied (not bounded by the file size), so a
            // value near u64::MAX plus the bytes filled so far could overflow on
            // a looping short read. Reject that explicitly, mirroring the
            // io_uring backend, rather than wrapping to a wrong low offset.
            let off = offset.checked_add(filled as u64).ok_or_else(|| {
                io::Error::new(io::ErrorKind::InvalidInput, "read offset overflow")
            })?;

            let n = {
                #[cfg(unix)]
                {
                    use std::os::unix::fs::FileExt;
                    match FileExt::read_at(self, remaining, off) {
                        Ok(n) => n,
                        Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
                        Err(e) => return Err(e.into()),
                    }
                }

                #[cfg(windows)]
                {
                    use std::os::windows::fs::FileExt;
                    match FileExt::seek_read(self, remaining, off) {
                        Ok(n) => n,
                        Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
                        Err(e) => return Err(e.into()),
                    }
                }

                #[cfg(not(any(unix, windows)))]
                {
                    let _ = (remaining, off);
                    return Err(io::Error::new(
                        io::ErrorKind::Unsupported,
                        "read_at is not supported on this platform",
                    ));
                }
            };

            if n == 0 {
                break; // EOF
            }
            filled += n;
        }

        Ok(filled)
    }

    fn lock_exclusive(&self) -> io::Result<()> {
        sys::lock_exclusive(self).map_err(io::Error::from)
    }

    fn try_lock_exclusive(&self) -> io::Result<bool> {
        sys::try_lock_exclusive(self).map_err(io::Error::from)
    }

    fn hint(&self, hint: FileHint) -> io::Result<()> {
        sys::fadvise(self, hint).map_err(io::Error::from)
    }

    fn start_writeback(&self, offset: u64, len: u64) -> io::Result<()> {
        #[cfg(target_os = "linux")]
        {
            use std::os::unix::io::AsRawFd;
            let (Ok(offset), Ok(len)) = (i64::try_from(offset), i64::try_from(len)) else {
                return Err(io::Error::new(
                    io::ErrorKind::InvalidInput,
                    "writeback range past the largest file offset",
                ));
            };
            // SAFETY: `fd` is a valid open descriptor for the lifetime of `self`.
            let rc = unsafe {
                libc::sync_file_range(self.as_raw_fd(), offset, len, libc::SYNC_FILE_RANGE_WRITE)
            };
            if rc == -1 {
                Err(std::io::Error::last_os_error().into())
            } else {
                Ok(())
            }
        }
        // Elsewhere there is no call that starts writeback without waiting
        // for it: the final sync writes everything.
        #[cfg(not(target_os = "linux"))]
        {
            let _ = (offset, len);
            Ok(())
        }
    }

    fn set_write_lifetime(&self, lifetime: super::WriteLifetime) -> io::Result<()> {
        #[cfg(target_os = "linux")]
        {
            use std::os::unix::io::AsRawFd;
            set_write_lifetime_fd(self.as_raw_fd(), lifetime)
        }
        // Linux is the only platform with a per-file write lifetime hint.
        #[cfg(not(target_os = "linux"))]
        {
            let _ = lifetime;
            Ok(())
        }
    }
}

/// `fcntl(fd, F_SET_RW_HINT, &hint)` with the hint `lifetime` maps to.
///
/// The constants are Linux uapi `include/uapi/linux/fcntl.h` (`F_SET_RW_HINT`
/// is `F_LINUX_SPECIFIC_BASE + 12`, the `RWH_WRITE_LIFE_*` values follow),
/// which `libc` does not export. The argument is a pointer to a `u64`, as that
/// header declares.
#[cfg(target_os = "linux")]
fn set_write_lifetime_fd(fd: i32, lifetime: super::WriteLifetime) -> io::Result<()> {
    const F_SET_RW_HINT: libc::c_int = 1024 + 12;
    let hint: u64 = match lifetime {
        super::WriteLifetime::Short => 2,
        super::WriteLifetime::Medium => 3,
        super::WriteLifetime::Long => 4,
        super::WriteLifetime::Extreme => 5,
    };
    // SAFETY: `fd` is a valid open descriptor for the duration of the call,
    // and `hint` outlives it; the kernel only reads the `u64` it points to.
    let rc = unsafe { libc::fcntl(fd, F_SET_RW_HINT, &raw const hint) };
    if rc == -1 {
        Err(std::io::Error::last_os_error().into())
    } else {
        Ok(())
    }
}

// ---------------------------------------------------------------------------
// Fs for StdFs
// ---------------------------------------------------------------------------

impl Fs for StdFs {
    fn open(&self, path: &Path, opts: &FsOpenOptions) -> io::Result<Box<dyn FsFile>> {
        let mut builder = OpenOptions::new();
        builder
            .read(opts.read)
            .write(opts.write)
            .create(opts.create)
            .create_new(opts.create_new)
            .truncate(opts.truncate)
            .append(opts.append);

        // Gate matches the `mod open_flags;` declaration in `fs/mod.rs`
        // - the submodule only exists when `feature = "std"` is on.
        // Without the gate this site would fail to compile under
        // `--no-default-features --features alloc` even before the
        // wider std-bound surface of `StdFs` itself hits the trait
        // signatures; keeping the cfg in sync prevents adding a
        // resolution-time error on top of the type-checking ones
        // already tracked under the no-std migration epic.
        #[cfg(feature = "std")]
        super::open_flags::apply_open_flags(&mut builder, opts);

        let file = builder.open(path)?;
        // No open waits off Unix, so nothing is left to clear there.
        #[cfg(all(feature = "std", unix))]
        super::open_flags::finish_open(&file, opts)?;
        Ok(Box::new(file))
    }

    fn create_dir_all(&self, path: &Path) -> io::Result<()> {
        std::fs::create_dir_all(path).map_err(io::Error::from)
    }

    fn create_dir(&self, path: &Path) -> io::Result<()> {
        std::fs::create_dir(path).map_err(io::Error::from)
    }

    fn read_dir(&self, path: &Path) -> io::Result<Vec<FsDirEntry>> {
        // Fail-fast on bad entries is intentional: non-UTF-8 filenames in an
        // lsm-tree data directory indicate filesystem corruption (see FsDirEntry docs).
        std::fs::read_dir(path)?
            .map(|res| {
                let entry = res?;
                let file_type = entry.file_type()?;
                let file_name_os = entry.file_name();
                let file_name = file_name_os.into_string().map_err(|os| {
                    #[expect(
                        clippy::unnecessary_debug_formatting,
                        reason = "OsString has no Display impl - Debug is required"
                    )]
                    let msg = format!("non-UTF-8 filename in directory {}: {os:?}", path.display());
                    io::Error::new(io::ErrorKind::InvalidData, msg)
                })?;
                Ok(FsDirEntry {
                    path: entry.path(),
                    file_name,
                    is_dir: file_type.is_dir(),
                })
            })
            .collect()
    }

    fn remove_file(&self, path: &Path) -> io::Result<()> {
        retry_transient_hold(|| std::fs::remove_file(path)).map_err(io::Error::from)
    }

    fn remove_dir_all(&self, path: &Path) -> io::Result<()> {
        std::fs::remove_dir_all(path).map_err(io::Error::from)
    }

    fn remove_dir(&self, path: &Path) -> io::Result<()> {
        std::fs::remove_dir(path).map_err(io::Error::from)
    }

    fn same_file(&self, a: &Path, b: &Path) -> io::Result<bool> {
        // Filesystem OBJECT identity, not canonical spellings: bind-mount
        // aliases of one directory keep their two mount-point spellings
        // through `canonicalize`, and a "distinct" verdict authorizes
        // deleting what is the same underlying file through another name —
        // destroying the retained copy too. A probe failure on either side
        // is PROPAGATED, never guessed as "distinct".
        #[cfg(unix)]
        {
            use std::os::unix::fs::MetadataExt;
            let ma = std::fs::metadata(a).map_err(io::Error::from)?;
            let mb = std::fs::metadata(b).map_err(io::Error::from)?;
            Ok(ma.dev() == mb.dev() && ma.ino() == mb.ino())
        }
        // The volume serial number and the 128-bit file ID: two hard links of
        // one file share them while their canonical paths differ.
        #[cfg(windows)]
        {
            let ia = hard_link_count_sys::file_identity(a).map_err(io::Error::from)?;
            let ib = hard_link_count_sys::file_identity(b).map_err(io::Error::from)?;
            Ok(ia == ib)
        }
        // No file-identity API elsewhere: canonical spellings still resolve
        // symlinks, `..`, and case folding.
        #[cfg(not(any(unix, windows)))]
        {
            let ca = std::fs::canonicalize(a).map_err(io::Error::from)?;
            let cb = std::fs::canonicalize(b).map_err(io::Error::from)?;
            Ok(ca == cb)
        }
    }

    fn read_link(&self, path: &Path) -> io::Result<Option<PathBuf>> {
        // On Windows `is_symlink` is true for every name-surrogate reparse
        // point, a junction (`IO_REPARSE_TAG_MOUNT_POINT`) included, and
        // `std::fs::read_link` reads both tags.
        match std::fs::symlink_metadata(path) {
            Ok(meta) if meta.file_type().is_symlink() => {
                std::fs::read_link(path).map(Some).map_err(io::Error::from)
            }
            Ok(_) => Ok(None),
            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
            Err(e) => Err(io::Error::from(e)),
        }
    }

    fn symlink(&self, target: &Path, link: &Path, to_dir: bool) -> io::Result<()> {
        #[cfg(unix)]
        let made = {
            // Unix records no kind with a link.
            let _ = to_dir;
            std::os::unix::fs::symlink(target, link)
        };
        // Windows records the kind with the link, whatever the target is
        // now, so the caller names it.
        #[cfg(windows)]
        let made = if to_dir {
            std::os::windows::fs::symlink_dir(target, link)
        } else {
            std::os::windows::fs::symlink_file(target, link)
        };
        #[cfg(not(any(unix, windows)))]
        let made: std::io::Result<()> = {
            let _ = (target, link, to_dir);
            Err(std::io::Error::from(std::io::ErrorKind::Unsupported))
        };
        made.map_err(io::Error::from)
    }

    fn symlink_to_dir(&self, path: &Path) -> io::Result<bool> {
        #[cfg(windows)]
        {
            use std::os::windows::fs::FileTypeExt;
            Ok(std::fs::symlink_metadata(path)?
                .file_type()
                .is_symlink_dir())
        }
        #[cfg(not(windows))]
        {
            let _ = path;
            Ok(false)
        }
    }

    fn rename(&self, from: &Path, to: &Path) -> io::Result<()> {
        // `std::fs::rename` satisfies the trait's replace contract on every
        // platform: on Windows it maps to `MoveFileExW` with
        // `MOVEFILE_REPLACE_EXISTING` (POSIX-semantics rename on newer
        // toolchains), so an existing destination FILE — open handles
        // included — is atomically replaced, same as on Unix.
        retry_transient_hold(|| std::fs::rename(from, to)).map_err(io::Error::from)
    }

    fn metadata(&self, path: &Path) -> io::Result<FsMetadata> {
        let m = std::fs::metadata(path)?;
        Ok(FsMetadata {
            len: m.len(),
            is_dir: m.is_dir(),
            is_file: m.is_file(),
        })
    }

    #[cfg(unix)]
    fn available_space(&self, path: &Path) -> io::Result<u64> {
        super::statvfs_available_space(path).map_err(io::Error::from)
    }

    #[cfg(windows)]
    fn available_space(&self, path: &Path) -> io::Result<u64> {
        available_space_sys::disk_free_available(path).map_err(io::Error::from)
    }

    #[cfg(unix)]
    fn allocated_size(&self, path: &Path) -> io::Result<Option<u64>> {
        Ok(super::unix_allocated_size(path)?)
    }

    fn sync_directory(&self, path: &Path) -> io::Result<()> {
        #[cfg(not(target_os = "windows"))]
        {
            let dir = File::open(path)?;
            if !dir.metadata()?.is_dir() {
                return Err(io::Error::new(
                    io::ErrorKind::InvalidInput,
                    "sync_directory: path is not a directory",
                ));
            }
            dir.sync_all().map_err(io::Error::from)
        }

        // Windows cannot fsync directories - no-op, same as crate::file::fsync_directory.
        #[cfg(target_os = "windows")]
        {
            let _ = path;
            Ok(())
        }
    }

    fn sync_directory_with(&self, path: &Path, mode: SyncMode) -> io::Result<()> {
        #[cfg(not(target_os = "windows"))]
        {
            let dir = File::open(path)?;
            if !dir.metadata()?.is_dir() {
                return Err(io::Error::new(
                    io::ErrorKind::InvalidInput,
                    "sync_directory: path is not a directory",
                ));
            }
            sync_file_with(&dir, mode)
        }

        // Windows cannot fsync directories - no-op.
        #[cfg(target_os = "windows")]
        {
            let _ = (path, mode);
            Ok(())
        }
    }

    fn sync_device(&self, path: &Path) -> io::Result<()> {
        // `F_FULLFSYNC` flushes the whole drive cache, whatever descriptor of
        // the device it is issued on.
        #[cfg(target_os = "macos")]
        {
            File::open(path)?.sync_all().map_err(io::Error::from)
        }
        #[cfg(not(target_os = "macos"))]
        {
            let _ = path;
            Ok(())
        }
    }

    fn exists(&self, path: &Path) -> io::Result<bool> {
        path.try_exists().map_err(io::Error::from)
    }

    fn hard_link(&self, src: &Path, dst: &Path) -> io::Result<()> {
        // Pure hard link. On a cross-device target this returns the EXDEV
        // error instead of silently byte-copying: the copy belongs to the
        // caller that actually wants a cross-filesystem copy
        // (checkpoint's `link_or_copy_cross_fs`), which detects the
        // cross-device error via `is_cross_device` and runs its own
        // SyncMode-aware streamed copy. Keeping the copy in one place means
        // the copied file's durability honors `Config::sync_mode` instead
        // of always paying `F_FULLFSYNC` here.
        //
        // Normalise a cross-device failure to `ErrorKind::CrossesDevices`
        // while the errno is still readable on the std error: once it folds
        // into `crate::io::Error` at this trait boundary the errno is gone,
        // so the caller's kind-based `is_cross_device` would otherwise miss
        // the platforms where std reports EXDEV without the matching kind.
        std::fs::hard_link(src, dst).map_err(|e| {
            if is_std_cross_device(&e) {
                io::Error::from_kind(io::ErrorKind::CrossesDevices)
            } else {
                e.into()
            }
        })
    }

    fn backend_id(&self) -> Option<u64> {
        Some(super::KERNEL_BACKEND_ID)
    }

    #[cfg(unix)]
    fn volume_id(&self, path: &Path) -> Option<u64> {
        super::unix_volume_id(path)
    }

    fn hard_link_count(&self, path: &Path) -> io::Result<u64> {
        #[cfg(unix)]
        {
            use std::os::unix::fs::MetadataExt;
            Ok(std::fs::metadata(path)?.nlink())
        }
        // Windows: no portable stable-Rust nlink. NTFS hard links are real, so
        // truncating without a confirmed link count could zero a checkpoint's
        // hard-linked copy — the same hazard the Unix guard prevents. Report
        // "unsupported" so the reclaim path conservatively SKIPS the
        // synchronous truncate and relies on the async unlink alone (correct,
        // just without the immediate-footprint fast path). Narrowing the
        // contract this way is deliberate; raising it to truncate-on-Windows
        // would need a winapi `GetFileInformationByHandle` link-count query.
        #[cfg(not(unix))]
        {
            let _ = path;
            Err(io::Error::new(
                io::ErrorKind::Unsupported,
                "hard_link_count requires a Unix target",
            ))
        }
    }

    /// macOS: detects APFS (the common macOS filesystem with copy-on-write,
    /// reflink, and native snapshots) via `statfs(2)`'s `f_fstypename`. Other
    /// macOS filesystems (HFS+, exFAT, SMB/NFS mounts) and any detection failure
    /// report the conservative default. (Linux has its own `statfs` `f_type`
    /// detection in the `target_os = "linux"` variant of this method below;
    /// other targets fall back to the trait's conservative default.)
    #[cfg(target_os = "macos")]
    fn capabilities(&self, path: &Path) -> super::FsCapabilities {
        macos_caps::capabilities(path)
    }

    /// macOS: O(1) clone via `clonefile(2)` on APFS. When the filesystem
    /// declines the clone (non-APFS mount, cross-device) it falls back to the
    /// shared streamed byte copy, so the clone still succeeds (just without
    /// block sharing). A genuine I/O error propagates.
    #[cfg(target_os = "macos")]
    fn reflink_file(&self, src: &Path, dst: &Path) -> io::Result<()> {
        match macos_caps::clonefile(src, dst) {
            Ok(()) => Ok(()),
            Err(e) if macos_caps::clone_should_fall_back(&e) => {
                super::copy_file_streamed(self, src, dst)
            }
            Err(e) => Err(e.into()),
        }
    }

    /// Linux: detect the filesystem via `statfs(2)`'s `f_type` magic. Btrfs and
    /// ZFS report the full profile (integrity, scrub, copy-on-write, reflink,
    /// snapshot); XFS reports copy-on-write plus reflink (reflink only when the
    /// volume was formatted `reflink=1`, which `reflink_file` handles by falling
    /// back). Other filesystems and any detection failure report the
    /// conservative default.
    #[cfg(target_os = "linux")]
    fn capabilities(&self, path: &Path) -> super::FsCapabilities {
        linux_caps::capabilities(path)
    }

    /// Windows: no directory can be synced (see `sync_directory`), so the
    /// directory sync is reported as doing nothing; every other capability
    /// is the conservative default.
    #[cfg(target_os = "windows")]
    fn capabilities(&self, _path: &Path) -> super::FsCapabilities {
        super::FsCapabilities {
            directory_sync_noop: true,
            ..super::FsCapabilities::default()
        }
    }

    /// Linux: O(1) clone via `ioctl(FICLONE)` (Btrfs, XFS-reflink). Falls back
    /// to the shared streamed copy when the filesystem declines the clone
    /// (non-reflink FS, cross-device); a genuine I/O error propagates.
    #[cfg(target_os = "linux")]
    fn reflink_file(&self, src: &Path, dst: &Path) -> io::Result<()> {
        match linux_caps::ficlone(src, dst) {
            Ok(()) => Ok(()),
            Err(e) if linux_caps::clone_should_fall_back(&e) => {
                super::copy_file_streamed(self, src, dst)
            }
            Err(e) => Err(e.into()),
        }
    }

    /// Linux: clear per-file copy-on-write (`FS_NOCOW_FL` via
    /// `ioctl(FS_IOC_SETFLAGS)`) so write-once SST files on Btrfs avoid the
    /// copy-on-write fragmentation penalty. The flag only takes effect on a
    /// still-empty file,
    /// so callers invoke this right after creating the SST. A no-op on
    /// filesystems that do not support the inode-flags ioctl.
    #[cfg(target_os = "linux")]
    fn try_disable_cow(&self, path: &Path) -> io::Result<()> {
        linux_caps::try_disable_cow(path).map_err(io::Error::from)
    }

    /// Linux: reclaim a mid-file byte range via `fallocate(PUNCH_HOLE)`. Gated
    /// by callers on [`super::FsCapabilities::punch_hole`], which `capabilities`
    /// sets for ext4 / xfs / btrfs / zfs / tmpfs. macOS and other targets keep
    /// the trait default ([`io::ErrorKind::Unsupported`]).
    #[cfg(target_os = "linux")]
    fn punch_hole(&self, path: &Path, offset: u64, len: u64) -> io::Result<()> {
        linux_caps::punch_hole(path, offset, len).map_err(io::Error::from)
    }

    /// Linux: extent-local hole probe via `lseek(SEEK_DATA)`. Only this target
    /// can punch, so only here can a hole exist to attribute; elsewhere the
    /// trait default ("cannot tell") is also the truthful answer.
    #[cfg(target_os = "linux")]
    fn extent_is_hole(&self, path: &Path, offset: u64, len: u64) -> io::Result<Option<bool>> {
        Ok(linux_caps::extent_is_hole(path, offset, len)?)
    }

    /// Linux: contained-hole probe via `lseek(SEEK_HOLE)` (see the trait docs —
    /// an unaligned punch leaves its zero-filled edges allocated, so the
    /// whole-range probe above answers `false` on a genuinely punched range).
    #[cfg(target_os = "linux")]
    fn extent_contains_hole(&self, path: &Path, offset: u64, len: u64) -> io::Result<Option<bool>> {
        Ok(linux_caps::extent_contains_hole(path, offset, len)?)
    }
}

/// Detects `EXDEV` (cross-device link) errors across platforms.
///
/// Unix exposes the raw `EXDEV` (errno 18 on Linux/macOS/BSDs); we also
/// accept [`io::ErrorKind::CrossesDevices`] (stable since Rust 1.85) and
/// [`io::ErrorKind::Unsupported`] which Windows surfaces when a volume
/// configuration disallows hard links altogether.
///
/// We deliberately do NOT treat [`io::ErrorKind::PermissionDenied`] as a
/// cross-device signal even though Windows can sometimes surface
/// cross-volume link failures that way. Misclassifying genuine ACL /
/// permission errors as "different filesystem" would silently turn a
/// "you can't read this" into a full byte copy, hiding real security
/// misconfigurations. Operators hitting that edge case on Windows can
/// fix it explicitly (move the target volume, adjust ACLs); the
/// fall-back is opt-out by design.
// `pub(crate)`, surfaced crate-wide via the `pub(crate) use` re-export in
// `fs/mod.rs` so `checkpoint::link_or_copy_cross_fs` can detect cross-device
// errors. The crate-scoped visibility (not `pub`) keeps this off any public
// surface even if `std_fs` is ever exported. clippy's `redundant_pub_crate`
// fires only because the enclosing module is currently private; the re-export
// genuinely needs crate visibility, so the lint is a false positive here.
/// Cross-device detection on a raw `std::io::Error`, where the errno is still
/// available. Used by [`StdFs::hard_link`] to NORMALISE a cross-device link
/// failure into [`crate::io::ErrorKind::CrossesDevices`] before the error
/// crosses the `Fs` trait boundary into [`crate::io::Error`] (which carries no
/// errno). After normalisation the kind-based [`is_cross_device`] is enough.
fn is_std_cross_device(err: &std::io::Error) -> bool {
    #[cfg(unix)]
    {
        // POSIX `EXDEV` ("invalid cross-device link"). The raw value is
        // 18 on every supported Unix target (Linux, macOS, FreeBSD,
        // OpenBSD, NetBSD, Android - see `errno.h` on each platform).
        // We declare it as a named constant rather than depending on
        // `libc`: the crate deliberately avoids a `libc` dependency
        // (see the `flock` extern in this file for the same pattern),
        // and EXDEV's value is locked by ABI compatibility.
        const EXDEV: i32 = 18;
        if err.raw_os_error() == Some(EXDEV) {
            return true;
        }
    }
    matches!(
        err.kind(),
        std::io::ErrorKind::CrossesDevices | std::io::ErrorKind::Unsupported
    )
}

/// Cross-device detection on a [`crate::io::Error`] (post-`Fs`-boundary, no
/// errno). [`StdFs::hard_link`] normalises the raw errno case to the
/// `CrossesDevices` kind, so a kind check is authoritative here.
#[expect(
    clippy::redundant_pub_crate,
    reason = "re-exported crate-wide via fs::mod; pub(crate) communicates the intended scope"
)]
pub(crate) fn is_cross_device(err: &io::Error) -> bool {
    matches!(
        err.kind(),
        io::ErrorKind::CrossesDevices | io::ErrorKind::Unsupported
    )
}

// ---------------------------------------------------------------------------
// Platform-specific file locking
// ---------------------------------------------------------------------------

#[cfg(unix)]
mod sys {
    use super::FileHint;
    use std::ffi::c_int;
    use std::fs::File;
    use std::io;
    use std::os::unix::io::AsRawFd;

    // Declare flock directly to avoid requiring libc as a direct dependency.
    const LOCK_EX: c_int = 2;
    // Non-blocking modifier: flock returns EWOULDBLOCK instead of waiting when
    // the lock is already held.
    const LOCK_NB: c_int = 4;

    // SAFETY: declaration matches the POSIX `flock` ABI on Unix targets.
    unsafe extern "C" {
        fn flock(fd: c_int, operation: c_int) -> c_int;
    }

    pub(super) fn lock_exclusive(file: &File) -> io::Result<()> {
        let fd = file.as_raw_fd();

        loop {
            // SAFETY: fd is a valid file descriptor owned by `file`.
            #[expect(unsafe_code, reason = "flock FFI call with valid fd")]
            let ret = unsafe { flock(fd, LOCK_EX) };

            if ret == 0 {
                return Ok(());
            }

            let err = std::io::Error::last_os_error();
            if err.kind() == std::io::ErrorKind::Interrupted {
                continue;
            }
            return Err(err);
        }
    }

    pub(super) fn try_lock_exclusive(file: &File) -> io::Result<bool> {
        let fd = file.as_raw_fd();

        loop {
            // SAFETY: fd is a valid file descriptor owned by `file`.
            #[expect(unsafe_code, reason = "flock FFI call with valid fd")]
            let ret = unsafe { flock(fd, LOCK_EX | LOCK_NB) };

            if ret == 0 {
                return Ok(true);
            }

            // EINTR can interrupt even the non-blocking call; retry. EWOULDBLOCK
            // (the lock is held elsewhere) is the expected contention signal and
            // maps to `Ok(false)`, not an error.
            let err = std::io::Error::last_os_error();
            if err.kind() == std::io::ErrorKind::Interrupted {
                continue;
            }
            if err.kind() == std::io::ErrorKind::WouldBlock {
                return Ok(false);
            }
            return Err(err);
        }
    }

    // POSIX_FADV_* values are stable across glibc / musl / *BSD libc.
    // macOS has no posix_fadvise - routed to a no-op `fadvise` below.
    #[cfg(not(target_os = "macos"))]
    const POSIX_FADV_NORMAL: c_int = 0;
    #[cfg(not(target_os = "macos"))]
    const POSIX_FADV_RANDOM: c_int = 1;
    #[cfg(not(target_os = "macos"))]
    const POSIX_FADV_SEQUENTIAL: c_int = 2;
    #[cfg(not(target_os = "macos"))]
    const POSIX_FADV_DONTNEED: c_int = 4;

    // EINVAL == 22 on every Linux / *BSD target we ship to. Used to
    // map "kernel rejected this hint" to Ok(()) per the FsFile::hint
    // contract - hints are advisory, EINVAL means "the kernel didn't
    // like this advice code", not "the file is broken".
    #[cfg(not(target_os = "macos"))]
    const EINVAL: c_int = 22;

    // libc symbol selection (the 32-bit glibc trap):
    //
    // Plain `posix_fadvise` on 32-bit glibc takes a 32-bit off_t
    // unless the caller opts into `_FILE_OFFSET_BITS=64` - and Rust
    // FFI declarations don't get that define. Passing i64 args into
    // a 32-bit-off_t syscall wrapper corrupts the stack.
    //
    // `posix_fadvise64` always takes int64_t but is a glibc-only
    // symbol; musl doesn't export it (the kernel sees the same
    // syscall, but musl has no userspace wrapper). bionic (Android)
    // does export it.
    //
    // Resolution by target:
    // - 32-bit glibc Linux + 32-bit Android: posix_fadvise64
    //   (otherwise stack corruption)
    // - musl on any pointer width: posix_fadvise (musl is sane on
    //   32-bit too - its plain wrapper takes 64-bit off_t)
    // - 64-bit Linux/Android: posix_fadvise (off_t is always 64-bit)
    // - BSDs (freebsd/netbsd/dragonfly): posix_fadvise (off_t is
    //   always 64-bit, no *64 variant exists)
    #[cfg(all(
        any(target_os = "linux", target_os = "android"),
        target_pointer_width = "32",
        any(target_env = "gnu", target_env = "")
    ))]
    // SAFETY: matches glibc / bionic posix_fadvise64 ABI on 32-bit.
    unsafe extern "C" {
        fn posix_fadvise64(fd: c_int, offset: i64, len: i64, advice: c_int) -> c_int;
    }

    #[cfg(all(
        not(target_os = "macos"),
        not(all(
            any(target_os = "linux", target_os = "android"),
            target_pointer_width = "32",
            any(target_env = "gnu", target_env = "")
        ))
    ))]
    // SAFETY: matches POSIX posix_fadvise ABI (off_t is 64-bit on
    // 64-bit Linux/Android, BSD, and on musl regardless of pointer
    // width).
    unsafe extern "C" {
        fn posix_fadvise(fd: c_int, offset: i64, len: i64, advice: c_int) -> c_int;
    }

    #[cfg(not(target_os = "macos"))]
    pub(super) fn fadvise(file: &File, hint: FileHint) -> io::Result<()> {
        let advice = match hint {
            FileHint::Default => POSIX_FADV_NORMAL,
            FileHint::Sequential => POSIX_FADV_SEQUENTIAL,
            FileHint::Random => POSIX_FADV_RANDOM,
            FileHint::WriteOnce => POSIX_FADV_DONTNEED,
        };
        // offset=0 + len=0 = "apply to the whole file" per
        // posix_fadvise(2). posix_fadvise{,64} returns the errno
        // DIRECTLY (not via errno; 0 on success, positive errno on
        // failure).
        let fd = file.as_raw_fd();
        // SAFETY: fd is a valid file descriptor owned by `file`;
        // offset / len / advice are all valid inputs.
        #[expect(unsafe_code, reason = "posix_fadvise FFI call with valid fd")]
        let ret = unsafe {
            #[cfg(all(
                any(target_os = "linux", target_os = "android"),
                target_pointer_width = "32",
                any(target_env = "gnu", target_env = "")
            ))]
            {
                posix_fadvise64(fd, 0, 0, advice)
            }
            #[cfg(not(all(
                any(target_os = "linux", target_os = "android"),
                target_pointer_width = "32",
                any(target_env = "gnu", target_env = "")
            )))]
            {
                posix_fadvise(fd, 0, 0, advice)
            }
        };
        // EINVAL = kernel doesn't recognise this advice code (or the
        // fd isn't a regular file - e.g. pipe / socket / character
        // device). Hints are advisory; treat as a no-op per the
        // FsFile::hint contract.
        if ret == 0 || ret == EINVAL {
            Ok(())
        } else {
            Err(std::io::Error::from_raw_os_error(ret))
        }
    }

    // macOS has no posix_fadvise. The closest primitives are
    // `fcntl(F_RDADVISE)` (sequential prefetch hint, requires a byte
    // range - not useful for the whole-file hints we want) and
    // `fcntl(F_NOCACHE)` (toggle uncached I/O - too blunt for our use
    // case). Treat as a no-op for now; the performance benefit on
    // macOS is small enough that wiring a half-equivalent isn't worth
    // the complexity.
    #[cfg(target_os = "macos")]
    #[expect(
        clippy::unnecessary_wraps,
        reason = "FsFile::hint signature requires io::Result<()>; macOS branch is a no-op until we wire fcntl(F_RDADVISE)"
    )]
    pub(super) fn fadvise(_file: &File, _hint: FileHint) -> io::Result<()> {
        Ok(())
    }
}

#[cfg(windows)]
mod sys {
    use super::FileHint;
    use std::fs::File;
    use std::io;
    use std::os::windows::io::AsRawHandle;

    #[expect(
        clippy::unnecessary_wraps,
        reason = "FsFile::hint signature requires io::Result<()>; Windows branch is a no-op until we thread the hint through FsOpenOptions for CreateFile flags"
    )]
    pub(super) fn fadvise(_file: &File, _hint: FileHint) -> io::Result<()> {
        // Windows has no direct posix_fadvise equivalent. The closest
        // primitive (`FILE_FLAG_SEQUENTIAL_SCAN` / `_RANDOM_ACCESS`)
        // must be set at CreateFile time and can't be changed for an
        // already-open handle. Treat as a no-op - if Windows
        // performance becomes a concern we'd thread the hint through
        // FsOpenOptions instead and set the flag at open time.
        Ok(())
    }

    pub(super) fn lock_exclusive(file: &File) -> io::Result<()> {
        use core::ptr;

        // https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-lockfileex
        const LOCKFILE_EXCLUSIVE_LOCK: u32 = 0x0000_0002;

        // SAFETY: declaration matches the Windows `LockFileEx` ABI and `Overlapped` layout.
        // `allow` not `expect`: the foreign-fn name `LockFileEx` is not flagged by
        // `non_snake_case` on the Windows toolchain, so `expect` would be unfulfilled
        // and warn on every build; `allow` documents the intentional API-matching name
        // without depending on whether the lint fires.
        #[allow(non_snake_case, reason = "FFI name matches Windows API")]
        unsafe extern "system" {
            fn LockFileEx(
                h_file: *mut std::ffi::c_void,
                dw_flags: u32,
                dw_reserved: u32,
                n_number_of_bytes_to_lock_low: u32,
                n_number_of_bytes_to_lock_high: u32,
                lp_overlapped: *mut Overlapped,
            ) -> i32;
        }

        #[repr(C)]
        struct Overlapped {
            internal: usize,
            internal_high: usize,
            offset: u32,
            offset_high: u32,
            h_event: *mut std::ffi::c_void,
        }

        let handle = file.as_raw_handle();
        let mut overlapped = Overlapped {
            internal: 0,
            internal_high: 0,
            offset: 0,
            offset_high: 0,
            h_event: ptr::null_mut(),
        };

        // SAFETY: handle is a valid file handle owned by `file`.
        #[expect(unsafe_code, reason = "LockFileEx FFI call with valid handle")]
        let ret = unsafe {
            LockFileEx(
                handle,
                LOCKFILE_EXCLUSIVE_LOCK,
                0,
                u32::MAX,
                u32::MAX,
                &raw mut overlapped,
            )
        };

        if ret == 0 {
            return Err(std::io::Error::last_os_error());
        }
        Ok(())
    }

    pub(super) fn try_lock_exclusive(file: &File) -> io::Result<bool> {
        use core::ptr;

        const LOCKFILE_EXCLUSIVE_LOCK: u32 = 0x0000_0002;
        // Non-blocking: fail immediately with ERROR_LOCK_VIOLATION instead of
        // waiting when the region is already locked.
        const LOCKFILE_FAIL_IMMEDIATELY: u32 = 0x0000_0001;
        // Windows system error code returned when the lock is already held.
        const ERROR_LOCK_VIOLATION: i32 = 33;

        // SAFETY: declaration matches the Windows `LockFileEx` ABI and `Overlapped` layout.
        #[allow(non_snake_case, reason = "FFI name matches Windows API")]
        unsafe extern "system" {
            fn LockFileEx(
                h_file: *mut std::ffi::c_void,
                dw_flags: u32,
                dw_reserved: u32,
                n_number_of_bytes_to_lock_low: u32,
                n_number_of_bytes_to_lock_high: u32,
                lp_overlapped: *mut Overlapped,
            ) -> i32;
        }

        #[repr(C)]
        struct Overlapped {
            internal: usize,
            internal_high: usize,
            offset: u32,
            offset_high: u32,
            h_event: *mut std::ffi::c_void,
        }

        let handle = file.as_raw_handle();
        let mut overlapped = Overlapped {
            internal: 0,
            internal_high: 0,
            offset: 0,
            offset_high: 0,
            h_event: ptr::null_mut(),
        };

        // SAFETY: handle is a valid file handle owned by `file`.
        #[expect(unsafe_code, reason = "LockFileEx FFI call with valid handle")]
        let ret = unsafe {
            LockFileEx(
                handle,
                LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY,
                0,
                u32::MAX,
                u32::MAX,
                &raw mut overlapped,
            )
        };

        if ret == 0 {
            let err = std::io::Error::last_os_error();
            // The "already locked" case is the expected contention signal and
            // maps to `Ok(false)`, mirroring the unix EWOULDBLOCK path.
            if err.raw_os_error() == Some(ERROR_LOCK_VIOLATION) {
                return Ok(false);
            }
            return Err(err);
        }
        Ok(true)
    }
}

#[cfg(not(any(unix, windows)))]
mod sys {
    use super::FileHint;
    use std::fs::File;
    use std::io;

    pub(super) fn lock_exclusive(_file: &File) -> io::Result<()> {
        Err(io::Error::new(
            io::ErrorKind::Unsupported,
            "file locking is not supported on this platform",
        ))
    }

    pub(super) fn try_lock_exclusive(_file: &File) -> io::Result<bool> {
        Err(io::Error::new(
            io::ErrorKind::Unsupported,
            "file locking is not supported on this platform",
        ))
    }

    #[expect(
        clippy::unnecessary_wraps,
        reason = "FsFile::hint signature requires io::Result<()>; unsupported platforms have no fallible path"
    )]
    pub(super) fn fadvise(_file: &File, _hint: FileHint) -> io::Result<()> {
        // Unsupported platform - silently ignore. Hints are advisory.
        Ok(())
    }
}

// ---------------------------------------------------------------------------
// macOS filesystem capability detection (statfs) + clonefile reflink
// ---------------------------------------------------------------------------

#[cfg(target_os = "macos")]
mod macos_caps {
    use crate::fs::FsCapabilities;
    use core::mem::MaybeUninit;
    use std::ffi::{CStr, CString};
    use std::io;
    use std::os::unix::ffi::OsStrExt;
    use std::path::Path;

    fn to_cstring(path: &Path) -> io::Result<CString> {
        CString::new(path.as_os_str().as_bytes()).map_err(|_| {
            io::Error::new(
                io::ErrorKind::InvalidInput,
                "path contains an interior NUL byte",
            )
        })
    }

    /// Filesystem type name for the mount backing `path` (e.g. `"apfs"`),
    /// read from `statfs(2)`'s `f_fstypename`.
    fn fs_type_name(path: &Path) -> io::Result<String> {
        let c = to_cstring(path)?;
        let mut buf = MaybeUninit::<libc::statfs>::uninit();
        // SAFETY: `c` is a valid NUL-terminated C string; on success (rc == 0)
        // the kernel fully initializes the `statfs` buffer.
        #[expect(unsafe_code, reason = "statfs FFI to read the filesystem type name")]
        let rc = unsafe { libc::statfs(c.as_ptr(), buf.as_mut_ptr()) };
        if rc != 0 {
            return Err(std::io::Error::last_os_error());
        }
        // SAFETY: statfs returned 0, so `buf` is initialized; `f_fstypename`
        // is a NUL-terminated `c_char` array.
        #[expect(unsafe_code, reason = "read initialized statfs.f_fstypename")]
        let name = unsafe {
            let st = buf.assume_init();
            CStr::from_ptr(st.f_fstypename.as_ptr())
                .to_string_lossy()
                .into_owned()
        };
        Ok(name)
    }

    /// APFS is the one common macOS filesystem with copy-on-write, reflink, and
    /// native snapshots. Detection failure falls back to the conservative
    /// default (capabilities are an optimization, never a correctness
    /// requirement).
    pub(super) fn capabilities(path: &Path) -> FsCapabilities {
        if matches!(fs_type_name(path).as_deref(), Ok("apfs")) {
            FsCapabilities {
                copy_on_write: true,
                reflink: true,
                native_snapshot: true,
                ..FsCapabilities::default()
            }
        } else {
            FsCapabilities::default()
        }
    }

    /// O(1) data clone via `clonefile(2)` (APFS block sharing).
    pub(super) fn clonefile(src: &Path, dst: &Path) -> io::Result<()> {
        let s = to_cstring(src)?;
        let d = to_cstring(dst)?;
        // SAFETY: both args are valid NUL-terminated C strings; flags = 0.
        #[expect(unsafe_code, reason = "clonefile FFI for an O(1) APFS clone")]
        let rc = unsafe { libc::clonefile(s.as_ptr(), d.as_ptr(), 0) };
        if rc != 0 {
            return Err(std::io::Error::last_os_error());
        }
        Ok(())
    }

    /// Whether a `clonefile` failure means "this filesystem / placement cannot
    /// clone" (so fall back to a byte copy) rather than a real I/O failure.
    pub(super) fn clone_should_fall_back(err: &io::Error) -> bool {
        // ENOTSUP/EOPNOTSUPP (45): non-APFS mount or clone unsupported.
        // EXDEV (18): cross-device - cannot clone across filesystems.
        const ENOTSUP: i32 = 45;
        const EXDEV: i32 = 18;
        matches!(err.raw_os_error(), Some(ENOTSUP | EXDEV))
            || matches!(
                err.kind(),
                io::ErrorKind::Unsupported | io::ErrorKind::CrossesDevices
            )
    }
}

// ---------------------------------------------------------------------------
// Linux filesystem capability detection (statfs) + FICLONE reflink + NoCoW
// ---------------------------------------------------------------------------

#[cfg(target_os = "linux")]
mod linux_caps {
    use crate::fs::FsCapabilities;
    use std::fs::{File, OpenOptions};
    use std::io;
    use std::os::unix::io::AsRawFd;
    use std::path::Path;

    // statfs-based capability detection only compiles on 64-bit (the magic
    // numbers exceed i32::MAX); these imports + helper feed only that path, so
    // gate them to avoid unused-import / dead_code warnings on 32-bit targets.
    #[cfg(target_pointer_width = "64")]
    use core::mem::MaybeUninit;
    #[cfg(target_pointer_width = "64")]
    use std::ffi::CString;
    #[cfg(target_pointer_width = "64")]
    use std::os::unix::ffi::OsStrExt;

    #[cfg(target_pointer_width = "64")]
    fn to_cstring(path: &Path) -> io::Result<CString> {
        CString::new(path.as_os_str().as_bytes()).map_err(|_| {
            io::Error::new(
                io::ErrorKind::InvalidInput,
                "path contains an interior NUL byte",
            )
        })
    }

    /// Capabilities derived from `statfs(2)`'s `f_type` magic for the mount
    /// backing `path` (see [`crate::fs::linux_fs_type_capabilities`]); any
    /// detection failure reports the conservative default.
    ///
    /// Gated to 64-bit targets: `statfs.f_type` is `__fsword_t` (`c_long`),
    /// and the magic numbers exceed `i32::MAX`, so a clean comparison needs the
    /// 64-bit field. 32-bit Linux is not a storage-engine deployment target;
    /// it reports the conservative default.
    #[cfg(target_pointer_width = "64")]
    pub(super) fn capabilities(path: &Path) -> FsCapabilities {
        let Ok(c) = to_cstring(path) else {
            return FsCapabilities::default();
        };
        let mut buf = MaybeUninit::<libc::statfs>::uninit();
        // SAFETY: `c` is a valid NUL-terminated C string; on success (rc == 0)
        // the kernel fully initializes the `statfs` buffer.
        #[expect(unsafe_code, reason = "statfs FFI to read f_type")]
        let rc = unsafe { libc::statfs(c.as_ptr(), buf.as_mut_ptr()) };
        if rc != 0 {
            return FsCapabilities::default();
        }
        // SAFETY: statfs returned 0, so `buf` is initialized. `f_type` is
        // `c_long` (i64) on glibc but `c_ulong` (u64) on musl, so normalize to
        // i64 before matching the magic constants (their bit patterns are
        // identical across the cast).
        #[expect(unsafe_code, reason = "read initialized statfs.f_type")]
        #[allow(
            clippy::unnecessary_cast,
            clippy::cast_possible_wrap,
            reason = "f_type is i64 on glibc (no-op cast) and u64 on musl (cast required)"
        )]
        let f_type = unsafe { buf.assume_init() }.f_type as i64;
        crate::fs::linux_fs_type_capabilities(f_type)
    }

    /// Deallocates `[offset, offset+len)` inside the file at `path` via
    /// `fallocate(FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE)`: frees the
    /// physical blocks, keeps the logical length, the hole reads back as zeros.
    ///
    /// 64-bit only: `off_t` is `i64` here, matching the `i64::try_from` range
    /// guard. 32-bit Linux reports `punch_hole = false` (see `capabilities`), so
    /// the mode never engages there; the stub below keeps the call site building.
    #[cfg(target_pointer_width = "64")]
    pub(super) fn punch_hole(path: &Path, offset: u64, len: u64) -> io::Result<()> {
        // off_t is i64 on 64-bit Linux; a file offset / length that exceeds
        // i64::MAX is not a real SST extent, so reject rather than wrap.
        let off = i64::try_from(offset)
            .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "punch offset exceeds i64"))?;
        let length = i64::try_from(len)
            .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "punch len exceeds i64"))?;
        let f = OpenOptions::new().write(true).open(path)?;
        // SAFETY: `f` owns a valid writable fd for the duration of the call;
        // the flags + range are plain integers.
        #[expect(
            unsafe_code,
            reason = "fallocate(PUNCH_HOLE) FFI for in-place extent reclaim"
        )]
        let rc = unsafe {
            libc::fallocate(
                f.as_raw_fd(),
                libc::FALLOC_FL_PUNCH_HOLE | libc::FALLOC_FL_KEEP_SIZE,
                off,
                length,
            )
        };
        if rc != 0 {
            return Err(io::Error::last_os_error());
        }
        Ok(())
    }

    /// Whether `[offset, offset + len)` is entirely unallocated, via
    /// `lseek(SEEK_DATA)`: it returns the next byte at or after `offset` that is
    /// backed by data, so the extent is a hole exactly when that byte lies at or
    /// past its end. `ENXIO` means no data follows at all — a hole to EOF.
    ///
    /// Unlike a file-wide allocation total this attributes the hole to THIS
    /// range, which is what tells a reclaimed data block from one destroyed by
    /// corruption (both read as zeros).
    #[cfg(target_pointer_width = "64")]
    pub(super) fn extent_is_hole(path: &Path, offset: u64, len: u64) -> io::Result<Option<bool>> {
        if len == 0 {
            return Ok(Some(false));
        }
        let off = i64::try_from(offset)
            .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "offset exceeds i64"))?;
        let end = offset
            .checked_add(len)
            .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "extent end overflows"))?;
        let f = File::open(path)?;
        // SAFETY: `f` owns a valid readable fd for the duration of the call; the
        // offset and whence are plain integers.
        #[expect(unsafe_code, reason = "lseek(SEEK_DATA) FFI for an extent hole probe")]
        let next_data = unsafe { libc::lseek(f.as_raw_fd(), off, libc::SEEK_DATA) };
        if next_data < 0 {
            let err = io::Error::last_os_error();
            // No data at or after `offset`: the rest of the file is a hole.
            if err.raw_os_error() == Some(libc::ENXIO) {
                return Ok(Some(true));
            }
            // A filesystem without SEEK_DATA cannot answer; do not guess.
            if matches!(err.raw_os_error(), Some(libc::EINVAL | libc::ENOTSUP)) {
                return Ok(None);
            }
            return Err(err);
        }
        Ok(Some(u64::try_from(next_data).unwrap_or(u64::MAX) >= end))
    }

    /// As [`extent_is_hole`], but answers whether the range CONTAINS a hole:
    /// `lseek(SEEK_HOLE)` finds the next hole at or after `offset`, and a
    /// result before the range end proves a contained hole. The implicit
    /// every-file-ends-in-a-hole EOF state is excluded by clamping the range
    /// end to the file length — a fully allocated file seeks to EOF, which is
    /// never below the clamped end.
    #[cfg(target_pointer_width = "64")]
    pub(super) fn extent_contains_hole(
        path: &Path,
        offset: u64,
        len: u64,
    ) -> io::Result<Option<bool>> {
        if len == 0 {
            return Ok(Some(false));
        }
        let off = i64::try_from(offset)
            .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "offset exceeds i64"))?;
        let end = offset
            .checked_add(len)
            .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "extent end overflows"))?;
        let f = File::open(path)?;
        let file_len = f.metadata()?.len();
        // SAFETY: `f` owns a valid readable fd for the duration of the call; the
        // offset and whence are plain integers.
        #[expect(
            unsafe_code,
            reason = "lseek(SEEK_HOLE) FFI for a contained-hole probe"
        )]
        let next_hole = unsafe { libc::lseek(f.as_raw_fd(), off, libc::SEEK_HOLE) };
        if next_hole < 0 {
            let err = io::Error::last_os_error();
            // Offset at or past EOF: nothing of the range is backed by the
            // file, so no hole INSIDE it can be attributed.
            if err.raw_os_error() == Some(libc::ENXIO) {
                return Ok(Some(false));
            }
            // A filesystem without SEEK_HOLE cannot answer; do not guess.
            if matches!(err.raw_os_error(), Some(libc::EINVAL | libc::ENOTSUP)) {
                return Ok(None);
            }
            return Err(err);
        }
        Ok(Some(
            u64::try_from(next_hole).unwrap_or(u64::MAX) < end.min(file_len),
        ))
    }

    /// 32-bit Linux: `punch_hole` is unsupported there, so no hole can exist to
    /// attribute; "cannot tell" is the truthful answer.
    #[cfg(not(target_pointer_width = "64"))]
    #[expect(
        clippy::unnecessary_wraps,
        reason = "the signature of the 64-bit probe the same caller uses"
    )]
    pub(super) fn extent_is_hole(
        _path: &Path,
        _offset: u64,
        _len: u64,
    ) -> io::Result<Option<bool>> {
        Ok(None)
    }

    /// 32-bit Linux: as [`extent_is_hole`] — no punch support, "cannot tell".
    #[cfg(not(target_pointer_width = "64"))]
    #[expect(
        clippy::unnecessary_wraps,
        reason = "the signature of the 64-bit probe the same caller uses"
    )]
    pub(super) fn extent_contains_hole(
        _path: &Path,
        _offset: u64,
        _len: u64,
    ) -> io::Result<Option<bool>> {
        Ok(None)
    }

    /// 32-bit Linux: `punch_hole` capability is reported false, so this is
    /// unreachable in practice; surface `Unsupported` to keep the contract.
    #[cfg(not(target_pointer_width = "64"))]
    pub(super) fn punch_hole(_path: &Path, _offset: u64, _len: u64) -> io::Result<()> {
        Err(io::Error::new(
            io::ErrorKind::Unsupported,
            "punch_hole unsupported on 32-bit",
        ))
    }

    /// 32-bit Linux: not a storage-engine deployment target; the `statfs`
    /// `f_type` magic comparison needs the 64-bit field, so report the
    /// conservative default.
    #[cfg(not(target_pointer_width = "64"))]
    pub(super) fn capabilities(_path: &Path) -> FsCapabilities {
        FsCapabilities::default()
    }

    /// O(1) data clone via `ioctl(FICLONE)`. Creates `dst` (failing if it
    /// exists), then clones `src` into it; on clone failure the empty `dst` is
    /// removed so a caller's copy fallback can re-create it.
    pub(super) fn ficlone(src: &Path, dst: &Path) -> io::Result<()> {
        let src_f = File::open(src)?;
        let dst_f = OpenOptions::new().write(true).create_new(true).open(dst)?;
        // SAFETY: FICLONE takes the source fd as its argument; both fds are
        // valid and owned by `src_f` / `dst_f` for the duration of the call.
        #[expect(unsafe_code, reason = "ioctl(FICLONE) for an O(1) reflink clone")]
        let rc = unsafe {
            libc::ioctl(
                dst_f.as_raw_fd(),
                // ioctl's request param is c_ulong on glibc but c_int on musl;
                // infer the target type so FICLONE casts to whichever applies.
                libc::FICLONE as _,
                src_f.as_raw_fd(),
            )
        };
        if rc != 0 {
            let err = std::io::Error::last_os_error();
            drop(dst_f);
            // Best-effort cleanup of the empty target so the copy fallback's
            // `create_new` does not trip over it.
            let _ = std::fs::remove_file(dst);
            return Err(err);
        }
        Ok(())
    }

    /// Whether a `ficlone` failure means "this filesystem / placement cannot
    /// reflink" (fall back to a byte copy) rather than a real I/O failure.
    pub(super) fn clone_should_fall_back(err: &io::Error) -> bool {
        // EOPNOTSUPP/ENOTSUP (95): FS has no reflink. EXDEV (18): cross-device.
        // EINVAL (22): kernel/FS rejected the clone (e.g. non-reflink XFS).
        const EOPNOTSUPP: i32 = 95;
        const EXDEV: i32 = 18;
        const EINVAL: i32 = 22;
        matches!(err.raw_os_error(), Some(EOPNOTSUPP | EXDEV | EINVAL))
            || matches!(
                err.kind(),
                io::ErrorKind::Unsupported | io::ErrorKind::CrossesDevices
            )
    }

    /// Clears per-file `CoW` (`FS_NOCOW_FL`) on the asm-generic ioctl
    /// architectures (the only ones where these constant values are correct).
    /// A no-op on other architectures and on filesystems without the
    /// inode-flags ioctl.
    #[cfg(any(
        target_arch = "x86",
        target_arch = "x86_64",
        target_arch = "arm",
        target_arch = "aarch64",
        target_arch = "riscv32",
        target_arch = "riscv64",
        target_arch = "loongarch64",
        target_arch = "s390x"
    ))]
    pub(super) fn try_disable_cow(path: &Path) -> io::Result<()> {
        // FS_IOC_{GET,SET}FLAGS encode sizeof(long) in the request, which is 4
        // on the 32-bit targets above and 8 on the 64-bit ones: libc's values
        // carry the target's width. FS_NOCOW_FL is the inode flag bit.
        const FS_IOC_GETFLAGS: libc::Ioctl = libc::FS_IOC_GETFLAGS;
        const FS_IOC_SETFLAGS: libc::Ioctl = libc::FS_IOC_SETFLAGS;
        const FS_NOCOW_FL: libc::c_int = 0x0080_0000;

        let f = OpenOptions::new().read(true).write(true).open(path)?;
        let mut flags: libc::c_int = 0;
        // SAFETY: valid fd; GETFLAGS writes one `int` through the pointer arg.
        #[expect(unsafe_code, reason = "ioctl(FS_IOC_GETFLAGS) to read inode flags")]
        #[allow(
            clippy::unnecessary_cast,
            clippy::cast_possible_truncation,
            reason = "ioctl request is c_ulong on glibc (no-op) but c_int on musl; \
                      the 32-bit request code's low bits are preserved by truncation"
        )]
        let rc = unsafe { libc::ioctl(f.as_raw_fd(), FS_IOC_GETFLAGS as _, &raw mut flags) };
        if rc != 0 {
            return ignore_if_unsupported(std::io::Error::last_os_error());
        }
        if flags & FS_NOCOW_FL != 0 {
            return Ok(()); // already NoCoW
        }
        flags |= FS_NOCOW_FL;
        // SAFETY: valid fd; SETFLAGS reads one `int` through the pointer arg.
        #[expect(unsafe_code, reason = "ioctl(FS_IOC_SETFLAGS) to set FS_NOCOW_FL")]
        #[allow(
            clippy::unnecessary_cast,
            clippy::cast_possible_truncation,
            reason = "ioctl request is c_ulong on glibc (no-op) but c_int on musl; \
                      the 32-bit request code's low bits are preserved by truncation"
        )]
        let rc = unsafe { libc::ioctl(f.as_raw_fd(), FS_IOC_SETFLAGS as _, &raw const flags) };
        if rc != 0 {
            return ignore_if_unsupported(std::io::Error::last_os_error());
        }
        Ok(())
    }

    #[cfg(not(any(
        target_arch = "x86",
        target_arch = "x86_64",
        target_arch = "arm",
        target_arch = "aarch64",
        target_arch = "riscv32",
        target_arch = "riscv64",
        target_arch = "loongarch64",
        target_arch = "s390x"
    )))]
    pub(super) fn try_disable_cow(_path: &Path) -> io::Result<()> {
        // The FS_IOC_* constants above are only valid for the asm-generic ioctl
        // encoding; on other arches leave `CoW` alone (optimization, not
        // correctness).
        Ok(())
    }

    /// Maps "filesystem does not support the inode-flags ioctl" errors to a
    /// no-op success - disabling `CoW` is an optimization, never required.
    #[cfg(any(
        target_arch = "x86",
        target_arch = "x86_64",
        target_arch = "arm",
        target_arch = "aarch64",
        target_arch = "riscv32",
        target_arch = "riscv64",
        target_arch = "loongarch64",
        target_arch = "s390x"
    ))]
    fn ignore_if_unsupported(err: io::Error) -> io::Result<()> {
        // ENOTTY (25): ioctl not supported by this FS. EOPNOTSUPP/ENOTSUP (95):
        // flag not supported. EPERM/EINVAL: FS rejected the flag.
        const ENOTTY: i32 = 25;
        const EOPNOTSUPP: i32 = 95;
        const EINVAL: i32 = 22;
        if matches!(err.raw_os_error(), Some(ENOTTY | EOPNOTSUPP | EINVAL))
            || matches!(err.kind(), io::ErrorKind::Unsupported)
        {
            Ok(())
        } else {
            Err(err)
        }
    }
}

// ---------------------------------------------------------------------------
// Hard-link count probe (Windows)
// ---------------------------------------------------------------------------

#[cfg(windows)]
mod hard_link_count_sys {
    use std::io;
    use std::os::windows::io::AsRawHandle as _;

    /// `BY_HANDLE_FILE_INFORMATION` (Win32): 13 little-endian `DWORD`s. The
    /// `FILETIME` pairs are flattened to `[u32; 2]` to keep the declaration
    /// dependency-free; the layout is identical.
    #[repr(C)]
    #[expect(non_snake_case, reason = "Win32 API struct")]
    #[derive(Default)]
    struct ByHandleFileInformation {
        dwFileAttributes: u32,
        ftCreationTime: [u32; 2],
        ftLastAccessTime: [u32; 2],
        ftLastWriteTime: [u32; 2],
        dwVolumeSerialNumber: u32,
        nFileSizeHigh: u32,
        nFileSizeLow: u32,
        nNumberOfLinks: u32,
        nFileIndexHigh: u32,
        nFileIndexLow: u32,
    }

    // SAFETY (ABI): the signature matches the Win32
    // `GetFileInformationByHandle` contract — a file handle plus an out
    // pointer, returning a non-zero `BOOL` on success.
    #[allow(non_snake_case, reason = "Win32 API signature")]
    unsafe extern "system" {
        fn GetFileInformationByHandle(
            hFile: *mut core::ffi::c_void,
            lpFileInformation: *mut ByHandleFileInformation,
        ) -> i32;
    }

    pub(super) fn hard_link_count(file: &std::fs::File) -> io::Result<u64> {
        let mut info = ByHandleFileInformation::default();
        // SAFETY: the handle comes from a live `File` borrow; `info` is a
        // valid out pointer; fields are read only on success.
        #[expect(
            unsafe_code,
            reason = "GetFileInformationByHandle for the NTFS link count"
        )]
        let rc = unsafe { GetFileInformationByHandle(file.as_raw_handle(), &raw mut info) };
        if rc == 0 {
            return Err(io::Error::last_os_error());
        }
        Ok(u64::from(info.nNumberOfLinks))
    }

    /// `FILE_ID_INFO` (Win32): the volume serial number and the 128-bit
    /// file ID, unique on the volume on `ReFS` too, where the 64-bit index of
    /// `BY_HANDLE_FILE_INFORMATION` is not.
    #[repr(C)]
    #[expect(non_snake_case, reason = "Win32 API struct")]
    #[derive(Default)]
    struct FileIdInfo {
        VolumeSerialNumber: u64,
        FileId: [u8; 16],
    }

    /// `FILE_INFO_BY_HANDLE_CLASS::FileIdInfo`.
    const FILE_ID_INFO_CLASS: i32 = 18;
    /// The size of `FILE_ID_INFO` the call is told, checked against the
    /// struct at compile time.
    const FILE_ID_INFO_SIZE: u32 = 24;
    const _: () = assert!(core::mem::size_of::<FileIdInfo>() == FILE_ID_INFO_SIZE as usize);
    /// Opens a directory as well as a file, for its attributes only.
    const FILE_FLAG_BACKUP_SEMANTICS: u32 = 0x0200_0000;

    // SAFETY (ABI): the signature matches the Win32
    // `GetFileInformationByHandleEx` contract — a handle, an information
    // class, an out buffer and its size, returning a non-zero `BOOL` on
    // success. `allow`, not `expect`: `non_snake_case` does not fire on a
    // foreign block, so an expectation would be unfulfilled.
    #[allow(non_snake_case, reason = "Win32 API signature")]
    unsafe extern "system" {
        fn GetFileInformationByHandleEx(
            hFile: *mut core::ffi::c_void,
            FileInformationClass: i32,
            lpFileInformation: *mut core::ffi::c_void,
            dwBufferSize: u32,
        ) -> i32;
    }

    /// The identity of the file or directory `path` names: its volume and
    /// file ID. The handle asks for no data access, so a file another
    /// handle holds open, and a directory, can still be identified.
    pub(super) fn file_identity(path: &std::path::Path) -> io::Result<(u64, [u8; 16])> {
        use std::os::windows::fs::OpenOptionsExt as _;
        let file = std::fs::OpenOptions::new()
            .access_mode(0)
            .custom_flags(FILE_FLAG_BACKUP_SEMANTICS)
            .open(path)?;
        let mut info = FileIdInfo::default();
        // SAFETY: the handle comes from a live `File`; `info` is a valid out
        // buffer of `FILE_ID_INFO_SIZE` bytes; fields are read only on success.
        #[expect(
            unsafe_code,
            reason = "GetFileInformationByHandleEx for the file identity"
        )]
        let rc = unsafe {
            GetFileInformationByHandleEx(
                file.as_raw_handle(),
                FILE_ID_INFO_CLASS,
                (&raw mut info).cast(),
                FILE_ID_INFO_SIZE,
            )
        };
        if rc == 0 {
            return Err(io::Error::last_os_error());
        }
        Ok((info.VolumeSerialNumber, info.FileId))
    }
}

// ---------------------------------------------------------------------------
// Filesystem free-space probe
// ---------------------------------------------------------------------------

#[cfg(windows)]
mod available_space_sys {
    use std::io;
    use std::os::windows::ffi::OsStrExt;
    use std::path::Path;

    // GetDiskFreeSpaceExW: free bytes available to the calling user on the
    // volume containing `path`.
    //
    // SAFETY (ABI): the signature matches the Win32 `GetDiskFreeSpaceExW`
    // contract — a wide directory-name pointer plus three optional `u64` out
    // pointers, returning a non-zero `BOOL` on success. Edition 2024 requires
    // foreign declarations in an `unsafe extern` block.
    #[allow(non_snake_case, reason = "Win32 API signature")]
    unsafe extern "system" {
        fn GetDiskFreeSpaceExW(
            lpDirectoryName: *const u16,
            lpFreeBytesAvailableToCaller: *mut u64,
            lpTotalNumberOfBytes: *mut u64,
            lpTotalNumberOfFreeBytes: *mut u64,
        ) -> i32;
    }

    pub(super) fn disk_free_available(path: &Path) -> io::Result<u64> {
        // NUL-terminated wide string of the path. Reject an interior NUL first:
        // Win32 treats the first NUL as the string terminator, so a path with an
        // embedded NUL would silently probe a truncated path (a different volume)
        // and feed admission / storage_stats free space for the wrong filesystem.
        // Mirrors the unix statvfs helper's `CString::new` rejection.
        let mut wide: Vec<u16> = path.as_os_str().encode_wide().collect();
        if wide.contains(&0) {
            return Err(io::Error::new(
                io::ErrorKind::InvalidInput,
                "path contains an interior NUL byte",
            ));
        }
        wide.push(0);
        let mut avail: u64 = 0;
        // SAFETY: `wide` is a valid NUL-terminated UTF-16 string; the three out
        // pointers are valid for the call; we read `avail` only on success.
        #[expect(unsafe_code, reason = "GetDiskFreeSpaceExW FFI for free space")]
        let rc = unsafe {
            GetDiskFreeSpaceExW(
                wide.as_ptr(),
                &raw mut avail,
                core::ptr::null_mut(),
                core::ptr::null_mut(),
            )
        };
        if rc == 0 {
            return Err(io::Error::last_os_error());
        }
        Ok(avail)
    }
}

// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------

#[cfg(test)]
#[allow(
    clippy::unwrap_used,
    clippy::indexing_slicing,
    clippy::useless_vec,
    reason = "test code"
)]
mod tests;