qdrant-edge 0.7.0

use std::borrow::Cow;
use std::path::PathBuf;
use std::sync::Arc;

use fs_err as fs;

use super::{BLOCK_SIZE, DiskCache, DiskCacheConfig, DiskCacheFs, DiskCacheFsContext};
use crate::common::generic_consts::Sequential;
use crate::common::mmap::AdviceSetting;
use crate::common::universal_io::{
    OpenOptions, Populate, ReadRange, UniversalRead, UniversalReadFileOps, UniversalReadFs,
};

fn make_test_data(n_bytes: usize) -> Vec<u8> {
    (0..n_bytes).map(|i| (i % 251) as u8).collect()
}

struct Scenario {
    _tmp: tempfile::TempDir,
    remote_path: PathBuf,
    data: Vec<u8>,
    config: Arc<DiskCacheConfig>,
}

impl Scenario {
    fn new(n_bytes: usize) -> Self {
        let tmp = tempfile::Builder::new()
            .prefix("disk_cache_tests")
            .tempdir()
            .unwrap();
        let remote_dir = tmp.path().join("remote");
        let local_dir = tmp.path().join("local");
        fs::create_dir_all(&remote_dir).unwrap();
        fs::create_dir_all(&local_dir).unwrap();

        let remote_path = remote_dir.join("data.bin");
        let data = make_test_data(n_bytes);
        fs::write(&remote_path, &data).unwrap();

        Self {
            _tmp: tmp,
            remote_path,
            data,
            config: Arc::new(DiskCacheConfig::new(remote_dir, local_dir).unwrap()),
        }
    }

    fn expected_local_path(&self) -> PathBuf {
        self.config.local_path_for(&self.remote_path).unwrap()
    }

    fn open<R>(&self, prefill: bool) -> DiskCache<R>
    where
        R: UniversalRead + Clone,
        R::Fs: Clone + Send + Sync,
        <R::Fs as UniversalReadFileOps>::ContextConfig: Default,
        <R::Fs as UniversalReadFs>::OpenExtra: Clone + Send + Sync,
        R::OwnedReadPipeline<u8, std::ops::Range<u32>>: Send,
    {
        let populate = if prefill {
            Populate::PreferBackground
        } else {
            Populate::No
        };

        let fs = DiskCacheFs::<R>::from_context(DiskCacheFsContext {
            config: self.config.clone(),
            remote: Default::default(),
        })
        .unwrap();
        fs.open(
            &self.remote_path,
            OpenOptions {
                writeable: false,
                populate,
                need_sequential: false,
                advice: AdviceSetting::Global,
            },
            Default::default(),
        )
        .unwrap()
    }

    /// Append `additional_bytes` bytes to the remote file in-place.
    /// Returns the full new remote contents.
    fn grow_remote(&mut self, additional_bytes: usize) -> Vec<u8> {
        use std::io::Write;

        let old_len = self.data.len();
        let new_data = make_test_data(old_len + additional_bytes);
        let mut file = fs::OpenOptions::new()
            .append(true)
            .open(&self.remote_path)
            .unwrap();
        file.write_all(&new_data[old_len..]).unwrap();
        self.data = new_data.clone();
        new_data
    }

    /// Shrink the remote file in-place to `new_len` bytes.
    fn truncate_remote(&mut self, new_len: usize) {
        let file = fs::OpenOptions::new()
            .write(true)
            .open(&self.remote_path)
            .unwrap();
        file.set_len(new_len as u64).unwrap();
        self.data.truncate(new_len);
    }
}

#[duplicate::duplicate_item(
    tests_mod       R               cfg_predicate               _PREFILL;
    [tests_prefill] [MmapFile]      [cfg(all())]                [true];
    [tests_mmap]    [MmapFile]      [cfg(all())]                [false];
    [tests_uring]   [IoUringFile]   [cfg(target_os = "linux")]  [false];
)]
#[cfg_predicate]
#[cfg(test)]
mod tests_mod {
    use std::io::ErrorKind;
    use std::sync::atomic::Ordering;

    use super::*;
    #[cfg_predicate]
    use crate::common::universal_io::R;

    const PREFILL: bool = _PREFILL;

    #[test]
    fn basic_read_returns_remote_bytes() {
        let scn = Scenario::new(BLOCK_SIZE * 3 + 100);
        let file = scn.open::<R>(PREFILL);

        // Read inside the first block.
        let bytes = file
            .read::<Sequential, u8>(ReadRange {
                byte_offset: 10,
                length: 20,
            })
            .unwrap();
        assert_eq!(&*bytes, &scn.data[10..30]);

        // Last block includes the 100-byte tail.
        let last = scn.data.len() as u64;
        let bytes = file
            .read::<Sequential, u8>(ReadRange {
                byte_offset: last - 50,
                length: 50,
            })
            .unwrap();
        assert_eq!(&*bytes, &scn.data[scn.data.len() - 50..]);
    }

    #[test]
    fn read_spanning_multiple_blocks_is_contiguous() {
        let scn = Scenario::new(BLOCK_SIZE * 3 + 100);
        let file = scn.open::<R>(PREFILL);

        let start = (BLOCK_SIZE - 50) as u64;
        let len = (BLOCK_SIZE + 100) as u64;
        let bytes = file
            .read::<Sequential, u8>(ReadRange {
                byte_offset: start,
                length: len,
            })
            .unwrap();
        let start = start as usize;
        let end = start + len as usize;
        assert!(matches!(bytes, Cow::Borrowed(_)));
        assert_eq!(bytes.as_ref(), &scn.data[start..end]);
    }

    #[test]
    fn local_file_is_created_on_first_read() {
        let scn = Scenario::new(BLOCK_SIZE * 2);
        let expected_local = scn.expected_local_path();

        let file = scn.open::<R>(PREFILL);

        // Before the first read, the local file doesn't exist yet.
        assert!(
            !expected_local.exists(),
            "local file should not exist before first read: {}",
            expected_local.display(),
        );

        // Trigger one read. This must bring up the local file.
        let _ = file
            .read::<Sequential, u8>(ReadRange {
                byte_offset: 0,
                length: 1,
            })
            .unwrap();

        assert!(
            expected_local.exists(),
            "local file should exist after first read"
        );
        assert_eq!(
            fs::metadata(&expected_local).unwrap().len(),
            scn.data.len() as u64,
            "local file should be sized to the remote",
        );
    }

    #[test]
    fn populate_fetches_every_block() {
        let scn = Scenario::new(BLOCK_SIZE * 3 + 100);
        let file = scn.open::<R>(PREFILL);

        file.populate().unwrap();

        let bytes = file
            .read::<Sequential, u8>(ReadRange {
                byte_offset: 0,
                length: scn.data.len() as u64,
            })
            .unwrap();
        assert_eq!(&*bytes, &scn.data[..]);
    }

    #[test]
    fn read_past_end_returns_out_of_bounds() {
        let scn = Scenario::new(1024);
        let file = scn.open::<R>(PREFILL);

        let err = file
            .read::<Sequential, u8>(ReadRange {
                byte_offset: 1000,
                length: 100,
            })
            .unwrap_err();
        assert!(
            matches!(
                err,
                crate::common::universal_io::UniversalIoError::OutOfBounds { .. }
            ),
            "expected OutOfBounds, got {err:?}",
        );
    }

    /// Reopen with no prior reads leaves the local mirror untouched.
    #[test]
    #[cfg_attr(target_os = "windows", ignore)] // FIXME: don't ignore, fix reopen on windows
    fn reopen_without_prior_reads_keeps_local_uninitialized() {
        let scn = Scenario::new(BLOCK_SIZE * 2);
        let mut cache = scn.open::<R>(PREFILL);
        let expected_local = scn.expected_local_path();
        assert!(!expected_local.exists());

        cache.reopen().unwrap();

        // unless it was scheduled for prefill
        if PREFILL {
            assert!(expected_local.exists());
            assert!(cache.local.get().is_some());
        } else {
            assert!(!expected_local.exists());
            assert!(cache.local.get().is_none());
        }
    }

    /// Reopen on an unchanged remote must not resize, repopulate, or mutate
    /// the fetched bitmap.
    #[test]
    #[cfg_attr(target_os = "windows", ignore)] // FIXME: don't ignore, fix reopen on windows
    fn reopen_no_growth_does_not_repopulate() {
        let scn = Scenario::new(BLOCK_SIZE * 3);
        let mut cache = scn.open::<R>(PREFILL);

        let _ = cache
            .read::<Sequential, u8>(ReadRange {
                byte_offset: 0,
                length: 1,
            })
            .unwrap();

        let (len_before, populated_before, fetched_before) = {
            let local = cache.local.get().expect("local initialized after read");
            (
                local.mmap().len::<u8>().unwrap(),
                local.fully_populated.load(Ordering::Acquire),
                local.fetched.lock().clone(),
            )
        };

        cache.reopen().unwrap();

        let local = cache.local.get().expect("local must still be initialized");
        assert_eq!(local.mmap().len::<u8>().unwrap(), len_before);
        assert_eq!(
            local.fully_populated.load(Ordering::Acquire),
            populated_before,
        );
        assert_eq!(local.fetched.lock().clone(), fetched_before);
    }

    /// Reopening over a shrunk remote must fail with `UnexpectedEof`.
    #[test]
    #[cfg_attr(target_os = "windows", ignore)] // FIXME: don't ignore, fix reopen on windows
    fn reopen_with_smaller_remote_fails() {
        let mut scn = Scenario::new(BLOCK_SIZE * 4);
        let mut cache = scn.open::<R>(PREFILL);

        let _ = cache
            .read::<Sequential, u8>(ReadRange {
                byte_offset: 0,
                length: 1,
            })
            .unwrap();

        scn.truncate_remote(BLOCK_SIZE * 2);

        let err = cache.reopen().unwrap_err();
        assert!(
            matches!(
                &err,
                crate::common::universal_io::UniversalIoError::Io(io_err)
                    if io_err.kind() == ErrorKind::UnexpectedEof,
            ),
            "expected Io(UnexpectedEof), got: {err:?}",
        );
    }

    /// Reads into the new section must fail before reopen (local mirror is at
    /// the old length) and succeed after reopen.
    #[test]
    #[cfg_attr(target_os = "windows", ignore)] // FIXME: don't ignore, fix reopen on windows
    fn reopen_growth_visible_after_reopen() {
        let mut scn = Scenario::new(BLOCK_SIZE * 2);
        let mut cache = scn.open::<R>(PREFILL);

        let original_len = scn.data.len() as u64;

        let _ = cache
            .read::<Sequential, u8>(ReadRange {
                byte_offset: 0,
                length: 1,
            })
            .unwrap();

        let new_data = scn.grow_remote(BLOCK_SIZE);

        let err = cache
            .read::<Sequential, u8>(ReadRange {
                byte_offset: original_len,
                length: BLOCK_SIZE as u64,
            })
            .unwrap_err();
        assert!(
            matches!(
                err,
                crate::common::universal_io::UniversalIoError::OutOfBounds { .. },
            ),
            "expected OutOfBounds before reopen, got: {err:?}",
        );

        cache.reopen().unwrap();

        let bytes = cache
            .read::<Sequential, u8>(ReadRange {
                byte_offset: original_len,
                length: BLOCK_SIZE as u64,
            })
            .unwrap();
        assert_eq!(&*bytes, &new_data[original_len as usize..]);
    }

    /// When the remote grows and the original tail block was only partially
    /// populated, reopen must invalidate that block so the next read re-fetches
    /// it instead of returning the zero-filled bytes left by `set_len`.
    #[test]
    #[cfg_attr(target_os = "windows", ignore)] // FIXME: don't ignore, fix reopen on windows
    fn reopen_growth_refetches_partial_tail_block() {
        // Non-block-aligned remote: block 1 holds only 100 real bytes.
        let mut scn = Scenario::new(BLOCK_SIZE + 100);
        let mut cache = scn.open::<R>(PREFILL);

        // Touch the partial tail so block 1 ends up in the `fetched` bitmap
        // (its fetch is clamped to the old EOF).
        let _ = cache
            .read::<Sequential, u8>(ReadRange {
                byte_offset: BLOCK_SIZE as u64,
                length: 1,
            })
            .unwrap();

        // Grow remote past the old tail block boundary.
        let new_data = scn.grow_remote(BLOCK_SIZE);

        cache.reopen().unwrap();

        // Read covers both the originally-partial range [BLOCK_SIZE..old_len)
        // and the newly-grown tail [old_len..BLOCK_SIZE*2). Without the
        // invalidation, the second half would be zeros from `set_len`.
        let bytes = cache
            .read::<Sequential, u8>(ReadRange {
                byte_offset: BLOCK_SIZE as u64,
                length: BLOCK_SIZE as u64,
            })
            .unwrap();
        assert_eq!(&*bytes, &new_data[BLOCK_SIZE..BLOCK_SIZE * 2]);
    }
}