hyperfile 0.10.2

//! Compaction on S3: `fs_relocate`, `fs_content_cno` and `fs_content_cno_fast`.
//!
//! `functional_compaction` covers when a publish is marked as moving blocks only.
//! This covers what only S3 shows: relocation reads blocks with the read path's
//! coalescing — a run of blocks in one object is one request, not one per block —
//! and the reader that does not open the file agrees with the one that does.
//!
//! ```bash
//! cargo test --release --test integration_s3_compaction -- --ignored --test-threads=1
//! ```

// The WAL build refuses relocation, and leaves what only that test uses unused.
#![cfg_attr(feature = "wal", allow(unused_imports, dead_code))]

mod common;
use common::*;

use hyperfile::config::HyperFileRuntimeConfig;
use hyperfile::file::flags::FileFlags;
use hyperfile::file::hyper::Hyper;
use hyperfile::file::mode::FileMode;

const BS: usize = 4096;
const MIB: usize = 1 << 20;

fn runtime() -> HyperFileRuntimeConfig {
    let mut r = HyperFileRuntimeConfig::default();
    r.data_cache_dirty_max_bytes_threshold = 1 << 40;
    r.data_cache_dirty_max_blocks_threshold = (1 << 40) / BS;
    r.data_cache_dirty_max_flush_interval = u64::MAX / 2;
    r
}

/// 8 MiB written in one flush, relocated whole and flushed again: few requests
/// for 2048 blocks, the content checkpoint unchanged by both readers, the data
/// the same from a cold handle; then a write moves it on.
#[cfg(not(feature = "wal"))]
#[tokio::test]
#[ignore]
async fn relocation_reads_in_few_requests_and_keeps_the_content_checkpoint() {
    let _ = env_logger::try_init();
    let client = make_client().await;
    let tf = TestFile::new(&client).await;
    let data: Vec<u8> = (0..8 * MIB).map(|i| (i / BS) as u8 ^ 0x2a).collect();
    {
        let mut h = Hyper::fs_open_or_create_with_default_opt(&client, tf.uri(), FileFlags::rdwr(), FileMode::default_file())
            .await.expect("create");
        h.fs_write(0, &data).await.expect("write");
        h.fs_flush().await.expect("flush");
        let _ = h.fs_release().await;
    }
    let before = Hyper::fs_content_cno_fast(&client, tf.uri()).await.expect("content_cno_fast");

    let mut h = Hyper::fs_open_opt(&client, tf.uri(), FileFlags::rdwr(), &runtime()).await.expect("open");
    let opened = h.fs_content_cno().expect("content_cno");
    h.read_timing().reset();
    let n = h.fs_relocate(&[(0, 2048)]).await.expect("relocate");
    let reads = h.read_timing().snapshot();
    let relocated_cno = h.fs_flush().await.expect("flush");
    let after_open = h.fs_content_cno().expect("content_cno");
    let _ = h.fs_release().await;
    let after_fast = Hyper::fs_content_cno_fast(&client, tf.uri()).await.expect("content_cno_fast");

    let mut c = Hyper::fs_open(&client, tf.uri(), FileFlags::rdonly()).await.expect("reopen");
    let mut back = vec![0u8; data.len()];
    let got = c.fs_read(0, &mut back).await.expect("read");
    back.truncate(got);
    let _ = c.fs_release().await;

    let mut w = Hyper::fs_open(&client, tf.uri(), FileFlags::rdwr()).await.expect("open to write");
    w.fs_write(0, b"changed").await.expect("write");
    w.fs_flush().await.expect("flush");
    let _ = w.fs_release().await;
    let changed = Hyper::fs_content_cno_fast(&client, tf.uri()).await.expect("content_cno_fast");
    tf.cleanup(&client).await;

    assert_eq!(n, 2048, "not every block was relocated");
    assert!(reads.data_gets <= 4, "2048 blocks in one object took {} requests", reads.data_gets);
    assert_eq!(opened, before, "the two readers disagree on an unchanged file");
    assert!(relocated_cno > before, "the relocation did not publish");
    assert_eq!(after_open, before, "content_cno moved on a relocation");
    assert_eq!(after_fast, before, "content_cno_fast moved on a relocation");
    assert!(back == data, "relocated blocks read back differently");
    assert!(changed > after_fast, "content_cno did not move on a write");
}