hyperfile 0.10.4

#![cfg(all(target_os = "linux", not(feature = "blocking")))]
//! Memory held by writes not yet flushed, on in-memory staging.
//!
//! Until a flush, written data sits in heap blocks of `data_block_size`, and what
//! they cost is what the process's resident set grows by. That is more than the
//! bytes written if the allocator pads each block: a block on a page boundary
//! leaves glibc's malloc no room for the header it keeps in front of every
//! allocation but the page before, where no other such block fits, and a 4 KiB
//! block then holds 8 KiB.
//!
//! One test, alone in its binary, so that nothing else allocates or frees while it
//! reads the resident set. Linux only: that is read from `/proc/self/status`.
//!
//! ```bash
//! cargo test --test functional_dirty_memory
//! ```

use btree_ondisk::{MemoryBlockLoader, NullNodeCache};
use hyperfile::config::{HyperFileConfigBuilder, HyperFileRuntimeConfig};
use hyperfile::file::file::HyperFile;
use hyperfile::file::flags::{FileFlags, HyperFileFlags};
use hyperfile::file::mode::{FileMode, HyperFileMode};
use hyperfile::staging::Staging;
use hyperfile::staging::config::StagingConfig;
use hyperfile::staging::memory::MemoryStaging;

type MemFile<'a> = HyperFile<'a, MemoryStaging, MemoryBlockLoader<hyperfile::BlockPtr>, NullNodeCache>;

const MIB: usize = 1024 * 1024;
/// Enough that whatever else moves the resident set is small beside it.
const DATA: usize = 128 * MIB;

fn resident() -> usize {
    let status = std::fs::read_to_string("/proc/self/status").expect("read /proc/self/status");
    let kib: usize = status.lines()
        .find(|l| l.starts_with("VmRSS:")).expect("VmRSS in /proc/self/status")
        .split_whitespace().nth(1).unwrap()
        .parse().unwrap();
    kib * 1024
}

/// Writes `DATA` without a flush and compares what the resident set grew by.
///
/// It grows by the data and the index over it, a few percent; with page-aligned
/// blocks it grew by twice the data. It also has to grow by at least the data, or
/// the writes were not held and this measured nothing.
#[tokio::test]
async fn unflushed_writes_hold_about_their_own_size_in_memory() {
    let mut runtime = HyperFileRuntimeConfig::default();
    // Nothing flushes, or writes a partial, before the reading.
    runtime.data_cache_dirty_max_bytes_threshold = 2 * DATA;
    runtime.data_cache_dirty_max_blocks_threshold = 2 * DATA / 4096;
    runtime.data_cache_dirty_max_flush_interval = u64::MAX / 2;
    runtime.segment_buffer_size = 2 * DATA;

    let staging_config = StagingConfig::new_memory("dirty-memory");
    let config = HyperFileConfigBuilder::new()
        .with_staging_config(&staging_config)
        .with_runtime_config(&runtime)
        .build();
    let staging = MemoryStaging::new(staging_config, runtime);
    let mut file = MemFile::new(
        staging.clone(),
        staging.to_block_loader(),
        NullNodeCache,
        config,
        HyperFileFlags::from_flags(FileFlags::rdwr()),
        HyperFileMode::from_mode(FileMode::default_file()),
    ).await.expect("create on memory staging");

    // Not zeroes, which a write can map to no block at all.
    let chunk = vec![0x5au8; MIB];
    let _ = file.write(0, &chunk).await.expect("first write");
    let before = resident();
    for i in 1..=DATA / MIB {
        let _ = file.write(i * MIB, &chunk).await.expect("write");
    }
    let grew = resident().saturating_sub(before);
    let ratio = grew as f64 / DATA as f64;
    assert!(ratio > 0.9,
        "{} MiB of writes grew the resident set by only {} MiB, so they were not all held",
        DATA / MIB, grew / MIB);
    assert!(ratio < 1.5,
        "{} MiB of unflushed writes grew the resident set by {} MiB ({:.2}x the data)",
        DATA / MIB, grew / MIB, ratio);
}