hyperfile 0.6.9

A file-like interface abstraction layer provides random read and write capabilities for S3
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68

use std::io::Result;
use crate::SegmentId;
use crate::segment;
use crate::config::HyperFileMetaConfig;
use crate::staging::config::StagingConfig;
use crate::inode::{OnDiskState, FlushInodeFlag};

pub trait StagingIntercept<Staging>: Send + Sync {
    /// Called before `flush_inode` writes the inode to storage.
    ///
    /// Returning `Err(_)` causes `flush_inode` itself to fail with that error,
    /// which is useful for fault-injection tests. Default is no-op.
    fn before_flush_inode(&self, staging: &Staging, payload: &[u8], flag: FlushInodeFlag) -> std::pin::Pin<Box<dyn Future<Output = Result<()>> + '_ + Send>> {
        let _ = (staging, payload, flag);
        Box::pin(async { Ok(()) })
    }
    fn after_flush_inode(&self, staging: &Staging, payload: &[u8], flag: FlushInodeFlag) -> std::pin::Pin<Box<dyn Future<Output = Result<()>> + '_ + Send>>;
    fn after_remove_inode(&self, staging: &Staging) -> std::pin::Pin<Box<dyn Future<Output = Result<()>> + '_ + Send>>;

    /// Called before `SegmentReadWrite::done` uploads the segment bytes.
    ///
    /// Returning `Err(_)` causes `done` to fail with that error, which is
    /// useful for simulating segment-write failures in tests. Default is
    /// no-op.
    fn before_segment_done(&self, staging: &Staging, segid: crate::SegmentId, buf: &[u8], len: usize) -> std::pin::Pin<Box<dyn Future<Output = Result<()>> + '_ + Send>> {
        let _ = (staging, segid, buf, len);
        Box::pin(async { Ok(()) })
    }
}

pub trait Staging<L> {
    /// Read-side counters for this file. Shared by every clone of the
    /// handle.
    fn read_timing(&self) -> &crate::file::ReadTiming;

    fn load_inode(&self, buf: &mut [u8]) -> impl Future<Output = Result<Option<OnDiskState>>>;
    fn load_inode_from_segment(&self, buf: &mut [u8], segid: u64) -> impl Future<Output = Result<Option<OnDiskState>>>;
    fn load_segment_timestamp(&self, segid: u64) -> impl Future<Output = Result<(i64, i64)>>;
    fn flush_inode(&self, buf: &[u8], inode_state: &Option<OnDiskState>, flag: FlushInodeFlag) -> impl Future<Output = Result<Option<OnDiskState>>> + Send;
    fn remove_inode(&self, inode_state: &Option<OnDiskState>) -> impl Future<Output = Result<()>>;
    fn load_data_block(&self, segid: SegmentId, staging_off: usize, offset: usize, block_size: usize, buf: &mut [u8]) -> impl Future<Output = Result<()>> + Send;
    /// Coalesced range read: fetch `buf.len()` bytes starting at
    /// byte offset `s3_off` of the segment file for `segid`,
    /// writing the bytes directly into `buf`. Used by the Level-A
    /// read coalescing planner to pack many block-sized fetches
    /// into one ranged S3 GET. The caller has already done all
    /// bmap lookups; this is a thin wrapper around the
    /// staging-layer ranged GET.
    fn load_range(&self, segid: SegmentId, s3_off: usize, buf: &mut [u8]) -> impl Future<Output = Result<()>> + Send;
    fn new_segwr(&self, segid: SegmentId, hyper_file_config: &HyperFileMetaConfig) -> segment::Writer<Self> where Self: Sized;
    fn dir_filename(&self) -> (&str, &str);
    fn root_path(&self) -> &str;
    fn config(&self) -> &StagingConfig;
    fn unlink(&self) -> impl Future<Output = Result<()>>;
    fn to_block_loader(&self) -> L;
    fn interceptor(&mut self, i: impl StagingIntercept<Self> + 'static) where Self: Sized;
}

pub mod config;
pub mod s3;
/// In-memory staging for tests. Not available under `blocking`: it holds
/// `btree_ondisk`'s `MemoryBlockLoader`, which is `Rc`-based when
/// `btree-ondisk/rc` is on and so cannot satisfy this trait's `Send`
/// futures.
#[cfg(not(feature = "blocking"))]
pub mod memory;
#[cfg(feature="bench")]
pub mod bench;