mod format;
mod io;
mod layout;
mod read;
mod snapshot;
mod state;
mod transaction;
use std::collections::{HashMap, VecDeque};
use std::fs::File;
use std::io as std_io;
use std::ops::{Deref, DerefMut};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU64, AtomicUsize};
use std::sync::{Arc, Condvar, Mutex, Weak};
use pi_append_log::file::{DefaultFileLayout, FileAppendLog, FileAppendLogBuilder, FileClosed};
use pi_append_log::{
AppendLog, AppendLogBuilder, AppendLogVisitor, AppendOptions, BlockEncoder, BlockVisitContext,
DefaultBlockCodec, Layout, ReadOrder,
};
use crate::{
AsyncBlockDevice, BlockDeviceCapabilities, BlockDeviceInfo, BlockDeviceMetrics,
BlockDeviceMetricsSnapshot, BlockId, BlockStore, BlockStoreError, DeviceId,
};
use read::{FileReadBuf, ReadLease};
use snapshot::FileSnapshot;
use state::State;
use transaction::FileTransaction;
enum InitStage {
ActiveFile(
u8,
),
DataDirectory,
RootDirectory,
LogFile,
LogDirectory,
CheckpointAppend,
}
trait InitHooks {
fn fill_device_id(&self, device_id: &mut [u8; 16]) -> std_io::Result<()>;
fn sync_file(&self, file: &File, stage: InitStage) -> std_io::Result<()>;
fn sync_directory(&self, path: &Path, stage: InitStage) -> std_io::Result<()>;
async fn append_checkpoint<S>(
&self,
storage: &S,
block: Vec<u8>,
stage: InitStage,
) -> std_io::Result<()>
where
S: AppendLog<Block = Vec<u8>>;
}
struct RealHooks;
impl InitHooks for RealHooks {
fn fill_device_id(&self, device_id: &mut [u8; 16]) -> std_io::Result<()> {
getrandom::getrandom(device_id).map_err(|error| std_io::Error::other(error.to_string()))
}
fn sync_file(&self, file: &File, stage: InitStage) -> std_io::Result<()> {
if let InitStage::ActiveFile(level) = stage {
let _ = level;
}
file.sync_all()
}
fn sync_directory(&self, path: &Path, _stage: InitStage) -> std_io::Result<()> {
io::sync_directory(path)
}
async fn append_checkpoint<S>(
&self,
storage: &S,
block: Vec<u8>,
stage: InitStage,
) -> std_io::Result<()>
where
S: AppendLog<Block = Vec<u8>>,
{
let InitStage::CheckpointAppend = stage else {
return Err(std_io::Error::new(
std_io::ErrorKind::InvalidInput,
"invalid checkpoint append stage",
));
};
storage
.append(block, AppendOptions { durable: true })
.await
.map(|_| ())
}
}
const SUPPORTED_BLOCK_SIZES: [u64; 15] = [
4 * 1024,
6 * 1024,
8 * 1024,
12 * 1024,
16 * 1024,
24 * 1024,
32 * 1024,
48 * 1024,
64 * 1024,
96 * 1024,
128 * 1024,
192 * 1024,
256 * 1024,
384 * 1024,
512 * 1024,
];
type FileLog = FileAppendLog<DefaultFileLayout>;
const STORE_READY: u8 = 0;
const STORE_RECOVERY_REQUIRED: u8 = 1;
pub(super) struct RetirementNotifier {
generation: Mutex<u64>,
wake: Condvar,
stopped: AtomicBool,
owners: AtomicUsize,
}
impl RetirementNotifier {
fn new() -> Self {
Self {
generation: Mutex::new(0),
wake: Condvar::new(),
stopped: AtomicBool::new(false),
owners: AtomicUsize::new(0),
}
}
pub(super) fn notify(&self) {
if let Ok(mut generation) = self.generation.lock() {
*generation = generation.wrapping_add(1);
}
self.wake.notify_one();
}
fn stop(&self) {
self.stopped
.store(true, std::sync::atomic::Ordering::Release);
self.notify();
}
pub(super) fn generation_for_test(&self) -> u64 {
self.generation.lock().map_or(0, |generation| *generation)
}
pub(super) fn wait_for_change(
&self,
last_seen: u64,
timeout: std::time::Duration,
) -> (u64, bool) {
let mut generation = match self.generation.lock() {
Ok(generation) => generation,
Err(poisoned) => poisoned.into_inner(),
};
if *generation == last_seen && !self.stopped.load(std::sync::atomic::Ordering::Acquire) {
match self.wake.wait_timeout(generation, timeout) {
Ok((next, _)) => generation = next,
Err(poisoned) => generation = poisoned.into_inner().0,
}
}
(
*generation,
self.stopped.load(std::sync::atomic::Ordering::Acquire),
)
}
}
pub(super) struct MaintenanceNotifier {
generation: Mutex<u64>,
wake: Condvar,
stopped: AtomicBool,
owners: AtomicUsize,
}
impl MaintenanceNotifier {
fn new() -> Self {
Self {
generation: Mutex::new(0),
wake: Condvar::new(),
stopped: AtomicBool::new(false),
owners: AtomicUsize::new(0),
}
}
pub(super) fn notify(&self) {
if let Ok(mut generation) = self.generation.lock() {
*generation = generation.wrapping_add(1);
}
self.wake.notify_one();
}
fn stop(&self) {
self.stopped
.store(true, std::sync::atomic::Ordering::Release);
self.notify();
}
pub(super) fn wait_for_change(
&self,
last_seen: u64,
timeout: std::time::Duration,
) -> (u64, bool) {
let mut generation = match self.generation.lock() {
Ok(value) => value,
Err(poisoned) => poisoned.into_inner(),
};
if *generation == last_seen && !self.stopped.load(std::sync::atomic::Ordering::Acquire) {
generation = match self.wake.wait_timeout(generation, timeout) {
Ok((value, _)) => value,
Err(poisoned) => poisoned.into_inner().0,
};
}
(
*generation,
self.stopped.load(std::sync::atomic::Ordering::Acquire),
)
}
}
#[cfg(test)]
pub(super) struct MaintenanceWorkerBarriers {
state: Mutex<[bool; 10]>,
outcome: Mutex<Option<transaction::MaintenanceOutcome>>,
wake: Condvar,
}
#[cfg(test)]
impl MaintenanceWorkerBarriers {
fn new() -> Self {
Self {
state: Mutex::new([false; 10]),
outcome: Mutex::new(None),
wake: Condvar::new(),
}
}
}
pub(super) struct WorkerOwner {
control: Arc<RetirementNotifier>,
maintenance_control: Arc<MaintenanceNotifier>,
}
impl WorkerOwner {
fn new(
control: Arc<RetirementNotifier>,
maintenance_control: Arc<MaintenanceNotifier>,
) -> Self {
control
.owners
.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
maintenance_control
.owners
.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
Self {
control,
maintenance_control,
}
}
}
impl Clone for WorkerOwner {
fn clone(&self) -> Self {
Self::new(self.control.clone(), self.maintenance_control.clone())
}
}
impl Drop for WorkerOwner {
fn drop(&mut self) {
if self
.control
.owners
.fetch_sub(1, std::sync::atomic::Ordering::AcqRel)
== 1
{
self.control.stop();
}
if self
.maintenance_control
.owners
.fetch_sub(1, std::sync::atomic::Ordering::AcqRel)
== 1
{
self.maintenance_control.stop();
}
}
}
pub(super) struct RuntimeState {
pub(super) current: Arc<State>,
pub(super) snapshot_states: Vec<Weak<State>>,
pub(super) read_leases: HashMap<PathBuf, Vec<Weak<ReadLease>>>,
pub(super) retired: Vec<PathBuf>,
}
impl RuntimeState {
fn new(state: State) -> Self {
Self {
current: Arc::new(state),
snapshot_states: Vec::new(),
read_leases: HashMap::new(),
retired: Vec::new(),
}
}
fn register_snapshot(&mut self, state: &Arc<State>) {
self.snapshot_states
.retain(|snapshot| snapshot.strong_count() != 0);
if !self
.snapshot_states
.iter()
.any(|snapshot| snapshot.ptr_eq(&Arc::downgrade(state)))
{
self.snapshot_states.push(Arc::downgrade(state));
}
}
fn select_read_source(&mut self, block: BlockId) -> Result<Arc<State>, BlockStoreError> {
self.snapshot_states
.retain(|snapshot| snapshot.strong_count() != 0);
if state_block_is_live(&self.current, block)? {
return Ok(self.current.clone());
}
for snapshot in &self.snapshot_states {
let Some(state) = snapshot.upgrade() else {
continue;
};
if state_block_is_live(&state, block)? {
return Ok(state);
}
}
Err(BlockStoreError::BlockNotFound)
}
fn register_read_lease(
&mut self,
path: PathBuf,
notifier: Arc<RetirementNotifier>,
) -> Arc<ReadLease> {
let leases = self.read_leases.entry(path.clone()).or_default();
leases.retain(|lease| lease.strong_count() != 0);
let lease = Arc::new(ReadLease::new(path, notifier));
leases.push(Arc::downgrade(&lease));
lease
}
pub(super) fn publish(&mut self, state: State, retired: Vec<PathBuf>) {
self.snapshot_states
.retain(|snapshot| snapshot.strong_count() != 0);
self.read_leases.retain(|_, leases| {
leases.retain(|lease| lease.strong_count() != 0);
!leases.is_empty()
});
self.current = Arc::new(state);
for path in retired {
if !self.retired.contains(&path) {
self.retired.push(path);
}
}
}
#[allow(dead_code)]
pub(super) fn retired_queue_len(&self) -> usize {
self.retired.len()
}
pub(super) fn state_references_path(root: &Path, state: &State, path: &Path) -> bool {
state.segments.keys().any(|&(level, version)| {
layout::data_dir(root).join(format!("{level:x}.{version:07x}")) == path
}) || state
.large_blocks
.keys()
.any(|id| layout::data_dir(root).join(format!("l{id:015x}")) == path)
}
}
impl Deref for RuntimeState {
type Target = State;
fn deref(&self) -> &Self::Target {
&self.current
}
}
impl DerefMut for RuntimeState {
fn deref_mut(&mut self) -> &mut Self::Target {
Arc::make_mut(&mut self.current)
}
}
const LOG_PHASE_ACTIVE: u8 = 0;
#[cfg_attr(not(test), allow(dead_code))]
const LOG_PHASE_NEEDS_CHECKPOINT: u8 = 1;
#[cfg_attr(test, allow(dead_code))]
pub(super) const MAINTENANCE_ROTATION_THRESHOLD: u64 = 4 * 1024 * 1024;
#[cfg_attr(not(test), allow(dead_code))]
pub(super) struct MaintenanceState {
pub(super) requests: u64,
pub(super) completed_count: u64,
pub(super) archive_queue: VecDeque<FileClosed>,
}
impl MaintenanceState {
fn with_archive_queue(archive_queue: VecDeque<FileClosed>) -> Self {
Self {
requests: 0,
completed_count: 0,
archive_queue,
}
}
}
pub(super) struct StoreInner {
root: PathBuf,
state: Mutex<RuntimeState>,
pending_allocations: Mutex<HashMap<u64, u64>>,
log: FileLog,
writer_active: AtomicBool,
transaction_identity: AtomicU64,
next_log_sequence: AtomicU64,
log_phase: AtomicU8,
#[cfg_attr(not(test), allow(dead_code))]
maintenance: Mutex<MaintenanceState>,
next_snapshot_identity: AtomicU64,
identity: u64,
status: AtomicU8,
retirement_notifier: Arc<RetirementNotifier>,
#[cfg_attr(not(test), allow(dead_code))]
maintenance_notifier: Arc<MaintenanceNotifier>,
#[cfg(test)]
maintenance_barriers: Arc<MaintenanceWorkerBarriers>,
#[cfg(test)]
test_maintenance: Mutex<transaction::TestMaintenanceState>,
}
impl StoreInner {
pub(super) fn recovery_required(&self) -> bool {
self.status.load(std::sync::atomic::Ordering::Acquire) == STORE_RECOVERY_REQUIRED
}
pub(super) fn require_recovery(&self) {
self.status.store(
STORE_RECOVERY_REQUIRED,
std::sync::atomic::Ordering::Release,
);
}
#[cfg_attr(test, allow(dead_code))]
pub(super) fn schedule_maintenance_for_active_size(
&self,
active_size: u64,
) -> Result<(), BlockStoreError> {
if active_size < MAINTENANCE_ROTATION_THRESHOLD {
return Ok(());
}
let notify = match self.maintenance.lock() {
Ok(mut state) => {
let notify = state.requests == 0;
state.requests = state.requests.saturating_add(1);
notify
}
Err(_) => {
self.require_recovery();
return Err(BlockStoreError::RecoveryRequired);
}
};
if notify {
self.maintenance_notifier.notify();
}
Ok(())
}
pub(super) fn allocate_log_sequence(&self) -> Result<u64, BlockStoreError> {
loop {
let current = self
.next_log_sequence
.load(std::sync::atomic::Ordering::Acquire);
let Some(next) = current.checked_add(1) else {
self.require_recovery();
return Err(BlockStoreError::RecoveryRequired);
};
if self
.next_log_sequence
.compare_exchange_weak(
current,
next,
std::sync::atomic::Ordering::AcqRel,
std::sync::atomic::Ordering::Acquire,
)
.is_ok()
{
return Ok(current);
}
}
}
}
pub struct FileBlockStore {
inner: Arc<StoreInner>,
worker_owner: WorkerOwner,
}
fn allocate_store_identity() -> Result<u64, BlockStoreError> {
static NEXT_STORE_ID: AtomicU64 = AtomicU64::new(1);
loop {
let current = NEXT_STORE_ID.load(std::sync::atomic::Ordering::Acquire);
if current == 0 || current == u64::MAX {
return Err(BlockStoreError::RecoveryRequired);
}
if NEXT_STORE_ID
.compare_exchange_weak(
current,
current + 1,
std::sync::atomic::Ordering::AcqRel,
std::sync::atomic::Ordering::Acquire,
)
.is_ok()
{
return Ok(current);
}
}
}
impl FileBlockStore {
pub async fn open(path: impl AsRef<Path>) -> Result<Self, BlockStoreError> {
Self::open_with_hooks(path.as_ref(), &RealHooks).await
}
async fn open_with_hooks<H: InitHooks>(
root: &Path,
hooks: &H,
) -> Result<Self, BlockStoreError> {
match io::classify_root(root).map_err(BlockStoreError::Io)? {
io::RootKind::Missing => Self::initialize(root, true, hooks).await,
io::RootKind::Empty => Self::initialize(root, false, hooks).await,
io::RootKind::Existing => Self::recover(root).await,
}
}
async fn initialize<H: InitHooks>(
root: &Path,
create_root: bool,
hooks: &H,
) -> Result<Self, BlockStoreError> {
let mut device_id = [0_u8; 16];
hooks
.fill_device_id(&mut device_id)
.map_err(BlockStoreError::Io)?;
if create_root {
io::create_root(root).map_err(BlockStoreError::Io)?;
}
let mut created = io::CreatedPaths::new(root, create_root);
let result = Self::initialize_created(root, device_id, &mut created, hooks).await;
if result.is_err() {
created.cleanup();
}
result
}
async fn initialize_created<H: InitHooks>(
root: &Path,
device_id: [u8; 16],
created: &mut io::CreatedPaths,
hooks: &H,
) -> Result<Self, BlockStoreError> {
io::create_initial_files(root, created, hooks).map_err(BlockStoreError::Io)?;
io::create_initial_log_file(root, created, hooks).map_err(BlockStoreError::Io)?;
let codec = DefaultBlockCodec::default();
let mut visitor = RecoveryVisitor::default();
let build = FileAppendLogBuilder::new(
layout::log_dir(root),
DefaultFileLayout::new(layout::log_dir(root)),
)
.build(&codec, ReadOrder::Forward, &mut visitor)
.await
.map_err(BlockStoreError::Io)?;
let payload = format::encode_initial_checkpoint(device_id);
let block = codec.encode(1, 0, &payload).map_err(BlockStoreError::Io)?;
hooks
.append_checkpoint(&build.storage, block, InitStage::CheckpointAppend)
.await
.map_err(BlockStoreError::Io)?;
Self::from_parts(
root,
State::initial(device_id),
build.storage,
2,
VecDeque::new(),
)
}
fn from_parts(
root: &Path,
state: State,
log: FileLog,
next_sequence: u64,
archive_queue: VecDeque<FileClosed>,
) -> Result<Self, BlockStoreError> {
let identity = allocate_store_identity()?;
let notifier = Arc::new(RetirementNotifier::new());
let maintenance_notifier = Arc::new(MaintenanceNotifier::new());
let inner = Arc::new(StoreInner {
root: root.to_path_buf(),
state: Mutex::new(RuntimeState::new(state)),
pending_allocations: Mutex::new(HashMap::new()),
log,
writer_active: AtomicBool::new(false),
transaction_identity: AtomicU64::new(1),
next_log_sequence: AtomicU64::new(next_sequence),
log_phase: AtomicU8::new(LOG_PHASE_ACTIVE),
maintenance: Mutex::new(MaintenanceState::with_archive_queue(archive_queue)),
next_snapshot_identity: AtomicU64::new(1),
identity,
status: AtomicU8::new(STORE_READY),
retirement_notifier: notifier.clone(),
maintenance_notifier: maintenance_notifier.clone(),
#[cfg(test)]
maintenance_barriers: Arc::new(MaintenanceWorkerBarriers::new()),
#[cfg(test)]
test_maintenance: Mutex::new(transaction::TestMaintenanceState::new()),
});
let worker_owner = WorkerOwner::new(notifier.clone(), maintenance_notifier.clone());
#[cfg(not(test))]
{
let worker_inner = inner.clone();
let worker_notifier = notifier.clone();
std::thread::Builder::new()
.name(String::from("pi-block-device-retirement"))
.spawn(move || transaction::retirement_worker_loop(worker_inner, worker_notifier))
.map_err(BlockStoreError::Io)?;
}
{
let worker_inner = inner.clone();
let worker_notifier = maintenance_notifier.clone();
std::thread::Builder::new()
.name(String::from("pi-block-device-maintenance"))
.spawn(move || transaction::maintenance_worker_loop(worker_inner, worker_notifier))
.map_err(BlockStoreError::Io)?;
}
Ok(Self {
worker_owner,
inner,
})
}
async fn recover(root: &Path) -> Result<Self, BlockStoreError> {
if !io::validate_root_layout(root).map_err(BlockStoreError::Io)? {
return Err(BlockStoreError::RecoveryFailed);
}
let codec = DefaultBlockCodec::default();
let mut visitor = RecoveryVisitor::default();
let build_result = FileAppendLogBuilder::new(
layout::log_dir(root),
DefaultFileLayout::new(layout::log_dir(root)),
)
.build(&codec, ReadOrder::Forward, &mut visitor)
.await;
if visitor.unsupported_format {
return Err(BlockStoreError::UnsupportedFormat);
}
let build = build_result.map_err(|_| BlockStoreError::RecoveryFailed)?;
let state = visitor.state.ok_or(BlockStoreError::RecoveryFailed)?;
if visitor.max_sequence == u64::MAX {
return Err(BlockStoreError::RecoveryFailed);
}
io::validate_segment_files(root, &state).map_err(|_| BlockStoreError::RecoveryFailed)?;
io::remove_orphan_data_files(root, &state).map_err(|_| BlockStoreError::RecoveryFailed)?;
let log_root = layout::log_dir(root);
let log_layout = DefaultFileLayout::new(&log_root);
let mut active_path = None;
for entry in std::fs::read_dir(&log_root).map_err(|_| BlockStoreError::RecoveryFailed)? {
let path = entry.map_err(|_| BlockStoreError::RecoveryFailed)?.path();
if log_layout.parse_active_name(&path).is_some() && active_path.replace(path).is_some()
{
return Err(BlockStoreError::RecoveryFailed);
}
}
let active_path = active_path.ok_or(BlockStoreError::RecoveryFailed)?;
let active_is_empty = std::fs::metadata(active_path)
.map_err(|_| BlockStoreError::RecoveryFailed)?
.len()
== 0;
let mut next_sequence = visitor.max_sequence + 1;
if active_is_empty && !build.recovered_closed.is_empty() {
let sequence = next_sequence;
next_sequence = sequence
.checked_add(1)
.ok_or(BlockStoreError::RecoveryFailed)?;
let block = format::encode_checkpoint(&state, sequence)
.map_err(|_| BlockStoreError::RecoveryFailed)?;
build
.storage
.append(block, AppendOptions { durable: true })
.await
.map_err(|_| BlockStoreError::RecoveryFailed)?;
}
Self::from_parts(
root,
state,
build.storage,
next_sequence,
build.recovered_closed.into(),
)
}
}
#[derive(Default)]
struct RecoveryVisitor {
state: Option<State>,
chain_device_id: Option<[u8; 16]>,
structure_has_record: bool,
unsupported_format: bool,
max_sequence: u64,
last_sequence: Option<u64>,
}
impl AppendLogVisitor for RecoveryVisitor {
fn visit(&mut self, block: &[u8], context: BlockVisitContext) -> std_io::Result<bool> {
let record = match format::decode_record_block(block) {
Ok(record) => record,
Err(format::DecodeError::UnsupportedFormat) => {
self.unsupported_format = true;
return Err(std_io::Error::new(
std_io::ErrorKind::InvalidData,
"unsupported record format",
));
}
Err(format::DecodeError::Recovery(error)) => return Err(error),
};
if context.is_first_in_structure {
self.structure_has_record = false;
}
let sequence = match &record {
format::Record::Checkpoint { sequence, .. }
| format::Record::Delta { sequence, .. } => *sequence,
};
if self
.last_sequence
.is_some_and(|last_sequence| sequence <= last_sequence)
{
return Err(std_io::Error::new(
std_io::ErrorKind::InvalidData,
"log record sequence is not strictly increasing",
));
}
self.last_sequence = Some(sequence);
self.max_sequence = self.max_sequence.max(sequence);
match record {
format::Record::Checkpoint { state, .. } => {
if self.structure_has_record || !context.is_first_in_structure {
return Err(std_io::Error::new(
std_io::ErrorKind::InvalidData,
"checkpoint is not first in structure",
));
}
if let Some(device_id) = self.chain_device_id {
if state.device_id != device_id {
return Err(std_io::Error::new(
std_io::ErrorKind::InvalidData,
"checkpoint DeviceId changed across structures",
));
}
} else {
self.chain_device_id = Some(state.device_id);
}
if let Some(current) = &self.state
&& (state.next_segment_version < current.next_segment_version
|| state.next_large_block_id < current.next_large_block_id)
{
return Err(std_io::Error::new(
std_io::ErrorKind::InvalidData,
"checkpoint allocation high water mark regressed",
));
}
self.state = Some(state);
}
format::Record::Delta { delta, .. } => {
if !self.structure_has_record || context.is_first_in_structure {
return Err(std_io::Error::new(
std_io::ErrorKind::InvalidData,
"DeltaRecord cannot start a structure",
));
}
self.state
.as_mut()
.ok_or_else(|| {
std_io::Error::new(
std_io::ErrorKind::InvalidData,
"DeltaRecord precedes checkpoint",
)
})?
.apply(delta)?;
}
}
self.structure_has_record = true;
Ok(false)
}
}
fn state_block_is_live(state: &State, block: BlockId) -> Result<bool, BlockStoreError> {
if block.level() == 15 {
let id = block
.large_block_id()
.ok_or(BlockStoreError::InvalidBlockId)?;
return Ok(state.large_blocks.contains_key(&id));
}
let level = block.level();
let version = block
.segment_version()
.ok_or(BlockStoreError::InvalidBlockId)?;
let index = u64::from(block.block_index().ok_or(BlockStoreError::InvalidBlockId)?);
let Some(record) = state.segments.get(&(level, version)) else {
return Ok(false);
};
if index >= record.frontier {
return Ok(false);
}
let word = usize::try_from(index / 64).map_err(|_| BlockStoreError::InvalidBlockId)?;
Ok(record
.bitmap
.get(word)
.is_some_and(|bits| bits & (1_u64 << (index % 64)) != 0))
}
impl AsyncBlockDevice for FileBlockStore {
type ReadBuf = FileReadBuf;
async fn read(&self, block: BlockId) -> Result<Self::ReadBuf, BlockStoreError> {
use std::io::{Read, Seek, SeekFrom};
if self.inner.recovery_required() {
return Err(BlockStoreError::RecoveryRequired);
}
let raw = u64::from(block);
if self
.inner
.pending_allocations
.lock()
.map_err(|_| {
self.inner.require_recovery();
BlockStoreError::RecoveryRequired
})?
.contains_key(&raw)
{
return Err(BlockStoreError::BlockNotCommitted);
}
let path = if block.level() == 15 {
let id = block
.large_block_id()
.ok_or(BlockStoreError::InvalidBlockId)?;
layout::data_dir(&self.inner.root).join(format!("l{id:015x}"))
} else {
let level = block.level();
let version = block
.segment_version()
.ok_or(BlockStoreError::InvalidBlockId)?;
layout::data_dir(&self.inner.root).join(format!("{level:x}.{version:07x}"))
};
let (source, lease) = {
let mut runtime = self.inner.state.lock().map_err(|_| {
self.inner.require_recovery();
BlockStoreError::RecoveryRequired
})?;
let source = runtime.select_read_source(block)?;
let lease =
runtime.register_read_lease(path.clone(), self.inner.retirement_notifier.clone());
(source, lease)
};
if self.inner.recovery_required() {
return Err(BlockStoreError::RecoveryRequired);
}
if block.level() == 15 {
let id = block
.large_block_id()
.ok_or(BlockStoreError::InvalidBlockId)?;
let logical_len = *source
.large_blocks
.get(&id)
.ok_or(BlockStoreError::BlockNotFound)?;
let file = File::open(&path).map_err(BlockStoreError::Io)?;
if file.metadata().map_err(BlockStoreError::Io)?.len() != logical_len {
return Err(BlockStoreError::BlockCorrupted);
}
let length =
usize::try_from(logical_len).map_err(|_| BlockStoreError::InvalidBlockId)?;
let map =
unsafe { memmap2::MmapOptions::new().map(&file) }.map_err(BlockStoreError::Io)?;
return Ok(FileReadBuf::mapped(map, 0..length, lease));
}
let level = block.level();
let version = block
.segment_version()
.ok_or(BlockStoreError::InvalidBlockId)?;
let index = u64::from(block.block_index().ok_or(BlockStoreError::InvalidBlockId)?);
let record = source
.segments
.get(&(level, version))
.ok_or(BlockStoreError::BlockNotFound)?;
let slot = SUPPORTED_BLOCK_SIZES[usize::from(level)];
let offset = index
.checked_mul(slot)
.ok_or(BlockStoreError::InvalidBlockId)?;
let end = offset
.checked_add(slot)
.ok_or(BlockStoreError::InvalidBlockId)?;
let mut file = File::open(&path).map_err(BlockStoreError::Io)?;
if file.metadata().map_err(BlockStoreError::Io)?.len() < end {
return Err(BlockStoreError::BlockCorrupted);
}
let slot_len = usize::try_from(slot).map_err(|_| BlockStoreError::InvalidBlockId)?;
match record.state {
state::SegmentState::Active => {
let mut data = vec![0_u8; slot_len];
file.seek(SeekFrom::Start(offset))
.map_err(BlockStoreError::Io)?;
file.read_exact(&mut data).map_err(|error| {
if error.kind() == std::io::ErrorKind::UnexpectedEof {
BlockStoreError::BlockCorrupted
} else {
BlockStoreError::Io(error)
}
})?;
Ok(FileReadBuf::owned(data, lease))
}
state::SegmentState::Sealed => {
let start = usize::try_from(offset).map_err(|_| BlockStoreError::InvalidBlockId)?;
let end = usize::try_from(end).map_err(|_| BlockStoreError::InvalidBlockId)?;
let map = unsafe { memmap2::MmapOptions::new().map(&file) }
.map_err(BlockStoreError::Io)?;
Ok(FileReadBuf::mapped(map, start..end, lease))
}
}
}
}
impl BlockStore for FileBlockStore {
type Snapshot = FileSnapshot;
type Transaction = FileTransaction;
fn open_snapshot(&self) -> Result<Self::Snapshot, BlockStoreError> {
if self.inner.recovery_required() {
return Err(BlockStoreError::RecoveryRequired);
}
let identity = loop {
let current = self
.inner
.next_snapshot_identity
.load(std::sync::atomic::Ordering::Acquire);
let Some(next) = current.checked_add(1) else {
self.inner.require_recovery();
return Err(BlockStoreError::RecoveryRequired);
};
if self
.inner
.next_snapshot_identity
.compare_exchange_weak(
current,
next,
std::sync::atomic::Ordering::AcqRel,
std::sync::atomic::Ordering::Acquire,
)
.is_ok()
{
break current;
}
};
let state = {
let mut runtime = self.inner.state.lock().map_err(|_| {
self.inner.require_recovery();
BlockStoreError::RecoveryRequired
})?;
let state = runtime.current.clone();
runtime.register_snapshot(&state);
state
};
if self.inner.recovery_required() {
return Err(BlockStoreError::RecoveryRequired);
}
Ok(FileSnapshot::new(
state,
self.inner.identity,
identity,
self.inner.retirement_notifier.clone(),
))
}
async fn begin_write(&self) -> Result<Self::Transaction, BlockStoreError> {
if self.inner.recovery_required() {
return Err(BlockStoreError::RecoveryRequired);
}
if self
.inner
.writer_active
.compare_exchange(
false,
true,
std::sync::atomic::Ordering::AcqRel,
std::sync::atomic::Ordering::Acquire,
)
.is_err()
{
return Err(BlockStoreError::WriteTransactionBusy);
}
if self.inner.recovery_required() {
self.inner
.writer_active
.store(false, std::sync::atomic::Ordering::Release);
return Err(BlockStoreError::RecoveryRequired);
}
if self
.inner
.log_phase
.load(std::sync::atomic::Ordering::Acquire)
!= LOG_PHASE_ACTIVE
{
self.inner.require_recovery();
self.inner
.writer_active
.store(false, std::sync::atomic::Ordering::Release);
return Err(BlockStoreError::RecoveryRequired);
}
let identity = loop {
let current = self
.inner
.transaction_identity
.load(std::sync::atomic::Ordering::Acquire);
if current == 0 || current == u64::MAX {
self.inner.require_recovery();
self.inner
.writer_active
.store(false, std::sync::atomic::Ordering::Release);
return Err(BlockStoreError::RecoveryRequired);
}
if self
.inner
.transaction_identity
.compare_exchange_weak(
current,
current + 1,
std::sync::atomic::Ordering::AcqRel,
std::sync::atomic::Ordering::Acquire,
)
.is_ok()
{
break current;
}
};
let state = match self.inner.state.lock() {
Ok(state) => state.current.as_ref().clone(),
Err(_) => {
self.inner.require_recovery();
self.inner
.writer_active
.store(false, std::sync::atomic::Ordering::Release);
return Err(BlockStoreError::RecoveryRequired);
}
};
Ok(FileTransaction::new(
self.inner.clone(),
self.worker_owner.clone(),
identity,
state,
))
}
}
impl BlockDeviceInfo for FileBlockStore {
fn device_id(&self) -> DeviceId {
DeviceId(
self.inner
.state
.lock()
.expect("store state lock poisoned")
.current
.device_id,
)
}
fn base_block_size(&self) -> u64 {
4096
}
fn supported_block_sizes(&self) -> &'static [u64] {
&SUPPORTED_BLOCK_SIZES
}
fn capabilities(&self) -> BlockDeviceCapabilities {
BlockDeviceCapabilities {
compression: false,
encryption: false,
data_checksums: false,
}
}
}
impl BlockDeviceMetrics for FileBlockStore {
fn metrics(&self) -> BlockDeviceMetricsSnapshot {
BlockDeviceMetricsSnapshot {
used_bytes: 0,
free_bytes: 0,
available_bytes: None,
read_ops: 0,
read_bytes: 0,
write_ops: 0,
write_bytes: 0,
commit_ok: 0,
commit_failed: 0,
recovery_count: 0,
reclaimed_bytes: 0,
}
}
}
#[cfg(test)]
#[path = "../../tests/unit/file_initialization_fault_test.rs"]
mod fault_tests;