#[cfg(test)]
use std::any::Any;
use std::collections::BTreeSet;
use std::fs::OpenOptions;
use std::future::Future;
use std::io::{self, Seek, SeekFrom, Write};
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
use std::task::{Context, Poll};
use pi_append_log::{AppendLog, AppendOptions, BlockEncoder, DefaultBlockCodec};
use smallvec::SmallVec;
use super::FileBlockStore;
use super::format;
use super::layout;
use super::state::{Delta, State};
use super::{
FileLog, LOG_PHASE_ACTIVE, LOG_PHASE_NEEDS_CHECKPOINT, SUPPORTED_BLOCK_SIZES, StoreInner,
WorkerOwner,
};
use crate::{
AllocatedBlock, BlockId, BlockStoreError, CommitResult, MaintenanceResult, RecordSeq, RootName,
WriteInput, WriteTransaction,
};
#[repr(u8)]
#[derive(Clone, Copy, Eq, PartialEq)]
enum TransactionState {
Active,
Writing,
WriteCancelled,
WriteFailed,
DurableInFlight,
DetachedInFlight,
Finished,
}
struct TransactionLifecycle {
state: AtomicU8,
owns_writer_gate: AtomicBool,
}
impl TransactionLifecycle {
fn new() -> Self {
Self {
state: AtomicU8::new(TransactionState::Active as u8),
owns_writer_gate: AtomicBool::new(true),
}
}
fn state(&self) -> TransactionState {
match self.state.load(Ordering::Acquire) {
value if value == TransactionState::Active as u8 => TransactionState::Active,
value if value == TransactionState::Writing as u8 => TransactionState::Writing,
value if value == TransactionState::WriteCancelled as u8 => {
TransactionState::WriteCancelled
}
value if value == TransactionState::WriteFailed as u8 => TransactionState::WriteFailed,
value if value == TransactionState::DurableInFlight as u8 => {
TransactionState::DurableInFlight
}
value if value == TransactionState::DetachedInFlight as u8 => {
TransactionState::DetachedInFlight
}
_ => TransactionState::Finished,
}
}
fn set_state(&self, state: TransactionState) {
self.state.store(state as u8, Ordering::Release);
}
fn release_writer_gate(&self, inner: &StoreInner) {
if self.owns_writer_gate.swap(false, Ordering::AcqRel) {
inner.writer_active.store(false, Ordering::Release);
}
}
}
struct WriteBatchCancelGuard {
lifecycle: Arc<TransactionLifecycle>,
inner: Arc<StoreInner>,
armed: bool,
}
impl WriteBatchCancelGuard {
fn new(lifecycle: Arc<TransactionLifecycle>, inner: Arc<StoreInner>) -> Self {
lifecycle.set_state(TransactionState::Writing);
Self {
lifecycle,
inner,
armed: true,
}
}
fn disarm(&mut self) {
self.armed = false;
}
}
impl Drop for WriteBatchCancelGuard {
fn drop(&mut self) {
if self.armed
&& self
.lifecycle
.state
.compare_exchange(
TransactionState::Writing as u8,
TransactionState::WriteCancelled as u8,
Ordering::AcqRel,
Ordering::Acquire,
)
.is_ok()
{
self.lifecycle.release_writer_gate(&self.inner);
}
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct DurableAppendOutcome {
active_size: u64,
}
enum CommitAppendError {
#[cfg_attr(not(test), allow(dead_code))]
Failed(
io::Error,
),
Unknown(
io::Error,
),
}
struct LargeWriteError {
error: io::Error,
created: bool,
}
pub(super) trait DeleteHooks {
fn unlink(&self, path: &Path) -> io::Result<()>;
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(super) enum DrainOutcome {
Empty,
Guarded,
Removed,
RetryPending,
}
struct RealDeleteHooks;
impl DeleteHooks for RealDeleteHooks {
fn unlink(&self, path: &Path) -> io::Result<()> {
std::fs::remove_file(path)
}
}
#[cfg(test)]
#[allow(dead_code)]
#[derive(Clone)]
pub(super) struct DurableCheckpointToken {
_identity: Arc<()>,
}
#[cfg(test)]
#[allow(dead_code)]
impl DurableCheckpointToken {
fn durable_for_test() -> Self {
Self {
_identity: Arc::new(()),
}
}
}
#[cfg(test)]
#[allow(dead_code)]
trait MaintenanceObserver {
fn maintenance_notified(&self, store: &FileBlockStore);
}
#[cfg(test)]
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(super) enum MaintenancePhase {
Idle,
ArchiveRetry,
}
#[cfg(test)]
pub(super) struct TestMaintenanceState {
pub(super) pending: bool,
pub(super) notification_count: usize,
pub(super) retry_closed: Option<Box<dyn Any + Send>>,
pub(super) retry_token: Option<DurableCheckpointToken>,
pub(super) phase: MaintenancePhase,
}
#[cfg(test)]
impl TestMaintenanceState {
pub(super) fn new() -> Self {
Self {
pending: false,
notification_count: 0,
retry_closed: None,
retry_token: None,
phase: MaintenancePhase::Idle,
}
}
}
#[cfg_attr(not(test), allow(dead_code))]
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(super) enum MaintenanceOutcome {
Idle,
Completed,
ArchiveRetryPending,
Fenced,
}
#[cfg(test)]
#[allow(dead_code)]
trait MaintenanceHooks {
type Closed: Any + Send;
fn rotate(
&self,
log: &FileLog,
) -> impl Future<Output = Result<Self::Closed, BlockStoreError>> + Send;
fn checkpoint(
&self,
closed: &Self::Closed,
state: State,
) -> impl Future<Output = Result<(), BlockStoreError>> + Send;
fn checkpoint_durable(
&self,
closed: &Self::Closed,
state: State,
) -> impl Future<Output = Result<DurableCheckpointToken, BlockStoreError>> + Send
where
Self: Sync,
Self::Closed: Sync,
{
async move {
self.checkpoint(closed, state).await?;
Ok(DurableCheckpointToken::durable_for_test())
}
}
fn archive(
&self,
closed: Self::Closed,
) -> impl Future<Output = Result<(), BlockStoreError>> + Send;
fn archive_durable_borrowed<'a>(
&'a self,
closed: &'a Self::Closed,
token: &'a DurableCheckpointToken,
) -> impl Future<Output = Result<(), BlockStoreError>> + Send + 'a
where
Self::Closed: Sync;
}
trait TransactionHooks {
fn write_ordinary(
&self,
path: &Path,
offset: u64,
data: &[u8],
slot_size: usize,
) -> io::Result<()>;
fn write_large(&self, path: &Path, data: &[u8]) -> io::Result<()>;
fn write_large_owned(&self, path: &Path, data: &[u8]) -> Result<(), LargeWriteError> {
self.write_large(path, data)
.map_err(|error| LargeWriteError {
error,
created: true,
})
}
fn create_replacement_active(&self, path: &Path) -> io::Result<()> {
let file = OpenOptions::new().write(true).create_new(true).open(path)?;
drop(file);
Ok(())
}
fn remove_replacement_active(&self, path: &Path) -> io::Result<()> {
std::fs::remove_file(path)
}
async fn after_block_write(&self, _index: usize) -> io::Result<()> {
Ok(())
}
fn preflight_payload_limit(&self) -> usize {
DefaultBlockCodec::default().max_payload_len()
}
fn sync_file(&self, path: &Path) -> io::Result<()>;
fn sync_data_directory(&self, path: &Path) -> io::Result<()>;
async fn before_durable_handoff(&self) -> io::Result<()> {
Ok(())
}
fn append_delta(
&self,
log: &FileLog,
block: Vec<u8>,
) -> impl Future<Output = Result<(), CommitAppendError>> + Send;
fn append_delta_with_outcome(
&self,
log: &FileLog,
block: Vec<u8>,
) -> impl Future<Output = Result<DurableAppendOutcome, CommitAppendError>> + Send
where
Self: Sync,
{
async move {
self.append_delta(log, block).await?;
Ok(DurableAppendOutcome { active_size: 0 })
}
}
}
#[cfg_attr(not(test), allow(dead_code))]
struct DurableAppendFence {
inner: Arc<StoreInner>,
armed: bool,
}
#[cfg_attr(not(test), allow(dead_code))]
impl DurableAppendFence {
fn new(inner: Arc<StoreInner>) -> Self {
Self { inner, armed: true }
}
fn disarm(&mut self) {
self.armed = false;
}
}
impl Drop for DurableAppendFence {
fn drop(&mut self) {
if self.armed {
self.inner.require_recovery();
}
}
}
struct PreDurableCommitCancelGuard {
inner: Arc<StoreInner>,
lifecycle: Arc<TransactionLifecycle>,
allocated: BTreeSet<u64>,
transaction_identity: u64,
created_paths: Vec<PathBuf>,
armed: bool,
}
impl PreDurableCommitCancelGuard {
fn new(transaction: &FileTransaction) -> Self {
Self {
inner: transaction.inner.clone(),
lifecycle: transaction.lifecycle.clone(),
allocated: transaction.allocated.clone(),
transaction_identity: transaction.identity,
created_paths: transaction
.created_large
.iter()
.chain(&transaction.created_segments)
.cloned()
.collect(),
armed: true,
}
}
fn disarm(&mut self) {
self.armed = false;
}
}
impl Drop for PreDurableCommitCancelGuard {
fn drop(&mut self) {
if !self.armed {
return;
}
let mut cleanup_failed = false;
for path in &self.created_paths {
if let Err(error) = std::fs::remove_file(path)
&& error.kind() != io::ErrorKind::NotFound
{
cleanup_failed = true;
}
}
match self.inner.pending_allocations.lock() {
Ok(mut pending) => {
for raw in &self.allocated {
if pending.get(raw) == Some(&self.transaction_identity) {
pending.remove(raw);
}
}
}
Err(_) => cleanup_failed = true,
}
if cleanup_failed {
self.inner.require_recovery();
}
self.lifecycle.set_state(TransactionState::Finished);
self.lifecycle.release_writer_gate(&self.inner);
}
}
type DetachedTask = Pin<Box<dyn Future<Output = Result<(), BlockStoreError>> + Send + 'static>>;
#[allow(dead_code)]
trait DetachedCommitRunner {
type JoinHandle: Future<Output = Result<(), BlockStoreError>> + Send;
fn spawn(&self, task: DetachedTask) -> Result<Self::JoinHandle, BlockStoreError>;
}
struct StdThreadCommitRunner;
struct StdThreadJoinHandle {
receiver: Pin<Box<futures_channel::oneshot::Receiver<Result<(), BlockStoreError>>>>,
}
impl Future for StdThreadJoinHandle {
type Output = Result<(), BlockStoreError>;
fn poll(self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll<Self::Output> {
let this = self.get_mut();
match this.receiver.as_mut().poll(context) {
Poll::Ready(Ok(result)) => Poll::Ready(result),
Poll::Ready(Err(_)) => Poll::Ready(Err(BlockStoreError::RecoveryRequired)),
Poll::Pending => Poll::Pending,
}
}
}
impl DetachedCommitRunner for StdThreadCommitRunner {
type JoinHandle = StdThreadJoinHandle;
fn spawn(&self, task: DetachedTask) -> Result<Self::JoinHandle, BlockStoreError> {
let (sender, receiver) = futures_channel::oneshot::channel();
std::thread::Builder::new()
.name(String::from("pi-block-device-commit"))
.spawn(move || {
let result = futures_lite::future::block_on(task);
let _ = sender.send(result);
})
.map_err(BlockStoreError::Io)?;
Ok(StdThreadJoinHandle {
receiver: Box::pin(receiver),
})
}
}
struct DetachedTaskGuard {
inner: Arc<StoreInner>,
lifecycle: Arc<TransactionLifecycle>,
allocated: BTreeSet<u64>,
transaction_identity: u64,
armed: bool,
}
impl DetachedTaskGuard {
fn new(
inner: Arc<StoreInner>,
lifecycle: Arc<TransactionLifecycle>,
allocated: BTreeSet<u64>,
transaction_identity: u64,
) -> Self {
Self {
inner,
lifecycle,
allocated,
transaction_identity,
armed: true,
}
}
fn clear_pending(&self) {
if let Ok(mut pending) = self.inner.pending_allocations.lock() {
for raw in &self.allocated {
if pending.get(raw) == Some(&self.transaction_identity) {
pending.remove(raw);
}
}
} else {
self.inner.require_recovery();
}
}
fn finish(&mut self) {
self.clear_pending();
self.lifecycle.set_state(TransactionState::Finished);
self.lifecycle.release_writer_gate(&self.inner);
self.armed = false;
}
}
impl Drop for DetachedTaskGuard {
fn drop(&mut self) {
if self.armed {
self.inner.require_recovery();
self.clear_pending();
self.lifecycle.set_state(TransactionState::Finished);
self.lifecycle.release_writer_gate(&self.inner);
}
}
}
struct DetachedCommitWait<J> {
join_handle: Pin<Box<J>>,
inner: Arc<StoreInner>,
armed: bool,
}
impl<J> DetachedCommitWait<J> {
fn new(join_handle: J, inner: Arc<StoreInner>) -> Self {
Self {
join_handle: Box::pin(join_handle),
inner,
armed: true,
}
}
}
impl<J> Future for DetachedCommitWait<J>
where
J: Future<Output = Result<(), BlockStoreError>> + Send,
{
type Output = Result<(), BlockStoreError>;
fn poll(self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll<Self::Output> {
let this = self.get_mut();
match this.join_handle.as_mut().poll(context) {
Poll::Ready(result) => {
if matches!(result, Err(BlockStoreError::RecoveryRequired)) {
this.inner.require_recovery();
}
this.armed = false;
Poll::Ready(result)
}
Poll::Pending => Poll::Pending,
}
}
}
impl<J> Drop for DetachedCommitWait<J> {
fn drop(&mut self) {
if self.armed {
self.inner.require_recovery();
}
}
}
struct OwnedCommitContext<H> {
inner: Arc<StoreInner>,
candidate: State,
created_large: Vec<PathBuf>,
created_segments: Vec<PathBuf>,
block: Vec<u8>,
lifecycle: Arc<TransactionLifecycle>,
hooks: Arc<H>,
allocated: BTreeSet<u64>,
transaction_identity: u64,
retired_paths: Vec<PathBuf>,
#[cfg(test)]
real_maintenance: Option<(u64, Option<Arc<dyn MaintenanceObserver + Send + Sync>>)>,
worker_owner: WorkerOwner,
}
struct RealTransactionHooks;
impl TransactionHooks for RealTransactionHooks {
fn write_ordinary(
&self,
path: &Path,
offset: u64,
data: &[u8],
slot_size: usize,
) -> io::Result<()> {
let mut file = OpenOptions::new().write(true).open(path)?;
file.seek(SeekFrom::Start(offset))?;
file.write_all(data)?;
if data.len() < slot_size {
file.write_all(&vec![0_u8; slot_size - data.len()])?;
}
Ok(())
}
fn write_large(&self, path: &Path, data: &[u8]) -> io::Result<()> {
self.write_large_owned(path, data)
.map_err(|error| error.error)
}
fn write_large_owned(&self, path: &Path, data: &[u8]) -> Result<(), LargeWriteError> {
let mut file = OpenOptions::new()
.write(true)
.create_new(true)
.open(path)
.map_err(|error| LargeWriteError {
error,
created: false,
})?;
file.write_all(data).map_err(|error| LargeWriteError {
error,
created: true,
})
}
fn sync_file(&self, path: &Path) -> io::Result<()> {
OpenOptions::new()
.read(true)
.write(true)
.open(path)?
.sync_all()
}
fn sync_data_directory(&self, path: &Path) -> io::Result<()> {
super::io::sync_directory(path)
}
async fn append_delta(&self, log: &FileLog, block: Vec<u8>) -> Result<(), CommitAppendError> {
self.append_delta_with_outcome(log, block).await.map(|_| ())
}
async fn append_delta_with_outcome(
&self,
log: &FileLog,
block: Vec<u8>,
) -> Result<DurableAppendOutcome, CommitAppendError> {
log.append(block, AppendOptions { durable: true })
.await
.map(|active_size| DurableAppendOutcome { active_size })
.map_err(CommitAppendError::Unknown)
}
}
pub struct FileTransaction {
inner: Arc<StoreInner>,
store_identity: u64,
identity: u64,
base: State,
candidate: State,
allocated: BTreeSet<u64>,
written: BTreeSet<u64>,
released: BTreeSet<u64>,
created_large: Vec<PathBuf>,
created_segments: Vec<PathBuf>,
modified_files: BTreeSet<PathBuf>,
requested_root_deletions: BTreeSet<u64>,
lifecycle: Arc<TransactionLifecycle>,
worker_owner: WorkerOwner,
}
#[cfg_attr(test, allow(dead_code))]
pub(super) fn retirement_worker_loop(
inner: Arc<StoreInner>,
notifier: Arc<super::RetirementNotifier>,
) {
const BACKOFFS: [std::time::Duration; 4] = [
std::time::Duration::from_millis(10),
std::time::Duration::from_millis(25),
std::time::Duration::from_millis(50),
std::time::Duration::from_millis(100),
];
let mut backoff = 0_usize;
let mut last_seen = notifier.generation_for_test();
loop {
let outcome = inner.drain_retired_once_with_hooks(&RealDeleteHooks);
if notifier.stopped.load(Ordering::Acquire) && matches!(outcome, DrainOutcome::Empty) {
break;
}
let delay = match outcome {
DrainOutcome::Removed => {
backoff = 0;
continue;
}
DrainOutcome::Empty => std::time::Duration::from_secs(60),
DrainOutcome::Guarded | DrainOutcome::RetryPending => {
let delay = BACKOFFS[backoff.min(BACKOFFS.len() - 1)];
backoff = (backoff + 1).min(BACKOFFS.len() - 1);
delay
}
};
let (generation, _stopped) = notifier.wait_for_change(last_seen, delay);
last_seen = generation;
}
}
pub(super) fn maintenance_worker_loop(
inner: Arc<StoreInner>,
notifier: Arc<super::MaintenanceNotifier>,
) {
let mut last_seen = 0_u64;
let mut backoff = 0_usize;
const BACKOFFS: [std::time::Duration; 4] = [
std::time::Duration::from_millis(10),
std::time::Duration::from_millis(25),
std::time::Duration::from_millis(50),
std::time::Duration::from_millis(100),
];
loop {
#[cfg(test)]
{
let barriers = &inner.maintenance_barriers;
if let Ok(mut state) = barriers.state.lock() {
state[6] = true;
barriers.wake.notify_all();
}
}
let outcome = futures_lite::future::block_on(run_real_maintenance_once_inner(&inner));
#[cfg(test)]
if matches!(
outcome,
MaintenanceOutcome::Completed | MaintenanceOutcome::Fenced
) {
let barriers = &inner.maintenance_barriers;
if barriers.state.lock().is_ok_and(|state| state[5]) {
if let Ok(mut completed) = barriers.outcome.lock() {
*completed = Some(outcome);
}
if let Ok(mut state) = barriers.state.lock() {
state[4] = true;
state[5] = false;
barriers.wake.notify_all();
}
}
}
let pending = inner.maintenance.lock().map_or(true, |state| {
state.requests != 0 || !state.archive_queue.is_empty()
});
let stopped = notifier.stopped.load(Ordering::Acquire);
if stopped && !pending {
break;
}
let delay = match outcome {
MaintenanceOutcome::Completed => {
backoff = 0;
continue;
}
MaintenanceOutcome::Fenced => break,
MaintenanceOutcome::Idle if !pending => std::time::Duration::from_secs(60),
MaintenanceOutcome::Idle | MaintenanceOutcome::ArchiveRetryPending => {
let delay = BACKOFFS[backoff.min(BACKOFFS.len() - 1)];
backoff = (backoff + 1).min(BACKOFFS.len() - 1);
delay
}
};
let (generation, _) = notifier.wait_for_change(last_seen, delay);
last_seen = generation;
}
}
impl StoreInner {
#[allow(dead_code)]
fn retired_queue_len(&self) -> usize {
match self.state.lock() {
Ok(runtime) => runtime.retired_queue_len(),
Err(_) => {
self.require_recovery();
0
}
}
}
fn drain_retired_once_with_hooks<H: DeleteHooks>(&self, hooks: &H) -> DrainOutcome {
let mut runtime = match self.state.lock() {
Ok(runtime) => runtime,
Err(_) => {
self.require_recovery();
return DrainOutcome::RetryPending;
}
};
runtime
.snapshot_states
.retain(|snapshot| snapshot.strong_count() != 0);
runtime.read_leases.retain(|_, leases| {
leases.retain(|lease| lease.strong_count() != 0);
!leases.is_empty()
});
let Some(path) = runtime.retired.first().cloned() else {
return DrainOutcome::Empty;
};
let current_guarded =
super::RuntimeState::state_references_path(&self.root, &runtime.current, &path);
let snapshot_guarded = runtime.snapshot_states.iter().any(|snapshot| {
snapshot.upgrade().is_some_and(|state| {
super::RuntimeState::state_references_path(&self.root, &state, &path)
})
});
let lease_guarded = runtime
.read_leases
.get(&path)
.is_some_and(|leases| leases.iter().any(|lease| lease.upgrade().is_some()));
if current_guarded || snapshot_guarded || lease_guarded {
return DrainOutcome::Guarded;
}
match hooks.unlink(&path) {
Ok(()) => {}
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
Err(_) => return DrainOutcome::RetryPending,
}
if super::io::sync_directory(&layout::data_dir(&self.root)).is_err() {
return DrainOutcome::RetryPending;
}
runtime.retired.remove(0);
DrainOutcome::Removed
}
}
async fn run_real_maintenance_once_inner(inner: &Arc<StoreInner>) -> MaintenanceOutcome {
let store = FileBlockStoreInternal { inner };
store.run().await
}
struct FileBlockStoreInternal<'a> {
inner: &'a Arc<StoreInner>,
}
impl FileBlockStoreInternal<'_> {
async fn run(&self) -> MaintenanceOutcome {
run_real_maintenance_core(self.inner).await
}
}
#[cfg(test)]
fn wait_real_maintenance_barrier_inner(inner: &Arc<StoreInner>, arrived: usize, released: usize) {
let barriers = &inner.maintenance_barriers;
let mut state = match barriers.state.lock() {
Ok(state) => state,
Err(poisoned) => poisoned.into_inner(),
};
if (arrived == 7 && !state[9]) || (arrived != 7 && !state[5]) {
return;
}
state[arrived] = true;
barriers.wake.notify_all();
while !state[released] {
state = match barriers.wake.wait(state) {
Ok(state) => state,
Err(poisoned) => poisoned.into_inner(),
};
}
}
impl FileBlockStore {
#[cfg_attr(not(test), allow(dead_code))]
async fn run_real_maintenance_once(&self) -> MaintenanceOutcome {
run_real_maintenance_core(&self.inner).await
}
}
async fn run_real_maintenance_core(inner: &Arc<StoreInner>) -> MaintenanceOutcome {
let retry = match inner.maintenance.lock() {
Ok(state) => state.archive_queue.front().cloned(),
Err(_) => {
inner.require_recovery();
return MaintenanceOutcome::Fenced;
}
};
if let Some(closed) = retry {
return match inner.log.archive(closed).await {
Ok(()) => match inner.maintenance.lock() {
Ok(mut state) => {
state.archive_queue.pop_front();
state.completed_count = state.completed_count.saturating_add(1);
MaintenanceOutcome::Completed
}
Err(_) => {
inner.require_recovery();
MaintenanceOutcome::Fenced
}
},
Err(_) => MaintenanceOutcome::ArchiveRetryPending,
};
}
let pending = match inner.maintenance.lock() {
Ok(mut state) => {
if state.requests == 0 {
false
} else {
state.requests -= 1;
true
}
}
Err(_) => {
inner.require_recovery();
return MaintenanceOutcome::Fenced;
}
};
if !pending {
return MaintenanceOutcome::Idle;
}
#[cfg(test)]
wait_real_maintenance_barrier_inner(inner, 0, 1);
if inner
.writer_active
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
.is_err()
{
return match inner.maintenance.lock() {
Ok(mut state) => {
state.requests = state.requests.saturating_add(1);
MaintenanceOutcome::Idle
}
Err(_) => {
inner.require_recovery();
MaintenanceOutcome::Fenced
}
};
}
#[cfg(test)]
wait_real_maintenance_barrier_inner(inner, 2, 3);
let latest = match inner.state.lock() {
Ok(runtime) => runtime.current.as_ref().clone(),
Err(_) => {
inner.require_recovery();
inner.writer_active.store(false, Ordering::Release);
return MaintenanceOutcome::Fenced;
}
};
if format::checkpoint_payload_len(&latest).is_err() {
inner.require_recovery();
inner.writer_active.store(false, Ordering::Release);
return MaintenanceOutcome::Fenced;
}
let closed = match inner.log.rotate().await {
Ok(Some(closed)) => {
inner
.log_phase
.store(LOG_PHASE_NEEDS_CHECKPOINT, Ordering::Release);
closed
}
Ok(None) => {
inner.writer_active.store(false, Ordering::Release);
return MaintenanceOutcome::Idle;
}
Err(_) => {
inner.require_recovery();
inner.writer_active.store(false, Ordering::Release);
return MaintenanceOutcome::Fenced;
}
};
let sequence = match inner.allocate_log_sequence() {
Ok(sequence) => sequence,
Err(_) => {
inner.writer_active.store(false, Ordering::Release);
return MaintenanceOutcome::Fenced;
}
};
let block = match format::encode_checkpoint(&latest, sequence) {
Ok(block) => block,
Err(_) => {
inner.require_recovery();
inner.writer_active.store(false, Ordering::Release);
return MaintenanceOutcome::Fenced;
}
};
if inner
.log
.append(block, AppendOptions { durable: true })
.await
.is_err()
{
inner.require_recovery();
inner.writer_active.store(false, Ordering::Release);
return MaintenanceOutcome::Fenced;
}
inner.log_phase.store(LOG_PHASE_ACTIVE, Ordering::Release);
inner.writer_active.store(false, Ordering::Release);
#[cfg(test)]
wait_real_maintenance_barrier_inner(inner, 7, 8);
match inner.log.archive(closed.clone()).await {
Ok(()) => match inner.maintenance.lock() {
Ok(mut state) => {
state.completed_count = state.completed_count.saturating_add(1);
MaintenanceOutcome::Completed
}
Err(_) => {
inner.require_recovery();
MaintenanceOutcome::Fenced
}
},
Err(_) => match inner.maintenance.lock() {
Ok(mut state) => {
state.archive_queue.push_back(closed);
MaintenanceOutcome::ArchiveRetryPending
}
Err(_) => {
inner.require_recovery();
MaintenanceOutcome::Fenced
}
},
}
}
#[cfg(test)]
#[allow(dead_code)]
impl FileBlockStore {
async fn wait_for_maintenance_worker_started_for_test(&self) {
self.wait_for_barrier_flag_for_test(6);
}
fn maintenance_business_owner_count_for_test(&self) -> usize {
self.inner
.maintenance_notifier
.owners
.load(Ordering::Acquire)
}
fn maintenance_notifier_stopped_for_test(&self) -> bool {
self.inner
.maintenance_notifier
.stopped
.load(Ordering::Acquire)
}
fn arm_real_maintenance_archive_barrier_for_test(&self) {
let barriers = &self.inner.maintenance_barriers;
if let Ok(mut state) = barriers.state.lock() {
state[7] = false;
state[8] = false;
state[9] = true;
}
}
async fn wait_for_real_maintenance_before_archive_for_test(&self) {
self.wait_for_barrier_flag_for_test(7);
}
fn release_real_maintenance_archive_for_test(&self) {
self.release_barrier_flag_for_test(8);
}
fn real_maintenance_completed_count_for_test(&self) -> u64 {
self.inner
.maintenance
.lock()
.map_or(0, |state| state.completed_count)
}
async fn wait_for_real_maintenance_completed_count_for_test(&self, target: u64) {
loop {
if self.real_maintenance_completed_count_for_test() >= target {
return;
}
let barriers = &self.inner.maintenance_barriers;
let state = match barriers.state.lock() {
Ok(state) => state,
Err(poisoned) => poisoned.into_inner(),
};
let _state = match barriers.wake.wait(state) {
Ok(state) => state,
Err(poisoned) => poisoned.into_inner(),
};
}
}
fn arm_real_maintenance_worker_barriers_for_test(&self) {
let barriers = &self.inner.maintenance_barriers;
if let Ok(mut state) = barriers.state.lock() {
*state = [
false, false, false, false, false, true, state[6], false, false, false,
];
}
if let Ok(mut outcome) = barriers.outcome.lock() {
*outcome = None;
}
}
async fn wait_for_real_maintenance_before_writer_gate_for_test(&self) {
self.wait_for_barrier_flag_for_test(0);
}
fn release_real_maintenance_before_writer_gate_for_test(&self) {
self.release_barrier_flag_for_test(1);
}
async fn wait_for_real_maintenance_after_writer_gate_before_state_capture_for_test(&self) {
self.wait_for_barrier_flag_for_test(2);
}
fn release_real_maintenance_state_capture_for_test(&self) {
self.release_barrier_flag_for_test(3);
}
async fn wait_for_real_maintenance_completed_for_test(&self) -> MaintenanceOutcome {
let barriers = &self.inner.maintenance_barriers;
let mut state = match barriers.state.lock() {
Ok(state) => state,
Err(poisoned) => poisoned.into_inner(),
};
while !state[4] {
state = match barriers.wake.wait(state) {
Ok(state) => state,
Err(poisoned) => poisoned.into_inner(),
};
}
drop(state);
barriers
.outcome
.lock()
.ok()
.and_then(|outcome| *outcome)
.unwrap_or(MaintenanceOutcome::Fenced)
}
fn wait_for_barrier_flag_for_test(&self, index: usize) {
let barriers = &self.inner.maintenance_barriers;
let mut state = match barriers.state.lock() {
Ok(state) => state,
Err(poisoned) => poisoned.into_inner(),
};
while !state[index] {
state = match barriers.wake.wait(state) {
Ok(state) => state,
Err(poisoned) => poisoned.into_inner(),
};
}
}
fn release_barrier_flag_for_test(&self, index: usize) {
let barriers = &self.inner.maintenance_barriers;
if let Ok(mut state) = barriers.state.lock() {
state[index] = true;
barriers.wake.notify_all();
}
}
fn schedule_real_maintenance_for_test(&self) {
match self.inner.maintenance.lock() {
Ok(mut state) => state.requests = state.requests.saturating_add(1),
Err(_) => self.inner.require_recovery(),
}
}
async fn run_real_maintenance_once_for_test(&self) -> MaintenanceOutcome {
self.run_real_maintenance_once().await
}
fn maintenance_rotation_threshold_for_test(&self) -> u64 {
u64::MAX
}
fn maintenance_pending_for_test(&self) -> bool {
match self.inner.test_maintenance.lock() {
Ok(state) => state.pending,
Err(_) => {
self.inner.require_recovery();
false
}
}
}
fn maintenance_notification_count_for_test(&self) -> usize {
match self.inner.test_maintenance.lock() {
Ok(state) => state.notification_count,
Err(_) => {
self.inner.require_recovery();
0
}
}
}
async fn run_maintenance_once_with_hooks<H>(&self, hooks: &H) -> MaintenanceOutcome
where
H: MaintenanceHooks + Sync,
H::Closed: Sync,
{
let retry = match self.inner.test_maintenance.lock() {
Ok(mut state) => {
if state.phase == MaintenancePhase::ArchiveRetry {
match (state.retry_closed.take(), state.retry_token.take()) {
(Some(closed), Some(token)) => Some((closed, token)),
_ => {
self.inner.require_recovery();
return MaintenanceOutcome::Fenced;
}
}
} else if !state.pending {
return MaintenanceOutcome::Idle;
} else {
None
}
}
Err(_) => {
self.inner.require_recovery();
return MaintenanceOutcome::Fenced;
}
};
if self
.inner
.writer_active
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
.is_err()
{
if let Some((closed, token)) = retry {
match self.inner.test_maintenance.lock() {
Ok(mut state) => {
state.retry_closed = Some(closed);
state.retry_token = Some(token);
}
Err(_) => {
self.inner.require_recovery();
return MaintenanceOutcome::Fenced;
}
}
}
return MaintenanceOutcome::Idle;
}
if let Some((closed_box, token)) = retry {
let Some(closed) = closed_box.downcast_ref::<H::Closed>() else {
self.inner.require_recovery();
self.inner.writer_active.store(false, Ordering::Release);
return MaintenanceOutcome::Fenced;
};
let result = hooks.archive_durable_borrowed(closed, &token).await;
self.inner.writer_active.store(false, Ordering::Release);
return match result {
Ok(()) => match self.inner.test_maintenance.lock() {
Ok(mut state) => {
state.pending = false;
state.phase = MaintenancePhase::Idle;
MaintenanceOutcome::Completed
}
Err(_) => {
self.inner.require_recovery();
MaintenanceOutcome::Fenced
}
},
Err(_) => match self.inner.test_maintenance.lock() {
Ok(mut state) => {
state.retry_closed = Some(closed_box);
state.retry_token = Some(token);
state.phase = MaintenancePhase::ArchiveRetry;
MaintenanceOutcome::ArchiveRetryPending
}
Err(_) => {
self.inner.require_recovery();
MaintenanceOutcome::Fenced
}
},
};
}
let latest = match self.inner.state.lock() {
Ok(runtime) => runtime.current.as_ref().clone(),
Err(_) => {
self.inner.require_recovery();
self.inner.writer_active.store(false, Ordering::Release);
return MaintenanceOutcome::Fenced;
}
};
let closed = match hooks.rotate(&self.inner.log).await {
Ok(closed) => closed,
Err(_) => {
self.inner.require_recovery();
self.inner.writer_active.store(false, Ordering::Release);
return MaintenanceOutcome::Fenced;
}
};
let token = match hooks.checkpoint_durable(&closed, latest).await {
Ok(token) => token,
Err(_) => {
self.inner.require_recovery();
self.inner.writer_active.store(false, Ordering::Release);
return MaintenanceOutcome::Fenced;
}
};
let archive_result = hooks.archive_durable_borrowed(&closed, &token).await;
self.inner.writer_active.store(false, Ordering::Release);
match archive_result {
Ok(()) => match self.inner.test_maintenance.lock() {
Ok(mut state) => {
state.pending = false;
state.phase = MaintenancePhase::Idle;
MaintenanceOutcome::Completed
}
Err(_) => {
self.inner.require_recovery();
MaintenanceOutcome::Fenced
}
},
Err(_) => match self.inner.test_maintenance.lock() {
Ok(mut state) => {
state.retry_closed = Some(Box::new(closed));
state.retry_token = Some(token);
state.phase = MaintenancePhase::ArchiveRetry;
MaintenanceOutcome::ArchiveRetryPending
}
Err(_) => {
self.inner.require_recovery();
MaintenanceOutcome::Fenced
}
},
}
}
}
impl FileBlockStore {
#[allow(dead_code)]
pub(super) fn retired_queue_len(&self) -> usize {
self.inner.retired_queue_len()
}
#[allow(dead_code)]
pub(super) fn drain_retired_once(&self) -> DrainOutcome {
self.inner.drain_retired_once_with_hooks(&RealDeleteHooks)
}
#[allow(dead_code)]
pub(super) fn drain_retired_once_with_hooks<H: DeleteHooks>(&self, hooks: &H) -> DrainOutcome {
self.inner.drain_retired_once_with_hooks(hooks)
}
}
impl FileTransaction {
pub(super) fn new(
inner: Arc<StoreInner>,
worker_owner: WorkerOwner,
identity: u64,
base: State,
) -> Self {
let store_identity = inner.identity;
Self {
inner,
store_identity,
identity,
candidate: base.clone(),
base,
allocated: BTreeSet::new(),
written: BTreeSet::new(),
released: BTreeSet::new(),
created_large: Vec::new(),
created_segments: Vec::new(),
modified_files: BTreeSet::new(),
requested_root_deletions: BTreeSet::new(),
lifecycle: Arc::new(TransactionLifecycle::new()),
worker_owner,
}
}
fn ensure_active(&self) -> Result<(), BlockStoreError> {
if self.inner.recovery_required() {
return Err(BlockStoreError::RecoveryRequired);
}
if self.lifecycle.state() == TransactionState::Active {
Ok(())
} else {
Err(BlockStoreError::WriteTransactionClosed)
}
}
fn release_writer_gate(&self) {
self.lifecycle.release_writer_gate(&self.inner);
}
fn clear_pending(&self) {
if let Ok(mut pending) = self.inner.pending_allocations.lock() {
for raw in &self.allocated {
if pending.get(raw) == Some(&self.identity) {
pending.remove(raw);
}
}
} else {
self.inner.require_recovery();
}
}
fn finish(&self) {
self.clear_pending();
self.lifecycle.set_state(TransactionState::Finished);
self.release_writer_gate();
}
fn fail_write(&self) {
self.lifecycle.set_state(TransactionState::WriteFailed);
self.release_writer_gate();
}
fn abort_definite_failure(&mut self, error: BlockStoreError) -> BlockStoreError {
let mut cleanup_failed = false;
for path in self.created_large.iter().chain(&self.created_segments) {
if let Err(cleanup_error) = std::fs::remove_file(path)
&& cleanup_error.kind() != io::ErrorKind::NotFound
{
cleanup_failed = true;
}
}
self.created_large.clear();
self.created_segments.clear();
if cleanup_failed {
self.inner.require_recovery();
}
self.finish();
if cleanup_failed {
BlockStoreError::RecoveryRequired
} else {
error
}
}
fn base_block_is_live(&self, raw: u64) -> bool {
let Ok(block) = BlockId::try_from(raw) else {
return false;
};
if block.level() == 15 {
return block
.large_block_id()
.is_some_and(|id| self.base.large_blocks.contains_key(&id));
}
let Some(version) = block.segment_version() else {
return false;
};
let Some(index) = block.block_index().map(u64::from) else {
return false;
};
self.base
.segments
.get(&(block.level(), version))
.and_then(|record| record.bitmap.get(usize::try_from(index / 64).ok()?))
.is_some_and(|word| word & (1_u64 << (index % 64)) != 0)
}
fn validate_final_roots(&self) -> Result<(), BlockStoreError> {
if self
.candidate
.roots
.iter()
.any(|target| self.released.contains(target))
{
return Err(BlockStoreError::InvalidWriteInput);
}
if self.candidate.bindings.values().any(|target| {
!self.candidate.roots.contains(target)
&& !self.requested_root_deletions.contains(target)
}) {
return Err(BlockStoreError::NamedBindingTargetNotFound);
}
if self.requested_root_deletions.iter().any(|target| {
self.candidate
.bindings
.values()
.any(|bound| bound == target)
}) {
return Err(BlockStoreError::RootStillNamed);
}
if self
.requested_root_deletions
.iter()
.any(|target| !self.base.roots.contains(target))
{
return Err(BlockStoreError::RootNotFound);
}
if self
.candidate
.roots
.difference(&self.base.roots)
.any(|raw| !self.written.contains(raw) && !self.base_block_is_live(*raw))
{
return Err(BlockStoreError::RootNotFound);
}
Ok(())
}
fn delta(&self) -> Delta {
let segment_upserts = self
.candidate
.segments
.iter()
.filter(|(key, record)| {
self.base.segments.get(key).is_none_or(|base| {
base.frontier != record.frontier
|| base.live_count != record.live_count
|| base.bitmap != record.bitmap
|| base.state != record.state
})
})
.map(|(_, record)| record.clone())
.collect();
let large_adds = self
.candidate
.large_blocks
.iter()
.filter(|(id, _)| !self.base.large_blocks.contains_key(id))
.map(|(&id, &length)| (id, length))
.collect();
Delta {
next_segment_version: self.candidate.next_segment_version,
next_large_block_id: self.candidate.next_large_block_id,
segment_upserts,
segment_removes: self
.base
.segments
.keys()
.filter(|key| !self.candidate.segments.contains_key(*key))
.copied()
.collect(),
large_adds,
large_removes: self
.base
.large_blocks
.keys()
.filter(|id| !self.candidate.large_blocks.contains_key(*id))
.copied()
.collect(),
root_adds: self
.candidate
.roots
.difference(&self.base.roots)
.copied()
.collect(),
root_removes: self
.base
.roots
.difference(&self.candidate.roots)
.copied()
.collect(),
binding_removes: self
.base
.bindings
.keys()
.filter(|name| !self.candidate.bindings.contains_key(*name))
.cloned()
.collect(),
binding_upserts: self
.candidate
.bindings
.iter()
.filter(|(name, target)| self.base.bindings.get(*name) != Some(*target))
.map(|(name, &target)| (name.clone(), target))
.collect(),
}
}
fn retired_paths(&self) -> Vec<PathBuf> {
let data = layout::data_dir(&self.inner.root);
let segments = self
.base
.segments
.keys()
.filter(|key| !self.candidate.segments.contains_key(*key))
.map(|&(level, version)| data.join(format!("{level:x}.{version:07x}")));
let large = self
.base
.large_blocks
.keys()
.filter(|id| !self.candidate.large_blocks.contains_key(*id))
.map(|id| data.join(format!("l{id:015x}")));
segments.chain(large).collect()
}
async fn write_batch_with_hooks<H: TransactionHooks>(
&mut self,
blocks: SmallVec<[AllocatedBlock; 8]>,
hooks: &H,
) -> Result<(), BlockStoreError> {
self.ensure_active()?;
let mut cancel_guard =
WriteBatchCancelGuard::new(self.lifecycle.clone(), self.inner.clone());
let result = (async {
let mut seen = BTreeSet::new();
let mut parts = Vec::with_capacity(blocks.len());
for block in blocks {
let (block_id, input, store_identity, transaction_identity) = block.into_parts();
let raw = u64::from(block_id);
if store_identity != self.store_identity
|| transaction_identity != self.identity
|| !self.allocated.contains(&raw)
{
return Err(BlockStoreError::AllocatedBlockNotOwned);
}
if self.written.contains(&raw) || !seen.insert(raw) {
return Err(BlockStoreError::AllocatedBlockAlreadyWritten);
}
parts.push((block_id, input));
}
for (index, (block_id, input)) in parts.into_iter().enumerate() {
if block_id.level() == 15 {
let id = block_id
.large_block_id()
.ok_or(BlockStoreError::InvalidBlockId)?;
let path = layout::data_dir(&self.inner.root).join(format!("l{id:015x}"));
if let Err(error) = hooks.write_large_owned(&path, &input.data) {
if error.created {
self.created_large.push(path);
}
return Err(BlockStoreError::Io(error.error));
}
self.created_large.push(path.clone());
self.modified_files.insert(path);
} else {
let level = block_id.level();
let version = block_id
.segment_version()
.ok_or(BlockStoreError::InvalidBlockId)?;
let index = u64::from(
block_id
.block_index()
.ok_or(BlockStoreError::InvalidBlockId)?,
);
let slot = SUPPORTED_BLOCK_SIZES[usize::from(level)];
let offset = index.checked_mul(slot).ok_or(BlockStoreError::NoSpace)?;
let path =
layout::data_dir(&self.inner.root).join(format!("{level:x}.{version:07x}"));
hooks.write_ordinary(&path, offset, &input.data, slot as usize)?;
self.modified_files.insert(path);
}
hooks.after_block_write(index).await?;
self.written.insert(u64::from(block_id));
}
Ok(())
})
.await;
cancel_guard.disarm();
if let Err(error) = result {
self.fail_write();
return Err(error);
}
self.lifecycle.set_state(TransactionState::Active);
Ok(())
}
#[cfg(test)]
async fn commit_with_runner_and_hooks<H, R>(
self,
hooks: Arc<H>,
runner: &R,
) -> Result<DetachedCommitWait<R::JoinHandle>, BlockStoreError>
where
H: TransactionHooks + Send + Sync + 'static,
R: DetachedCommitRunner,
{
self.commit_with_runner_and_hooks_real_maintenance(hooks, runner, None)
.await
}
#[allow(dead_code)]
async fn commit_with_runner_and_hooks_real_maintenance<H, R>(
mut self,
hooks: Arc<H>,
runner: &R,
#[cfg(test)] real_maintenance_config: Option<(
u64,
Option<Arc<dyn MaintenanceObserver + Send + Sync>>,
)>,
#[cfg(not(test))] _real_maintenance_config: Option<(
u64,
Option<Arc<dyn std::marker::Sync + Send>>,
)>,
) -> Result<DetachedCommitWait<R::JoinHandle>, BlockStoreError>
where
H: TransactionHooks + Send + Sync + 'static,
R: DetachedCommitRunner,
{
if self.inner.recovery_required() {
self.finish();
return Err(BlockStoreError::RecoveryRequired);
}
if self.lifecycle.state() == TransactionState::WriteFailed {
let error = self.abort_definite_failure(BlockStoreError::AllocatedBlockNotWritten);
return Err(error);
}
self.ensure_active()?;
if self.written.len() != self.allocated.len() {
let error = self.abort_definite_failure(BlockStoreError::AllocatedBlockNotWritten);
return Err(error);
}
if let Err(error) = self.validate_final_roots() {
let error = self.abort_definite_failure(error);
return Err(error);
}
if self.candidate.validate().is_err() {
let error = self.abort_definite_failure(BlockStoreError::CommitFailed);
return Err(error);
}
let delta = self.delta();
if let Err(error) =
format::delta_payload_len_with_limit(&delta, hooks.preflight_payload_limit())
{
let error = self.abort_definite_failure(error);
return Err(error);
}
if let Err(error) = format::checkpoint_payload_len_with_limit(
&self.candidate,
hooks.preflight_payload_limit(),
) {
let error = self.abort_definite_failure(error);
return Err(error);
}
for path in self.modified_files.clone() {
if hooks.sync_file(&path).is_err() {
let error = self.abort_definite_failure(BlockStoreError::CommitFailed);
return Err(error);
}
}
if hooks
.sync_data_directory(&layout::data_dir(&self.inner.root))
.is_err()
{
let error = self.abort_definite_failure(BlockStoreError::CommitFailed);
return Err(error);
}
let payload = format::encode_delta(&delta);
let sequence = self.inner.allocate_log_sequence()?;
let block = match DefaultBlockCodec::default().encode(sequence, 0, &payload) {
Ok(block) => block,
Err(_) => {
let error = self.abort_definite_failure(BlockStoreError::CommitFailed);
return Err(error);
}
};
if hooks.before_durable_handoff().await.is_err() {
let error = self.abort_definite_failure(BlockStoreError::CommitFailed);
return Err(error);
}
if self.inner.log_phase.load(Ordering::Acquire) != LOG_PHASE_ACTIVE {
self.inner.require_recovery();
let error = self.abort_definite_failure(BlockStoreError::RecoveryRequired);
return Err(error);
}
self.lifecycle.set_state(TransactionState::DetachedInFlight);
let context = OwnedCommitContext {
inner: self.inner.clone(),
candidate: self.candidate.clone(),
created_large: std::mem::take(&mut self.created_large),
created_segments: std::mem::take(&mut self.created_segments),
block,
lifecycle: self.lifecycle.clone(),
hooks,
allocated: self.allocated.clone(),
transaction_identity: self.identity,
retired_paths: self.retired_paths(),
#[cfg(test)]
real_maintenance: real_maintenance_config,
worker_owner: self.worker_owner.clone(),
};
let inner = context.inner.clone();
let task: DetachedTask = Box::pin(async move {
let OwnedCommitContext {
inner,
candidate,
created_large,
created_segments,
block,
lifecycle,
hooks,
allocated,
transaction_identity,
retired_paths,
#[cfg(test)]
real_maintenance,
worker_owner,
} = context;
let _worker_owner = worker_owner;
let mut task_guard =
DetachedTaskGuard::new(inner.clone(), lifecycle, allocated, transaction_identity);
let result = match hooks.append_delta_with_outcome(&inner.log, block).await {
Ok(outcome) => {
let published = match inner.state.lock() {
Ok(mut state) => {
state.publish(candidate, retired_paths);
true
}
Err(_) => false,
};
if !published {
inner.require_recovery();
Err(BlockStoreError::RecoveryRequired)
} else {
inner.retirement_notifier.notify();
task_guard.finish();
#[cfg(test)]
if let Some((threshold, observer)) = real_maintenance
&& outcome.active_size >= threshold
{
let notify = match inner.maintenance.lock() {
Ok(mut state) => {
let notify = state.requests == 0;
state.requests = state.requests.saturating_add(1);
notify
}
Err(_) => {
inner.require_recovery();
return Err(BlockStoreError::RecoveryRequired);
}
};
if notify {
if let Some(observer) = observer {
observer.maintenance_notified(&FileBlockStore {
inner: inner.clone(),
worker_owner: _worker_owner.clone(),
});
}
inner.maintenance_notifier.notify();
}
}
#[cfg(not(test))]
inner.schedule_maintenance_for_active_size(outcome.active_size)?;
Ok(())
}
}
Err(CommitAppendError::Failed(error)) => {
drop(error);
let mut cleanup_failed = false;
for path in created_large.iter().chain(&created_segments) {
if let Err(cleanup_error) = std::fs::remove_file(path)
&& cleanup_error.kind() != io::ErrorKind::NotFound
{
cleanup_failed = true;
}
}
if cleanup_failed {
inner.require_recovery();
Err(BlockStoreError::RecoveryRequired)
} else {
Err(BlockStoreError::CommitFailed)
}
}
Err(CommitAppendError::Unknown(error)) => {
drop(error);
inner.require_recovery();
Err(BlockStoreError::CommitStatusUnknown)
}
};
task_guard.finish();
result
});
let join_handle = match runner.spawn(task) {
Ok(join_handle) => join_handle,
Err(_) => {
self.inner.require_recovery();
self.clear_pending();
self.lifecycle.set_state(TransactionState::Finished);
self.lifecycle.release_writer_gate(&inner);
return Err(BlockStoreError::RecoveryRequired);
}
};
Ok(DetachedCommitWait::new(join_handle, inner))
}
#[cfg(test)]
async fn commit_with_runner_hooks_and_real_maintenance_threshold_for_test<H, R>(
self,
hooks: Arc<H>,
runner: &R,
threshold: u64,
observer: Option<Arc<dyn MaintenanceObserver + Send + Sync>>,
) -> Result<DetachedCommitWait<R::JoinHandle>, BlockStoreError>
where
H: TransactionHooks + Send + Sync + 'static,
R: DetachedCommitRunner,
{
self.commit_with_runner_and_hooks_real_maintenance(
hooks,
runner,
Some((threshold, observer)),
)
.await
}
#[cfg(test)]
fn schedule_real_maintenance_after_publish(
&self,
observer: Option<&dyn MaintenanceObserver>,
) -> Result<(), BlockStoreError> {
let notify = match self.inner.maintenance.lock() {
Ok(mut state) => {
let notify = state.requests == 0;
state.requests = state.requests.saturating_add(1);
notify
}
Err(_) => {
self.inner.require_recovery();
return Err(BlockStoreError::RecoveryRequired);
}
};
if notify {
if let Some(observer) = observer {
observer.maintenance_notified(&FileBlockStore {
inner: self.inner.clone(),
worker_owner: self.worker_owner.clone(),
});
}
self.inner.maintenance_notifier.notify();
}
Ok(())
}
#[cfg(test)]
async fn commit_with_hooks_and_maintenance_threshold<H: TransactionHooks + Sync>(
&mut self,
hooks: &H,
threshold: u64,
observer: Option<&dyn MaintenanceObserver>,
) -> Result<CommitResult, BlockStoreError> {
self.commit_with_hooks_internal(hooks, Some((threshold, observer)), None)
.await
}
#[cfg(test)]
async fn commit_with_hooks_and_real_maintenance_threshold_for_test<
H: TransactionHooks + Sync,
>(
&mut self,
hooks: &H,
threshold: u64,
observer: Option<&dyn MaintenanceObserver>,
) -> Result<CommitResult, BlockStoreError> {
self.commit_with_hooks_internal(hooks, None, Some((threshold, observer)))
.await
}
#[cfg_attr(not(test), allow(dead_code))]
async fn commit_with_hooks<H: TransactionHooks + Sync>(
&mut self,
hooks: &H,
) -> Result<CommitResult, BlockStoreError> {
self.commit_with_hooks_internal(hooks, None, None).await
}
async fn commit_with_hooks_internal<H: TransactionHooks + Sync>(
&mut self,
hooks: &H,
#[cfg(test)] maintenance: Option<(u64, Option<&dyn MaintenanceObserver>)>,
#[cfg(test)] real_maintenance: Option<(u64, Option<&dyn MaintenanceObserver>)>,
#[cfg(not(test))] _maintenance: Option<(u64, Option<&dyn std::marker::Sync>)>,
#[cfg(not(test))] _real_maintenance: Option<(u64, Option<&dyn std::marker::Sync>)>,
) -> Result<CommitResult, BlockStoreError> {
if self.inner.recovery_required() {
self.finish();
return Err(BlockStoreError::RecoveryRequired);
}
if self.lifecycle.state() == TransactionState::WriteFailed {
let error = self.abort_definite_failure(BlockStoreError::AllocatedBlockNotWritten);
return Err(error);
}
self.ensure_active()?;
if self.written.len() != self.allocated.len() {
let error = self.abort_definite_failure(BlockStoreError::AllocatedBlockNotWritten);
return Err(error);
}
if let Err(error) = self.validate_final_roots() {
let error = self.abort_definite_failure(error);
return Err(error);
}
if self.candidate.validate().is_err() {
let error = self.abort_definite_failure(BlockStoreError::CommitFailed);
return Err(error);
}
let delta = self.delta();
if let Err(error) =
format::delta_payload_len_with_limit(&delta, hooks.preflight_payload_limit())
{
let error = self.abort_definite_failure(error);
return Err(error);
}
if let Err(error) = format::checkpoint_payload_len_with_limit(
&self.candidate,
hooks.preflight_payload_limit(),
) {
let error = self.abort_definite_failure(error);
return Err(error);
}
for path in self.modified_files.clone() {
if hooks.sync_file(&path).is_err() {
let error = self.abort_definite_failure(BlockStoreError::CommitFailed);
return Err(error);
}
}
if hooks
.sync_data_directory(&layout::data_dir(&self.inner.root))
.is_err()
{
let error = self.abort_definite_failure(BlockStoreError::CommitFailed);
return Err(error);
}
let payload = format::encode_delta(&delta);
let sequence = match self.inner.allocate_log_sequence() {
Ok(sequence) => sequence,
Err(error) => {
let error = self.abort_definite_failure(error);
return Err(error);
}
};
let business_record_seq = match RecordSeq::try_from_u64(sequence) {
Ok(record_seq) => record_seq,
Err(_) => {
self.inner.require_recovery();
let error = self.abort_definite_failure(BlockStoreError::RecoveryRequired);
return Err(error);
}
};
let block = match DefaultBlockCodec::default().encode(sequence, 0, &payload) {
Ok(block) => block,
Err(_) => {
let error = self.abort_definite_failure(BlockStoreError::CommitFailed);
return Err(error);
}
};
let mut pre_durable_cancel = PreDurableCommitCancelGuard::new(self);
let handoff_result = hooks.before_durable_handoff().await;
pre_durable_cancel.disarm();
if handoff_result.is_err() {
let error = self.abort_definite_failure(BlockStoreError::CommitFailed);
return Err(error);
}
if self.inner.log_phase.load(Ordering::Acquire) != LOG_PHASE_ACTIVE {
self.inner.require_recovery();
let error = self.abort_definite_failure(BlockStoreError::RecoveryRequired);
return Err(error);
}
self.lifecycle.set_state(TransactionState::DurableInFlight);
let mut durable_fence = DurableAppendFence::new(self.inner.clone());
let append_result = hooks
.append_delta_with_outcome(&self.inner.log, block)
.await;
durable_fence.disarm();
let maintenance_active_size = match append_result {
Ok(outcome) => outcome.active_size,
Err(CommitAppendError::Failed(error)) => {
drop(error);
let error = self.abort_definite_failure(BlockStoreError::CommitFailed);
return Err(error);
}
Err(CommitAppendError::Unknown(error)) => {
drop(error);
self.inner.require_recovery();
self.created_large.clear();
self.created_segments.clear();
self.finish();
return Err(BlockStoreError::CommitStatusUnknown);
}
};
#[cfg(not(test))]
let _ = maintenance_active_size;
self.created_large.clear();
self.created_segments.clear();
let publish_failed = match self.inner.state.lock() {
Ok(mut state) => {
state.publish(self.candidate.clone(), self.retired_paths());
self.inner.retirement_notifier.notify();
false
}
Err(_) => true,
};
if publish_failed {
self.inner.require_recovery();
self.finish();
return Err(BlockStoreError::RecoveryRequired);
}
self.finish();
#[cfg(not(test))]
self.inner
.schedule_maintenance_for_active_size(maintenance_active_size)?;
#[cfg(test)]
if let Some((threshold, observer)) = real_maintenance
&& maintenance_active_size >= threshold
{
self.schedule_real_maintenance_after_publish(observer)?;
}
#[cfg(test)]
if let Some((threshold, observer)) = maintenance
&& maintenance_active_size >= threshold
{
let notify = match self.inner.test_maintenance.lock() {
Ok(mut state) => {
if state.pending {
false
} else {
state.pending = true;
state.notification_count += 1;
true
}
}
Err(_) => {
self.inner.require_recovery();
return Err(BlockStoreError::RecoveryRequired);
}
};
if notify && let Some(observer) = observer {
observer.maintenance_notified(&FileBlockStore {
inner: self.inner.clone(),
worker_owner: self.worker_owner.clone(),
});
}
}
Ok(CommitResult::new(
business_record_seq,
MaintenanceResult::NotNeeded,
))
}
fn release_blocks_with_hooks<H: TransactionHooks>(
&mut self,
blocks: Vec<BlockId>,
hooks: &H,
) -> Result<(), BlockStoreError> {
self.ensure_active()?;
let pending = self.inner.pending_allocations.lock().map_err(|_| {
self.inner.require_recovery();
BlockStoreError::RecoveryRequired
})?;
let mut batch = BTreeSet::new();
for block in &blocks {
let raw = u64::from(*block);
if pending.contains_key(&raw) {
return Err(BlockStoreError::BlockNotCommitted);
}
if self.released.contains(&raw) || !batch.insert(raw) {
return Err(BlockStoreError::BlockAlreadyReleased);
}
if block.level() == 15 {
let id = block
.large_block_id()
.ok_or(BlockStoreError::BlockNotFound)?;
if !self.base.large_blocks.contains_key(&id) {
return Err(BlockStoreError::BlockNotFound);
}
continue;
}
let version = block
.segment_version()
.ok_or(BlockStoreError::BlockNotFound)?;
let index = u64::from(block.block_index().ok_or(BlockStoreError::BlockNotFound)?);
let record = self
.base
.segments
.get(&(block.level(), version))
.ok_or(BlockStoreError::BlockNotFound)?;
if index >= record.frontier {
return Err(BlockStoreError::BlockNotFound);
}
let word = usize::try_from(index / 64)
.ok()
.and_then(|word| record.bitmap.get(word))
.ok_or(BlockStoreError::BlockNotFound)?;
if word & (1_u64 << (index % 64)) == 0 {
return Err(BlockStoreError::BlockAlreadyReleased);
}
}
drop(pending);
let mut candidate = self.candidate.clone();
for block in &blocks {
if block.level() == 15 {
let id = block
.large_block_id()
.ok_or(BlockStoreError::BlockNotFound)?;
candidate.large_blocks.remove(&id);
continue;
}
let version = block
.segment_version()
.ok_or(BlockStoreError::BlockNotFound)?;
let index = u64::from(block.block_index().ok_or(BlockStoreError::BlockNotFound)?);
let record = candidate
.segments
.get_mut(&(block.level(), version))
.ok_or(BlockStoreError::BlockNotFound)?;
let word = usize::try_from(index / 64).map_err(|_| BlockStoreError::BlockNotFound)?;
record.bitmap[word] &= !(1_u64 << (index % 64));
record.live_count -= 1;
}
let empty_segments: Vec<(u8, u32, super::state::SegmentState)> = candidate
.segments
.iter()
.filter(|(_, record)| record.live_count == 0 && record.frontier != 0)
.map(|(&(level, version), record)| (level, version, record.state))
.collect();
let mut replacement_paths = Vec::new();
for (level, version, state) in empty_segments {
candidate.segments.remove(&(level, version));
if state == super::state::SegmentState::Active {
let next_version = candidate.next_segment_version;
if next_version > 0x0fff_ffff {
return Err(BlockStoreError::SegmentVersionExhausted);
}
let path = layout::data_dir(&self.inner.root)
.join(format!("{level:x}.{next_version:07x}"));
replacement_paths.push(path);
candidate.segments.insert(
(level, next_version),
super::state::SegmentRecord {
level,
state: super::state::SegmentState::Active,
version: next_version,
frontier: 0,
live_count: 0,
bitmap: Vec::new(),
},
);
candidate.active_versions[usize::from(level)] = next_version;
candidate.next_segment_version = next_version + 1;
}
}
let mut created = Vec::new();
for path in &replacement_paths {
match hooks.create_replacement_active(path) {
Ok(()) => created.push(path.clone()),
Err(error) => {
let mut cleanup_failed = false;
for created_path in &created {
if hooks.remove_replacement_active(created_path).is_err() {
cleanup_failed = true;
}
}
if cleanup_failed {
self.inner.require_recovery();
return Err(BlockStoreError::RecoveryRequired);
}
return Err(BlockStoreError::Io(error));
}
}
}
self.candidate = candidate;
self.released.extend(batch);
for path in created {
self.modified_files.insert(path.clone());
self.created_segments.push(path);
}
Ok(())
}
#[cfg(test)]
fn set_active_frontier_for_test(&mut self, level: u8, frontier: u64) -> io::Result<()> {
let version = self.candidate.active_versions[usize::from(level)];
let record = self
.candidate
.segments
.get_mut(&(level, version))
.ok_or_else(|| io::Error::other("active segment missing"))?;
record.frontier = frontier;
record.live_count = 0;
record.bitmap = vec![0; usize::try_from(frontier.div_ceil(64)).map_err(io::Error::other)?];
let path = layout::data_dir(&self.inner.root).join(format!("{level:x}.{version:07x}"));
OpenOptions::new()
.write(true)
.open(path)?
.set_len(frontier * SUPPORTED_BLOCK_SIZES[usize::from(level)])
}
#[cfg(test)]
fn poison_state_lock_for_test(&self) {
let inner = self.inner.clone();
let _ = std::thread::spawn(move || {
let _guard = inner.state.lock().expect("state lock");
panic!("poison state lock for test");
})
.join();
}
}
impl WriteTransaction for FileTransaction {
fn alloc(&mut self, input: WriteInput) -> Result<AllocatedBlock, BlockStoreError> {
self.ensure_active()?;
if input.data.is_empty() {
return Err(BlockStoreError::InvalidWriteInput);
}
let level = SUPPORTED_BLOCK_SIZES
.iter()
.position(|size| input.data.len() <= *size as usize)
.map(|level| level as u8)
.unwrap_or(15);
let block = if level == 15 {
let id = self.candidate.next_large_block_id;
if id >= (1_u64 << 60) - 1 {
return Err(BlockStoreError::LargeBlockIdExhausted);
}
self.candidate.next_large_block_id = id + 1;
self.candidate
.large_blocks
.insert(id, input.data.len() as u64);
BlockId::from_raw_unchecked((id << 4) | 15)
} else {
let slot_size = SUPPORTED_BLOCK_SIZES[usize::from(level)];
let max_slots = (256_u64 * 1024 * 1024) / slot_size;
let mut version = self.candidate.active_versions[usize::from(level)];
let frontier = self
.candidate
.segments
.get(&(level, version))
.ok_or(BlockStoreError::RecoveryRequired)?
.frontier;
if frontier == max_slots {
let next_version = self.candidate.next_segment_version;
if next_version > 0x0fff_ffff {
return Err(BlockStoreError::SegmentVersionExhausted);
}
self.candidate
.segments
.get_mut(&(level, version))
.ok_or(BlockStoreError::RecoveryRequired)?
.state = super::state::SegmentState::Sealed;
let path = layout::data_dir(&self.inner.root)
.join(format!("{level:x}.{next_version:07x}"));
let file = OpenOptions::new()
.write(true)
.create_new(true)
.open(&path)?;
drop(file);
self.created_segments.push(path.clone());
self.modified_files.insert(path);
self.candidate.segments.insert(
(level, next_version),
super::state::SegmentRecord {
level,
state: super::state::SegmentState::Active,
version: next_version,
frontier: 0,
live_count: 0,
bitmap: Vec::new(),
},
);
self.candidate.active_versions[usize::from(level)] = next_version;
self.candidate.next_segment_version = next_version + 1;
version = next_version;
}
let record = self
.candidate
.segments
.get_mut(&(level, version))
.ok_or(BlockStoreError::RecoveryRequired)?;
let index = record.frontier;
if index >= max_slots || index > u64::from(u32::MAX) {
return Err(BlockStoreError::NoSpace);
}
record.frontier += 1;
record.live_count += 1;
let word = usize::try_from(index / 64).map_err(|_| BlockStoreError::NoSpace)?;
if record.bitmap.len() <= word {
record.bitmap.resize(word + 1, 0);
}
record.bitmap[word] |= 1_u64 << (index % 64);
BlockId::from_raw_unchecked(
(index << 32) | (u64::from(version) << 4) | u64::from(level),
)
};
let raw = u64::from(block);
self.allocated.insert(raw);
match self.inner.pending_allocations.lock() {
Ok(mut pending) => {
pending.insert(raw, self.identity);
}
Err(_) => {
self.inner.require_recovery();
return Err(BlockStoreError::RecoveryRequired);
}
}
Ok(AllocatedBlock::new(
block,
input,
self.store_identity,
self.identity,
))
}
async fn write_batch(
&mut self,
blocks: SmallVec<[AllocatedBlock; 8]>,
) -> Result<(), BlockStoreError> {
self.write_batch_with_hooks(blocks, &RealTransactionHooks)
.await
}
fn add_root_version(&mut self, block: BlockId) -> Result<(), BlockStoreError> {
self.ensure_active()?;
let raw = u64::from(block);
self.candidate.roots.insert(raw);
self.requested_root_deletions.remove(&raw);
Ok(())
}
fn create_named_binding(
&mut self,
name: RootName,
target: BlockId,
) -> Result<(), BlockStoreError> {
self.ensure_active()?;
let name = name.as_str().to_owned();
if self.candidate.bindings.contains_key(&name) {
return Err(BlockStoreError::RootNameAlreadyBound);
}
self.candidate.bindings.insert(name, u64::from(target));
Ok(())
}
fn remove_named_binding(&mut self, name: &RootName) -> Result<(), BlockStoreError> {
self.ensure_active()?;
if self.candidate.bindings.remove(name.as_str()).is_none() {
return Err(BlockStoreError::NamedBindingNotFound);
}
Ok(())
}
fn delete_root_version(&mut self, target: BlockId) -> Result<(), BlockStoreError> {
self.ensure_active()?;
let raw = u64::from(target);
if !self.requested_root_deletions.insert(raw) {
return Err(BlockStoreError::RootAlreadyDeleted);
}
self.candidate.roots.remove(&raw);
Ok(())
}
fn release_blocks(&mut self, blocks: Vec<BlockId>) -> Result<(), BlockStoreError> {
self.release_blocks_with_hooks(blocks, &RealTransactionHooks)
}
async fn commit(&mut self) -> Result<CommitResult, BlockStoreError> {
self.commit_with_hooks(&RealTransactionHooks).await
}
}
impl Drop for FileTransaction {
fn drop(&mut self) {
if self.lifecycle.state() == TransactionState::DurableInFlight {
self.inner.require_recovery();
self.created_large.clear();
self.created_segments.clear();
self.finish();
} else if self.lifecycle.state() == TransactionState::DetachedInFlight {
} else if self.lifecycle.state() != TransactionState::Finished {
for path in self.created_large.iter().chain(&self.created_segments) {
let _ = std::fs::remove_file(path);
}
self.finish();
}
}
}
impl From<io::Error> for BlockStoreError {
fn from(error: io::Error) -> Self {
Self::Io(error)
}
}
#[cfg(test)]
#[path = "../../tests/unit/file_transaction_fault_test.rs"]
mod fault_tests;