use std::io;
use std::mem;
use std::panic;
use std::panic::AssertUnwindSafe;
use std::sync::Arc;
use std::sync::Condvar;
use std::sync::Mutex;
use std::sync::MutexGuard;
use std::sync::atomic::AtomicU8;
use std::sync::atomic::AtomicUsize;
use std::sync::atomic::Ordering;
use std::thread;
use std::time::Duration;
use std::time::Instant;
use asyncband::event::AutoResetEvent;
use asyncband::semaphore::OwnedSemaphorePermit;
use asyncband::semaphore::Semaphore;
use thread::JoinHandle;
use tokio::runtime::Handle;
use crate::IoRole;
use crate::cache::runtime::metrics::RuntimeMetrics;
use crate::config::CacheConfig;
use crate::config::runtime::IoMode;
use crate::config::runtime::IoPoolTopology;
use crate::config::runtime::RuntimeOptions;
use crate::config::runtime::read_io_wait_capacity;
use crate::config::runtime::read_io_wait_timeout;
use crate::hashing::route_hash;
use crate::io::background::BackgroundIoAttempt;
use crate::io::background::IoRecovery;
use crate::io::engine::IoBuffer;
use crate::io::engine::IoEngine;
use crate::io::engine::IoOperation;
use crate::io::engine::ReadSlot;
use crate::io::engine::ReadSlotWaiter;
use crate::io::engine::StoredIoError;
use crate::io::engine::build_file_engine;
use crate::io::engine::submit_background_io;
use crate::io::file::DataFileHandles;
use crate::io::fill_control::FLUSH_RETRY;
use crate::io::fill_control::FillController;
use crate::io::fill_control::FlushCharge;
use crate::managed_memory::BufferLease;
use crate::managed_memory::CACHE_THREAD_STACK_BYTES;
use crate::managed_memory::ManagedMemory;
use crate::managed_memory::ManagedMemoryLimits;
use crate::memory::MemoryLookup;
use crate::memory::MemoryReadToken;
use crate::memory::MemoryStore;
use crate::memory::MemoryValue;
use crate::region::RegionRotation;
use crate::region::RegionStageValue;
use crate::region::RegionStore;
use crate::region::RegionValue;
use crate::region::is_read_pressure;
use crate::region::reader::PendingRead;
use crate::region::reader::ReadCompletion;
use crate::region::reader::ReadDesc;
use crate::region::reader::describe_read;
use crate::region::record::MAX_KEY_SIZE;
use crate::region::record::codec::hash_key;
use crate::region::record::codec::required_record_bytes;
use crate::region::recovery::DataSuperblock;
use crate::region::staging::AppendStaging;
use crate::region::staging::StagingError;
use crate::snapshot::CacheHealth;
use crate::snapshot::CacheIoDirectionSnapshot;
use crate::snapshot::CacheIoSnapshot;
use crate::snapshot::CacheSnapshot;
use crate::snapshot::DetailedCacheSnapshot;
use crate::stats::recording::Recorder;
use crate::stats::recording::RequestGuard;
pub mod metrics;
const WRITE_FLUSH_DELAY: Duration = Duration::from_millis(1);
const STAGING_RETRY_DELAY: Duration = Duration::from_micros(50);
const LIFECYCLE_RUNNING: u8 = 0;
const LIFECYCLE_DRAINING: u8 = 1;
const LIFECYCLE_FAILED: u8 = 2;
const MUTATION_DRAINING: usize = 1_usize << (usize::BITS - 1);
const MUTATION_CLOSED: usize = 1_usize << (usize::BITS - 2);
const MUTATION_FENCED: usize = MUTATION_DRAINING | MUTATION_CLOSED;
const MUTATION_COUNT_MASK: usize = !MUTATION_FENCED;
const MUTATION_ENTER_ATTEMPTS: usize = 8;
struct LifecycleDrainingGuard<'a> {
lifecycle: &'a AtomicU8,
operations: &'a MutationGate,
}
impl<'a> LifecycleDrainingGuard<'a> {
fn enter(lifecycle: &'a AtomicU8, operations: &'a MutationGate) -> Option<Self> {
lifecycle
.compare_exchange(
LIFECYCLE_RUNNING,
LIFECYCLE_DRAINING,
Ordering::AcqRel,
Ordering::Acquire,
)
.is_ok()
.then_some(Self {
lifecycle,
operations,
})
}
}
impl Drop for LifecycleDrainingGuard<'_> {
fn drop(&mut self) {
if !self.operations.is_closed() {
let _ = self.lifecycle.compare_exchange(
LIFECYCLE_DRAINING,
LIFECYCLE_RUNNING,
Ordering::AcqRel,
Ordering::Acquire,
);
}
}
}
struct MutationGate {
state: AtomicUsize,
quiescent: Mutex<()>,
quiescent_changed: Condvar,
async_changed: AutoResetEvent,
}
impl MutationGate {
fn new() -> Self {
Self {
state: AtomicUsize::new(0),
quiescent: Mutex::new(()),
quiescent_changed: Condvar::new(),
async_changed: AutoResetEvent::new(),
}
}
fn try_enter(&self) -> Option<MutationGuard<'_>> {
let mut state = self.state.load(Ordering::Acquire);
for _ in 0..MUTATION_ENTER_ATTEMPTS {
if state & MUTATION_FENCED != 0 || state & MUTATION_COUNT_MASK == MUTATION_COUNT_MASK {
return None;
}
match self.state.compare_exchange_weak(
state,
state + 1,
Ordering::Acquire,
Ordering::Relaxed,
) {
Ok(_) => return Some(MutationGuard { gate: self }),
Err(observed) => state = observed,
}
}
None
}
fn begin_drain(&self) -> io::Result<MutationDrainGuard<'_>> {
let previous = self.state.fetch_or(MUTATION_DRAINING, Ordering::AcqRel);
if previous & MUTATION_FENCED != 0 {
return Err(io::Error::new(
io::ErrorKind::WouldBlock,
"cache drain is already in progress",
));
}
Ok(MutationDrainGuard { gate: self })
}
fn start_close(&self) {
self.state.fetch_or(MUTATION_CLOSED, Ordering::AcqRel);
}
fn is_closed(&self) -> bool {
self.state.load(Ordering::Acquire) & MUTATION_CLOSED != 0
}
fn active_mutations(&self) -> usize {
self.state.load(Ordering::Acquire) & MUTATION_COUNT_MASK
}
fn wait_quiescent(&self) -> io::Result<()> {
let mut quiescent = self
.quiescent
.lock()
.map_err(|_| poisoned_runtime_error())?;
while self.active_mutations() != 0 {
quiescent = self
.quiescent_changed
.wait(quiescent)
.map_err(|_| poisoned_runtime_error())?;
}
Ok(())
}
async fn wait_quiescent_async(&self) {
while self.active_mutations() != 0 {
self.async_changed.wait().await;
}
}
fn mutation_finished(&self) {
let previous = self.state.fetch_sub(1, Ordering::Release);
debug_assert_ne!(previous & MUTATION_COUNT_MASK, 0);
if previous & MUTATION_COUNT_MASK == 1 && previous & MUTATION_FENCED != 0 {
let quiescent = self
.quiescent
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
self.quiescent_changed.notify_all();
drop(quiescent);
self.async_changed.set();
}
}
}
struct MutationGuard<'a> {
gate: &'a MutationGate,
}
impl Drop for MutationGuard<'_> {
fn drop(&mut self) {
self.gate.mutation_finished();
}
}
struct MutationDrainGuard<'a> {
gate: &'a MutationGate,
}
impl MutationDrainGuard<'_> {
#[cfg(test)]
fn wait(&self) -> io::Result<()> {
self.gate.wait_quiescent()
}
async fn wait_async(&self) {
self.gate.wait_quiescent_async().await;
}
}
impl Drop for MutationDrainGuard<'_> {
fn drop(&mut self) {
let previous = self
.gate
.state
.fetch_and(!MUTATION_DRAINING, Ordering::Release);
debug_assert_ne!(previous & MUTATION_DRAINING, 0);
}
}
const WAKE_DATA: u8 = 1;
const WAKE_URGENT: u8 = 2;
const WAKE_ROTATE: u8 = 4;
pub enum CacheRead {
L1(MemoryValue),
L2(RegionValue),
PromotedL2(MemoryValue),
}
enum PreparedGet {
Complete(Option<CacheRead>),
Pending(PendingGet),
Waiting(WaitingGet),
}
struct PendingGet {
engine: Arc<IoEngine>,
read: PendingRead,
read_token: MemoryReadToken,
hash: u64,
}
struct WaitingGet {
engine: Arc<IoEngine>,
slot_waiter: ReadSlotWaiter,
desc: ReadDesc,
read_token: MemoryReadToken,
hash: u64,
deadline: Instant,
waiter_permit: OwnedSemaphorePermit,
}
struct ReservedGet {
engine: Arc<IoEngine>,
slot: ReadSlot,
desc: ReadDesc,
read_token: MemoryReadToken,
hash: u64,
}
struct CompletedGet {
read: ReadCompletion,
read_token: MemoryReadToken,
hash: u64,
}
impl PendingGet {
#[cfg(test)]
fn wait(self) -> CompletedGet {
let Self {
engine,
read,
read_token,
hash,
} = self;
CompletedGet {
read: read.wait(engine.as_ref()),
read_token,
hash,
}
}
async fn wait_async(self, tokio_handle: &Handle) -> CompletedGet {
let Self {
engine,
read,
read_token,
hash,
} = self;
CompletedGet {
read: read.wait_async(engine, tokio_handle).await,
read_token,
hash,
}
}
}
impl WaitingGet {
async fn reserve_async(self, tokio_handle: &Handle) -> io::Result<ReservedGet> {
let Self {
engine,
slot_waiter,
desc,
read_token,
hash,
deadline,
waiter_permit,
} = self;
let slot = slot_waiter.reserve_until(deadline, tokio_handle).await?;
drop(waiter_permit);
Ok(ReservedGet {
engine,
slot,
desc,
read_token,
hash,
})
}
}
impl CacheRead {
pub fn value(&self) -> &[u8] {
match self {
Self::L1(value) | Self::PromotedL2(value) => value.as_ref(),
Self::L2(value) => value.value(),
}
}
pub const fn is_l1(&self) -> bool {
matches!(self, Self::L1(_))
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ShutdownDisposition {
Quiescent,
UnfencedWrite,
}
#[derive(Clone)]
pub struct CacheRuntime {
data: DataSuperblock,
state: Arc<RuntimeState>,
workers: Arc<Mutex<Option<RuntimeWorkers>>>,
}
struct RuntimeWorkers {
state: Arc<RuntimeState>,
append_workers: Vec<JoinHandle<io::Result<()>>>,
reclaim_workers: Vec<JoinHandle<io::Result<()>>>,
}
struct RuntimeState {
regions: Arc<RegionStore>,
read_engines: Box<[Arc<IoEngine>]>,
read_lane_cursor: AtomicUsize,
read_waiters: Option<Arc<Semaphore>>,
read_wait_timeout: Duration,
write_engines: Box<[Arc<IoEngine>]>,
reclaim_engines: Box<[Arc<IoEngine>]>,
reclaim_control: ReclaimControl,
reclaim_io_timeout: Duration,
io_recovery: IoRecovery,
fill_control: Option<FillController>,
managed_memory: Arc<ManagedMemory>,
metrics: Arc<RuntimeMetrics>,
memory: Arc<MemoryStore>,
staging: Arc<AppendStaging>,
operations: Arc<MutationGate>,
append_controls: Box<[Arc<AppendWorkerControl>]>,
write_flush_threshold_bytes: usize,
align_reads_for_direct_io: bool,
activity_counters: bool,
#[cfg(test)]
after_io_snapshot: Mutex<Option<Box<dyn FnOnce() + Send>>>,
}
#[derive(Default)]
struct ReclaimControlState {
generation: u64,
stop: bool,
}
struct ReclaimControl {
state: Mutex<ReclaimControlState>,
changed: Condvar,
}
impl ReclaimControl {
fn new() -> Self {
Self {
state: Mutex::new(ReclaimControlState::default()),
changed: Condvar::new(),
}
}
fn notify(&self) -> io::Result<()> {
let mut state = self.state.lock().map_err(|_| poisoned_runtime_error())?;
state.generation = state.generation.wrapping_add(1);
self.changed.notify_all();
Ok(())
}
fn stop(&self) -> io::Result<()> {
let mut state = self.state.lock().map_err(|_| poisoned_runtime_error())?;
state.stop = true;
self.changed.notify_all();
Ok(())
}
fn is_stopped(&self) -> io::Result<bool> {
Ok(self
.state
.lock()
.map_err(|_| poisoned_runtime_error())?
.stop)
}
fn wait(&self, observed_generation: &mut u64) -> io::Result<bool> {
let mut state = self.state.lock().map_err(|_| poisoned_runtime_error())?;
while state.generation == *observed_generation && !state.stop {
state = self
.changed
.wait(state)
.map_err(|_| poisoned_runtime_error())?;
}
if state.stop {
return Ok(false);
}
*observed_generation = state.generation;
Ok(true)
}
}
impl RuntimeState {
fn background_io_attempt(&self) -> BackgroundIoAttempt<'_> {
BackgroundIoAttempt::new(&self.io_recovery, self.fill_control.as_ref())
}
fn stop_fill_and_recovery(&self) {
self.io_recovery.stop();
if let Some(fill) = &self.fill_control {
fill.stop();
}
}
fn write_engine_for(&self, route: u64) -> &Arc<IoEngine> {
&self.write_engines[route_hash(route, self.write_engines.len())]
}
fn try_reserve_read(&self, route: u64) -> io::Result<(Arc<IoEngine>, ReadSlot)> {
try_reserve_read_lane(&self.read_engines, route, &self.read_lane_cursor)
}
fn reject_write(&self) -> io::Error {
if self.activity_counters {
self.metrics.record_write_rejection();
}
io::Error::new(io::ErrorKind::WouldBlock, "write path is busy")
}
fn try_queue_read(
&self,
route: u64,
desc: ReadDesc,
read_token: MemoryReadToken,
) -> io::Result<WaitingGet> {
let waiters = self
.read_waiters
.as_ref()
.expect("non-zero read wait timeout creates a waiter bound");
let waiter_permit = Arc::clone(waiters).try_acquire_owned(1).ok_or_else(|| {
io::Error::new(io::ErrorKind::WouldBlock, "L2 read wait queue is full")
})?;
let engine = Arc::clone(&self.read_engines[route_hash(route, self.read_engines.len())]);
let slot_waiter = engine.read_slot_waiter();
let deadline = Instant::now()
.checked_add(self.read_wait_timeout)
.unwrap_or_else(Instant::now);
Ok(WaitingGet {
engine,
slot_waiter,
desc,
read_token,
hash: route,
deadline,
waiter_permit,
})
}
fn engines(&self) -> impl Iterator<Item = &Arc<IoEngine>> {
self.read_engines
.iter()
.chain(self.write_engines.iter())
.chain(self.reclaim_engines.iter())
}
}
fn try_reserve_read_lane(
engines: &[Arc<IoEngine>],
route: u64,
pressure_cursor: &AtomicUsize,
) -> io::Result<(Arc<IoEngine>, ReadSlot)> {
let reserve = |lane: usize| -> io::Result<(Arc<IoEngine>, ReadSlot)> {
let slot = engines[lane].try_reserve_read()?;
Ok((Arc::clone(&engines[lane]), slot))
};
let lane_count = engines.len();
debug_assert_ne!(lane_count, 0);
let primary = route_hash(route, lane_count);
match reserve(primary) {
Ok(reservation) => Ok(reservation),
Err(error) if lane_count == 1 || !is_read_pressure(error.kind()) => Err(error),
Err(primary_error) => {
let offset = 1 + pressure_cursor.fetch_add(1, Ordering::Relaxed) % (lane_count - 1);
let alternate = (primary + offset) % lane_count;
match reserve(alternate) {
Ok(reservation) => Ok(reservation),
Err(error) if !is_read_pressure(error.kind()) => Err(error),
Err(_) => Err(primary_error),
}
}
}
}
fn should_wake_write(previous_bytes: usize, current_bytes: usize, threshold: usize) -> bool {
previous_bytes == 0 || (previous_bytes < threshold && current_bytes >= threshold)
}
#[derive(Default)]
struct AppendWorkerState {
wake_flags: u8,
drain_requested: u64,
drain_completed: u64,
stop: bool,
failure: Option<StoredIoError>,
}
struct AppendWorkerControl {
state: Mutex<AppendWorkerState>,
changed: Condvar,
async_changed: AutoResetEvent,
}
impl AppendWorkerControl {
fn new() -> Self {
Self {
state: Mutex::new(AppendWorkerState::default()),
changed: Condvar::new(),
async_changed: AutoResetEvent::new(),
}
}
fn notify(&self, flags: u8) -> io::Result<()> {
self.notify_inner(flags, false)
}
fn notify_if_running(&self, flags: u8) -> io::Result<()> {
self.notify_inner(flags, true)
}
fn notify_inner(&self, flags: u8, ignore_stopped: bool) -> io::Result<()> {
let mut state = self.lock()?;
if let Some(failure) = &state.failure {
return Err(failure.restore());
}
if state.stop {
return if ignore_stopped {
Ok(())
} else {
Err(closed_runtime_error())
};
}
let was_idle = state.wake_flags == 0;
state.wake_flags |= flags;
if was_idle {
self.changed.notify_one();
}
Ok(())
}
fn request_drain(&self) -> io::Result<u64> {
let mut state = self.lock()?;
if state.stop {
return Err(closed_runtime_error());
}
state.drain_requested = state
.drain_requested
.checked_add(1)
.ok_or_else(|| io::Error::other("shard drain generation exhausted"))?;
let generation = state.drain_requested;
self.changed.notify_one();
Ok(generation)
}
fn stop(&self) -> io::Result<()> {
let (mut state, poisoned) = match self.state.lock() {
Ok(state) => (state, false),
Err(error) => (error.into_inner(), true),
};
state.stop = true;
self.changed.notify_all();
if poisoned {
Err(poisoned_runtime_error())
} else if let Some(failure) = &state.failure {
Err(failure.restore())
} else {
Ok(())
}
}
fn wait_for_drain(&self, generation: u64) -> io::Result<()> {
let mut state = self.lock()?;
while state.drain_completed < generation && state.failure.is_none() {
state = self
.changed
.wait(state)
.map_err(|_| poisoned_runtime_error())?;
}
if let Some(failure) = &state.failure {
return Err(failure.restore());
}
Ok(())
}
async fn wait_for_drain_async(&self, generation: u64) -> io::Result<()> {
loop {
{
let state = self.lock()?;
if let Some(failure) = &state.failure {
return Err(failure.restore());
}
if state.drain_completed >= generation {
return Ok(());
}
}
self.async_changed.wait().await;
}
}
fn fail(&self, error: &io::Error) {
let mut state = self
.state
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
state
.failure
.get_or_insert_with(|| StoredIoError::capture(error));
drop(state);
self.changed.notify_all();
self.async_changed.set();
}
fn lock(&self) -> io::Result<MutexGuard<'_, AppendWorkerState>> {
self.state.lock().map_err(|_| poisoned_runtime_error())
}
}
impl CacheRuntime {
pub fn regions(&self) -> &Arc<RegionStore> {
&self.state.regions
}
pub fn stats_recorder(&self) -> &Recorder {
&self.state.metrics.stats
}
pub fn start(
regions: Arc<RegionStore>,
data: DataSuperblock,
handles: DataFileHandles,
config: CacheConfig,
) -> io::Result<Self> {
let storage = config.storage();
if data.geometry != storage.geometry()
|| regions.region_count() != storage.region_count() as usize
|| regions.index_slot_count() != storage.index_slots()
|| regions.shard_count() != config.runtime().append_shards as usize
{
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"recovered layout does not match the cache configuration",
));
}
let options = config.runtime();
let reclaim_worker_count = IoPoolTopology::reclaim(options.io_engine).max_in_flight();
regions.configure_reclaim_workers(reclaim_worker_count)?;
regions.set_index_activity_counters_enabled(options.stats.activity_counters);
let metrics = Arc::new(RuntimeMetrics::new(
regions.shard_count(),
options.stats,
config.storage().peak_disk_bytes(),
)?);
let operations = Arc::new(MutationGate::new());
let workers = start_workers(
regions,
data,
handles,
config,
metrics,
operations,
reclaim_worker_count,
)?;
let state = Arc::clone(&workers.state);
Ok(Self {
data,
state,
workers: Arc::new(Mutex::new(Some(workers))),
})
}
pub fn start_close(&self) {
self.state.stop_fill_and_recovery();
self.state.operations.start_close();
}
pub fn put(&self, key: &[u8], value: &[u8]) -> io::Result<u64> {
self.put_with_l1::<true>(key, value)
}
pub fn put_l2(&self, key: &[u8], value: &[u8]) -> io::Result<u64> {
self.put_with_l1::<false>(key, value)
}
fn put_with_l1<const ADMIT_L1: bool>(&self, key: &[u8], value: &[u8]) -> io::Result<u64> {
validate_key(key)?;
let record_bytes = required_record_bytes(key.len(), value.len())
.map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error.to_string()))?;
if u64::from(record_bytes) > self.data.geometry.region_size {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"encoded cache record exceeds one Region",
));
}
let state = &self.state;
if state.io_recovery.is_recovering() {
return Err(state.reject_write());
}
let hash = hash_key(self.data.hash_seed, key);
let shard_id = state.regions.append_shard(hash);
let control = &state.append_controls[shard_id];
let activity = state
.activity_counters
.then(|| state.metrics.activity_for_hash(hash));
let Some(operation) = state.operations.try_enter() else {
return Err(state.reject_write());
};
if let Some(fill) = &state.fill_control
&& !fill.try_admit_fill()
{
return Err(state.reject_write());
}
let staged = state.regions.try_stage_value(
&state.staging,
shard_id,
hash,
record_bytes,
key,
value,
)?;
match staged {
RegionStageValue::Staged {
seqno,
previous_bytes,
current_bytes,
} => {
if ADMIT_L1 {
state.memory.publish(hash, key, value, seqno);
} else {
state.memory.delete(hash, key, seqno);
}
if should_wake_write(
previous_bytes,
current_bytes,
state.write_flush_threshold_bytes,
) {
control.notify(WAKE_DATA)?;
}
if let Some(activity) = activity {
RuntimeMetrics::increment(&activity.puts);
RuntimeMetrics::add(&activity.written_bytes, value.len());
}
Ok(seqno)
}
RegionStageValue::Contended => {
drop(operation);
Err(state.reject_write())
}
RegionStageValue::NeedsFlush => {
reject_staged_write(state, control, WAKE_URGENT, operation)
}
RegionStageValue::NeedsRotation => {
reject_staged_write(state, control, WAKE_ROTATE | WAKE_URGENT, operation)
}
}
}
pub fn delete(&self, key: &[u8]) -> io::Result<u64> {
validate_key(key)?;
let state = &self.state;
let hash = hash_key(self.data.hash_seed, key);
let activity = state
.activity_counters
.then(|| state.metrics.activity_for_hash(hash));
let Some(operation) = state.operations.try_enter() else {
return Err(state.reject_write());
};
let Some(seqno) = state.regions.try_delete_value(hash)? else {
drop(operation);
return Err(state.reject_write());
};
state.memory.delete(hash, key, seqno);
if let Some(activity) = activity {
RuntimeMetrics::increment(&activity.deletes);
}
Ok(seqno)
}
#[cfg(test)]
pub fn get(&self, key: &[u8]) -> io::Result<Option<CacheRead>> {
match self.prepare_get(key, None)? {
PreparedGet::Complete(value) => Ok(value),
PreparedGet::Pending(pending) => self.finish_get(pending.wait(), key),
PreparedGet::Waiting(_) => Err(io::Error::other(
"bounded read waiting requires the async get path",
)),
}
}
pub async fn get_async(
&self,
key: &[u8],
tokio_handle: &Handle,
guard: Option<&mut RequestGuard<'_>>,
) -> io::Result<Option<CacheRead>> {
match self.prepare_get(key, guard)? {
PreparedGet::Complete(value) => Ok(value),
PreparedGet::Pending(pending) => {
self.finish_get(pending.wait_async(tokio_handle).await, key)
}
PreparedGet::Waiting(waiting) => {
let wait_started = self.state.activity_counters.then(Instant::now);
let reserved = waiting.reserve_async(tokio_handle).await;
if let Some(wait_started) = wait_started {
self.state.metrics.record_read_wait(wait_started.elapsed());
}
let reserved = reserved.inspect_err(|error| self.record_read_wait_error(error))?;
let Some(pending) = self.submit_reserved_get(reserved)? else {
return Ok(None);
};
self.finish_get(pending.wait_async(tokio_handle).await, key)
}
}
}
fn record_read_wait_error(&self, error: &io::Error) {
if !self.state.activity_counters {
return;
}
if is_read_pressure(error.kind()) {
self.state.metrics.record_read_overload();
} else {
RuntimeMetrics::increment(&self.state.metrics.io_failures);
}
}
fn submit_reserved_get(&self, reserved: ReservedGet) -> io::Result<Option<PendingGet>> {
let ReservedGet {
engine,
slot,
desc,
read_token,
hash,
} = reserved;
let state = &self.state;
let Some(buffer) = state.managed_memory.try_read_buffer(desc.read_len) else {
if state.activity_counters {
state.metrics.record_read_overload();
}
return Err(io::Error::new(
io::ErrorKind::OutOfMemory,
"L2 read could not reserve its bounded buffer after waiting",
));
};
match state
.regions
.submit_value_read(engine.as_ref(), slot, buffer, desc)
{
Ok(read) => Ok(Some(PendingGet {
engine,
read,
read_token,
hash,
})),
Err(_) if !state.regions.is_healthy() => {
if state.activity_counters {
RuntimeMetrics::increment(&state.metrics.io_failures);
RuntimeMetrics::increment(&state.metrics.activity_for_hash(hash).l2_misses);
}
Ok(None)
}
Err(error) => {
self.record_read_wait_error(&error);
Err(error)
}
}
}
fn prepare_get(
&self,
key: &[u8],
guard: Option<&mut RequestGuard<'_>>,
) -> io::Result<PreparedGet> {
let state = &self.state;
if key.len() > MAX_KEY_SIZE {
if state.activity_counters {
let activity = state.metrics.activity(0);
RuntimeMetrics::increment(&activity.l1_misses);
RuntimeMetrics::increment(&activity.l2_misses);
}
return Ok(PreparedGet::Complete(None));
}
let hash = hash_key(self.data.hash_seed, key);
let activity = state
.activity_counters
.then(|| state.metrics.activity_for_hash(hash));
if !state.regions.is_healthy() {
if let Some(activity) = activity {
RuntimeMetrics::increment(&activity.l1_misses);
RuntimeMetrics::increment(&activity.l2_misses);
}
return Ok(PreparedGet::Complete(None));
}
let read_token = match state.memory.lookup(hash, key) {
MemoryLookup::Hit(value) => {
if let Some(activity) = activity {
RuntimeMetrics::increment(&activity.l1_hits);
RuntimeMetrics::add(&activity.served_bytes, value.len());
}
return Ok(PreparedGet::Complete(Some(CacheRead::L1(value))));
}
MemoryLookup::Miss(token) => {
if let Some(guard) = guard {
guard.enter_l2();
}
if let Some(activity) = activity {
RuntimeMetrics::increment(&activity.l1_misses);
}
token
}
};
let Some(candidate) = state.regions.begin_point_read(hash) else {
if let Some(activity) = activity {
RuntimeMetrics::increment(&activity.l2_misses);
}
return Ok(PreparedGet::Complete(None));
};
let desc = match describe_read(
self.data.geometry,
hash,
candidate,
state.align_reads_for_direct_io,
) {
Ok(desc) => desc,
Err(error) => {
state
.regions
.enter_miss_only_with_error("record_read_descriptor_invalid", &error);
if state.activity_counters {
RuntimeMetrics::increment(&state.metrics.io_failures);
}
if let Some(activity) = activity {
RuntimeMetrics::increment(&activity.l2_misses);
}
return Ok(PreparedGet::Complete(None));
}
};
let (engine, slot) = match state.try_reserve_read(hash) {
Ok(reservation) => reservation,
Err(error)
if error.kind() == io::ErrorKind::WouldBlock
&& !state.read_wait_timeout.is_zero() =>
{
let waiting = state
.try_queue_read(hash, desc, read_token)
.inspect_err(|_| {
if state.activity_counters {
state.metrics.record_read_overload();
}
})?;
return Ok(PreparedGet::Waiting(waiting));
}
Err(error) if is_read_pressure(error.kind()) => {
if let Some(activity) = activity {
RuntimeMetrics::increment(&activity.l2_misses);
RuntimeMetrics::increment(&activity.l2_read_busy_misses);
}
return Ok(PreparedGet::Complete(None));
}
Err(error) => {
state
.regions
.enter_miss_only_with_error("read_engine_reservation_failed", &error);
if state.activity_counters {
RuntimeMetrics::increment(&state.metrics.io_failures);
}
if let Some(activity) = activity {
RuntimeMetrics::increment(&activity.l2_misses);
}
return Ok(PreparedGet::Complete(None));
}
};
let Some(buffer) = state.managed_memory.try_read_buffer(desc.read_len) else {
if !state.read_wait_timeout.is_zero() {
if state.activity_counters {
state.metrics.record_read_overload();
}
return Err(io::Error::new(
io::ErrorKind::OutOfMemory,
"L2 read could not reserve its bounded buffer",
));
}
if let Some(activity) = activity {
RuntimeMetrics::increment(&activity.l2_misses);
RuntimeMetrics::increment(&activity.l2_read_memory_misses);
}
return Ok(PreparedGet::Complete(None));
};
match state
.regions
.submit_value_read(engine.as_ref(), slot, buffer, desc)
{
Ok(read) => Ok(PreparedGet::Pending(PendingGet {
engine,
read,
read_token,
hash,
})),
Err(_) if !state.regions.is_healthy() => {
if let Some(activity) = activity {
RuntimeMetrics::increment(&state.metrics.io_failures);
RuntimeMetrics::increment(&activity.l2_misses);
}
Ok(PreparedGet::Complete(None))
}
Err(error) if is_read_pressure(error.kind()) => {
if !state.read_wait_timeout.is_zero() {
if state.activity_counters {
state.metrics.record_read_overload();
}
return Err(error);
}
if let Some(activity) = activity {
RuntimeMetrics::increment(&activity.l2_misses);
RuntimeMetrics::increment(&activity.l2_read_busy_misses);
}
Ok(PreparedGet::Complete(None))
}
Err(error) => {
if state.activity_counters {
RuntimeMetrics::increment(&state.metrics.io_failures);
}
Err(error)
}
}
}
fn finish_get(&self, completed: CompletedGet, key: &[u8]) -> io::Result<Option<CacheRead>> {
let state = &self.state;
let CompletedGet {
read,
read_token,
hash,
} = completed;
let activity = state
.activity_counters
.then(|| state.metrics.activity_for_hash(hash));
let result = state.regions.finish_value_read(read, key);
match result {
Err(_) if !state.regions.is_healthy() => {
if let Some(activity) = activity {
RuntimeMetrics::increment(&state.metrics.io_failures);
RuntimeMetrics::increment(&activity.l2_misses);
}
Ok(None)
}
Ok(Some(value)) => {
let promoted =
state
.memory
.promote(read_token, hash, key, value.value(), value.seqno());
let read = if let Some(promoted) = promoted {
if let Some(activity) = activity {
RuntimeMetrics::increment(&activity.l1_promotions);
}
CacheRead::PromotedL2(promoted)
} else {
CacheRead::L2(value)
};
if let Some(activity) = activity {
RuntimeMetrics::increment(&activity.l2_hits);
RuntimeMetrics::add(&activity.served_bytes, read.value().len());
}
Ok(Some(read))
}
Ok(None) => {
if let Some(activity) = activity {
RuntimeMetrics::increment(&activity.l2_misses);
}
Ok(None)
}
Err(error) if is_read_pressure(error.kind()) => {
if !state.read_wait_timeout.is_zero() {
self.record_read_wait_error(&error);
return Err(error);
}
if let Some(activity) = activity {
RuntimeMetrics::increment(&activity.l2_misses);
RuntimeMetrics::increment(&activity.l2_read_busy_misses);
}
Ok(None)
}
Err(error) => {
if state.activity_counters {
RuntimeMetrics::increment(&state.metrics.io_failures);
}
Err(error)
}
}
}
#[cfg(test)]
pub fn drain(&self) -> io::Result<()> {
let state = &self.state;
let operations = state.operations.begin_drain()?;
operations.wait()?;
let _draining = LifecycleDrainingGuard::enter(&state.metrics.lifecycle, &state.operations);
drain_shards(state)
}
pub async fn drain_async(&self) -> io::Result<()> {
let state = &self.state;
let operations = state.operations.begin_drain()?;
operations.wait_async().await;
let _draining = LifecycleDrainingGuard::enter(&state.metrics.lifecycle, &state.operations);
drain_shards_async(state).await
}
pub fn detailed_snapshot(&self) -> io::Result<DetailedCacheSnapshot> {
let state = &self.state;
Ok(DetailedCacheSnapshot {
summary: self.snapshot(),
write_buffer_rejections: state
.metrics
.write_buffer_rejections
.load(Ordering::Relaxed),
l1: state.memory.detailed_snapshot()?,
index: state.regions.index_snapshot()?,
region: state.regions.region_snapshot()?,
})
}
pub fn snapshot(&self) -> CacheSnapshot {
let state = &self.state;
let mut snapshot = state.metrics.snapshot(
state.regions.is_healthy(),
state.activity_counters,
state.managed_memory.snapshot(),
state.memory.metrics_snapshot(),
);
if snapshot.health == CacheHealth::Running && state.io_recovery.is_recovering() {
snapshot.health = CacheHealth::Recovering;
}
if let Some(fill) = &state.fill_control {
snapshot.fill_control = fill.snapshot();
}
snapshot.io = aggregate_io_stats(
&state.read_engines,
&state.write_engines,
&state.reclaim_engines,
);
snapshot
}
pub fn shutdown(&self) -> io::Result<ShutdownDisposition> {
self.start_close();
self.state.operations.wait_quiescent()?;
let _ = self.state.metrics.lifecycle.compare_exchange(
LIFECYCLE_RUNNING,
LIFECYCLE_DRAINING,
Ordering::AcqRel,
Ordering::Acquire,
);
let workers = self
.workers
.lock()
.map_err(|_| poisoned_runtime_error())?
.take()
.ok_or_else(closed_runtime_error)?;
stop_workers(workers)
}
#[cfg(test)]
pub fn reserve_read_slot_for_test(&self) -> ReadSlot {
self.state
.try_reserve_read(0)
.map(|(_, slot)| slot)
.expect("test read slot is available")
}
#[cfg(test)]
pub fn poison_append_worker_for_test(&self, shard_id: usize) {
let shard = self
.state
.append_controls
.get(shard_id)
.expect("test shard exists");
let result = panic::catch_unwind(AssertUnwindSafe(|| {
let _state = shard.state.lock().unwrap();
panic!("poison shard gate");
}));
assert!(result.is_err());
}
}
fn aggregate_io_stats(
read_engines: &[Arc<IoEngine>],
write_engines: &[Arc<IoEngine>],
reclaim_engines: &[Arc<IoEngine>],
) -> CacheIoSnapshot {
let mut aggregate = CacheIoSnapshot::default();
for (engine_index, engine) in read_engines.iter().chain(write_engines).enumerate() {
let snapshot = engine.stats();
if engine_index < read_engines.len() {
add_io_direction(&mut aggregate.read, snapshot.requests);
} else {
add_io_direction(&mut aggregate.write, snapshot.requests);
}
if engine_index == 0 {
aggregate.read.buffered = snapshot.file_io.read.buffered;
aggregate.read.direct = snapshot.file_io.read.direct;
aggregate.write.buffered = snapshot.file_io.write.buffered;
aggregate.write.direct = snapshot.file_io.write.direct;
}
}
for engine in reclaim_engines {
add_io_direction(&mut aggregate.read, engine.stats().requests);
}
aggregate
}
fn add_io_direction(aggregate: &mut CacheIoDirectionSnapshot, snapshot: CacheIoDirectionSnapshot) {
aggregate.requests_submitted = aggregate
.requests_submitted
.saturating_add(snapshot.requests_submitted);
aggregate.requests_succeeded = aggregate
.requests_succeeded
.saturating_add(snapshot.requests_succeeded);
aggregate.requests_cancelled = aggregate
.requests_cancelled
.saturating_add(snapshot.requests_cancelled);
aggregate.requests_failed = aggregate
.requests_failed
.saturating_add(snapshot.requests_failed);
aggregate.requests_in_flight = aggregate
.requests_in_flight
.saturating_add(snapshot.requests_in_flight);
aggregate.requests_in_flight_peak = aggregate
.requests_in_flight_peak
.saturating_add(snapshot.requests_in_flight_peak);
aggregate.slot_wait_ns = aggregate.slot_wait_ns.saturating_add(snapshot.slot_wait_ns);
aggregate.request_time_ns = aggregate
.request_time_ns
.saturating_add(snapshot.request_time_ns);
}
fn abort_worker_start(
state: &RuntimeState,
append_workers: Vec<JoinHandle<io::Result<()>>>,
reclaim_workers: Vec<JoinHandle<io::Result<()>>>,
) {
let _ = state.reclaim_control.stop();
for worker in reclaim_workers {
let _ = worker.join();
}
for shard in &state.append_controls {
let _ = shard.stop();
}
for worker in append_workers {
let _ = worker.join();
}
state.staging.close();
for engine in state.engines() {
let _ = engine.shutdown();
}
}
fn start_workers(
regions: Arc<RegionStore>,
data: DataSuperblock,
handles: DataFileHandles,
config: CacheConfig,
metrics: Arc<RuntimeMetrics>,
operations: Arc<MutationGate>,
reclaim_worker_count: usize,
) -> io::Result<RuntimeWorkers> {
let shard_count = regions.shard_count();
let options = config.runtime();
let l1_entry_capacity = config.l1_entry_capacity();
let memory_limit = options.managed_memory_limit_bytes;
let managed_memory = Arc::new(ManagedMemory::try_new(ManagedMemoryLimits {
memory_limit_bytes: memory_limit,
reserved_memory_bytes: config.reserved_memory_bytes(),
})?);
let region_bytes = usize::try_from(data.geometry.region_size)
.map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "Region size is too large"))?;
let staging = Arc::new(AppendStaging::try_new(
shard_count,
region_bytes,
data.geometry.region_size,
&managed_memory,
)?);
let memory = Arc::new(MemoryStore::new(
options.l1_capacity_bytes,
l1_entry_capacity,
options.l1_shards,
options.l1_eviction_policy,
options.stats.activity_counters,
)?);
let mut reclaim_buffers = Vec::new();
reclaim_buffers
.try_reserve_exact(reclaim_worker_count)
.map_err(|_| {
io::Error::new(
io::ErrorKind::OutOfMemory,
"cannot allocate Region reclaim buffer owners",
)
})?;
for _ in 0..reclaim_worker_count {
reclaim_buffers.push(
managed_memory
.try_read_buffer(region_bytes)
.ok_or_else(|| {
io::Error::new(
io::ErrorKind::OutOfMemory,
"cannot allocate a fixed Region reclaim buffer",
)
})?,
);
}
let reclaim_handles = handles.try_clone()?;
let write_handles = handles.try_clone()?;
let read_wait_timeout = read_io_wait_timeout(options);
let read_wait_enabled = !read_wait_timeout.is_zero();
let read_engines = build_engine_pool(
handles,
options,
IoPoolTopology::read(options.io_engine),
read_wait_enabled,
)?;
let read_waiters =
read_wait_enabled.then(|| Arc::new(Semaphore::new(read_io_wait_capacity(options))));
let write_engines = build_engine_pool(
write_handles,
options,
IoPoolTopology::write(options.io_engine),
false,
)?;
let reclaim_engines = build_engine_pool(
reclaim_handles,
options,
IoPoolTopology::reclaim(options.io_engine),
false,
)?;
for (engines, role) in [
(&read_engines, IoRole::Read),
(&write_engines, IoRole::Write),
(&reclaim_engines, IoRole::Reclaim),
] {
for engine in engines.iter() {
if let Some(timing) = metrics.stats.io_timing(role) {
engine.set_latency_recorder(timing);
}
}
}
let mut append_controls = Vec::new();
append_controls
.try_reserve_exact(shard_count)
.map_err(|_| {
io::Error::new(io::ErrorKind::OutOfMemory, "cannot allocate shard controls")
})?;
append_controls.resize_with(shard_count, || Arc::new(AppendWorkerControl::new()));
let fill_control = FillController::new(
options.fill_control,
shard_count + reclaim_worker_count,
data.geometry.region_size,
)?;
let state = Arc::new(RuntimeState {
regions,
read_engines,
read_lane_cursor: AtomicUsize::new(0),
read_waiters,
read_wait_timeout,
write_engines,
reclaim_engines,
reclaim_control: ReclaimControl::new(),
reclaim_io_timeout: options.reclaim_io_timeout,
io_recovery: IoRecovery::new(options.io_recovery_timeout),
fill_control,
managed_memory,
metrics,
memory,
staging,
operations,
append_controls: append_controls.into_boxed_slice(),
write_flush_threshold_bytes: options.write_flush_threshold_bytes,
align_reads_for_direct_io: options.io_mode == IoMode::Direct,
activity_counters: options.stats.activity_counters,
#[cfg(test)]
after_io_snapshot: Mutex::new(None),
});
let reclaim_on_start = state.regions.reclaim_needed()?;
let mut reclaim_workers = Vec::new();
reclaim_workers
.try_reserve_exact(reclaim_worker_count)
.map_err(|_| {
io::Error::new(io::ErrorKind::OutOfMemory, "cannot allocate worker handles")
})?;
let mut append_workers = Vec::new();
append_workers.try_reserve_exact(shard_count).map_err(|_| {
io::Error::new(io::ErrorKind::OutOfMemory, "cannot allocate worker handles")
})?;
for shard_id in 0..shard_count {
let worker_state = Arc::clone(&state);
match thread::Builder::new()
.name(format!("cache2-shard-{shard_id}"))
.stack_size(CACHE_THREAD_STACK_BYTES)
.spawn(move || append_worker(worker_state, shard_id))
{
Ok(worker) => append_workers.push(worker),
Err(error) => {
abort_worker_start(&state, append_workers, reclaim_workers);
return Err(error);
}
}
}
for (worker_id, buffer) in reclaim_buffers.into_iter().enumerate() {
let reclaim_shared = Arc::clone(&state);
match thread::Builder::new()
.name(format!("cache2-reclaim-{worker_id}"))
.stack_size(CACHE_THREAD_STACK_BYTES)
.spawn(move || reclaim_worker(reclaim_shared, buffer, worker_id, reclaim_worker_count))
{
Ok(worker) => reclaim_workers.push(worker),
Err(error) => {
abort_worker_start(&state, append_workers, reclaim_workers);
return Err(error);
}
}
}
if reclaim_on_start {
state.reclaim_control.notify()?;
}
Ok(RuntimeWorkers {
state,
append_workers,
reclaim_workers,
})
}
fn build_engine_pool(
handles: DataFileHandles,
options: &RuntimeOptions,
topology: IoPoolTopology,
read_wait_enabled: bool,
) -> io::Result<Box<[Arc<IoEngine>]>> {
let mut source = Some(handles);
let engine_count = topology.engine_count();
let mut engines = Vec::new();
engines
.try_reserve_exact(engine_count)
.map_err(|_| io::Error::new(io::ErrorKind::OutOfMemory, "cannot allocate I/O workers"))?;
for engine in 0..engine_count {
let worker_handles = if engine + 1 == engine_count {
source.take().expect("last I/O worker owns file set")
} else {
source.as_ref().expect("I/O file set exists").try_clone()?
};
engines.push(build_file_engine(
worker_handles,
topology.engine_config(engine),
options.stats.activity_counters,
read_wait_enabled,
)?);
}
Ok(engines.into_boxed_slice())
}
fn append_worker(state: Arc<RuntimeState>, shard_id: usize) -> io::Result<()> {
let control = Arc::clone(&state.append_controls[shard_id]);
let result = panic::catch_unwind(AssertUnwindSafe(|| {
append_worker_result(&state, shard_id, &control)
}));
let error = match result {
Ok(Ok(())) => return Ok(()),
Ok(Err(error)) => error,
Err(_) => io::Error::other("shard worker panicked"),
};
if state.activity_counters {
RuntimeMetrics::increment(&state.metrics.io_failures);
}
let first_failure = state
.metrics
.lifecycle
.swap(LIFECYCLE_FAILED, Ordering::AcqRel)
!= LIFECYCLE_FAILED;
if first_failure {
log::error!(
target: "cache2::health",
event = "cache_append_worker_failed",
shard_id,
error:% = error;
"cache shard worker failed"
);
}
state.regions.enter_miss_only();
state.stop_fill_and_recovery();
control.fail(&error);
for shard in &state.append_controls {
if !Arc::ptr_eq(shard, &control) {
shard.fail(&error);
}
}
Err(error)
}
struct ReinsertShardCursor {
first: usize,
stride: usize,
shard_count: usize,
next: usize,
}
impl ReinsertShardCursor {
fn new(worker_id: usize, worker_count: usize, shard_count: usize) -> Self {
debug_assert!(worker_count != 0);
debug_assert!(worker_id < worker_count);
debug_assert!(worker_count <= shard_count);
Self {
first: worker_id,
stride: worker_count,
shard_count,
next: worker_id,
}
}
fn take(&mut self) -> usize {
let shard = self.next;
self.next = shard
.checked_add(self.stride)
.filter(|next| *next < self.shard_count)
.unwrap_or(self.first);
shard
}
}
fn reclaim_worker(
state: Arc<RuntimeState>,
buffer: BufferLease,
worker_id: usize,
worker_count: usize,
) -> io::Result<()> {
let result = panic::catch_unwind(AssertUnwindSafe(|| {
reclaim_worker_result(&state, buffer, worker_id, worker_count)
}));
let error = match result {
Ok(Ok(())) => return Ok(()),
Ok(Err(error)) => error,
Err(_) => io::Error::other("Region reclaim worker panicked"),
};
if state.activity_counters {
RuntimeMetrics::increment(&state.metrics.io_failures);
}
state
.metrics
.lifecycle
.store(LIFECYCLE_FAILED, Ordering::Release);
state.regions.enter_miss_only();
state.stop_fill_and_recovery();
log::error!(
target: "cache2::health",
event = "cache_reclaim_worker_failed",
worker_id,
error:% = error;
"cache Region reclaim worker failed"
);
for shard in &state.append_controls {
shard.fail(&error);
}
Err(error)
}
fn reclaim_worker_result(
state: &RuntimeState,
mut buffer: BufferLease,
worker_id: usize,
worker_count: usize,
) -> io::Result<()> {
let engine = &state.reclaim_engines[route_hash(worker_id as u64, state.reclaim_engines.len())];
let mut observed_generation = 0_u64;
let mut reinsert_shards =
ReinsertShardCursor::new(worker_id, worker_count, state.append_controls.len());
while state.reclaim_control.wait(&mut observed_generation)? {
loop {
if state.reclaim_control.is_stopped()? {
return Ok(());
}
let mut attempt = state.background_io_attempt();
let Some(receipt) = state.regions.begin_reclaim()? else {
break;
};
let used = usize::try_from(receipt.used_offset).map_err(|_| {
io::Error::new(
io::ErrorKind::InvalidData,
"reclaim length does not fit usize",
)
})?;
if used != 0 {
let io_buffer = match IoBuffer::for_read(buffer, used) {
Ok(buffer) => buffer,
Err(error) => return Err(error.error),
};
let absolute = state.regions.reclaim_absolute(receipt)?;
let request = submit_background_io(
engine.as_ref(),
IoOperation::read(io_buffer, absolute),
state.reclaim_io_timeout,
&mut attempt,
)
.map_err(|error| error.into_lease().0)?;
let completion = request
.wait_background(engine.as_ref(), &mut attempt)
.map_err(|error| error.into_lease().0)?;
let (result, returned) = completion.into_lease();
let transferred = result?;
let returned = returned.ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidData,
"reclaim completion lost its buffer",
)
})?;
if transferred != used {
return Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"reclaim Region read was short",
));
}
buffer = returned;
}
let bytes = buffer.prepared(used).ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidData,
"reclaim buffer is not initialized",
)
})?;
let reinsert_shard = reinsert_shards.take();
let preserve_hot = state.regions.reclaim_can_reinsert()?
&& !state
.fill_control
.as_ref()
.is_some_and(|fill| fill.suppress_reinsertion());
let reinsert_operation = if preserve_hot {
state.operations.try_enter()
} else {
None
};
let stats = state.regions.scan_reclaim(
receipt,
bytes,
reinsert_operation.is_some(),
|record| {
Ok(matches!(
state
.regions
.try_stage_reinsert(&state.staging, reinsert_shard, record,)?,
RegionStageValue::Staged { .. }
))
},
)?;
if stats.reinsert_records != 0 {
let generation = state.append_controls[reinsert_shard].request_drain()?;
state.append_controls[reinsert_shard].wait_for_drain(generation)?;
}
state.regions.complete_reclaim(receipt)?;
attempt.finish();
drop(reinsert_operation);
if state.activity_counters {
state.metrics.record_reclaim(stats);
}
log::debug!(
target: "cache2::reclaim",
event = "cache_region_reclaimed",
worker_id,
region_id = receipt.region_id,
reinsert_shard,
preserve_hot,
bytes = stats.bytes_read,
records_scanned = stats.records_scanned,
records_removed = stats.records_removed,
reinsert_records = stats.reinsert_records,
reinsert_bytes = stats.reinsert_bytes,
reinsert_skipped = stats.reinsert_skipped,
reinsert_budget_skipped = stats.reinsert_budget_skipped;
"cache Region reclaimed"
);
for shard in &state.append_controls {
shard.notify_if_running(WAKE_ROTATE)?;
}
}
}
Ok(())
}
fn append_worker_result(
state: &RuntimeState,
shard_id: usize,
control: &AppendWorkerControl,
) -> io::Result<()> {
let mut deadline = None;
let mut rotate_pending = false;
loop {
let (flags, drain_generation, stop, timed_out) = wait_for_shard_work(control, deadline)?;
let draining = drain_generation != 0 || stop;
let force_flush = flags & WAKE_URGENT != 0 || timed_out || draining || rotate_pending;
let rotate = flags & WAKE_ROTATE != 0;
match state.staging.shard_fill_snapshot(shard_id) {
Ok(Some(fill)) => {
if deadline.is_none() {
deadline = Some(Instant::now() + WRITE_FLUSH_DELAY);
}
if force_flush || fill.bytes >= state.write_flush_threshold_bytes {
let essential =
flags & (WAKE_URGENT | WAKE_ROTATE) != 0 || draining || rotate_pending;
let bytes = fill.bytes as u64;
let records = u32::try_from(fill.records).unwrap_or(u32::MAX);
let charge = match &state.fill_control {
None => Some(FlushCharge {
records: 0,
byte_units: 0,
}),
Some(control) => {
control.try_acquire_flush_budget(bytes, records, essential)
}
};
if let Some(charge) = charge {
let engine = state.write_engine_for(shard_id as u64);
match state.regions.flush_staging_shard(
&state.staging,
engine.as_ref(),
shard_id,
state.background_io_attempt(),
)? {
Some(_) => deadline = None,
None => {
if let Some(control) = &state.fill_control {
control.refund_flush_budget(charge);
}
deadline = Some(Instant::now() + STAGING_RETRY_DELAY);
}
}
} else {
deadline = Some(Instant::now() + FLUSH_RETRY);
}
}
}
Ok(None) => {
deadline = None;
}
Err(StagingError::WouldBlock) => {
deadline = Some(Instant::now() + STAGING_RETRY_DELAY);
}
Err(error) => return Err(staging_runtime_error(error)),
}
if rotate || rotate_pending {
match state.regions.rotate_shard(shard_id)? {
RegionRotation::Idle => rotate_pending = false,
RegionRotation::Pending => rotate_pending = true,
RegionRotation::Rotated => {
rotate_pending = false;
if state.activity_counters {
RuntimeMetrics::increment(&state.metrics.region_rotations);
}
state.reclaim_control.notify()?;
}
}
}
if draining {
match state.staging.shard_fill_snapshot(shard_id) {
Ok(Some(_)) => {
let engine = state.write_engine_for(shard_id as u64);
state.regions.flush_staging_shard(
&state.staging,
engine.as_ref(),
shard_id,
state.background_io_attempt(),
)?;
}
Ok(None) => {}
Err(StagingError::WouldBlock) => {
deadline = Some(Instant::now() + STAGING_RETRY_DELAY);
continue;
}
Err(error) => return Err(staging_runtime_error(error)),
}
complete_shard_drain(control, drain_generation)?;
if stop {
return Ok(());
}
}
}
}
fn wait_for_shard_work(
control: &AppendWorkerControl,
deadline: Option<Instant>,
) -> io::Result<(u8, u64, bool, bool)> {
let mut state = control.lock()?;
let mut timed_out = false;
while state.wake_flags == 0
&& state.drain_requested == state.drain_completed
&& !state.stop
&& state.failure.is_none()
{
if let Some(deadline) = deadline {
let Some(remaining) = deadline.checked_duration_since(Instant::now()) else {
timed_out = true;
break;
};
let (next, timeout) = control
.changed
.wait_timeout(state, remaining)
.map_err(|_| poisoned_runtime_error())?;
state = next;
if timeout.timed_out()
&& state.wake_flags == 0
&& state.drain_requested == state.drain_completed
{
timed_out = true;
break;
}
} else {
state = control
.changed
.wait(state)
.map_err(|_| poisoned_runtime_error())?;
}
}
if let Some(failure) = &state.failure {
return Err(failure.restore());
}
let flags = mem::take(&mut state.wake_flags);
let drain_generation = if state.drain_requested > state.drain_completed {
state.drain_requested
} else {
0
};
Ok((flags, drain_generation, state.stop, timed_out))
}
fn reject_staged_write(
state: &RuntimeState,
control: &AppendWorkerControl,
flags: u8,
operation: MutationGuard<'_>,
) -> io::Result<u64> {
control.notify(flags)?;
drop(operation);
if state.activity_counters {
RuntimeMetrics::increment(&state.metrics.write_buffer_rejections);
}
Err(state.reject_write())
}
fn complete_shard_drain(control: &AppendWorkerControl, generation: u64) -> io::Result<()> {
let mut state = control.lock()?;
state.drain_completed = state.drain_completed.max(generation);
control.changed.notify_all();
drop(state);
control.async_changed.set();
Ok(())
}
fn request_shard_drains(state: &RuntimeState) -> io::Result<(Vec<Option<u64>>, Option<io::Error>)> {
let mut generations = Vec::new();
generations
.try_reserve_exact(state.append_controls.len())
.map_err(|_| io::Error::new(io::ErrorKind::OutOfMemory, "cannot allocate drain fence"))?;
let mut first_error = None;
for shard in &state.append_controls {
match shard.request_drain() {
Ok(generation) => generations.push(Some(generation)),
Err(error) => {
first_error.get_or_insert(error);
generations.push(None);
}
}
}
Ok((generations, first_error))
}
#[cfg(test)]
fn drain_shards(state: &RuntimeState) -> io::Result<()> {
let (generations, mut first_error) = request_shard_drains(state)?;
for (shard, generation) in state.append_controls.iter().zip(generations) {
if let Some(generation) = generation
&& let Err(error) = shard.wait_for_drain(generation)
{
first_error.get_or_insert(error);
}
}
first_error.map_or(Ok(()), Err)
}
async fn drain_shards_async(state: &RuntimeState) -> io::Result<()> {
let (generations, mut first_error) = request_shard_drains(state)?;
for (shard, generation) in state.append_controls.iter().zip(generations) {
if let Some(generation) = generation
&& let Err(error) = shard.wait_for_drain_async(generation).await
{
first_error.get_or_insert(error);
}
}
first_error.map_or(Ok(()), Err)
}
fn stop_workers(mut workers: RuntimeWorkers) -> io::Result<ShutdownDisposition> {
workers.state.stop_fill_and_recovery();
let mut worker_error = None;
for shard in &workers.state.append_controls {
if let Err(error) = shard.stop() {
worker_error.get_or_insert(error);
}
}
for worker in workers.append_workers.drain(..) {
let result = worker
.join()
.unwrap_or_else(|_| Err(io::Error::other("shard worker panicked")));
if let Err(error) = result {
worker_error.get_or_insert(error);
}
}
if let Err(error) = workers.state.reclaim_control.stop() {
worker_error.get_or_insert(error);
}
for worker in workers.reclaim_workers.drain(..) {
let result = worker
.join()
.unwrap_or_else(|_| Err(io::Error::other("Region reclaim worker panicked")));
if let Err(error) = result {
worker_error.get_or_insert(error);
}
}
workers.state.staging.close();
for engine in workers.state.engines() {
engine.stop_accepting_requests();
}
let in_flight = workers
.state
.engines()
.map(|engine| engine.in_flight())
.sum::<usize>();
#[cfg(test)]
{
let after_snapshot = workers.state.after_io_snapshot.lock().unwrap().take();
if let Some(after_snapshot) = after_snapshot {
after_snapshot();
}
}
let writes_in_flight = workers
.state
.engines()
.map(|engine| engine.writes_in_flight())
.sum::<usize>();
let unfenced_before = workers
.state
.engines()
.any(|engine| engine.has_unfenced_writes());
let skip_shutdown = in_flight != 0 || unfenced_before;
let shutdown = if skip_shutdown {
Ok(())
} else {
let mut result = Ok(());
for engine in workers.state.engines() {
if let Err(error) = engine.shutdown()
&& result.is_ok()
{
result = Err(error);
}
}
result
};
let unfenced = unfenced_before
|| workers
.state
.engines()
.any(|engine| engine.has_unfenced_writes());
let result = worker_error.map_or(Ok(()), Err).and(shutdown);
if skip_shutdown || unfenced {
if unfenced {
for engine in workers.state.engines() {
mem::forget(Arc::clone(engine));
}
} else {
for engine in workers.state.engines() {
if engine.in_flight() != 0 {
reap_engine_after_target_fence(engine);
} else {
let _ = engine.shutdown();
}
}
}
if writes_in_flight != 0 || unfenced {
return Ok(ShutdownDisposition::UnfencedWrite);
}
}
result.map(|()| ShutdownDisposition::Quiescent)
}
fn reap_engine_after_target_fence(engine: &Arc<IoEngine>) {
let reaper_engine = Arc::clone(engine);
let spawn = thread::Builder::new()
.name("cache2-io-reaper".to_owned())
.stack_size(CACHE_THREAD_STACK_BYTES)
.spawn(move || {
let _ = reaper_engine.shutdown();
});
if spawn.is_err() {
mem::forget(Arc::clone(engine));
}
}
fn validate_key(key: &[u8]) -> io::Result<()> {
if key.len() > MAX_KEY_SIZE {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"cache key exceeds the 4 KiB limit",
));
}
Ok(())
}
fn staging_runtime_error(error: StagingError) -> io::Error {
io::Error::new(io::ErrorKind::InvalidData, error)
}
fn poisoned_runtime_error() -> io::Error {
io::Error::new(
io::ErrorKind::InvalidData,
"runtime synchronization is poisoned",
)
}
fn closed_runtime_error() -> io::Error {
io::Error::new(io::ErrorKind::NotConnected, "cache runtime is closed")
}
#[cfg(test)]
mod tests {
use std::pin;
use std::sync::Barrier;
use std::sync::atomic::AtomicBool;
use std::sync::mpsc;
use std::task::Context;
use std::task::Poll;
use std::task::Wake;
use std::task::Waker;
use tokio::task;
use super::*;
use crate::FillControlOptions;
use crate::FillLimits;
use crate::L1EvictionPolicy;
use crate::StatsOptions;
use crate::cache::session::CacheSession;
use crate::config::runtime::IoEngineOptions;
use crate::config::runtime::PosixIoOptions;
use crate::config::runtime::ReadAdmission;
use crate::fixtures::TestDirectory;
use crate::fixtures::TestFile;
use crate::fixtures::test_data_superblock;
use crate::io::engine::IoEngine;
use crate::io::file::DIRECT_IO_ALIGNMENT;
use crate::region::index::packed::IndexEntry;
use crate::region::index::packed::PackedLocation;
use crate::region::record::RECORD_ALIGNMENT;
use crate::region::record::RECORD_HEADER_SIZE;
use crate::region::recovery::DATA_REGION_AREA_OFFSET;
struct ShardStateLockProbe {
control: Arc<AppendWorkerControl>,
observed_unlocked: AtomicBool,
}
impl Wake for ShardStateLockProbe {
fn wake(self: Arc<Self>) {
self.observed_unlocked
.store(self.control.state.try_lock().is_ok(), Ordering::Release);
}
}
#[test]
fn read_lane_uses_one_bounded_alternate_on_primary_pressure() {
let file = TestFile::new("read-lane");
let io = file.io();
let engines: Box<[Arc<IoEngine>]> = vec![
Arc::new(IoEngine::for_test(Arc::clone(&io), 1).unwrap()),
Arc::new(IoEngine::for_test(Arc::clone(&io), 1).unwrap()),
]
.into_boxed_slice();
let pressure_cursor = AtomicUsize::new(0);
let primary = engines[0].try_reserve_read().unwrap();
let (selected, alternate) = try_reserve_read_lane(&engines, 0, &pressure_cursor).unwrap();
assert!(Arc::ptr_eq(&selected, &engines[1]));
let error = match try_reserve_read_lane(&engines, 0, &pressure_cursor) {
Ok(_) => panic!("both read lanes are already reserved"),
Err(error) => error,
};
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
drop(alternate);
drop(primary);
drop(selected);
engines[0].stop_accepting_requests();
let (selected, alternate) = try_reserve_read_lane(&engines, 0, &pressure_cursor).unwrap();
assert!(Arc::ptr_eq(&selected, &engines[1]));
drop(alternate);
drop(selected);
for engine in &engines {
engine.shutdown().unwrap();
}
drop(engines);
drop(io);
}
#[test]
fn hot_read_route_rotates_pressure_fallback_across_all_lanes() {
let file = TestFile::new("read-lane-rotation");
let io = file.io();
let engines: Box<[Arc<IoEngine>]> = (0..4)
.map(|_| Arc::new(IoEngine::for_test(Arc::clone(&io), 1).unwrap()))
.collect::<Vec<_>>()
.into_boxed_slice();
let pressure_cursor = AtomicUsize::new(0);
let primary = engines[0].try_reserve_read().unwrap();
for expected in 1..4 {
let (selected, slot) = try_reserve_read_lane(&engines, 0, &pressure_cursor).unwrap();
assert!(Arc::ptr_eq(&selected, &engines[expected]));
drop(slot);
drop(selected);
}
drop(primary);
for engine in &engines {
engine.shutdown().unwrap();
}
drop(engines);
drop(io);
}
#[test]
fn write_wake_is_only_needed_for_new_batches_and_threshold_crossings() {
assert!(should_wake_write(0, 64, 4096));
assert!(!should_wake_write(64, 128, 4096));
assert!(should_wake_write(4032, 4096, 4096));
assert!(!should_wake_write(4096, 4160, 4096));
}
fn write_pressure_store(region_size: u64, append_shards: u32) -> (CacheSession, TestDirectory) {
runtime_store(
region_size,
RuntimeOptions {
append_shards,
l1_capacity_bytes: 0,
fill_control: FillControlOptions::Adaptive(FillLimits::new(1_048_576, 10_000)),
stats: StatsOptions {
activity_counters: true,
..Default::default()
},
..RuntimeOptions::default()
},
)
}
fn runtime_store(region_size: u64, options: RuntimeOptions) -> (CacheSession, TestDirectory) {
let directory = TestDirectory::new("write-pressure");
let data = test_data_superblock(region_size, 4);
let store =
CacheSession::for_test_with_options(directory.paths.clone(), data, 8, options).unwrap();
(store, directory)
}
#[test]
fn served_bytes_count_the_value_returned_after_promotion() {
for policy in [L1EvictionPolicy::Clock, L1EvictionPolicy::S3Fifo] {
for (l1_capacity, activity_counters, old_length, new_length) in [
(4096, true, 64, Some(128)),
(4096, true, 128, Some(64)),
(4096, true, 64, None),
(0, true, 64, Some(128)),
(4096, false, 64, Some(128)),
] {
let (mut store, _directory) = runtime_store(
8192,
RuntimeOptions {
append_shards: 1,
l1_capacity_bytes: l1_capacity,
l1_shards: 1,
l1_eviction_policy: policy,
stats: StatsOptions {
activity_counters,
..Default::default()
},
..RuntimeOptions::default()
},
);
let plane = store.runtime().unwrap().clone();
let old_value = vec![1; old_length];
plane.put_l2(b"key", &old_value).unwrap();
plane.drain().unwrap();
let PreparedGet::Pending(pending) = plane.prepare_get(b"key", None).unwrap() else {
panic!("the initial value must be read from L2");
};
let completed = pending.wait();
let new_value = new_length.map(|length| vec![2; length]);
if let Some(value) = &new_value {
plane.put(b"key", value).unwrap();
}
let observed = plane.finish_get(completed, b"key").unwrap().unwrap();
let expected = if l1_capacity == 0 {
&old_value
} else {
new_value.as_ref().unwrap_or(&old_value)
};
assert_eq!(observed.value(), expected);
assert_eq!(
matches!(&observed, CacheRead::PromotedL2(_)),
l1_capacity != 0
);
let snapshot = plane.snapshot();
let returned_bytes = observed.value().len() as u64;
drop(observed);
store.close_warm().unwrap();
assert_eq!(snapshot.l1_hits, 0);
if l1_capacity == 0 {
assert_eq!(snapshot.l1_bypasses, 2);
}
assert_eq!(snapshot.l2_hits, u64::from(activity_counters));
assert_eq!(
snapshot.served_bytes,
if activity_counters { returned_bytes } else { 0 },
"served bytes must follow the final value with {policy:?}"
);
}
}
}
#[test]
fn promoted_value_retains_its_bytes_and_charge_during_concurrent_replacement() {
for policy in [L1EvictionPolicy::Clock, L1EvictionPolicy::S3Fifo] {
for activity_counters in [false, true] {
for (old_length, returned_length, final_length) in
[(0, 1, 0), (1, 0, 1), (64, 128, 128), (128, 64, 257)]
{
let (mut store, _directory) = runtime_store(
8192,
RuntimeOptions {
append_shards: 1,
l1_capacity_bytes: 4096,
l1_shards: 1,
l1_eviction_policy: policy,
stats: StatsOptions {
activity_counters,
..Default::default()
},
..RuntimeOptions::default()
},
);
let plane = store.runtime().unwrap().clone();
let key = b"key";
let old_value = vec![1; old_length];
let returned_value = vec![2; returned_length];
let final_value = vec![3; final_length];
plane.put_l2(key, &old_value).unwrap();
plane.drain().unwrap();
let PreparedGet::Pending(pending) = plane.prepare_get(key, None).unwrap()
else {
panic!("the initial value must be read from L2");
};
let completed = pending.wait();
plane.put(key, &returned_value).unwrap();
let held = plane.finish_get(completed, key).unwrap().unwrap();
let CacheRead::PromotedL2(promoted) = &held else {
panic!("promotion must select the newer resident value");
};
let held_clone = promoted.clone();
assert_eq!(held.value(), returned_value);
let before = plane.state.memory.detailed_snapshot().unwrap();
assert_eq!(before.resident_entries, 1);
assert_eq!(before.retained_bytes, 0);
let promoted_charge = before.resident_bytes;
let seqno = plane
.regions()
.lock_manager_for_test()
.allocate_seqno()
.unwrap();
let concurrent = plane.clone();
let replacement = final_value.clone();
thread::spawn(move || {
let hash = hash_key(concurrent.data.hash_seed, key);
assert!(
concurrent
.state
.memory
.publish(hash, key, &replacement, seqno)
);
})
.join()
.unwrap();
assert_eq!(held.value(), returned_value);
assert_eq!(held_clone.as_ref(), returned_value);
let snapshot = plane.snapshot();
assert_eq!(
snapshot.served_bytes,
if activity_counters {
returned_length as u64
} else {
0
}
);
assert_eq!(snapshot.l1_hits, 0);
assert_eq!(snapshot.l2_hits, u64::from(activity_counters));
let l1 = plane.state.memory.detailed_snapshot().unwrap();
assert_eq!(l1.resident_entries, 1);
assert_eq!(l1.retained_bytes, promoted_charge);
assert_eq!(
l1.resident_bytes,
promoted_charge - returned_length + final_length
);
let current = plane.get(key).unwrap().unwrap();
assert_eq!(current.value(), final_value);
let snapshot = plane.snapshot();
assert_eq!(
snapshot.served_bytes,
if activity_counters {
(returned_length + final_length) as u64
} else {
0
}
);
assert_eq!(snapshot.l1_hits, u64::from(activity_counters));
assert_eq!(snapshot.l2_hits, u64::from(activity_counters));
drop(current);
drop(held);
assert_eq!(
plane
.state
.memory
.detailed_snapshot()
.unwrap()
.retained_bytes,
promoted_charge
);
drop(held_clone);
let released = plane.state.memory.detailed_snapshot().unwrap();
assert_eq!(released.resident_bytes, l1.resident_bytes);
assert_eq!(released.retained_bytes, 0);
store.close_warm().unwrap();
}
}
}
}
#[test]
fn contention_rejects_without_waking_but_full_buffers_flush_and_publish() {
let (mut store, _directory) = write_pressure_store(4096, 1);
let plane = store.runtime().unwrap().clone();
for manager_contention in [false, true] {
let shard = (!manager_contention).then(|| plane.regions().lock_append_for_test(0));
let manager = manager_contention.then(|| plane.regions().lock_manager_for_test());
let state = plane.state.append_controls[0].lock().unwrap();
let (send, receive) = mpsc::channel();
let writer_plane = plane.clone();
let writer = thread::spawn(move || {
send.send(writer_plane.put(b"contended", b"value")).unwrap();
});
let result = receive.recv_timeout(Duration::from_secs(1));
assert_eq!(state.wake_flags, 0);
drop(state);
drop(manager);
drop(shard);
writer.join().unwrap();
assert!(matches!(result, Ok(Err(error)) if error.kind() == io::ErrorKind::WouldBlock));
}
assert_eq!(plane.snapshot().write_rejections, 2);
assert_eq!(
plane.detailed_snapshot().unwrap().write_buffer_rejections,
0
);
let controller = plane.state.fill_control.as_ref().unwrap();
assert_eq!(controller.snapshot().outstanding_operations, 0);
let key = b"pressure";
let value = vec![7; 4096 - RECORD_HEADER_SIZE - key.len()];
assert!(matches!(
plane
.regions()
.try_stage_value(
&plane.state.staging,
0,
hash_key(plane.data.hash_seed, key),
4096,
key,
&value
)
.unwrap(),
RegionStageValue::Staged { .. }
));
assert_eq!(
plane.put(b"next", b"value").unwrap_err().kind(),
io::ErrorKind::WouldBlock
);
assert_eq!(
plane.detailed_snapshot().unwrap().write_buffer_rejections,
1
);
let deadline = Instant::now() + Duration::from_secs(2);
while plane
.regions()
.begin_point_read(hash_key(plane.data.hash_seed, key))
.is_none()
{
assert!(
Instant::now() < deadline,
"capacity pressure must flush and publish without an explicit drain"
);
thread::sleep(Duration::from_millis(1));
}
assert!(plane.get(key).unwrap().is_some());
store.close_warm().unwrap();
}
#[test]
fn full_staging_capacity_request_rotates_without_another_put_or_drain() {
let (mut store, _directory) = write_pressure_store(4096, 1);
let plane = store.runtime().unwrap().clone();
let key = b"full";
let value = vec![7; 4096 - RECORD_HEADER_SIZE - key.len()];
assert!(matches!(
plane
.regions()
.try_stage_value(
&plane.state.staging,
0,
hash_key(plane.data.hash_seed, key),
4096,
key,
&value,
)
.unwrap(),
RegionStageValue::Staged { .. }
));
assert_eq!(
plane.put(b"next", b"value").unwrap_err().kind(),
io::ErrorKind::WouldBlock
);
let deadline = Instant::now() + Duration::from_secs(2);
while plane.snapshot().region_rotations == 0 && Instant::now() < deadline {
thread::sleep(Duration::from_millis(1));
}
let rotations = plane.snapshot().region_rotations;
assert_eq!(plane.get(key).unwrap().unwrap().value(), value);
plane.state.append_controls[0].notify(WAKE_ROTATE).unwrap();
plane.drain().unwrap();
let snapshot = plane.detailed_snapshot().unwrap();
store.close_warm().unwrap();
assert_eq!(
rotations, 1,
"one rejection must flush and rotate the full Region"
);
assert_eq!(
snapshot.summary.region_rotations, 1,
"a duplicate wake must remain idle"
);
assert_eq!(snapshot.summary.write_rejections, 1);
assert_eq!(snapshot.write_buffer_rejections, 1);
}
#[test]
fn staging_record_limit_accounts_for_flush_padding_before_rotation() {
const REGION_BYTES: usize = 1024 * 1024;
for needs_rotation in [false, true] {
let (mut store, _directory) = write_pressure_store(REGION_BYTES as u64, 1);
let plane = store.runtime().unwrap().clone();
let key = b"key";
let hash = hash_key(plane.data.hash_seed, key);
let first = vec![1; 96 - RECORD_HEADER_SIZE - key.len()];
assert!(matches!(
plane
.regions()
.try_stage_value(&plane.state.staging, 0, hash, 96, key, &first)
.unwrap(),
RegionStageValue::Staged { .. }
));
let mut used = 96;
loop {
match plane
.regions()
.try_stage_value(&plane.state.staging, 0, hash, 64, key, b"value")
.unwrap()
{
RegionStageValue::Staged { .. } => used += 64,
RegionStageValue::NeedsFlush => break,
outcome => panic!("unexpected staging outcome: {outcome:?}"),
}
assert!(
used < REGION_BYTES,
"the record limit must leave Region capacity"
);
}
assert_eq!(used % 4096, 32);
let next_record_bytes = if needs_rotation {
REGION_BYTES - used
} else {
64
};
let next = vec![2; next_record_bytes - RECORD_HEADER_SIZE - key.len()];
assert_eq!(
plane.put(key, &next).unwrap_err().kind(),
io::ErrorKind::WouldBlock
);
let deadline = Instant::now() + Duration::from_secs(2);
loop {
let fill = plane.state.staging.shard_fill_snapshot(0);
assert!(matches!(fill, Ok(_) | Err(StagingError::WouldBlock)));
let rotations = plane.snapshot().region_rotations;
if matches!(fill, Ok(None)) && rotations == u64::from(needs_rotation) {
break;
}
assert!(
Instant::now() < deadline,
"record capacity must complete publication and the requested rotation"
);
thread::sleep(Duration::from_millis(1));
}
let published = plane.regions().begin_point_read(hash).is_some();
let rotations = plane.snapshot().region_rotations;
plane.drain().unwrap();
let final_rotations = plane.snapshot().region_rotations;
assert_eq!(plane.get(key).unwrap().unwrap().value(), b"value");
store.close_warm().unwrap();
assert!(published, "record capacity must wake the worker");
assert_eq!(rotations, u64::from(needs_rotation));
assert_eq!(final_rotations, rotations);
}
}
#[test]
fn staging_record_limit_distinguishes_exact_fit_from_one_unit_overflow() {
const REGION_BYTES: usize = 1024 * 1024;
const ALIGNMENT: usize = DIRECT_IO_ALIGNMENT;
let unit = RECORD_ALIGNMENT as usize;
for prefix in [0, ALIGNMENT] {
for extra in [0, unit, ALIGNMENT - unit] {
for overflow in [0, unit] {
let (mut store, _directory) = write_pressure_store(REGION_BYTES as u64, 1);
let plane = store.runtime().unwrap().clone();
if prefix != 0 {
let value = vec![8; prefix - RECORD_HEADER_SIZE - b"prefix".len()];
plane.put_l2(b"prefix", &value).unwrap();
plane.drain().unwrap();
}
let key = b"key";
let hash = hash_key(plane.data.hash_seed, key);
let first_bytes = 64 + extra;
let first = vec![1; first_bytes - RECORD_HEADER_SIZE - key.len()];
assert!(matches!(
plane
.regions()
.try_stage_value(
&plane.state.staging,
0,
hash,
first_bytes as u32,
key,
&first
)
.unwrap(),
RegionStageValue::Staged { .. }
));
let mut used = prefix + first_bytes;
loop {
match plane
.regions()
.try_stage_value(&plane.state.staging, 0, hash, 64, key, b"value")
.unwrap()
{
RegionStageValue::Staged { .. } => used += 64,
RegionStageValue::NeedsFlush => break,
outcome => panic!("unexpected staging outcome: {outcome:?}"),
}
assert!(
used < REGION_BYTES,
"the record limit must leave Region capacity"
);
}
let aligned = used.div_ceil(ALIGNMENT) * ALIGNMENT;
let record_bytes = REGION_BYTES - aligned + overflow;
let value = vec![2; record_bytes - RECORD_HEADER_SIZE - key.len()];
assert_eq!(
plane.put_l2(key, &value).unwrap_err().kind(),
io::ErrorKind::WouldBlock
);
let expected_rotations = u64::from(overflow != 0);
let deadline = Instant::now() + Duration::from_secs(3);
loop {
let rotations = plane.snapshot().region_rotations;
let fill = plane.state.staging.shard_fill_snapshot(0);
assert!(matches!(fill, Ok(_) | Err(StagingError::WouldBlock)));
if matches!(fill, Ok(None)) && rotations == expected_rotations {
break;
}
assert!(
Instant::now() < deadline,
"no progress: prefix={prefix}, extra={extra}, overflow={overflow}, rotations={rotations}"
);
thread::sleep(Duration::from_millis(1));
}
{
let manager = plane.regions().lock_manager_for_test();
let active = manager.active_regions()[0] as usize;
assert_eq!(
manager.regions()[active].completed_used,
if expected_rotations == 0 {
aligned as u64
} else {
0
}
);
}
let deadline = Instant::now() + Duration::from_secs(3);
loop {
match plane.put_l2(key, &value) {
Ok(_) => break,
Err(error) if error.kind() == io::ErrorKind::WouldBlock => {
assert!(
Instant::now() < deadline,
"write admission did not resume"
);
thread::sleep(Duration::from_millis(1));
}
Err(error) => panic!("unexpected write failure: {error}"),
}
}
plane.drain().unwrap();
assert_eq!(plane.get(key).unwrap().unwrap().value(), value);
assert_eq!(plane.snapshot().region_rotations, expected_rotations);
store.close_warm().unwrap();
}
}
}
}
#[test]
fn capacity_request_flushes_then_rotates_without_another_put_or_drain() {
let (mut store, _directory) = write_pressure_store(8192, 1);
let plane = store.runtime().unwrap().clone();
let first = vec![1; 4096 - RECORD_HEADER_SIZE - b"first".len()];
plane.put(b"first", &first).unwrap();
plane.drain().unwrap();
let tail = vec![2; 4064 - RECORD_HEADER_SIZE - b"tail".len()];
assert!(matches!(
plane
.regions()
.try_stage_value(
&plane.state.staging,
0,
hash_key(plane.data.hash_seed, b"tail"),
4064,
b"tail",
&tail,
)
.unwrap(),
RegionStageValue::Staged { .. }
));
assert_eq!(
plane.put(b"next", b"value").unwrap_err().kind(),
io::ErrorKind::WouldBlock
);
let deadline = Instant::now() + Duration::from_secs(2);
while plane.snapshot().region_rotations == 0 {
assert!(
Instant::now() < deadline,
"one capacity request must flush and rotate without another put or drain"
);
thread::sleep(Duration::from_millis(1));
}
assert_eq!(plane.get(b"first").unwrap().unwrap().value(), first);
assert_eq!(plane.get(b"tail").unwrap().unwrap().value(), tail);
plane.state.append_controls[0].notify(WAKE_ROTATE).unwrap();
plane.drain().unwrap();
assert_eq!(plane.snapshot().region_rotations, 1);
assert_eq!(plane.snapshot().write_rejections, 1);
store.close_warm().unwrap();
}
#[test]
fn primary_contention_rejects_without_probing_other_shards() {
let (mut store, _directory) = write_pressure_store(8192, 3);
let plane = store.runtime().unwrap().clone();
let key = b"fixed-route";
let value = [7; 64];
let primary = plane
.regions()
.append_shard(hash_key(plane.data.hash_seed, key));
let primary_gate = plane.regions().lock_append_for_test(primary);
let primary_wake = plane.state.append_controls[primary].lock().unwrap();
let (send, receive) = mpsc::channel();
let writer_plane = plane.clone();
let writer = thread::spawn(move || {
send.send(writer_plane.put(key, &value)).unwrap();
});
let result = receive.recv_timeout(Duration::from_secs(1));
assert_eq!(primary_wake.wake_flags, 0);
drop(primary_wake);
drop(primary_gate);
writer.join().unwrap();
assert_eq!(
result
.expect("contention must not wait for a mutation or wake lock")
.unwrap_err()
.kind(),
io::ErrorKind::WouldBlock
);
let controller = plane.state.fill_control.as_ref().unwrap();
assert_eq!(controller.snapshot().outstanding_operations, 0);
let snapshot = plane.detailed_snapshot().unwrap();
assert_eq!(snapshot.summary.puts, 0);
assert_eq!(snapshot.summary.write_rejections, 1);
assert_eq!(snapshot.write_buffer_rejections, 0);
assert_eq!(snapshot.summary.fill_control.outstanding_operations, 0);
plane.put(key, &value).unwrap();
plane.drain().unwrap();
assert_eq!(plane.get(key).unwrap().unwrap().value(), value);
let snapshot = plane.detailed_snapshot().unwrap();
assert_eq!(snapshot.summary.puts, 1);
assert_eq!(snapshot.summary.write_rejections, 1);
assert_eq!(snapshot.write_buffer_rejections, 0);
store.close_warm().unwrap();
}
#[test]
fn reclaimed_cache_rejects_primary_contention_without_trying_a_free_neighbor() {
let (mut store, _directory) = write_pressure_store(8192, 3);
let plane = store.runtime().unwrap().clone();
let key = b"stable-route";
let primary = plane
.regions()
.append_shard(hash_key(plane.data.hash_seed, key));
plane
.regions()
.lock_manager_for_test()
.request_rotation_for_test(primary)
.unwrap();
assert_eq!(
plane.regions().rotate_shard(primary).unwrap(),
RegionRotation::Rotated
);
let receipt = plane.regions().begin_reclaim().unwrap().unwrap();
plane.regions().complete_reclaim(receipt).unwrap();
let primary_gate = plane.regions().lock_append_for_test(primary);
let primary_wake = plane.state.append_controls[primary].lock().unwrap();
let (send, receive) = mpsc::channel();
let writer_plane = plane.clone();
let writer = thread::spawn(move || {
send.send(writer_plane.put(key, &[7; 64])).unwrap();
});
let result = receive.recv_timeout(Duration::from_secs(1));
assert_eq!(primary_wake.wake_flags, 0);
drop(primary_wake);
drop(primary_gate);
writer.join().unwrap();
assert_eq!(
result
.expect("contention must not wait for a mutation or wake lock")
.unwrap_err()
.kind(),
io::ErrorKind::WouldBlock
);
let snapshot = plane.detailed_snapshot().unwrap();
assert_eq!(snapshot.summary.puts, 0);
assert_eq!(snapshot.summary.write_rejections, 1);
assert_eq!(snapshot.write_buffer_rejections, 0);
assert_eq!(snapshot.summary.fill_control.outstanding_operations, 0);
assert_eq!(
plane
.regions()
.begin_point_read(hash_key(plane.data.hash_seed, key)),
None
);
plane.put(key, &[7; 64]).unwrap();
plane.drain().unwrap();
assert_eq!(plane.get(key).unwrap().unwrap().value(), &[7; 64]);
store.close_warm().unwrap();
}
#[test]
fn existing_alternate_records_survive_reclaim_and_warm_reopen() {
let (mut store, directory) = write_pressure_store(8192, 3);
let plane = store.runtime().unwrap().clone();
let key = b"fallback-route";
let value = [9; 64];
let data = plane.data;
let hash = hash_key(data.hash_seed, key);
let primary = plane.regions().append_shard(hash);
let alternate = (primary + 1) % plane.state.append_controls.len();
let alternate_region = plane.regions().lock_manager_for_test().active_regions()[alternate];
let record_bytes = required_record_bytes(key.len(), value.len()).unwrap();
assert!(matches!(
plane
.regions()
.try_stage_value(
&plane.state.staging,
alternate,
hash,
record_bytes,
key,
&value,
)
.unwrap(),
RegionStageValue::Staged { .. }
));
plane.drain().unwrap();
assert_eq!(
plane
.regions()
.begin_point_read(hash)
.unwrap()
.entry
.location
.region_id(),
alternate_region
);
assert_eq!(plane.get(key).unwrap().unwrap().value(), value);
plane
.regions()
.lock_manager_for_test()
.request_rotation_for_test(primary)
.unwrap();
assert_eq!(
plane.regions().rotate_shard(primary).unwrap(),
RegionRotation::Rotated
);
let receipt = plane.regions().begin_reclaim().unwrap().unwrap();
assert_ne!(receipt.region_id, alternate_region);
assert_eq!(receipt.physical_record_count, 0);
plane.regions().complete_reclaim(receipt).unwrap();
assert!(!plane.regions().reclaim_needed().unwrap());
assert_eq!(plane.get(key).unwrap().unwrap().value(), value);
assert_eq!(
plane
.regions()
.begin_point_read(hash)
.unwrap()
.entry
.location
.region_id(),
alternate_region
);
store.close_warm().unwrap();
drop(plane);
drop(store);
let options = RuntimeOptions {
append_shards: 3,
l1_capacity_bytes: 0,
fill_control: FillControlOptions::Adaptive(FillLimits::new(1_048_576, 10_000)),
stats: StatsOptions {
activity_counters: true,
..Default::default()
},
..RuntimeOptions::default()
};
let mut reopened =
CacheSession::for_test_with_options(directory.paths.clone(), data, 8, options).unwrap();
let plane = reopened.runtime().unwrap().clone();
assert_eq!(plane.get(key).unwrap().unwrap().value(), value);
assert_eq!(
plane
.regions()
.begin_point_read(hash)
.unwrap()
.entry
.location
.region_id(),
alternate_region
);
plane.put_l2(key, &[11; 64]).unwrap();
plane.drain().unwrap();
assert_eq!(plane.get(key).unwrap().unwrap().value(), &[11; 64]);
let primary_region = plane.regions().lock_manager_for_test().active_regions()[primary];
assert_eq!(
plane
.regions()
.begin_point_read(hash)
.unwrap()
.entry
.location
.region_id(),
primary_region
);
reopened.close_warm().unwrap();
drop(plane);
drop(reopened);
}
#[test]
fn permanent_close_is_not_reopened_by_an_active_drain() {
let gate = Arc::new(MutationGate::new());
let mutation = gate.try_enter().unwrap();
let drain = gate.begin_drain().unwrap();
let closing_gate = Arc::clone(&gate);
let close = thread::spawn(move || {
closing_gate.start_close();
closing_gate.wait_quiescent().unwrap();
});
while gate.state.load(Ordering::Acquire) & MUTATION_CLOSED == 0 {
thread::yield_now();
}
drop(mutation);
drain.wait().unwrap();
drop(drain);
close.join().unwrap();
assert!(gate.try_enter().is_none());
assert!(gate.begin_drain().is_err());
}
#[test]
fn closing_during_drain_does_not_restore_running_lifecycle() {
let lifecycle = AtomicU8::new(LIFECYCLE_RUNNING);
let operations = MutationGate::new();
let drain = operations.begin_drain().unwrap();
let lifecycle_drain = LifecycleDrainingGuard::enter(&lifecycle, &operations);
operations.start_close();
operations.wait_quiescent().unwrap();
drop(lifecycle_drain);
drop(drain);
assert_eq!(lifecycle.load(Ordering::Acquire), LIFECYCLE_DRAINING);
assert!(operations.try_enter().is_none());
}
#[tokio::test]
async fn mutation_gate_wakes_async_drain_without_blocking() {
let gate = Arc::new(MutationGate::new());
let mutation = gate.try_enter().unwrap();
let drain_gate = Arc::clone(&gate);
let drain = task::spawn(async move {
let drain = drain_gate.begin_drain().unwrap();
drain.wait_async().await;
assert!(drain_gate.try_enter().is_none());
assert!(
matches!(drain_gate.begin_drain(), Err(error) if error.kind() == io::ErrorKind::WouldBlock)
);
});
task::yield_now().await;
assert!(gate.try_enter().is_none());
drop(mutation);
drain.await.unwrap();
assert!(gate.try_enter().is_some());
}
#[tokio::test]
async fn cancelling_async_drain_reopens_mutation_admission() {
let gate = Arc::new(MutationGate::new());
for signal_before_cancel in [true, false] {
let mut mutation = Some(gate.try_enter().unwrap());
let drain_gate = Arc::clone(&gate);
let drain = task::spawn(async move {
let drain = drain_gate.begin_drain().unwrap();
drain.wait_async().await;
});
task::yield_now().await;
assert!(gate.try_enter().is_none());
if signal_before_cancel {
drop(mutation.take());
}
drain.abort();
assert!(drain.await.unwrap_err().is_cancelled());
assert!(gate.try_enter().is_some());
drop(mutation);
}
}
#[test]
fn shard_completion_and_failure_wake_after_releasing_state_lock() {
for failed in [false, true] {
let control = Arc::new(AppendWorkerControl::new());
let probe = Arc::new(ShardStateLockProbe {
control: Arc::clone(&control),
observed_unlocked: AtomicBool::new(false),
});
let waker = Waker::from(Arc::clone(&probe));
let mut context = Context::from_waker(&waker);
let mut wait = pin::pin!(control.wait_for_drain_async(1));
assert!(wait.as_mut().poll(&mut context).is_pending());
if failed {
control.fail(&io::Error::other("test shard failure"));
} else {
complete_shard_drain(&control, 1).unwrap();
}
assert!(probe.observed_unlocked.load(Ordering::Acquire));
let Poll::Ready(result) = wait.as_mut().poll(&mut context) else {
panic!("completion or failure must finish the pending drain");
};
assert_eq!(result.is_err(), failed);
}
}
#[test]
fn urgent_empty_shard_wake_is_consumed() {
let control = AppendWorkerControl::new();
control.notify(WAKE_URGENT).unwrap();
let (flags, drain_generation, stop, timed_out) =
wait_for_shard_work(&control, None).unwrap();
assert_eq!(flags, WAKE_URGENT);
assert_eq!(drain_generation, 0);
assert!(!stop);
assert!(!timed_out);
let (flags, drain_generation, stop, timed_out) =
wait_for_shard_work(&control, Some(Instant::now())).unwrap();
assert_eq!(flags, 0);
assert_eq!(drain_generation, 0);
assert!(!stop);
assert!(timed_out);
}
#[test]
fn failed_idle_shard_exits_without_another_work_notification() {
let control = Arc::new(AppendWorkerControl::new());
control.fail(&io::Error::new(io::ErrorKind::BrokenPipe, "shard failed"));
let worker_control = Arc::clone(&control);
let (completed, completion) = mpsc::channel();
let worker = thread::spawn(move || {
completed
.send(wait_for_shard_work(&worker_control, None))
.unwrap();
});
let result = completion.recv_timeout(Duration::from_secs(1));
if result.is_err() {
control.state.lock().unwrap().wake_flags = WAKE_URGENT;
control.changed.notify_all();
}
worker.join().unwrap();
assert_eq!(
result
.expect("terminal failure must stop an idle worker")
.unwrap_err()
.kind(),
io::ErrorKind::BrokenPipe
);
}
#[test]
fn close_does_not_need_a_new_drain_generation() {
let (mut session, _directory) = runtime_store(
4096,
RuntimeOptions {
append_shards: 1,
..RuntimeOptions::default()
},
);
let control = Arc::clone(&session.runtime().unwrap().state.append_controls[0]);
{
let mut state = control.state.lock().unwrap();
state.drain_requested = u64::MAX;
state.drain_completed = u64::MAX;
}
let (closed, completion) = mpsc::channel();
let close = thread::spawn(move || closed.send(session.close_warm()).unwrap());
let result = completion.recv_timeout(Duration::from_secs(5));
if result.is_err() {
let mut state = control.state.lock().unwrap();
state.stop = true;
state.drain_requested = 1;
state.drain_completed = 0;
drop(state);
control.changed.notify_all();
}
close.join().unwrap();
result
.expect("close must stop workers even when drain generations are exhausted")
.unwrap();
}
#[test]
fn reclaim_progress_does_not_fail_after_a_shard_stops() {
let control = AppendWorkerControl::new();
control.stop().unwrap();
control.notify_if_running(WAKE_ROTATE).unwrap();
assert_eq!(
control.notify(WAKE_ROTATE).unwrap_err().kind(),
io::ErrorKind::NotConnected
);
}
#[test]
fn stopped_shard_rejects_new_drains_and_completes_accepted_drains() {
let control = AppendWorkerControl::new();
let old = control.request_drain().unwrap();
complete_shard_drain(&control, old).unwrap();
let first = control.request_drain().unwrap();
control.stop().unwrap();
assert_eq!(
control.request_drain().unwrap_err().kind(),
io::ErrorKind::NotConnected
);
assert_eq!(control.lock().unwrap().drain_requested, first);
let mut wait = pin::pin!(control.wait_for_drain_async(first));
let mut context = Context::from_waker(Waker::noop());
assert!(wait.as_mut().poll(&mut context).is_pending());
complete_shard_drain(&control, first).unwrap();
assert!(matches!(
wait.as_mut().poll(&mut context),
Poll::Ready(Ok(()))
));
}
#[test]
fn one_reclaim_notification_reaches_every_worker() {
let control = Arc::new(ReclaimControl::new());
let ready = Arc::new(Barrier::new(3));
let mut workers = Vec::new();
for _ in 0..2 {
let control = Arc::clone(&control);
let ready = Arc::clone(&ready);
workers.push(thread::spawn(move || {
let mut observed_generation = 0;
ready.wait();
let notified = control.wait(&mut observed_generation).unwrap();
(notified, observed_generation)
}));
}
ready.wait();
control.notify().unwrap();
for worker in workers {
assert_eq!(worker.join().unwrap(), (true, 1));
}
}
#[test]
fn completion_timeouts_follow_read_wait_mode() {
let directory = TestDirectory::new("completion-timeout");
let data = test_data_superblock(4096, 2);
for wait in [Duration::ZERO, Duration::from_millis(1)] {
let config = RuntimeOptions {
io_engine: IoEngineOptions::Posix(PosixIoOptions {
read_workers: 1,
write_workers: 1,
reclaim_workers: 1,
}),
append_shards: 1,
l1_capacity_bytes: 0,
stats: StatsOptions {
activity_counters: true,
..Default::default()
},
read_admission: if wait.is_zero() {
ReadAdmission::Immediate
} else {
ReadAdmission::Wait {
timeout: wait,
max_waiters: None,
}
},
..RuntimeOptions::default()
};
let mut session =
CacheSession::for_test_with_options(directory.paths.clone(), data, 8, config)
.unwrap();
let runtime = session.runtime().unwrap().clone();
let MemoryLookup::Miss(read_token) = runtime.state.memory.lookup(7, b"key") else {
panic!("empty L1 must miss");
};
let result = runtime.finish_get(
CompletedGet {
read: ReadCompletion {
desc: ReadDesc {
hash: 7,
entry: IndexEntry {
location: PackedLocation::new(0, 0, 64).unwrap(),
},
region_generation: 1,
absolute: DATA_REGION_AREA_OFFSET,
read_len: 64,
record_range: 0..64,
},
result: Err(io::Error::from(io::ErrorKind::TimedOut)),
},
read_token,
hash: 7,
},
b"key",
);
let snapshot = runtime.snapshot();
assert!(runtime.regions().is_healthy());
session.close_fast().unwrap();
if wait.is_zero() {
assert!(matches!(result, Ok(None)));
assert_eq!(snapshot.l2_read_busy_misses, 1);
assert_eq!(snapshot.l2_read_overloads, 0);
} else {
assert!(matches!(result, Err(error) if error.kind() == io::ErrorKind::TimedOut));
assert_eq!(snapshot.l2_read_busy_misses, 0);
assert_eq!(snapshot.l2_read_overloads, 1);
}
}
}
#[test]
fn reclaim_workers_rotate_over_disjoint_append_shards() {
for shard_count in 1..=8 {
for worker_count in 1..=shard_count {
let mut owners = vec![None; shard_count];
for worker_id in 0..worker_count {
let mut cursor = ReinsertShardCursor::new(worker_id, worker_count, shard_count);
for shard in (worker_id..shard_count).step_by(worker_count) {
assert_eq!(cursor.take(), shard);
assert_eq!(owners[shard].replace(worker_id), None);
}
assert_eq!(cursor.take(), worker_id);
}
assert!(owners.iter().all(Option::is_some));
}
}
}
}
#[cfg(test)]
mod shutdown_tests;