use std::fmt;
use std::future::Future;
use std::io;
use std::ops::Deref;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::AtomicU64;
use std::sync::atomic::Ordering;
use std::time::Duration;
use std::time::Instant;
use std::time::SystemTime;
use std::time::UNIX_EPOCH;
use tokio::task::JoinError;
use crate::config::CacheConfig;
use crate::config::storage::KEY_HASH_SEED;
use crate::config::storage_fingerprint;
use crate::config::storage_geometry;
use crate::error::Error;
use crate::error::ErrorOperation;
use crate::error::from_io;
use crate::region::file_backend::FileRegionBackend;
use crate::region::file_backend::RegionFiles;
use crate::region::file_backend::SystemRegionFileSystem;
use crate::region::recovery::DataSuperblock;
use crate::region::recovery::PersistentId;
use crate::region::recovery::RECOVERY_IMAGE_INDEX_OFFSET;
use crate::region::recovery::recovery_image_index_len;
use crate::region::runtime::HybridValueRead;
use crate::region::runtime::RegionDataPlane;
use crate::region::store::RegionStore;
use crate::snapshot::CacheSnapshot;
use crate::snapshot::DetailedCacheSnapshot;
use crate::snapshot::StartupMode;
#[non_exhaustive]
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum CacheTier {
L1,
L2,
}
pub struct Value {
inner: HybridValueRead,
}
impl Deref for Value {
type Target = [u8];
fn deref(&self) -> &Self::Target {
self.inner.value()
}
}
impl AsRef<[u8]> for Value {
fn as_ref(&self) -> &[u8] {
self
}
}
impl Value {
pub const fn tier(&self) -> CacheTier {
if self.inner.is_l1() {
CacheTier::L1
} else {
CacheTier::L2
}
}
}
pub struct Cache {
closed: AtomicBool,
read_statistics: bool,
mutation_statistics: bool,
data_plane: RegionDataPlane,
owner: Arc<Mutex<RegionStore<FileRegionBackend<SystemRegionFileSystem>>>>,
startup: StartupMode,
path: PathBuf,
logical_disk_peak_bytes: u64,
tokio_handle: tokio::runtime::Handle,
}
impl fmt::Debug for Cache {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("Cache")
.field("startup", &self.startup_mode())
.finish_non_exhaustive()
}
}
impl Cache {
pub async fn open(path: impl AsRef<Path>, config: CacheConfig) -> Result<Self, Error> {
let handle = tokio::runtime::Handle::try_current().map_err(|error| {
from_io(
ErrorOperation::Open,
io::Error::new(io::ErrorKind::InvalidInput, error.to_string()),
)
})?;
Self::open_with_handle(path, config, handle).await
}
pub async fn open_with_handle(
path: impl AsRef<Path>,
config: CacheConfig,
tokio_handle: tokio::runtime::Handle,
) -> Result<Self, Error> {
let path = path.as_ref().to_path_buf();
let cache_handle = tokio_handle.clone();
let started = Instant::now();
let result = tokio_handle
.spawn_blocking(move || Self::open_blocking(path, config, cache_handle, started))
.await
.map_err(|error| {
from_io(
ErrorOperation::Open,
blocking_task_error("cache open", error),
)
})?;
public_result(ErrorOperation::Open, result)
}
fn open_blocking(
path: PathBuf,
config: CacheConfig,
tokio_handle: tokio::runtime::Handle,
started: Instant,
) -> io::Result<Cache> {
let capacity_bytes = config.storage().capacity_bytes();
let index_slots = config.storage().index_slots();
let index_bytes = u64::try_from(index_slots)
.ok()
.and_then(recovery_image_index_len)
.unwrap_or(0);
let result = Self::open_blocking_inner(path.clone(), config, tokio_handle);
match &result {
Ok(cache) => {
let startup = cache.startup_mode();
log::info!(
target: "cache2::lifecycle",
event = "cache_opened",
path:% = path.display(),
startup = startup_name(startup),
index_backing = index_backing_name(startup),
index_validation = index_validation_name(startup),
index_copy_on_write = matches!(startup, StartupMode::Warm),
capacity_bytes,
index_slots,
index_mapping_bytes = index_mapping_bytes(startup, index_bytes),
elapsed_us = elapsed_micros(started.elapsed());
"cache opened"
);
}
Err(error) => log::error!(
target: "cache2::lifecycle",
event = "cache_open_failed",
path:% = path.display(),
capacity_bytes,
index_slots,
elapsed_us = elapsed_micros(started.elapsed()),
error:% = error;
"cache open failed"
),
}
result
}
fn open_blocking_inner(
path: PathBuf,
config: CacheConfig,
tokio_handle: tokio::runtime::Handle,
) -> io::Result<Cache> {
let format_data = DataSuperblock {
generation: 1,
cache_uuid: next_persistent_id(),
data_identity: next_persistent_id(),
geometry: storage_geometry(config.storage()),
hash_seed: KEY_HASH_SEED,
config_fingerprint: storage_fingerprint(config.storage()),
};
let files = RegionFiles::new(
&path,
sidecar_path(&path, ".state"),
sidecar_path(&path, ".image"),
);
let index_slots = config.storage().index_slots();
let logical_disk_peak_bytes = config.storage().peak_disk_bytes();
let stats = config.runtime().stats;
let read_statistics = stats.request_counters
|| stats.l1_latency != crate::LatencyMode::Off
|| stats.l2_latency != crate::LatencyMode::Off;
let mutation_statistics =
stats.request_counters || stats.mutation_latency != crate::LatencyMode::Off;
let backend = FileRegionBackend::new(files, format_data, config);
let store = RegionStore::open(index_slots, backend)?;
let startup = store.startup();
let data_plane = store.data_plane_handle()?;
Ok(Cache {
closed: AtomicBool::new(false),
read_statistics,
mutation_statistics,
data_plane,
owner: Arc::new(Mutex::new(store)),
startup,
path,
logical_disk_peak_bytes,
tokio_handle,
})
}
pub const fn startup_mode(&self) -> StartupMode {
self.startup
}
pub fn put(&self, key: impl AsRef<[u8]>, value: impl AsRef<[u8]>) -> Result<u64, Error> {
self.mutate(ErrorOperation::Put, crate::RequestOperation::Put, || {
self.data_plane.put(key.as_ref(), value.as_ref())
})
}
pub fn put_l2(&self, key: impl AsRef<[u8]>, value: impl AsRef<[u8]>) -> Result<u64, Error> {
self.mutate(
ErrorOperation::PutL2,
crate::RequestOperation::PutL2,
|| self.data_plane.put_l2(key.as_ref(), value.as_ref()),
)
}
pub fn delete(&self, key: impl AsRef<[u8]>) -> Result<u64, Error> {
self.mutate(
ErrorOperation::Delete,
crate::RequestOperation::Delete,
|| self.data_plane.delete(key.as_ref()),
)
}
pub async fn get(&self, key: impl AsRef<[u8]> + Send) -> Result<Option<Value>, Error> {
if !self.read_statistics {
if self.is_closed() {
return Ok(None);
}
return public_result(
ErrorOperation::Get,
self.data_plane
.get_async(key.as_ref(), &self.tokio_handle, None)
.await,
)
.map(|value| value.map(|inner| Value { inner }));
}
let mut guard = self
.data_plane
.stats_recorder()
.begin(crate::RequestOperation::Get);
let result = if self.is_closed() {
Ok(None)
} else {
public_result(
ErrorOperation::Get,
self.data_plane
.get_async(key.as_ref(), &self.tokio_handle, Some(&mut guard))
.await,
)
.map(|value| value.map(|inner| Value { inner }))
};
let outcome = match &result {
Ok(Some(value)) if value.inner.is_l1() => crate::RequestOutcome::L1Hit,
Ok(Some(_)) => crate::RequestOutcome::L2Hit,
_ => crate::RequestOutcome::Miss,
};
guard.finish(&result, outcome);
result
}
pub async fn drain(&self) -> Result<(), Error> {
self.ensure_open(ErrorOperation::Drain)?;
public_result(ErrorOperation::Drain, self.data_plane.drain_async().await)
}
pub fn snapshot(&self) -> Result<CacheSnapshot, Error> {
self.ensure_open(ErrorOperation::Snapshot)?;
let mut snapshot = public_result(ErrorOperation::Snapshot, self.data_plane.snapshot())?;
snapshot.logical_disk_peak_bytes = self.logical_disk_peak_bytes;
Ok(snapshot)
}
pub fn stats_snapshot(&self) -> Result<crate::CacheStatsSnapshot, Error> {
Ok(self.data_plane.stats_recorder().snapshot(self.snapshot()?))
}
pub fn detailed_snapshot(&self) -> Result<DetailedCacheSnapshot, Error> {
self.ensure_open(ErrorOperation::DetailedSnapshot)?;
let mut snapshot = public_result(
ErrorOperation::DetailedSnapshot,
self.data_plane.detailed_snapshot(),
)?;
snapshot.summary.logical_disk_peak_bytes = self.logical_disk_peak_bytes;
Ok(snapshot)
}
pub fn close_fast(&self) -> impl Future<Output = Result<(), Error>> + Send + 'static {
self.close(false)
}
pub fn close_warm(&self) -> impl Future<Output = Result<(), Error>> + Send + 'static {
self.close(true)
}
#[inline]
fn mutate(
&self,
operation: ErrorOperation,
stats_operation: crate::RequestOperation,
mutation: impl FnOnce() -> io::Result<u64>,
) -> Result<u64, Error> {
let guard = self
.mutation_statistics
.then(|| self.data_plane.stats_recorder().begin(stats_operation));
let result = self
.ensure_open(operation)
.and_then(|()| public_result(operation, mutation()));
if let Some(guard) = guard {
guard.finish(&result, crate::RequestOutcome::Accepted);
}
result
}
#[inline(always)]
fn ensure_open(&self, operation: ErrorOperation) -> Result<(), Error> {
if self.is_closed() {
return public_result(operation, Err(cache_closed_error()));
}
Ok(())
}
#[inline(always)]
fn is_closed(&self) -> bool {
self.closed.load(Ordering::Acquire)
}
fn close(&self, warm: bool) -> impl Future<Output = Result<(), Error>> + Send + 'static {
let (operation, mode) = if warm {
(ErrorOperation::CloseWarm, "warm")
} else {
(ErrorOperation::CloseFast, "fast")
};
let tokio_handle = self.tokio_handle.clone();
let close = (!self.closed.swap(true, Ordering::AcqRel)).then(|| {
self.data_plane.start_close();
let owner = Arc::clone(&self.owner);
let path = self.path.clone();
let started = Instant::now();
tokio_handle.spawn_blocking(move || {
let result = owner
.lock()
.map_err(|_| cache_lifecycle_poisoned())
.and_then(|mut store| {
if warm {
store.close_warm()
} else {
store.close_fast()
}
});
log_cache_close(&path, mode, started.elapsed(), &result);
result
})
});
async move {
let result = match close {
Some(close) => match close.await {
Ok(result) => result,
Err(error) => Err(blocking_task_error("cache close", error)),
},
None => Err(cache_closed_error()),
};
public_result(operation, result)
}
}
}
fn cache_closed_error() -> io::Error {
io::Error::new(io::ErrorKind::NotConnected, "cache is closing or closed")
}
fn cache_lifecycle_poisoned() -> io::Error {
io::Error::new(
io::ErrorKind::InvalidData,
"cache lifecycle synchronization is poisoned",
)
}
fn startup_name(startup: StartupMode) -> &'static str {
match startup {
StartupMode::Cold => "cold",
StartupMode::Warm => "warm",
}
}
fn index_backing_name(startup: StartupMode) -> &'static str {
match startup {
StartupMode::Cold => anonymous_index_backing_name(),
StartupMode::Warm => "file_private_mmap",
}
}
const fn anonymous_index_backing_name() -> &'static str {
#[cfg(any(target_os = "linux", target_os = "macos"))]
{
"anonymous_mmap"
}
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
{
"heap"
}
}
fn index_validation_name(startup: StartupMode) -> &'static str {
match startup {
StartupMode::Cold => "not_required",
StartupMode::Warm => "lazy_crc32c",
}
}
fn index_mapping_bytes(startup: StartupMode, index_bytes: u64) -> u64 {
match startup {
StartupMode::Cold => index_bytes,
StartupMode::Warm => RECOVERY_IMAGE_INDEX_OFFSET.saturating_add(index_bytes),
}
}
fn elapsed_micros(elapsed: Duration) -> u64 {
u64::try_from(elapsed.as_micros()).unwrap_or(u64::MAX)
}
fn log_cache_close(path: &Path, mode: &'static str, elapsed: Duration, result: &io::Result<()>) {
match result {
Ok(()) => log::info!(
target: "cache2::lifecycle",
event = "cache_closed",
path:% = path.display(),
mode,
elapsed_us = elapsed_micros(elapsed);
"cache closed"
),
Err(error) => log::error!(
target: "cache2::lifecycle",
event = "cache_close_failed",
path:% = path.display(),
mode,
elapsed_us = elapsed_micros(elapsed),
error:% = error;
"cache close failed"
),
}
}
fn blocking_task_error(operation: &'static str, error: JoinError) -> io::Error {
io::Error::other(format!("{operation} task failed: {error}"))
}
fn sidecar_path(path: &Path, suffix: &str) -> PathBuf {
let mut value = path.as_os_str().to_owned();
value.push(suffix);
PathBuf::from(value)
}
fn public_result<T>(operation: ErrorOperation, result: io::Result<T>) -> Result<T, Error> {
result.map_err(|error| from_io(operation, error))
}
fn next_persistent_id() -> PersistentId {
static NEXT_ID: AtomicU64 = AtomicU64::new(1);
let counter = NEXT_ID.fetch_add(1, Ordering::Relaxed);
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_nanos();
let mut bytes = now.to_le_bytes();
let mix = counter ^ u64::from(std::process::id()).rotate_left(32);
for (target, source) in bytes[8..].iter_mut().zip(mix.to_le_bytes()) {
*target ^= source;
}
PersistentId::from_bytes(bytes).unwrap_or_else(|| {
PersistentId::from_bytes(counter.max(1).to_le_bytes().repeat(2).try_into().unwrap())
.expect("non-zero counter produces a cache identity")
})
}
#[cfg(test)]
mod tests {
use super::*;
fn assert_send_sync<T: Send + Sync>() {}
#[test]
fn public_cache_and_values_are_send_and_sync() {
assert_send_sync::<Cache>();
assert_send_sync::<Value>();
}
}