#![allow(dead_code)]
use serde::{Deserialize, Serialize};
use crate::search::storage_integrity::StorageState;
pub(crate) const CONTENTION_REPORT_SCHEMA_VERSION: u32 = 1;
const CONTENTION_REPORT_KIND: &str = "contention_diagnostic";
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum ContentionClass {
None,
BusyLocked,
BusyRecovery,
SnapshotConflict,
StaleWalSidecar,
StaleSearcherCache,
HostPressure,
}
impl ContentionClass {
pub(crate) fn stable_name(self) -> &'static str {
match self {
Self::None => "none",
Self::BusyLocked => "busy_locked",
Self::BusyRecovery => "busy_recovery",
Self::SnapshotConflict => "snapshot_conflict",
Self::StaleWalSidecar => "stale_wal_sidecar",
Self::StaleSearcherCache => "stale_searcher_cache",
Self::HostPressure => "host_pressure",
}
}
pub(crate) fn is_archive_loss(self) -> bool {
false
}
pub(crate) fn is_transient_lock(self) -> bool {
matches!(
self,
Self::BusyLocked | Self::BusyRecovery | Self::SnapshotConflict
)
}
pub(crate) fn retryability(self) -> Retryability {
match self {
Self::None => Retryability::NotRetryable,
Self::BusyLocked | Self::BusyRecovery | Self::SnapshotConflict => {
Retryability::RetryAfterBackoff
}
Self::StaleWalSidecar | Self::StaleSearcherCache => Retryability::RetryAfterInspection,
Self::HostPressure => Retryability::NotRetryable,
}
}
pub(crate) fn bounded_wait(self) -> Option<BoundedWaitGuidance> {
match self.retryability() {
Retryability::RetryAfterBackoff => Some(BoundedWaitGuidance::transient_lock()),
Retryability::RetryAfterInspection => Some(BoundedWaitGuidance::after_inspection()),
Retryability::NotRetryable => None,
}
}
pub(crate) fn to_storage_state(self) -> Option<StorageState> {
match self {
Self::None | Self::HostPressure => None,
Self::BusyLocked | Self::BusyRecovery | Self::SnapshotConflict => {
Some(StorageState::BusyOrLocked)
}
Self::StaleWalSidecar => Some(StorageState::WalSidecarSuspect),
Self::StaleSearcherCache => Some(StorageState::DerivedOnlyDrift),
}
}
pub(crate) fn recommended_command(self) -> Option<&'static str> {
match self {
Self::BusyLocked | Self::BusyRecovery | Self::SnapshotConflict => {
Some("cass status --json")
}
Self::StaleWalSidecar => Some("cass doctor check --json"),
Self::StaleSearcherCache => Some("cass status --json"),
Self::HostPressure => Some("cass status --json"),
Self::None => None,
}
}
pub(crate) fn explanation(self) -> &'static str {
match self {
Self::None => "no storage contention observed",
Self::BusyLocked => {
"another writer holds the lock; this is contention, not missing data — retry after a bounded backoff"
}
Self::BusyRecovery => {
"the database is recovering a hot WAL; this is transient, not corruption — retry after a bounded backoff"
}
Self::SnapshotConflict => {
"an MVCC snapshot/serialization conflict; the transaction can be retried after a bounded backoff"
}
Self::StaleWalSidecar => {
"a WAL/SHM sidecar is stale or orphaned; the canonical rows are intact — checkpoint/recover, do not treat as loss"
}
Self::StaleSearcherCache => {
"a cached searcher is serving an older generation; reload/invalidate it — the published index is not lost"
}
Self::HostPressure => {
"host resource pressure (disk/memory/load) is the proximate cause; waiting will not clear it — relieve host pressure"
}
}
}
}
pub(crate) fn classify_franken_error(
err: &crate::franken_sync::FrankenError,
) -> Option<ContentionClass> {
use crate::franken_sync::FrankenError as E;
match err {
E::Busy | E::DatabaseLocked { .. } | E::LockFailed { .. } => {
Some(ContentionClass::BusyLocked)
}
E::BusyRecovery => Some(ContentionClass::BusyRecovery),
E::BusySnapshot { .. } | E::WriteConflict { .. } | E::SerializationFailure { .. } => {
Some(ContentionClass::SnapshotConflict)
}
_ => None,
}
}
pub(crate) fn is_retryable_contention(err: &crate::franken_sync::FrankenError) -> bool {
classify_franken_error(err)
.is_some_and(|c| matches!(c.retryability(), Retryability::RetryAfterBackoff))
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum Retryability {
RetryAfterBackoff,
RetryAfterInspection,
NotRetryable,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) struct BoundedWaitGuidance {
pub max_attempts: u32,
pub initial_backoff_ms: u64,
pub max_backoff_ms: u64,
pub max_total_wait_ms: u64,
pub jittered: bool,
}
impl BoundedWaitGuidance {
pub(crate) fn transient_lock() -> Self {
Self {
max_attempts: 6,
initial_backoff_ms: 2,
max_backoff_ms: 256,
max_total_wait_ms: 2_000,
jittered: true,
}
}
pub(crate) fn after_inspection() -> Self {
Self {
max_attempts: 3,
initial_backoff_ms: 10,
max_backoff_ms: 100,
max_total_wait_ms: 500,
jittered: true,
}
}
pub(crate) fn is_bounded(&self) -> bool {
self.max_attempts > 0 && self.max_total_wait_ms > 0 && self.max_total_wait_ms < u64::MAX
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum CacheStaleness {
Fresh,
StaleGeneration,
ReloadFailed,
}
impl CacheStaleness {
pub(crate) fn classify(
published_generation: Option<&str>,
cached_generation: Option<&str>,
reload_ok: bool,
) -> Self {
if !reload_ok {
return Self::ReloadFailed;
}
match (published_generation, cached_generation) {
(Some(p), Some(c)) if p == c => Self::Fresh,
(None, _) | (_, None) => Self::Fresh,
_ => Self::StaleGeneration,
}
}
pub(crate) fn to_contention_class(self) -> ContentionClass {
match self {
Self::Fresh => ContentionClass::None,
Self::StaleGeneration | Self::ReloadFailed => ContentionClass::StaleSearcherCache,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
pub(crate) struct LockEvidence {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub holder_pid: Option<u32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub note: Option<String>,
pub source: String,
}
impl LockEvidence {
pub(crate) fn from_busy_timeout() -> Self {
Self {
holder_pid: None,
note: Some("busy lock observed; holder identity not available on this platform".into()),
source: "busy_timeout_expiry".to_string(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) struct ContentionReport {
pub schema_version: u32,
pub report_kind: String,
pub class: ContentionClass,
pub retryability: Retryability,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub bounded_wait: Option<BoundedWaitGuidance>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub storage_state: Option<StorageState>,
pub is_archive_loss: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub evidence: Option<LockEvidence>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub recommended_command: Option<String>,
pub explanation: String,
}
impl ContentionReport {
pub(crate) fn classify(class: ContentionClass, evidence: Option<LockEvidence>) -> Self {
Self {
schema_version: CONTENTION_REPORT_SCHEMA_VERSION,
report_kind: CONTENTION_REPORT_KIND.to_string(),
class,
retryability: class.retryability(),
bounded_wait: class.bounded_wait(),
storage_state: class.to_storage_state(),
is_archive_loss: class.is_archive_loss(),
evidence,
recommended_command: class.recommended_command().map(str::to_string),
explanation: class.explanation().to_string(),
}
}
pub(crate) fn from_franken_error(err: &crate::franken_sync::FrankenError) -> Option<Self> {
classify_franken_error(err).map(|class| Self::classify(class, None))
}
}
#[cfg(test)]
mod tests {
use super::*;
const ALL_CLASSES: &[ContentionClass] = &[
ContentionClass::None,
ContentionClass::BusyLocked,
ContentionClass::BusyRecovery,
ContentionClass::SnapshotConflict,
ContentionClass::StaleWalSidecar,
ContentionClass::StaleSearcherCache,
ContentionClass::HostPressure,
];
#[test]
fn classes_serialize_snake_case_and_are_stable() {
let pairs: &[(ContentionClass, &str)] = &[
(ContentionClass::None, "none"),
(ContentionClass::BusyLocked, "busy_locked"),
(ContentionClass::BusyRecovery, "busy_recovery"),
(ContentionClass::SnapshotConflict, "snapshot_conflict"),
(ContentionClass::StaleWalSidecar, "stale_wal_sidecar"),
(ContentionClass::StaleSearcherCache, "stale_searcher_cache"),
(ContentionClass::HostPressure, "host_pressure"),
];
for (variant, want) in pairs {
assert_eq!(
serde_json::to_string(variant).expect("serialize class"),
format!("\"{want}\"")
);
assert_eq!(variant.stable_name(), *want);
}
assert_eq!(pairs.len(), ALL_CLASSES.len());
}
#[test]
fn contention_is_never_archive_loss() {
for &class in ALL_CLASSES {
assert!(
!class.is_archive_loss(),
"{class:?} must not be archive loss"
);
let report = ContentionReport::classify(class, None);
assert!(!report.is_archive_loss, "{class:?} report archive loss");
}
}
#[test]
fn retryability_matches_class_semantics() {
assert_eq!(
ContentionClass::BusyLocked.retryability(),
Retryability::RetryAfterBackoff
);
assert_eq!(
ContentionClass::BusyRecovery.retryability(),
Retryability::RetryAfterBackoff
);
assert_eq!(
ContentionClass::SnapshotConflict.retryability(),
Retryability::RetryAfterBackoff
);
assert_eq!(
ContentionClass::StaleWalSidecar.retryability(),
Retryability::RetryAfterInspection
);
assert_eq!(
ContentionClass::StaleSearcherCache.retryability(),
Retryability::RetryAfterInspection
);
assert_eq!(
ContentionClass::HostPressure.retryability(),
Retryability::NotRetryable
);
assert_eq!(
ContentionClass::None.retryability(),
Retryability::NotRetryable
);
}
#[test]
fn bounded_wait_is_always_finite_and_present_only_when_retryable() {
for &class in ALL_CLASSES {
match class.retryability() {
Retryability::NotRetryable => {
assert!(class.bounded_wait().is_none(), "{class:?} should not wait");
}
_ => {
let wait = class
.bounded_wait()
.expect("retryable class has a wait policy");
assert!(wait.is_bounded(), "{class:?} wait must be bounded");
assert!(wait.max_total_wait_ms > 0 && wait.max_total_wait_ms < u64::MAX);
assert!(wait.max_attempts > 0);
}
}
}
}
#[test]
fn storage_state_mapping_is_consistent_with_taxonomy() {
assert_eq!(
ContentionClass::BusyLocked.to_storage_state(),
Some(StorageState::BusyOrLocked)
);
assert_eq!(
ContentionClass::SnapshotConflict.to_storage_state(),
Some(StorageState::BusyOrLocked)
);
assert_eq!(
ContentionClass::StaleWalSidecar.to_storage_state(),
Some(StorageState::WalSidecarSuspect)
);
assert_eq!(
ContentionClass::StaleSearcherCache.to_storage_state(),
Some(StorageState::DerivedOnlyDrift)
);
assert_eq!(ContentionClass::HostPressure.to_storage_state(), None);
assert_eq!(ContentionClass::None.to_storage_state(), None);
}
#[test]
fn recommended_commands_are_concrete_and_never_destructive() {
for &class in ALL_CLASSES {
if let Some(cmd) = class.recommended_command() {
assert!(cmd.starts_with("cass "), "must be concrete cass: {cmd}");
assert_ne!(cmd.trim(), "cass");
for bad in [
"rm ",
"rm -",
"delete ",
"DROP ",
"--purge",
"--force-clean",
] {
assert!(!cmd.contains(bad), "destructive token in {cmd}");
}
}
}
}
#[test]
fn cache_staleness_classification() {
assert_eq!(
CacheStaleness::classify(Some("g7"), Some("g7"), true),
CacheStaleness::Fresh
);
assert_eq!(
CacheStaleness::classify(Some("g8"), Some("g7"), true),
CacheStaleness::StaleGeneration
);
assert_eq!(
CacheStaleness::classify(Some("g8"), Some("g7"), false),
CacheStaleness::ReloadFailed
);
assert_eq!(
CacheStaleness::classify(None, Some("g7"), true),
CacheStaleness::Fresh
);
assert_eq!(
CacheStaleness::StaleGeneration.to_contention_class(),
ContentionClass::StaleSearcherCache
);
assert!(
!CacheStaleness::ReloadFailed
.to_contention_class()
.is_archive_loss()
);
}
#[test]
fn report_round_trips_through_json_with_invariant() {
let report = ContentionReport::classify(
ContentionClass::BusyLocked,
Some(LockEvidence::from_busy_timeout()),
);
let json = serde_json::to_string(&report).expect("serialize report");
assert!(json.contains("\"report_kind\":\"contention_diagnostic\""));
assert!(json.contains("\"class\":\"busy_locked\""));
assert!(json.contains("\"is_archive_loss\":false"));
assert!(json.contains("\"retryability\":\"retry_after_backoff\""));
let parsed: ContentionReport = serde_json::from_str(&json).expect("parse report");
assert_eq!(parsed, report);
}
#[test]
fn lock_evidence_is_platform_tolerant() {
let ev = LockEvidence::from_busy_timeout();
assert!(ev.holder_pid.is_none());
assert_eq!(ev.source, "busy_timeout_expiry");
assert!(ev.note.is_some());
}
#[test]
fn classify_franken_error_maps_busy_variants_and_skips_corruption() {
use crate::franken_sync::FrankenError as E;
assert_eq!(
classify_franken_error(&E::Busy),
Some(ContentionClass::BusyLocked)
);
assert_eq!(
classify_franken_error(&E::BusyRecovery),
Some(ContentionClass::BusyRecovery)
);
assert_eq!(
classify_franken_error(&E::DatabaseLocked {
path: std::path::PathBuf::from("/tmp/locked.db"),
}),
Some(ContentionClass::BusyLocked)
);
assert_eq!(
classify_franken_error(&E::LockFailed {
detail: "reserved lock held".to_string(),
}),
Some(ContentionClass::BusyLocked)
);
assert!(is_retryable_contention(&E::Busy));
assert!(is_retryable_contention(&E::BusyRecovery));
assert_eq!(classify_franken_error(&E::QueryReturnedNoRows), None);
assert!(!is_retryable_contention(&E::QueryReturnedNoRows));
}
}
#[cfg(test)]
mod contention_integration_tests {
use super::{ContentionClass, classify_franken_error};
use crate::franken_sync::compat::{RowExt, TransactionExt};
use crate::franken_sync::params as fparams;
use crate::storage::sqlite::{ConnectionManagerConfig, FrankenConnectionManager, WriterGuard};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use tempfile::TempDir;
fn try_increment(guard: &WriterGuard<'_>) -> anyhow::Result<()> {
let mut tx = guard.storage().raw().transaction()?;
tx.execute_compat(
"UPDATE counter SET v = v + ?1 WHERE id = 1",
fparams![1_i64],
)?;
tx.commit()?;
Ok(())
}
#[test]
fn concurrent_writers_on_hot_row_classify_as_retryable_contention_and_converge() {
let dir = TempDir::new().expect("temp dir");
let db_path = dir.path().join("contention.db");
let config = ConnectionManagerConfig {
reader_count: 2,
max_writers: 4,
};
let mgr = FrankenConnectionManager::new(&db_path, config).expect("open manager");
{
let mut guard = mgr.writer().expect("writer");
guard
.storage()
.raw()
.execute("CREATE TABLE counter (id INTEGER PRIMARY KEY, v INTEGER NOT NULL)")
.expect("create table");
guard
.storage()
.raw()
.execute("INSERT INTO counter (id, v) VALUES (1, 0)")
.expect("seed counter");
guard.mark_committed();
}
let num_threads = 6;
let incr_per_thread = 60;
let classified_conflicts = Arc::new(AtomicUsize::new(0));
let archive_loss_seen = Arc::new(AtomicUsize::new(0));
let non_contention_errors = Arc::new(AtomicUsize::new(0));
std::thread::scope(|s| {
for _ in 0..num_threads {
let m = &mgr;
let conflicts = Arc::clone(&classified_conflicts);
let losses = Arc::clone(&archive_loss_seen);
let unexpected = Arc::clone(&non_contention_errors);
s.spawn(move || {
for _ in 0..incr_per_thread {
let mut attempt: u32 = 0;
loop {
let mut guard = m.concurrent_writer().expect("concurrent writer");
let result = try_increment(&guard);
match result {
Ok(()) => {
guard.mark_committed();
break;
}
Err(err) => {
let franken = err
.downcast_ref::<crate::franken_sync::FrankenError>()
.or_else(|| {
err.root_cause()
.downcast_ref::<crate::franken_sync::FrankenError>()
});
let class = franken.and_then(classify_franken_error);
match class {
Some(c) => {
conflicts.fetch_add(1, Ordering::Relaxed);
if c.is_archive_loss() {
losses.fetch_add(1, Ordering::Relaxed);
}
attempt += 1;
assert!(
attempt < 500,
"bounded retry must converge, not spin"
);
let backoff = (1u64 << attempt.min(8)).min(256);
std::thread::sleep(Duration::from_millis(backoff));
}
None => {
unexpected.fetch_add(1, Ordering::Relaxed);
break;
}
}
}
}
}
}
});
}
});
assert_eq!(
non_contention_errors.load(Ordering::Relaxed),
0,
"all errors under write contention must be contention classes"
);
assert_eq!(
archive_loss_seen.load(Ordering::Relaxed),
0,
"no contention error may report archive loss"
);
let reader = mgr.reader();
let rows = reader
.query("SELECT v FROM counter WHERE id = 1")
.expect("read counter");
let final_v: i64 = rows[0].get_typed(0).expect("typed counter");
assert_eq!(
final_v,
(num_threads * incr_per_thread) as i64,
"every increment must be durably applied (no lost updates)"
);
let observed = classified_conflicts.load(Ordering::Relaxed);
eprintln!("hot-row contention: {observed} classified retryable conflicts");
assert!(
observed >= 1,
"hot-row contention should raise at least one classified conflict"
);
let busy = ContentionClass::SnapshotConflict;
assert!(busy.to_storage_state().is_some());
assert!(!busy.is_archive_loss());
}
}