use rusqlite::Connection;
use std::collections::HashMap;
use std::panic::{catch_unwind, AssertUnwindSafe};
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex, OnceLock};
use std::time::{Duration, Instant};
use tokio::sync::{mpsc, oneshot};
use khive_storage::error::{StorageError, WriterTaskRequestState};
use crate::error::SqliteError;
use crate::pool::{ConnectionPool, WriterAcquisitionCounters};
const WRITER_BEGIN_RETRY_DELAYS: [Duration; 2] =
[Duration::from_millis(5), Duration::from_millis(10)];
type WriteOp<R> = Box<dyn FnOnce(&Connection) -> Result<R, StorageError> + Send>;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct WriterStageObservation {
pub queue_wait_micros: u64,
pub transaction_acquire_micros: u64,
pub body_micros: u64,
pub commit_micros: u64,
pub total_micros: u64,
pub queue_depth_at_entry: u64,
pub observed_at_unix_ms: u64,
}
static WRITER_STAGE_OBSERVATIONS: OnceLock<
Mutex<HashMap<Option<PathBuf>, WriterStageObservation>>,
> = OnceLock::new();
fn writer_stage_observations() -> &'static Mutex<HashMap<Option<PathBuf>, WriterStageObservation>> {
WRITER_STAGE_OBSERVATIONS.get_or_init(|| Mutex::new(HashMap::new()))
}
fn writer_db_key_from_path(path: Option<&Path>) -> Option<PathBuf> {
path.map(Path::to_path_buf)
}
fn writer_db_key(pool: &ConnectionPool) -> Option<PathBuf> {
writer_db_key_from_path(pool.canonical_path())
}
fn duration_micros(duration: Duration) -> u64 {
duration.as_micros().min(u128::from(u64::MAX)) as u64
}
fn observed_at_unix_ms() -> u64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|duration| duration.as_millis() as u64)
.unwrap_or(0)
}
pub fn last_writer_stage_observation(pool: &ConnectionPool) -> Option<WriterStageObservation> {
writer_stage_observations()
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.get(&writer_db_key(pool))
.cloned()
}
struct WriteTelemetry {
backend_key: Option<PathBuf>,
db: String,
submitted_at: Instant,
queue_depth_at_entry: usize,
slow_write_threshold: Option<Duration>,
}
impl WriteTelemetry {
fn new(
backend_key: Option<PathBuf>,
db: String,
queue_depth_at_entry: usize,
slow_write_threshold: Option<Duration>,
) -> Self {
Self {
backend_key,
db,
submitted_at: Instant::now(),
queue_depth_at_entry,
slow_write_threshold,
}
}
fn queue_wait(&self) -> Duration {
self.submitted_at.elapsed()
}
fn finish(
self,
queue_wait: Duration,
transaction_acquire: Duration,
body: Duration,
commit: Duration,
) {
let total = self.submitted_at.elapsed();
let observation = WriterStageObservation {
queue_wait_micros: duration_micros(queue_wait),
transaction_acquire_micros: duration_micros(transaction_acquire),
body_micros: duration_micros(body),
commit_micros: duration_micros(commit),
total_micros: duration_micros(total),
queue_depth_at_entry: self.queue_depth_at_entry as u64,
observed_at_unix_ms: observed_at_unix_ms(),
};
writer_stage_observations()
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(self.backend_key, observation.clone());
if self
.slow_write_threshold
.is_some_and(|threshold| total >= threshold)
{
crate::timeout_sink::emit_slow_write(&self.db, &observation);
}
}
}
pub struct WriteRequest<R: Send + 'static> {
op: WriteOp<R>,
reply: oneshot::Sender<Result<R, StorageError>>,
top_level: bool,
telemetry: WriteTelemetry,
}
mod sealed {
pub trait Sealed {
fn execute_and_reply_reporting_terminal(
self: Box<Self>,
conn: &rusqlite::Connection,
tx_span: Option<khive_storage::tx_registry::TxHandle>,
queue_wait: std::time::Duration,
transaction_acquire: std::time::Duration,
) -> Option<khive_storage::error::WriterTaskRequestState>;
fn execute_and_reply_top_level_reporting_terminal(
self: Box<Self>,
conn: &rusqlite::Connection,
queue_wait: std::time::Duration,
) -> Option<khive_storage::error::WriterTaskRequestState>;
fn reply_error_after_begin(
self: Box<Self>,
err: khive_storage::error::StorageError,
queue_wait: std::time::Duration,
transaction_acquire: std::time::Duration,
);
}
}
pub trait AnyWriteRequest: sealed::Sealed + Send {
fn execute_and_reply(self: Box<Self>, conn: &Connection);
fn execute_and_reply_top_level(self: Box<Self>, conn: &Connection);
fn reply_error(self: Box<Self>, err: StorageError);
fn is_top_level(&self) -> bool;
fn queue_wait(&self) -> Duration;
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum RollbackDisposition {
RolledBack,
SideEffectsUnknown,
}
fn rollback_after_failure(conn: &Connection, failure_context: &'static str) -> RollbackDisposition {
match conn.execute_batch("ROLLBACK") {
Ok(()) if conn.is_autocommit() => RollbackDisposition::RolledBack,
Ok(()) => {
tracing::error!(
failure_context,
"writer transaction: ROLLBACK returned success but the connection is still in a \
transaction; request side effects are unknown"
);
RollbackDisposition::SideEffectsUnknown
}
Err(rollback_error) => {
tracing::error!(
error = %rollback_error,
failure_context,
"writer transaction: rollback after request failure failed; request side effects are \
unknown"
);
RollbackDisposition::SideEffectsUnknown
}
}
}
pub(crate) fn execute_wrapped_transaction<R, F>(
conn: &Connection,
commit_operation: &'static str,
operation: F,
) -> (Result<R, StorageError>, Option<WriterTaskRequestState>)
where
F: FnOnce(&Connection) -> Result<R, StorageError>,
{
let profiled = execute_wrapped_transaction_profiled(conn, commit_operation, operation);
(profiled.result, profiled.terminal_state)
}
struct ProfiledWrappedTransaction<R> {
result: Result<R, StorageError>,
terminal_state: Option<WriterTaskRequestState>,
body: Duration,
commit: Duration,
}
fn execute_wrapped_transaction_profiled<R, F>(
conn: &Connection,
commit_operation: &'static str,
operation: F,
) -> ProfiledWrappedTransaction<R>
where
F: FnOnce(&Connection) -> Result<R, StorageError>,
{
let body_started = Instant::now();
let operation_outcome = catch_unwind(AssertUnwindSafe(|| operation(conn)));
let body = body_started.elapsed();
match operation_outcome {
Ok(Ok(value)) => {
let commit_started = Instant::now();
let commit_outcome = conn.execute_batch("COMMIT");
let commit = commit_started.elapsed();
match commit_outcome {
Ok(()) if conn.is_autocommit() => ProfiledWrappedTransaction {
result: Ok(value),
terminal_state: None,
body,
commit,
},
Ok(()) => {
tracing::error!(
"writer transaction: COMMIT returned success but the connection is still in \
a transaction; request side effects are unknown"
);
let request_state = WriterTaskRequestState::SideEffectsUnknown;
ProfiledWrappedTransaction {
result: Err(writer_task_terminated(request_state)),
terminal_state: Some(request_state),
body,
commit,
}
}
Err(commit_error) => match rollback_after_failure(conn, "commit failure") {
RollbackDisposition::RolledBack => ProfiledWrappedTransaction {
result: Err(StorageError::WriterTaskRequestFailed {
request_state: WriterTaskRequestState::TransactionRolledBack,
source: Box::new(StorageError::Pool {
operation: commit_operation.into(),
message: commit_error.to_string(),
}),
}),
terminal_state: None,
body,
commit,
},
RollbackDisposition::SideEffectsUnknown => {
let request_state = WriterTaskRequestState::SideEffectsUnknown;
ProfiledWrappedTransaction {
result: Err(writer_task_terminated(request_state)),
terminal_state: Some(request_state),
body,
commit,
}
}
},
}
}
Ok(Err(operation_error)) => {
match rollback_after_failure(conn, "request operation failure") {
RollbackDisposition::RolledBack => ProfiledWrappedTransaction {
result: Err(StorageError::WriterTaskRequestFailed {
request_state: WriterTaskRequestState::TransactionRolledBack,
source: Box::new(operation_error),
}),
terminal_state: None,
body,
commit: Duration::ZERO,
},
RollbackDisposition::SideEffectsUnknown => {
let request_state = WriterTaskRequestState::SideEffectsUnknown;
ProfiledWrappedTransaction {
result: Err(writer_task_terminated(request_state)),
terminal_state: Some(request_state),
body,
commit: Duration::ZERO,
}
}
}
}
Err(_panic_payload) => {
let request_state = match rollback_after_failure(conn, "request panic") {
RollbackDisposition::RolledBack => WriterTaskRequestState::TransactionRolledBack,
RollbackDisposition::SideEffectsUnknown => {
WriterTaskRequestState::SideEffectsUnknown
}
};
ProfiledWrappedTransaction {
result: Err(writer_task_terminated(request_state)),
terminal_state: Some(request_state),
body,
commit: Duration::ZERO,
}
}
}
}
impl<R: Send + 'static> sealed::Sealed for WriteRequest<R> {
fn execute_and_reply_reporting_terminal(
self: Box<Self>,
conn: &Connection,
tx_span: Option<khive_storage::tx_registry::TxHandle>,
queue_wait: Duration,
transaction_acquire: Duration,
) -> Option<WriterTaskRequestState> {
let WriteRequest {
op,
reply,
telemetry,
..
} = *self;
let profiled = execute_wrapped_transaction_profiled(conn, "writer_task_commit", op);
drop(tx_span);
telemetry.finish(
queue_wait,
transaction_acquire,
profiled.body,
profiled.commit,
);
let _ = reply.send(profiled.result);
profiled.terminal_state
}
fn execute_and_reply_top_level_reporting_terminal(
self: Box<Self>,
conn: &Connection,
queue_wait: Duration,
) -> Option<WriterTaskRequestState> {
let WriteRequest {
op,
reply,
telemetry,
..
} = *self;
let body_started = Instant::now();
let outcome = catch_unwind(AssertUnwindSafe(|| op(conn)));
let body = body_started.elapsed();
telemetry.finish(queue_wait, Duration::ZERO, body, Duration::ZERO);
match outcome {
Ok(outcome) if conn.is_autocommit() => {
let _ = reply.send(outcome);
None
}
Ok(_outcome) => {
tracing::error!(
"writer task: top-level request returned with an open transaction; request \
side effects are unknown"
);
let request_state = WriterTaskRequestState::SideEffectsUnknown;
let _ = reply.send(Err(writer_task_terminated(request_state)));
Some(request_state)
}
Err(_panic_payload) => {
let request_state = WriterTaskRequestState::SideEffectsUnknown;
let _ = reply.send(Err(writer_task_terminated(request_state)));
Some(request_state)
}
}
}
fn reply_error_after_begin(
self: Box<Self>,
err: StorageError,
queue_wait: Duration,
transaction_acquire: Duration,
) {
let WriteRequest {
reply, telemetry, ..
} = *self;
telemetry.finish(
queue_wait,
transaction_acquire,
Duration::ZERO,
Duration::ZERO,
);
let _ = reply.send(Err(err));
}
}
impl<R: Send + 'static> AnyWriteRequest for WriteRequest<R> {
fn execute_and_reply(self: Box<Self>, conn: &Connection) {
let queue_wait = self.queue_wait();
let _ = sealed::Sealed::execute_and_reply_reporting_terminal(
self,
conn,
None,
queue_wait,
Duration::ZERO,
);
}
fn execute_and_reply_top_level(self: Box<Self>, conn: &Connection) {
let queue_wait = self.queue_wait();
let _ =
sealed::Sealed::execute_and_reply_top_level_reporting_terminal(self, conn, queue_wait);
}
fn reply_error(self: Box<Self>, err: StorageError) {
let queue_wait = self.queue_wait();
sealed::Sealed::reply_error_after_begin(self, err, queue_wait, Duration::ZERO);
}
fn is_top_level(&self) -> bool {
self.top_level
}
fn queue_wait(&self) -> Duration {
self.telemetry.queue_wait()
}
}
fn writer_task_terminated(request_state: WriterTaskRequestState) -> StorageError {
StorageError::WriterTaskTerminated { request_state }
}
fn writer_task_begin_error(error: rusqlite::Error, busy_timeout: Duration) -> StorageError {
if crate::timeout_sink::is_busy_or_locked(&error) {
StorageError::WriterTaskBusy {
timeout_ms: u64::try_from(busy_timeout.as_millis()).unwrap_or(u64::MAX),
}
} else {
StorageError::Pool {
operation: "writer_task_begin".into(),
message: error.to_string(),
}
}
}
#[derive(Clone, Debug)]
pub struct WriterTaskHandle {
tx: mpsc::Sender<Box<dyn AnyWriteRequest + Send>>,
backend_key: Option<PathBuf>,
db: String,
slow_write_threshold: Option<std::time::Duration>,
enqueue_timeout: std::time::Duration,
}
impl WriterTaskHandle {
async fn enqueue<R, F>(
&self,
op: F,
) -> Result<oneshot::Receiver<Result<R, StorageError>>, StorageError>
where
R: Send + 'static,
F: FnOnce(&Connection) -> Result<R, StorageError> + Send + 'static,
{
self.enqueue_inner(op, false).await
}
async fn enqueue_inner<R, F>(
&self,
op: F,
top_level: bool,
) -> Result<oneshot::Receiver<Result<R, StorageError>>, StorageError>
where
R: Send + 'static,
F: FnOnce(&Connection) -> Result<R, StorageError> + Send + 'static,
{
let (reply_tx, reply_rx) = oneshot::channel();
let telemetry = WriteTelemetry::new(
self.backend_key.clone(),
self.db.clone(),
self.queue_depth(),
self.slow_write_threshold,
);
let request = WriteRequest {
op: Box::new(op),
reply: reply_tx,
top_level,
telemetry,
};
self.tx
.send(Box::new(request))
.await
.map_err(|_| writer_task_terminated(WriterTaskRequestState::NotStarted))?;
Ok(reply_rx)
}
pub async fn send<R, F>(&self, op: F) -> Result<R, StorageError>
where
R: Send + 'static,
F: FnOnce(&Connection) -> Result<R, StorageError> + Send + 'static,
{
let reply_rx = self.enqueue(op).await?;
reply_rx
.await
.map_err(|_| writer_task_terminated(WriterTaskRequestState::SideEffectsUnknown))?
}
pub async fn send_with_timeout<R, F>(
&self,
op: F,
timeout: std::time::Duration,
) -> Result<R, StorageError>
where
R: Send + 'static,
F: FnOnce(&Connection) -> Result<R, StorageError> + Send + 'static,
{
let reply_rx = match tokio::time::timeout(timeout, self.enqueue(op)).await {
Ok(Ok(reply_rx)) => reply_rx,
Ok(Err(e)) => return Err(e),
Err(_elapsed) => {
let timeout_ms = timeout.as_millis() as u64;
crate::timeout_sink::emit_queue_saturation(&self.db, timeout_ms);
return Err(StorageError::WriteQueueFull { timeout_ms });
}
};
reply_rx
.await
.map_err(|_| writer_task_terminated(WriterTaskRequestState::SideEffectsUnknown))?
}
pub async fn send_bounded<R, F>(&self, op: F) -> Result<R, StorageError>
where
R: Send + 'static,
F: FnOnce(&Connection) -> Result<R, StorageError> + Send + 'static,
{
self.send_with_timeout(op, self.enqueue_timeout).await
}
pub async fn send_top_level<R, F>(&self, op: F) -> Result<R, StorageError>
where
R: Send + 'static,
F: FnOnce(&Connection) -> Result<R, StorageError> + Send + 'static,
{
let reply_rx = self.enqueue_inner(op, true).await?;
reply_rx
.await
.map_err(|_| writer_task_terminated(WriterTaskRequestState::SideEffectsUnknown))?
}
pub async fn send_top_level_bounded<R, F>(&self, op: F) -> Result<R, StorageError>
where
R: Send + 'static,
F: FnOnce(&Connection) -> Result<R, StorageError> + Send + 'static,
{
let reply_rx =
match tokio::time::timeout(self.enqueue_timeout, self.enqueue_inner(op, true)).await {
Ok(Ok(reply_rx)) => reply_rx,
Ok(Err(e)) => return Err(e),
Err(_elapsed) => {
let timeout_ms = self.enqueue_timeout.as_millis() as u64;
crate::timeout_sink::emit_queue_saturation(&self.db, timeout_ms);
return Err(StorageError::WriteQueueFull { timeout_ms });
}
};
reply_rx
.await
.map_err(|_| writer_task_terminated(WriterTaskRequestState::SideEffectsUnknown))?
}
pub fn queue_depth(&self) -> usize {
self.tx.max_capacity() - self.tx.capacity()
}
pub fn capacity(&self) -> usize {
self.tx.max_capacity()
}
}
pub fn spawn(pool: &ConnectionPool, capacity: usize) -> Result<WriterTaskHandle, SqliteError> {
let conn = pool.open_standalone_writer_untracked()?;
let acquisition_counters = pool.writer_acquisition_counters();
let busy_timeout = pool.config().busy_timeout;
let origin = pool.origin();
let backend_key = writer_db_key(pool);
let db = crate::timeout_sink::db_label(pool);
let (tx, rx) = mpsc::channel(capacity.max(1));
let join = tokio::spawn(run_writer_task(
conn,
rx,
origin,
db.clone(),
acquisition_counters,
busy_timeout,
));
pool.set_writer_task_join(join);
Ok(WriterTaskHandle {
tx,
backend_key,
db,
slow_write_threshold: crate::timeout_sink::slow_write_threshold(),
enqueue_timeout: std::time::Duration::from_millis(
pool.config().write_admission_deadline_ms,
),
})
}
async fn close_and_fail_queued_requests(rx: &mut mpsc::Receiver<Box<dyn AnyWriteRequest + Send>>) {
rx.close();
while let Some(request) = rx.recv().await {
request.reply_error(writer_task_terminated(WriterTaskRequestState::NotStarted));
}
}
fn begin_immediate_with_retry(
conn: &Connection,
acquisition_counters: &WriterAcquisitionCounters,
busy_timeout: Duration,
mut set_busy_timeout: impl FnMut(&Connection, Duration) -> rusqlite::Result<()>,
) -> (rusqlite::Result<()>, Duration, u32) {
let transaction_acquire_started = Instant::now();
let mut begin_attempt = 1_u32;
let mut retry_delays = WRITER_BEGIN_RETRY_DELAYS.into_iter();
let mut busy_timeout_lowered = false;
let begin_outcome = loop {
match conn.execute_batch("BEGIN IMMEDIATE") {
Ok(()) => break Ok(()),
Err(error) if crate::timeout_sink::is_busy_or_locked(&error) => {
acquisition_counters.record_writer_task_begin_busy();
let Some(delay) = retry_delays.next() else {
break Err(error);
};
let remaining_budget =
busy_timeout.saturating_sub(transaction_acquire_started.elapsed());
if remaining_budget.is_zero() {
break Err(error);
}
if let Err(set_err) = set_busy_timeout(conn, remaining_budget) {
tracing::warn!(
error = %set_err,
"writer task: failed to lower busy_timeout for BEGIN \
retry; surfacing the original busy refusal"
);
break Err(error);
}
busy_timeout_lowered = true;
acquisition_counters.record_writer_task_begin_busy_absorbed();
tracing::debug!(
attempt = begin_attempt,
backoff_ms = delay.as_millis() as u64,
budget_remaining_ms = remaining_budget.as_millis() as u64,
"writer task: BEGIN IMMEDIATE refused busy; retrying before \
request execution"
);
std::thread::sleep(delay);
begin_attempt = begin_attempt.saturating_add(1);
}
Err(error) => break Err(error),
}
};
let transaction_acquire = transaction_acquire_started.elapsed();
if busy_timeout_lowered {
if let Err(restore_err) = set_busy_timeout(conn, busy_timeout) {
tracing::warn!(
error = %restore_err,
"writer task: failed to restore busy_timeout after a BEGIN retry \
sequence"
);
}
}
(begin_outcome, transaction_acquire, begin_attempt)
}
async fn run_writer_task(
mut conn: Connection,
mut rx: mpsc::Receiver<Box<dyn AnyWriteRequest + Send>>,
origin: khive_storage::tx_registry::TxOrigin,
db: String,
acquisition_counters: Arc<WriterAcquisitionCounters>,
busy_timeout: Duration,
) {
while let Some(request) = rx.recv().await {
let queue_wait = request.queue_wait();
let origin = origin.clone();
let blocking_counters = Arc::clone(&acquisition_counters);
let outcome = tokio::task::spawn_blocking(move || {
let acquisition_counters = blocking_counters;
if !conn.is_autocommit() {
tracing::error!(
"writer task: connection is not in autocommit mode before request dispatch; \
retiring the poisoned writer without running the request"
);
let request_state = WriterTaskRequestState::NotStarted;
request.reply_error(writer_task_terminated(request_state));
return (conn, Some(request_state));
}
let terminal_state = if request.is_top_level() {
acquisition_counters.record_writer_task_acquisition();
sealed::Sealed::execute_and_reply_top_level_reporting_terminal(
request, &conn, queue_wait,
)
} else {
let tx_span = khive_storage::tx_registry::register_scoped(
Some("writer_task_tx".to_string()),
origin,
);
let (begin_outcome, transaction_acquire, begin_attempt) =
begin_immediate_with_retry(
&conn,
&acquisition_counters,
busy_timeout,
Connection::busy_timeout,
);
match begin_outcome {
Ok(()) => {
acquisition_counters.record_writer_task_acquisition();
sealed::Sealed::execute_and_reply_reporting_terminal(
request,
&conn,
Some(tx_span),
queue_wait,
transaction_acquire,
)
}
Err(e) => {
tracing::warn!(
error = %e,
attempts = begin_attempt,
"writer task: BEGIN IMMEDIATE failed; replying an \
error without running the request's operation"
);
drop(tx_span);
let begin_error = writer_task_begin_error(e, busy_timeout);
if !matches!(&begin_error, StorageError::WriterTaskBusy { .. }) {
acquisition_counters.record_writer_task_begin_error();
}
sealed::Sealed::reply_error_after_begin(
request,
begin_error,
queue_wait,
transaction_acquire,
);
None
}
}
};
(conn, terminal_state)
})
.await;
match outcome {
Ok((returned_conn, None)) => conn = returned_conn,
Ok((_returned_conn, Some(request_state))) => {
acquisition_counters.record_writer_task_request_failure();
if request_state == WriterTaskRequestState::SideEffectsUnknown {
acquisition_counters.record_writer_task_side_effects_unknown();
}
tracing::error!(
request_state = %request_state,
"writer task reached a terminal request or connection state; closing and \
failing the queue without restarting"
);
crate::timeout_sink::emit_writer_task_retirement(
&db,
&format!("terminal request state: {request_state}"),
);
close_and_fail_queued_requests(&mut rx).await;
return;
}
Err(join_err) => {
acquisition_counters.record_writer_task_request_failure();
tracing::error!(
error = %join_err,
"writer task blocking closure failed outside the request \
panic boundary; closing and failing the queue without restarting"
);
crate::timeout_sink::emit_writer_task_retirement(
&db,
&format!("blocking closure join failure: {join_err}"),
);
close_and_fail_queued_requests(&mut rx).await;
return;
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::pool::PoolConfig;
use rusqlite::hooks::{AuthAction, AuthContext, Authorization, TransactionOperation};
use serial_test::serial;
#[test]
fn begin_error_classification_is_code_based_and_narrow() {
for code in [rusqlite::ffi::SQLITE_BUSY, rusqlite::ffi::SQLITE_LOCKED] {
let error = rusqlite::Error::SqliteFailure(
rusqlite::ffi::Error::new(code),
Some("rendered text is irrelevant".to_string()),
);
assert!(matches!(
writer_task_begin_error(error, Duration::from_millis(175)),
StorageError::WriterTaskBusy { timeout_ms: 175 }
));
}
let structural = rusqlite::Error::SqliteFailure(
rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_CONSTRAINT),
Some("database is locked".to_string()),
);
assert!(matches!(
writer_task_begin_error(structural, Duration::from_millis(175)),
StorageError::Pool { ref operation, .. } if operation == "writer_task_begin"
));
}
use std::future::Future;
use std::pin::Pin;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::mpsc as std_mpsc;
use std::sync::{Arc, Mutex};
use std::task::{Context, Poll, Wake, Waker};
use std::time::Duration;
fn file_pool(path: &std::path::Path) -> ConnectionPool {
let cfg = PoolConfig {
path: Some(path.to_path_buf()),
..PoolConfig::for_test()
};
ConnectionPool::new(cfg).expect("pool open")
}
fn deny_commit_and_rollback(ctx: AuthContext<'_>) -> Authorization {
match ctx.action {
AuthAction::Transaction {
operation: TransactionOperation::Unknown | TransactionOperation::Rollback,
} => Authorization::Deny,
_ => Authorization::Allow,
}
}
fn deny_commit(ctx: AuthContext<'_>) -> Authorization {
match ctx.action {
AuthAction::Transaction {
operation: TransactionOperation::Unknown,
} => Authorization::Deny,
_ => Authorization::Allow,
}
}
fn deny_rollback(ctx: AuthContext<'_>) -> Authorization {
match ctx.action {
AuthAction::Transaction {
operation: TransactionOperation::Rollback,
} => Authorization::Deny,
_ => Authorization::Allow,
}
}
fn assert_writer_task_terminal_state<T: std::fmt::Debug>(
result: Result<T, StorageError>,
expected: WriterTaskRequestState,
) {
match result {
Err(StorageError::WriterTaskTerminated { request_state }) => {
assert_eq!(request_state, expected)
}
other => panic!("expected WriterTaskTerminated({expected:?}), got {other:?}"),
}
}
struct ParkedWake {
entered: std_mpsc::SyncSender<()>,
release: Mutex<std_mpsc::Receiver<()>>,
}
impl Wake for ParkedWake {
fn wake(self: Arc<Self>) {
self.entered
.send(())
.expect("reply sender must rendezvous with the test");
self.release
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.recv()
.expect("test must release the parked reply sender");
}
}
fn arm_parked_wake<F: Future>(
mut future: Pin<&mut F>,
) -> (std_mpsc::Receiver<()>, std_mpsc::Sender<()>) {
let (entered_tx, entered_rx) = std_mpsc::sync_channel(0);
let (release_tx, release_rx) = std_mpsc::channel();
let waker = Waker::from(Arc::new(ParkedWake {
entered: entered_tx,
release: Mutex::new(release_rx),
}));
let mut context = Context::from_waker(&waker);
assert!(
matches!(future.as_mut().poll(&mut context), Poll::Pending),
"writer send must remain pending until its operation replies"
);
(entered_rx, release_tx)
}
fn poll_ready<F: Future>(mut future: Pin<&mut F>) -> F::Output {
let mut context = Context::from_waker(Waker::noop());
match future.as_mut().poll(&mut context) {
Poll::Ready(output) => output,
Poll::Pending => panic!("reply wake must make the writer send ready"),
}
}
fn database_tx_view(pool: &ConnectionPool) -> khive_storage::tx_registry::TxOriginFilter {
match pool.origin() {
khive_storage::tx_registry::TxOrigin::Database(identity) => {
khive_storage::tx_registry::TxOriginFilter::Secondary(identity)
}
other => panic!("expected a file-backed database origin, got {other:?}"),
}
}
async fn wait_for_writer_span_to_close(view: &khive_storage::tx_registry::TxOriginFilter) {
tokio::time::timeout(Duration::from_secs(5), async {
while khive_storage::tx_registry::any_open_labeled(view, "writer_task_tx") {
tokio::task::yield_now().await;
}
})
.await
.expect("writer task transaction span must eventually close");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial(tx_registry)]
async fn successful_send_reply_waits_for_writer_tx_deregistration() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("writer_task_success_reply_lifecycle.db");
let pool = file_pool(&path);
let view = database_tx_view(&pool);
let handle = spawn(&pool, 8).expect("writer task spawn");
let (op_started_tx, op_started_rx) = std_mpsc::sync_channel(0);
let (op_release_tx, op_release_rx) = std_mpsc::channel();
let send = handle.send(move |_conn| {
op_started_tx
.send(())
.expect("operation must rendezvous with the test");
op_release_rx
.recv()
.expect("test must release the operation");
Ok::<_, StorageError>(())
});
tokio::pin!(send);
let (reply_entered_rx, reply_release_tx) = arm_parked_wake(send.as_mut());
op_started_rx
.recv_timeout(Duration::from_secs(5))
.expect("writer operation must start");
op_release_tx.send(()).expect("release writer operation");
reply_entered_rx
.recv_timeout(Duration::from_secs(5))
.expect("reply sender must wake the waiting caller");
let reply = poll_ready(send.as_mut());
let span_was_open_at_reply =
khive_storage::tx_registry::any_open_labeled(&view, "writer_task_tx");
reply_release_tx
.send(())
.expect("release parked reply sender");
wait_for_writer_span_to_close(&view).await;
reply.expect("committed operation reply");
assert!(
!span_was_open_at_reply,
"a successful caller reply must not become observable while its committed writer_task_tx span remains registered"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial(tx_registry)]
async fn begin_failure_reply_waits_for_writer_tx_deregistration() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("writer_task_begin_reply_lifecycle.db");
let cfg = PoolConfig {
path: Some(path),
busy_timeout: Duration::from_millis(150),
..PoolConfig::for_test()
};
let pool = ConnectionPool::new(cfg).unwrap();
let view = database_tx_view(&pool);
let handle = spawn(&pool, 8).expect("writer task spawn");
let lock_holder = pool.try_writer().expect("pool writer");
lock_holder
.conn()
.execute_batch("BEGIN IMMEDIATE")
.expect("hold database write lock");
let op_ran = Arc::new(AtomicBool::new(false));
let op_ran_in_request = Arc::clone(&op_ran);
let send = handle.send(move |_conn| {
op_ran_in_request.store(true, Ordering::SeqCst);
Ok::<_, StorageError>(())
});
tokio::pin!(send);
let (reply_entered_rx, reply_release_tx) = arm_parked_wake(send.as_mut());
reply_entered_rx
.recv_timeout(Duration::from_secs(5))
.expect("BEGIN failure must wake the waiting caller");
let reply = poll_ready(send.as_mut());
let span_was_open_at_reply =
khive_storage::tx_registry::any_open_labeled(&view, "writer_task_tx");
reply_release_tx
.send(())
.expect("release parked reply sender");
wait_for_writer_span_to_close(&view).await;
lock_holder
.conn()
.execute_batch("ROLLBACK")
.expect("release database write lock");
assert!(
matches!(
&reply,
Err(StorageError::WriterTaskBusy { timeout_ms }) if *timeout_ms == 150
),
"expected typed retryable writer-task contention, got {reply:?}"
);
assert!(!op_ran.load(Ordering::SeqCst));
assert!(
!span_was_open_at_reply,
"a BEGIN-failure caller reply must not become observable while its writer_task_tx span remains registered"
);
}
#[tokio::test]
#[serial(tx_registry)]
async fn begin_immediate_failure_replies_error_without_running_op() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("writer_task_begin_failure.db");
let cfg = PoolConfig {
path: Some(path.clone()),
busy_timeout: Duration::from_millis(150),
..PoolConfig::for_test()
};
let pool = ConnectionPool::new(cfg).unwrap();
{
let writer = pool.try_writer().unwrap();
writer
.conn()
.execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)")
.unwrap();
}
let handle = spawn(&pool, 8).expect("writer task should spawn on a file-backed pool");
let lock_holder = pool.try_writer().unwrap();
lock_holder.conn().execute_batch("BEGIN IMMEDIATE").unwrap();
let op_ran = Arc::new(AtomicBool::new(false));
let op_ran_clone = Arc::clone(&op_ran);
let result = handle
.send(move |conn| {
op_ran_clone.store(true, Ordering::SeqCst);
conn.execute("INSERT INTO t (id, v) VALUES (99, 'should-not-land')", [])
.map_err(|e| StorageError::Pool {
operation: "test_insert".into(),
message: e.to_string(),
})
})
.await;
assert!(
matches!(
&result,
Err(StorageError::WriterTaskBusy { timeout_ms }) if *timeout_ms == 150
),
"expected a typed retryable error on contended BEGIN IMMEDIATE, got {result:?}"
);
assert!(
!op_ran.load(Ordering::SeqCst),
"the request's operation closure must never run when BEGIN \
IMMEDIATE fails — running it would land a partial write in \
autocommit mode for a request the caller is told failed"
);
lock_holder.conn().execute_batch("ROLLBACK").unwrap();
drop(lock_holder);
handle
.send(|conn| {
conn.execute("INSERT INTO t (id, v) VALUES (100, 'next-request')", [])
.map_err(|e| StorageError::Pool {
operation: "test_insert_after_busy".into(),
message: e.to_string(),
})
})
.await
.expect("transient contention must not retire the writer task");
let reader = pool.reader().expect("reader");
let count: i64 = reader
.conn()
.query_row("SELECT COUNT(*) FROM t WHERE id IN (99, 100)", [], |row| {
row.get(0)
})
.unwrap();
assert_eq!(
count, 1,
"the failed request must not land, while the next request commits on the same task"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial(tx_registry)]
async fn transient_begin_contention_clears_within_budget_and_op_runs_once() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("writer_task_begin_transient_contention.db");
let pool = ConnectionPool::new(PoolConfig {
path: Some(path),
busy_timeout: Duration::from_millis(500),
..PoolConfig::for_test()
})
.unwrap();
{
let writer = pool.try_writer().unwrap();
writer
.conn()
.execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY)")
.unwrap();
}
let handle = spawn(&pool, 8).expect("writer task spawn");
let lock_holder = pool.try_writer().unwrap();
lock_holder.conn().execute_batch("BEGIN IMMEDIATE").unwrap();
let op_runs = Arc::new(AtomicUsize::new(0));
let op_runs_in_request = Arc::clone(&op_runs);
let send_future = handle.send(move |conn| {
op_runs_in_request.fetch_add(1, Ordering::SeqCst);
conn.execute("INSERT INTO t (id) VALUES (1)", [])
.map_err(|error| StorageError::Pool {
operation: "test_insert_after_transient_contention".into(),
message: error.to_string(),
})
});
let release_future = async {
tokio::time::sleep(Duration::from_millis(50)).await;
lock_holder.conn().execute_batch("ROLLBACK").unwrap();
};
let (result, ()) = tokio::join!(send_future, release_future);
assert_eq!(
result.expect("BEGIN IMMEDIATE succeeds once the transient lock clears"),
1
);
assert_eq!(
op_runs.load(Ordering::SeqCst),
1,
"the FnOnce request closure must execute exactly once"
);
let settled = pool.writer_acquisition_snapshot();
assert_eq!(
settled.writer_task_begin_busy, 0,
"contention absorbed inside SQLite's own busy_timeout wait must never \
surface as a Rust-level refusal"
);
assert_eq!(settled.writer_task_begin_busy_absorbed, 0);
let reader = pool.reader().unwrap();
let rows: i64 = reader
.conn()
.query_row("SELECT COUNT(*) FROM t", [], |row| row.get(0))
.unwrap();
assert_eq!(rows, 1, "exactly one closure execution commits one row");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial(tx_registry)]
async fn transient_begin_refusal_retries_once_and_restores_timeout() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("writer_task_begin_transient_contention.db");
let busy_timeout = Duration::from_secs(5);
let configured_timeout_ms = i64::try_from(busy_timeout.as_millis()).unwrap();
let pool = ConnectionPool::new(PoolConfig {
path: Some(path),
busy_timeout,
..PoolConfig::for_test()
})
.unwrap();
{
let writer = pool.try_writer().unwrap();
writer
.conn()
.execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY)")
.unwrap();
}
let handle = spawn(&pool, 8).expect("writer task spawn");
let begin_attempts = Arc::new(AtomicUsize::new(0));
let begin_attempts_in_setup = Arc::clone(&begin_attempts);
handle
.send_top_level(move |conn| {
conn.busy_handler(None)
.map_err(|error| StorageError::Internal(error.to_string()))?;
count_begin_attempts(conn, begin_attempts_in_setup)
.map_err(|error| StorageError::Internal(error.to_string()))
})
.await
.expect("install connection-local contention observers");
let lock_holder = pool.try_writer().unwrap();
lock_holder.conn().execute_batch("BEGIN IMMEDIATE").unwrap();
let op_runs = Arc::new(AtomicUsize::new(0));
let op_runs_in_request = Arc::clone(&op_runs);
let send_future = handle.send(move |conn| {
op_runs_in_request.fetch_add(1, Ordering::SeqCst);
conn.execute("INSERT INTO t (id) VALUES (1)", [])
.map_err(|error| StorageError::Pool {
operation: "test_insert_after_transient_contention".into(),
message: error.to_string(),
})?;
conn.query_row("PRAGMA busy_timeout", [], |row| row.get::<_, i64>(0))
.map_err(|error| StorageError::Internal(error.to_string()))
});
let release_future = async {
let observed = tokio::time::timeout(Duration::from_secs(2), async {
while pool.writer_acquisition_snapshot().writer_task_begin_busy == 0 {
tokio::time::sleep(Duration::from_millis(1)).await;
}
})
.await;
lock_holder.conn().execute_batch("ROLLBACK").unwrap();
observed.expect("first BEGIN refusal must be observed before releasing the lock");
};
let (result, ()) = tokio::join!(send_future, release_future);
assert_eq!(
result.expect("BEGIN IMMEDIATE succeeds once the transient lock clears"),
configured_timeout_ms,
"the configured timeout must be restored before the operation runs"
);
assert_eq!(
begin_attempts.load(Ordering::SeqCst),
2,
"the request must actually retry BEGIN"
);
assert_eq!(
op_runs.load(Ordering::SeqCst),
1,
"the FnOnce request closure must execute exactly once"
);
let settled = pool.writer_acquisition_snapshot();
assert_eq!(
settled.writer_task_begin_busy, 1,
"the first real BEGIN refusal must be observed"
);
assert_eq!(settled.writer_task_begin_busy_absorbed, 1);
let next_timeout = handle
.send_top_level(|conn| {
conn.query_row("PRAGMA busy_timeout", [], |row| row.get::<_, i64>(0))
.map_err(|error| StorageError::Internal(error.to_string()))
})
.await
.unwrap();
assert_eq!(
next_timeout, configured_timeout_ms,
"the next request must retain the configured timeout"
);
assert_eq!(
begin_attempts.load(Ordering::SeqCst),
2,
"timeout probes and top-level setup must not count as BEGIN attempts"
);
let reader = pool.reader().unwrap();
let rows: i64 = reader
.conn()
.query_row("SELECT COUNT(*) FROM t", [], |row| row.get(0))
.unwrap();
assert_eq!(rows, 1, "exactly one closure execution commits one row");
}
fn count_begin_attempts(conn: &Connection, attempts: Arc<AtomicUsize>) -> rusqlite::Result<()> {
conn.authorizer(Some(move |context: AuthContext<'_>| {
if matches!(
context.action,
AuthAction::Transaction {
operation: TransactionOperation::Begin
}
) {
attempts.fetch_add(1, Ordering::SeqCst);
}
Authorization::Allow
}))
}
#[test]
fn failed_busy_timeout_reduction_stops_before_a_second_begin() {
let dir = tempfile::tempdir().unwrap();
let pool = file_pool(&dir.path().join("writer_task_timeout_update_failure.db"));
let conn = pool.open_standalone_writer_untracked().unwrap();
let busy_timeout = Duration::from_secs(5);
conn.busy_handler(None).unwrap();
let original_timeout: i64 = conn
.query_row("PRAGMA busy_timeout", [], |row| row.get(0))
.unwrap();
let attempts = Arc::new(AtomicUsize::new(0));
count_begin_attempts(&conn, Arc::clone(&attempts)).unwrap();
let lock_holder = pool.try_writer().unwrap();
lock_holder.conn().execute_batch("BEGIN IMMEDIATE").unwrap();
let counters = pool.writer_acquisition_counters();
let mut timeout_updates = Vec::new();
let (result, _, reported_attempts) =
begin_immediate_with_retry(&conn, &counters, busy_timeout, |_, timeout| {
timeout_updates.push(timeout);
Err(rusqlite::Error::InvalidQuery)
});
let error = result.expect_err("failed timeout reduction must surface the busy refusal");
assert_eq!(
error.sqlite_error_code(),
Some(rusqlite::ErrorCode::DatabaseBusy),
"preserve the original BEGIN error, not the injected setter error"
);
assert_eq!(
attempts.load(Ordering::SeqCst),
1,
"failed reduction must not issue a second BEGIN"
);
assert_eq!(reported_attempts as usize, attempts.load(Ordering::SeqCst));
assert_eq!(timeout_updates.len(), 1, "a failed first update must not trigger retries or a spurious restoration call through the injected setter");
assert!(timeout_updates[0] < busy_timeout);
assert!(!timeout_updates[0].is_zero());
let unchanged_timeout: i64 = conn
.query_row("PRAGMA busy_timeout", [], |row| row.get(0))
.unwrap();
assert_eq!(
unchanged_timeout, original_timeout,
"the failed setter did not change the connection timeout"
);
assert!(
conn.is_autocommit(),
"the refused acquisition must not open a transaction"
);
let snapshot = pool.writer_acquisition_snapshot();
assert_eq!(snapshot.writer_task_begin_busy, 1);
assert_eq!(
snapshot.writer_task_begin_busy_absorbed, 0,
"an unretried refusal must not be counted as absorbed"
);
lock_holder.conn().execute_batch("ROLLBACK").unwrap();
drop(lock_holder);
conn.busy_timeout(busy_timeout).unwrap();
let (positive, _, reported_attempts) =
begin_immediate_with_retry(&conn, &counters, busy_timeout, Connection::busy_timeout);
positive.expect(
"the same connection and observer must see a valid BEGIN once contention clears",
);
assert_eq!(attempts.load(Ordering::SeqCst), 2);
assert_eq!(
reported_attempts, 1,
"attempt count is local to this request"
);
conn.execute_batch("ROLLBACK").unwrap();
}
#[tokio::test]
#[serial(tx_registry)]
async fn contended_begin_exhaustion_separates_absorbed_and_surfaced_refusals() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("writer_task_begin_busy_counter.db");
let cfg = PoolConfig {
path: Some(path.clone()),
busy_timeout: Duration::from_millis(150),
..PoolConfig::for_test()
};
let pool = ConnectionPool::new(cfg).unwrap();
{
let writer = pool.try_writer().unwrap();
writer
.conn()
.execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY)")
.unwrap();
}
let handle = spawn(&pool, 8).expect("writer task should spawn on a file-backed pool");
let before = pool.writer_acquisition_snapshot();
assert_eq!(
before.writer_task_begin_busy, 0,
"baseline: nothing has been refused yet"
);
assert_eq!(before.writer_task_begin_busy_absorbed, 0);
let lock_holder = pool.try_writer().unwrap();
lock_holder.conn().execute_batch("BEGIN IMMEDIATE").unwrap();
let result = handle
.send(|conn| {
conn.execute("INSERT INTO t (id) VALUES (1)", [])
.map_err(|e| StorageError::Pool {
operation: "test_insert".into(),
message: e.to_string(),
})
})
.await;
assert!(
matches!(&result, Err(StorageError::WriterTaskBusy { .. })),
"precondition: the request must actually be refused busy, got {result:?}"
);
let after = pool.writer_acquisition_snapshot();
assert_eq!(
after.writer_task_begin_busy, 1,
"the single refusal the caller was told about must still be counted"
);
assert_eq!(
after.writer_task_begin_busy_absorbed, 0,
"a refusal that already spent the whole shared budget waiting out \
SQLite's own busy handler must not be retried, so nothing is absorbed"
);
assert_eq!(
after.writer_task_begin_errors, 0,
"a busy refusal must not be counted as a non-busy BEGIN error"
);
assert_eq!(
after.timeouts, before.timeouts,
"a writer-task BEGIN refusal must not be mislabeled as a pool-mutex \
checkout timeout — separate ADR-135 F6 stages, separate counters"
);
lock_holder.conn().execute_batch("ROLLBACK").unwrap();
drop(lock_holder);
handle
.send(|conn| {
conn.execute("INSERT INTO t (id) VALUES (2)", [])
.map_err(|e| StorageError::Pool {
operation: "test_insert_after_busy".into(),
message: e.to_string(),
})
})
.await
.expect("the writer task survives transient contention");
let settled = pool.writer_acquisition_snapshot();
assert_eq!(
settled.writer_task_begin_busy, 1,
"a successful request must leave the refusal counter untouched"
);
assert_eq!(
settled.writer_task_begin_busy_absorbed, 0,
"an uncontended request must not move the absorbed counter"
);
assert!(
settled.writer_task_acquisitions > after.writer_task_acquisitions,
"and it must still register as a success"
);
}
#[tokio::test]
#[serial(tx_registry)]
async fn begin_retry_budget_makes_exactly_one_attempt_under_sustained_contention() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("writer_task_begin_retry_budget.db");
let busy_timeout = Duration::from_millis(150);
let cfg = PoolConfig {
path: Some(path),
busy_timeout,
..PoolConfig::for_test()
};
let pool = ConnectionPool::new(cfg).unwrap();
{
let writer = pool.try_writer().unwrap();
writer
.conn()
.execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY)")
.unwrap();
}
let handle = spawn(&pool, 8).expect("writer task should spawn on a file-backed pool");
let lock_holder = pool.try_writer().unwrap();
lock_holder.conn().execute_batch("BEGIN IMMEDIATE").unwrap();
let result = handle
.send(|conn| {
conn.execute("INSERT INTO t (id) VALUES (1)", [])
.map_err(|e| StorageError::Pool {
operation: "test_insert".into(),
message: e.to_string(),
})
})
.await;
assert!(
matches!(&result, Err(StorageError::WriterTaskBusy { .. })),
"precondition: the request must actually be refused busy, got {result:?}"
);
let counters = pool.writer_acquisition_snapshot();
assert_eq!(
counters.writer_task_begin_busy, 1,
"one busy refusal must surface after the shared budget is spent"
);
assert_eq!(
counters.writer_task_begin_busy_absorbed, 0,
"a refusal that already consumed the whole budget must not be retried"
);
lock_holder.conn().execute_batch("ROLLBACK").unwrap();
}
#[tokio::test]
#[serial(tx_registry)]
async fn writer_task_executes_op_and_commits() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("writer_task_commit.db");
let pool = file_pool(&path);
{
let writer = pool.try_writer().unwrap();
writer
.conn()
.execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)")
.unwrap();
}
let handle = spawn(&pool, 8).expect("writer task should spawn on a file-backed pool");
let affected = handle
.send(|conn| {
conn.execute("INSERT INTO t (id, v) VALUES (1, 'hello')", [])
.map_err(|e| StorageError::Pool {
operation: "test_insert".into(),
message: e.to_string(),
})
})
.await
.expect("op should succeed");
assert_eq!(affected, 1);
let reader = pool.reader().expect("reader");
let v: String = reader
.conn()
.query_row("SELECT v FROM t WHERE id = 1", [], |row| row.get(0))
.expect("row must be committed and visible to a reader");
assert_eq!(v, "hello");
let counters = pool.writer_acquisition_snapshot();
assert_eq!(counters.acquisitions, 2);
assert_eq!(counters.pooled_acquisitions, 1);
assert_eq!(counters.standalone_acquisitions, 0);
assert_eq!(counters.writer_task_acquisitions, 1);
assert_eq!(counters.timeouts, 0);
}
#[tokio::test]
async fn writer_task_connection_follows_checkpoint_ownership_claim() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("writer_task_autocheckpoint.db");
let pool = file_pool(&path);
let handle = pool
.writer_task_handle()
.expect("writer task should spawn")
.expect("file-backed pool resolves the write queue on");
let read_pages = |handle: &WriterTaskHandle| {
let handle = handle.clone();
async move {
handle
.send_top_level(|conn| {
conn.pragma_query_value(None, "wal_autocheckpoint", |row| {
row.get::<_, u32>(0)
})
.map_err(|e| StorageError::Pool {
operation: "test_wal_autocheckpoint".into(),
message: e.to_string(),
})
})
.await
.expect("query writer-task connection pragma")
}
};
assert_eq!(
read_pages(&handle).await,
crate::pool::FALLBACK_WAL_AUTOCHECKPOINT_PAGES
);
pool.claim_checkpoint_ownership().expect("claim ownership");
pool.propagate_checkpoint_claim_to_writer_task()
.await
.expect("propagate claim to the running writer task");
assert_eq!(read_pages(&handle).await, 0);
}
#[test]
fn spawn_fails_on_in_memory_pool() {
let cfg = PoolConfig {
path: None,
..PoolConfig::default()
};
let pool = ConnectionPool::new(cfg).unwrap();
let result = spawn(&pool, 8);
assert!(
result.is_err(),
"in-memory pools must reject spawn, not panic"
);
}
#[tokio::test]
async fn full_channel_applies_backpressure_not_immediate_error() {
let (tx, _rx) = mpsc::channel::<Box<dyn AnyWriteRequest + Send>>(1);
let handle = WriterTaskHandle {
tx,
backend_key: None,
db: "test".to_string(),
slow_write_threshold: None,
enqueue_timeout: Duration::from_secs(5),
};
let first = tokio::spawn({
let handle = handle.clone();
async move {
let _ = handle.send(|_conn| Ok::<(), StorageError>(())).await;
}
});
tokio::time::sleep(Duration::from_millis(20)).await;
let second = tokio::time::timeout(
Duration::from_millis(100),
handle.send(|_conn| Ok::<(), StorageError>(())),
)
.await;
assert!(
second.is_err(),
"a full channel must apply backpressure (send suspends) rather \
than erroring immediately — no try_send escape hatch per ADR-067"
);
first.abort();
}
#[tokio::test]
async fn send_with_timeout_maps_full_channel_to_write_queue_full() {
let (tx, _rx) = mpsc::channel::<Box<dyn AnyWriteRequest + Send>>(1);
let handle = WriterTaskHandle {
tx,
backend_key: None,
db: "test".to_string(),
slow_write_threshold: None,
enqueue_timeout: Duration::from_secs(5),
};
let first = tokio::spawn({
let handle = handle.clone();
async move {
let _ = handle.send(|_conn| Ok::<(), StorageError>(())).await;
}
});
tokio::time::sleep(Duration::from_millis(20)).await;
let result = handle
.send_with_timeout(
|_conn| Ok::<(), StorageError>(()),
Duration::from_millis(50),
)
.await;
match result {
Err(StorageError::WriteQueueFull { timeout_ms }) => assert_eq!(timeout_ms, 50),
other => panic!("expected WriteQueueFull, got {other:?}"),
}
first.abort();
}
#[tokio::test]
async fn configured_enqueue_timeout_rejects_only_unaccepted_request() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("configured_enqueue_timeout.db");
let cfg = PoolConfig {
path: Some(path.clone()),
write_admission_deadline_ms: 100,
..PoolConfig::for_test()
};
let pool = ConnectionPool::new(cfg).unwrap();
let handle = spawn(&pool, 1).expect("writer task should spawn on a file-backed pool");
let (started_tx, started_rx) = oneshot::channel::<()>();
let (release_tx, release_rx) = std_mpsc::channel::<()>();
let handle_a = handle.clone();
let a_task = tokio::spawn(async move {
handle_a
.send(move |_conn| {
let _ = started_tx.send(());
release_rx.recv().expect("test must release request A");
Ok::<(), StorageError>(())
})
.await
});
tokio::time::timeout(Duration::from_secs(5), started_rx)
.await
.expect("request A did not start")
.expect("request A dropped its start signal");
let b_reply_rx = tokio::time::timeout(
Duration::from_secs(5),
handle.enqueue(|_conn| Ok::<(), StorageError>(())),
)
.await
.expect("B must be accepted promptly")
.expect("B must be accepted: the one channel slot is free while A drains");
let c_ran = Arc::new(AtomicBool::new(false));
let c_ran_in_op = Arc::clone(&c_ran);
let c_result = handle
.send_bounded(move |_conn| {
c_ran_in_op.store(true, Ordering::SeqCst);
Ok::<(), StorageError>(())
})
.await;
match c_result {
Err(StorageError::WriteQueueFull { .. }) => {}
other => panic!("expected WriteQueueFull, got {other:?}"),
}
assert!(!c_ran.load(Ordering::SeqCst), "C must never run");
release_tx.send(()).expect("release request A");
tokio::time::timeout(Duration::from_secs(5), a_task)
.await
.expect("A did not complete")
.expect("A task join")
.expect("A must complete successfully");
tokio::time::timeout(Duration::from_secs(5), b_reply_rx)
.await
.expect("B did not reply")
.expect("B's reply channel must not be dropped")
.expect("B must complete successfully");
}
#[tokio::test]
#[serial(tx_registry)]
async fn send_with_timeout_returns_op_result_when_op_outlives_the_timeout() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("writer_task_slow_op.db");
let pool = file_pool(&path);
{
let writer = pool.try_writer().unwrap();
writer
.conn()
.execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)")
.unwrap();
}
let handle = spawn(&pool, 8).expect("writer task should spawn on a file-backed pool");
let result = handle
.send_with_timeout(
|conn| {
std::thread::sleep(Duration::from_millis(150));
conn.execute("INSERT INTO t (id, v) VALUES (1, 'slow')", [])
.map_err(|e| StorageError::Pool {
operation: "test_insert".into(),
message: e.to_string(),
})
},
Duration::from_millis(20),
)
.await;
let affected = result.expect(
"an accepted request must return its real result even when the \
op takes longer than the enqueue timeout, not WriteQueueFull",
);
assert_eq!(affected, 1);
let reader = pool.reader().expect("reader");
let v: String = reader
.conn()
.query_row("SELECT v FROM t WHERE id = 1", [], |row| row.get(0))
.expect("the slow op's write must have committed");
assert_eq!(v, "slow");
}
#[tokio::test]
#[serial(tx_registry)]
async fn operation_failure_with_successful_rollback_reports_finality_once_and_continues() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("writer_task_operation_rollback.db");
let pool = file_pool(&path);
{
let writer = pool.try_writer().unwrap();
writer
.conn()
.execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)")
.unwrap();
}
let handle = spawn(&pool, 8).expect("writer task spawn");
let executions = Arc::new(AtomicUsize::new(0));
let executions_in_op = Arc::clone(&executions);
let original_error = handle
.send(move |conn| -> Result<(), StorageError> {
executions_in_op.fetch_add(1, Ordering::SeqCst);
conn.execute("INSERT INTO t (id, v) VALUES (1, 'rolled-back')", [])
.map_err(|e| StorageError::Pool {
operation: "test_operation_error_insert".into(),
message: e.to_string(),
})?;
Err(StorageError::Internal(
"intentional operation failure".into(),
))
})
.await;
match &original_error {
Err(StorageError::WriterTaskRequestFailed {
request_state: WriterTaskRequestState::TransactionRolledBack,
source,
}) => assert!(
matches!(source.as_ref(), StorageError::Internal(message)
if message == "intentional operation failure"),
"the proven-rollback wrapper must retain the typed operation error: {source:?}"
),
other => panic!(
"a confirmed rollback must carry TransactionRolledBack and preserve the operation error, got {other:?}"
),
}
assert_eq!(
executions.load(Ordering::SeqCst),
1,
"finality propagation must not replay the request closure"
);
let affected = handle
.send(|conn| {
conn.execute("INSERT INTO t (id, v) VALUES (2, 'committed')", [])
.map_err(|e| StorageError::Pool {
operation: "test_operation_error_followup_insert".into(),
message: e.to_string(),
})
})
.await
.expect("the writer must continue after a confirmed rollback");
assert_eq!(affected, 1);
assert_eq!(
executions.load(Ordering::SeqCst),
1,
"serving a follow-up request must not replay the rolled-back closure"
);
let reader = pool.reader().expect("reader");
let rolled_back: i64 = reader
.conn()
.query_row("SELECT COUNT(*) FROM t WHERE id = 1", [], |row| row.get(0))
.unwrap();
let committed: i64 = reader
.conn()
.query_row("SELECT COUNT(*) FROM t WHERE id = 2", [], |row| row.get(0))
.unwrap();
assert_eq!(rolled_back, 0);
assert_eq!(committed, 1);
}
#[tokio::test]
#[serial(tx_registry)]
async fn commit_failure_with_successful_rollback_reports_finality_once_and_continues() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("writer_task_commit_rollback.db");
let pool = file_pool(&path);
{
let writer = pool.try_writer().unwrap();
writer
.conn()
.execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)")
.unwrap();
}
let handle = spawn(&pool, 8).expect("writer task spawn");
let executions = Arc::new(AtomicUsize::new(0));
let executions_in_op = Arc::clone(&executions);
let commit_error = handle
.send(move |conn| -> Result<usize, StorageError> {
executions_in_op.fetch_add(1, Ordering::SeqCst);
let affected = conn
.execute("INSERT INTO t (id, v) VALUES (1, 'rolled-back')", [])
.map_err(|e| StorageError::Pool {
operation: "test_commit_error_insert".into(),
message: e.to_string(),
})?;
conn.authorizer(Some(deny_commit))
.map_err(|e| StorageError::Pool {
operation: "test_install_authorizer".into(),
message: e.to_string(),
})?;
Ok(affected)
})
.await;
match &commit_error {
Err(StorageError::WriterTaskRequestFailed {
request_state: WriterTaskRequestState::TransactionRolledBack,
source,
}) => assert!(
matches!(source.as_ref(), StorageError::Pool { operation, .. }
if operation == "writer_task_commit"),
"the proven-rollback wrapper must retain the typed COMMIT error: {source:?}"
),
other => panic!(
"a confirmed rollback must carry TransactionRolledBack and preserve the COMMIT error, got {other:?}"
),
}
assert!(
commit_error
.as_ref()
.expect_err("COMMIT must be denied")
.is_retryable(),
"the existing retryable commit-error contract must remain unchanged after a \
confirmed rollback"
);
assert_eq!(
executions.load(Ordering::SeqCst),
1,
"finality propagation must not replay the request closure"
);
let affected = handle
.send(|conn| {
conn.authorizer(None::<fn(AuthContext<'_>) -> Authorization>)
.map_err(|e| StorageError::Pool {
operation: "test_remove_authorizer".into(),
message: e.to_string(),
})?;
conn.execute("INSERT INTO t (id, v) VALUES (2, 'committed')", [])
.map_err(|e| StorageError::Pool {
operation: "test_commit_error_followup_insert".into(),
message: e.to_string(),
})
})
.await
.expect("the writer must continue after the failed COMMIT is rolled back");
assert_eq!(affected, 1);
assert_eq!(
executions.load(Ordering::SeqCst),
1,
"serving a follow-up request must not replay the rolled-back closure"
);
let reader = pool.reader().expect("reader");
let rolled_back: i64 = reader
.conn()
.query_row("SELECT COUNT(*) FROM t WHERE id = 1", [], |row| row.get(0))
.unwrap();
let committed: i64 = reader
.conn()
.query_row("SELECT COUNT(*) FROM t WHERE id = 2", [], |row| row.get(0))
.unwrap();
assert_eq!(rolled_back, 0);
assert_eq!(committed, 1);
}
#[test]
fn top_level_request_returning_with_open_transaction_reports_side_effects_unknown() {
let conn = Connection::open_in_memory().expect("in-memory connection");
conn.execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY)")
.unwrap();
let (reply_tx, mut reply_rx) = oneshot::channel();
let request = WriteRequest {
op: Box::new(|conn| -> Result<usize, StorageError> {
conn.execute_batch("BEGIN IMMEDIATE")
.map_err(|e| StorageError::Pool {
operation: "test_top_level_begin".into(),
message: e.to_string(),
})?;
conn.execute("INSERT INTO t (id) VALUES (1)", [])
.map_err(|e| StorageError::Pool {
operation: "test_top_level_insert".into(),
message: e.to_string(),
})
}),
reply: reply_tx,
top_level: true,
telemetry: WriteTelemetry::new(None, "test".to_string(), 0, None),
};
let terminal_state = sealed::Sealed::execute_and_reply_top_level_reporting_terminal(
Box::new(request),
&conn,
Duration::ZERO,
);
assert_eq!(
terminal_state,
Some(WriterTaskRequestState::SideEffectsUnknown)
);
let reply = reply_rx
.try_recv()
.expect("active request must receive a typed terminal reply");
assert_writer_task_terminal_state(reply, WriterTaskRequestState::SideEffectsUnknown);
assert!(
!conn.is_autocommit(),
"the fixture must prove the post-request autocommit check observed an open transaction"
);
}
#[test]
fn commit_failure_with_failed_rollback_reports_side_effects_unknown() {
let conn = Connection::open_in_memory().expect("in-memory connection");
conn.execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY); BEGIN IMMEDIATE")
.unwrap();
let executions = Arc::new(AtomicUsize::new(0));
let executions_in_op = Arc::clone(&executions);
let (reply_tx, mut reply_rx) = oneshot::channel();
let request = WriteRequest {
op: Box::new(move |conn| -> Result<usize, StorageError> {
executions_in_op.fetch_add(1, Ordering::SeqCst);
let affected = conn
.execute("INSERT INTO t (id) VALUES (1)", [])
.map_err(|e| StorageError::Pool {
operation: "test_insert_before_commit_failure".into(),
message: e.to_string(),
})?;
conn.authorizer(Some(deny_commit_and_rollback))
.map_err(|e| StorageError::Pool {
operation: "test_install_authorizer".into(),
message: e.to_string(),
})?;
Ok(affected)
}),
reply: reply_tx,
top_level: false,
telemetry: WriteTelemetry::new(None, "test".to_string(), 0, None),
};
let terminal_state = sealed::Sealed::execute_and_reply_reporting_terminal(
Box::new(request),
&conn,
None,
Duration::ZERO,
Duration::ZERO,
);
assert_eq!(
terminal_state,
Some(WriterTaskRequestState::SideEffectsUnknown)
);
let reply = reply_rx
.try_recv()
.expect("active request must receive a typed terminal reply");
assert_writer_task_terminal_state(reply, WriterTaskRequestState::SideEffectsUnknown);
assert_eq!(executions.load(Ordering::SeqCst), 1);
assert!(
!conn.is_autocommit(),
"the denied COMMIT and ROLLBACK must leave the test connection poisoned"
);
}
#[tokio::test]
#[serial(tx_registry)]
async fn poisoned_connection_retires_before_queued_top_level_request() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("writer_task_rollback_poison.db");
let pool = file_pool(&path);
{
let writer = pool.try_writer().unwrap();
writer
.conn()
.execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)")
.unwrap();
}
let handle = spawn(&pool, 8).expect("writer task spawn");
let (started_tx, started_rx) = oneshot::channel::<()>();
let (release_tx, release_rx) = std_mpsc::channel::<()>();
let active = tokio::spawn({
let handle = handle.clone();
async move {
handle
.send(move |conn| -> Result<usize, StorageError> {
let affected = conn
.execute("INSERT INTO t (id, v) VALUES (1, 'active')", [])
.map_err(|e| StorageError::Pool {
operation: "test_active_insert".into(),
message: e.to_string(),
})?;
conn.authorizer(Some(deny_commit_and_rollback))
.map_err(|e| StorageError::Pool {
operation: "test_install_authorizer".into(),
message: e.to_string(),
})?;
let _ = started_tx.send(());
release_rx.recv().expect("test must release active op");
Ok(affected)
})
.await
}
});
tokio::time::timeout(Duration::from_secs(5), started_rx)
.await
.expect("active request did not start")
.expect("active request dropped its start signal");
let queued_ran = Arc::new(AtomicBool::new(false));
let queued_ran_in_op = Arc::clone(&queued_ran);
let queued_top_level = handle
.enqueue_inner(
move |conn| {
queued_ran_in_op.store(true, Ordering::SeqCst);
conn.execute("INSERT INTO t (id, v) VALUES (2, 'queued')", [])
.map_err(|e| StorageError::Pool {
operation: "test_queued_top_level_insert".into(),
message: e.to_string(),
})
},
true,
)
.await
.expect("top-level request must queue behind active request");
release_tx.send(()).expect("release active op");
let active_result = tokio::time::timeout(Duration::from_secs(5), active)
.await
.expect("active caller hung after rollback failure")
.expect("active caller task join");
assert_writer_task_terminal_state(
active_result,
WriterTaskRequestState::SideEffectsUnknown,
);
let queued_result = tokio::time::timeout(Duration::from_secs(5), queued_top_level)
.await
.expect("queued top-level caller hung after terminal failure")
.expect("terminal drain must preserve queued typed reply");
assert_writer_task_terminal_state(queued_result, WriterTaskRequestState::NotStarted);
assert!(
!queued_ran.load(Ordering::SeqCst),
"a top-level request must never run on the poisoned connection"
);
let future_ran = Arc::new(AtomicBool::new(false));
let future_ran_in_op = Arc::clone(&future_ran);
let future_result = handle
.send_top_level(move |_conn| {
future_ran_in_op.store(true, Ordering::SeqCst);
Ok::<(), StorageError>(())
})
.await;
assert_writer_task_terminal_state(future_result, WriterTaskRequestState::NotStarted);
assert!(!future_ran.load(Ordering::SeqCst));
}
#[test]
fn operation_failure_with_failed_rollback_reports_side_effects_unknown() {
let conn = Connection::open_in_memory().expect("in-memory connection");
conn.execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY); BEGIN IMMEDIATE")
.unwrap();
let executions = Arc::new(AtomicUsize::new(0));
let executions_in_op = Arc::clone(&executions);
let (reply_tx, mut reply_rx) = oneshot::channel();
let request = WriteRequest {
op: Box::new(move |conn| -> Result<(), StorageError> {
executions_in_op.fetch_add(1, Ordering::SeqCst);
conn.authorizer(Some(deny_rollback))
.map_err(|e| StorageError::Pool {
operation: "test_install_authorizer".into(),
message: e.to_string(),
})?;
Err(StorageError::Internal(
"intentional operation failure before denied rollback".into(),
))
}),
reply: reply_tx,
top_level: false,
telemetry: WriteTelemetry::new(None, "test".to_string(), 0, None),
};
let terminal_state = sealed::Sealed::execute_and_reply_reporting_terminal(
Box::new(request),
&conn,
None,
Duration::ZERO,
Duration::ZERO,
);
assert_eq!(
terminal_state,
Some(WriterTaskRequestState::SideEffectsUnknown)
);
let reply = reply_rx
.try_recv()
.expect("active request must receive a typed terminal reply");
assert_writer_task_terminal_state(reply, WriterTaskRequestState::SideEffectsUnknown);
assert_eq!(executions.load(Ordering::SeqCst), 1);
assert!(
!conn.is_autocommit(),
"the denied ROLLBACK must leave the test connection poisoned"
);
}
#[test]
fn wrapped_panic_with_failed_rollback_reports_side_effects_unknown() {
let conn = Connection::open_in_memory().expect("in-memory connection");
conn.execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY); BEGIN IMMEDIATE")
.unwrap();
let (reply_tx, mut reply_rx) = oneshot::channel();
let request = WriteRequest {
op: Box::new(|conn| -> Result<(), StorageError> {
conn.execute_batch("INSERT INTO t (id) VALUES (1); COMMIT")
.map_err(|e| StorageError::Pool {
operation: "test_force_rollback_failure".into(),
message: e.to_string(),
})?;
panic!("intentional panic after illicit commit");
}),
reply: reply_tx,
top_level: false,
telemetry: WriteTelemetry::new(None, "test".to_string(), 0, None),
};
let terminal_state = sealed::Sealed::execute_and_reply_reporting_terminal(
Box::new(request),
&conn,
None,
Duration::ZERO,
Duration::ZERO,
);
assert_eq!(
terminal_state,
Some(WriterTaskRequestState::SideEffectsUnknown)
);
let reply = reply_rx
.try_recv()
.expect("active request must receive a typed terminal reply");
assert_writer_task_terminal_state(reply, WriterTaskRequestState::SideEffectsUnknown);
let count: i64 = conn
.query_row("SELECT COUNT(*) FROM t", [], |row| row.get(0))
.unwrap();
assert_eq!(
count, 1,
"the fixture's committed side effect proves why the state must be unknown"
);
}
#[tokio::test]
#[serial(tx_registry)]
async fn wrapped_panic_rolls_back_and_terminally_fails_queue() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("writer_task_wrapped_panic.db");
let cfg = PoolConfig {
path: Some(path),
write_queue_enabled: Some(true),
write_queue_capacity: 8,
..PoolConfig::for_test()
};
let pool = ConnectionPool::new(cfg).unwrap();
{
let writer = pool.try_writer().unwrap();
writer
.conn()
.execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)")
.unwrap();
}
let handle = pool
.writer_task_handle()
.expect("writer task lookup")
.expect("file-backed queued pool must spawn its writer task");
assert_eq!(pool.writer_task_spawn_count(), 1);
let (started_tx, started_rx) = oneshot::channel::<()>();
let (release_tx, release_rx) = std_mpsc::channel::<()>();
let active = tokio::spawn({
let handle = handle.clone();
async move {
handle
.send(move |conn| -> Result<usize, StorageError> {
conn.execute("INSERT INTO t (id, v) VALUES (1, 'active')", [])
.map_err(|e| StorageError::Pool {
operation: "test_active_insert".into(),
message: e.to_string(),
})?;
let _ = started_tx.send(());
release_rx.recv().expect("test must release active op");
panic!("intentional wrapped writer request panic");
})
.await
}
});
tokio::time::timeout(Duration::from_secs(5), started_rx)
.await
.expect("active request did not start")
.expect("active request dropped its start signal");
let queued_one_ran = Arc::new(AtomicBool::new(false));
let queued_one_ran_in_op = Arc::clone(&queued_one_ran);
let queued_one = handle
.enqueue(move |conn| {
queued_one_ran_in_op.store(true, Ordering::SeqCst);
conn.execute("INSERT INTO t (id, v) VALUES (2, 'queued-one')", [])
.map_err(|e| StorageError::Pool {
operation: "test_queued_one_insert".into(),
message: e.to_string(),
})
})
.await
.expect("first queued request must be accepted");
let queued_two_ran = Arc::new(AtomicBool::new(false));
let queued_two_ran_in_op = Arc::clone(&queued_two_ran);
let queued_two = handle
.enqueue(move |_conn| {
queued_two_ran_in_op.store(true, Ordering::SeqCst);
Ok::<String, StorageError>("queued-two-ran".to_string())
})
.await
.expect("second queued request must be accepted");
assert_eq!(
handle.queue_depth(),
2,
"both heterogeneous requests must be buffered behind the active op"
);
release_tx.send(()).expect("release active op");
let active_result = tokio::time::timeout(Duration::from_secs(5), active)
.await
.expect("active caller hung after panic")
.expect("active caller task join");
assert_writer_task_terminal_state(
active_result,
WriterTaskRequestState::TransactionRolledBack,
);
let queued_one_result = tokio::time::timeout(Duration::from_secs(5), queued_one)
.await
.expect("first queued caller hung after terminal failure")
.expect("terminal drain must preserve first typed reply");
assert_writer_task_terminal_state(queued_one_result, WriterTaskRequestState::NotStarted);
let queued_two_result = tokio::time::timeout(Duration::from_secs(5), queued_two)
.await
.expect("second queued caller hung after terminal failure")
.expect("terminal drain must preserve second typed reply");
assert_writer_task_terminal_state(queued_two_result, WriterTaskRequestState::NotStarted);
assert!(!queued_one_ran.load(Ordering::SeqCst));
assert!(!queued_two_ran.load(Ordering::SeqCst));
let future_ran = Arc::new(AtomicBool::new(false));
let future_ran_in_op = Arc::clone(&future_ran);
let future_result = handle
.send(move |_conn| {
future_ran_in_op.store(true, Ordering::SeqCst);
Ok::<(), StorageError>(())
})
.await;
assert_writer_task_terminal_state(future_result, WriterTaskRequestState::NotStarted);
assert!(!future_ran.load(Ordering::SeqCst));
let cached_after_failure = pool
.writer_task_handle()
.expect("cached writer task lookup")
.expect("pool retains its terminal handle");
assert_eq!(
pool.writer_task_spawn_count(),
1,
"a terminal writer task must not be restarted behind callers' backs"
);
let cached_result = cached_after_failure
.send(|_conn| Ok::<(), StorageError>(()))
.await;
assert_writer_task_terminal_state(cached_result, WriterTaskRequestState::NotStarted);
let reader = pool.reader().expect("reader");
let count: i64 = reader
.conn()
.query_row("SELECT COUNT(*) FROM t", [], |row| row.get(0))
.unwrap();
assert_eq!(
count, 0,
"the active transaction must be rolled back and queued ops must never run"
);
}
#[tokio::test]
async fn top_level_panic_reports_unknown_and_fails_queue_without_running_it() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("writer_task_top_level_panic.db");
let pool = file_pool(&path);
{
let writer = pool.try_writer().unwrap();
writer
.conn()
.execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)")
.unwrap();
}
let handle = spawn(&pool, 8).expect("writer task spawn");
let (started_tx, started_rx) = oneshot::channel::<()>();
let (release_tx, release_rx) = std_mpsc::channel::<()>();
let active = tokio::spawn({
let handle = handle.clone();
async move {
handle
.send_top_level(move |conn| -> Result<usize, StorageError> {
conn.execute("INSERT INTO t (id, v) VALUES (10, 'autocommitted')", [])
.map_err(|e| StorageError::Pool {
operation: "test_top_level_insert".into(),
message: e.to_string(),
})?;
let _ = started_tx.send(());
release_rx.recv().expect("test must release top-level op");
panic!("intentional top-level writer request panic");
})
.await
}
});
tokio::time::timeout(Duration::from_secs(5), started_rx)
.await
.expect("top-level request did not start")
.expect("top-level request dropped its start signal");
let queued_ran = Arc::new(AtomicBool::new(false));
let queued_ran_in_op = Arc::clone(&queued_ran);
let queued = handle
.enqueue(move |conn| {
queued_ran_in_op.store(true, Ordering::SeqCst);
conn.execute("INSERT INTO t (id, v) VALUES (11, 'queued')", [])
.map_err(|e| StorageError::Pool {
operation: "test_top_level_queued_insert".into(),
message: e.to_string(),
})
})
.await
.expect("queued request must be accepted");
assert_eq!(handle.queue_depth(), 1);
release_tx.send(()).expect("release top-level op");
let active_result = tokio::time::timeout(Duration::from_secs(5), active)
.await
.expect("top-level caller hung after panic")
.expect("top-level caller task join");
assert_writer_task_terminal_state(
active_result,
WriterTaskRequestState::SideEffectsUnknown,
);
let queued_result = tokio::time::timeout(Duration::from_secs(5), queued)
.await
.expect("queued caller hung after top-level panic")
.expect("terminal drain must preserve queued typed reply");
assert_writer_task_terminal_state(queued_result, WriterTaskRequestState::NotStarted);
assert!(!queued_ran.load(Ordering::SeqCst));
let reader = pool.reader().expect("reader");
let active_count: i64 = reader
.conn()
.query_row("SELECT COUNT(*) FROM t WHERE id = 10", [], |row| row.get(0))
.unwrap();
let queued_count: i64 = reader
.conn()
.query_row("SELECT COUNT(*) FROM t WHERE id = 11", [], |row| row.get(0))
.unwrap();
assert_eq!(
active_count, 1,
"the completed top-level statement autocommits before the panic"
);
assert_eq!(queued_count, 0, "the queued request must never run");
}
#[tokio::test]
async fn closed_receiver_rejects_all_send_surfaces_as_not_started() {
let (tx, rx) = mpsc::channel::<Box<dyn AnyWriteRequest + Send>>(4);
drop(rx);
let handle = WriterTaskHandle {
tx,
backend_key: None,
db: "test".to_string(),
slow_write_threshold: None,
enqueue_timeout: Duration::from_secs(5),
};
let send_result = handle.send(|_conn| Ok::<(), StorageError>(())).await;
assert_writer_task_terminal_state(send_result, WriterTaskRequestState::NotStarted);
let timed_result = handle
.send_with_timeout(|_conn| Ok::<(), StorageError>(()), Duration::from_secs(1))
.await;
assert_writer_task_terminal_state(timed_result, WriterTaskRequestState::NotStarted);
let top_level_result = handle
.send_top_level(|_conn| Ok::<(), StorageError>(()))
.await;
assert_writer_task_terminal_state(top_level_result, WriterTaskRequestState::NotStarted);
}
#[tokio::test]
async fn accepted_request_lost_reply_is_side_effects_unknown() {
let (tx, mut rx) = mpsc::channel::<Box<dyn AnyWriteRequest + Send>>(1);
let handle = WriterTaskHandle {
tx,
backend_key: None,
db: "test".to_string(),
slow_write_threshold: None,
enqueue_timeout: Duration::from_secs(5),
};
let request_ran = Arc::new(AtomicBool::new(false));
let request_ran_in_op = Arc::clone(&request_ran);
let dropper = tokio::spawn(async move {
let request = rx.recv().await.expect("request must be accepted");
drop(request);
});
let result = tokio::time::timeout(
Duration::from_secs(5),
handle.send(move |_conn| {
request_ran_in_op.store(true, Ordering::SeqCst);
Ok::<(), StorageError>(())
}),
)
.await
.expect("caller hung after accepted request was dropped");
dropper.await.expect("dropper task join");
assert_writer_task_terminal_state(result, WriterTaskRequestState::SideEffectsUnknown);
assert!(!request_ran.load(Ordering::SeqCst));
}
#[cfg(unix)]
#[test]
fn writer_stage_backend_key_preserves_non_utf8_path_bytes() {
use std::ffi::OsString;
use std::os::unix::ffi::OsStringExt;
let path_a =
std::path::PathBuf::from(OsString::from_vec(b"/tmp/khive-writer-\x80.db".to_vec()));
let path_b =
std::path::PathBuf::from(OsString::from_vec(b"/tmp/khive-writer-\x81.db".to_vec()));
assert_eq!(
path_a.display().to_string(),
path_b.display().to_string(),
"fixture must reproduce the lossy display-label collision"
);
assert_ne!(
writer_db_key_from_path(Some(&path_a)),
writer_db_key_from_path(Some(&path_b)),
"backend keys must retain the canonical path's exact OS bytes"
);
}
#[tokio::test]
async fn writer_stage_sample_attributes_a_slow_body() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("writer_stage_sample.db");
let pool = file_pool(&path);
{
let writer = pool.try_writer().unwrap();
writer
.conn()
.execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY)")
.unwrap();
}
let handle = spawn(&pool, 8).unwrap();
handle
.send(|conn| {
std::thread::sleep(Duration::from_millis(400));
conn.execute("INSERT INTO t VALUES (1)", [])
.map_err(|error| StorageError::Pool {
operation: "writer_stage_sample".into(),
message: error.to_string(),
})
})
.await
.unwrap();
let sample = last_writer_stage_observation(&pool).expect("writer stage sample");
assert!(
sample.body_micros >= 350_000,
"the synthetic delay must land in the body stage: {sample:?}"
);
assert!(
sample.body_micros > sample.queue_wait_micros,
"fast queueing must not receive the body's delay: {sample:?}"
);
assert!(
sample.body_micros > sample.transaction_acquire_micros,
"an uncontended BEGIN must not receive the body's delay: {sample:?}"
);
assert!(
sample.body_micros > sample.commit_micros,
"a fast COMMIT must not receive the body's delay: {sample:?}"
);
assert!(sample.observed_at_unix_ms > 0);
}
#[test]
fn writer_queue_wait_excludes_blocking_pool_scheduling_delay() {
let runtime = tokio::runtime::Builder::new_multi_thread()
.worker_threads(1)
.max_blocking_threads(1)
.enable_all()
.build()
.expect("test runtime");
runtime.block_on(async {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("writer_dequeue_boundary.db");
let pool = file_pool(&path);
let handle = spawn(&pool, 8).unwrap();
let (blocker_started_tx, blocker_started_rx) = std_mpsc::sync_channel(0);
let (release_blocker_tx, release_blocker_rx) = std_mpsc::channel();
let blocker = tokio::task::spawn_blocking(move || {
blocker_started_tx.send(()).unwrap();
release_blocker_rx.recv().unwrap();
});
blocker_started_rx
.recv_timeout(Duration::from_secs(1))
.expect("sole blocking worker must be occupied");
let reply = handle
.enqueue(|_conn| Ok::<(), StorageError>(()))
.await
.expect("request must enter the bounded writer channel");
let dequeue_deadline = Instant::now() + Duration::from_secs(1);
while handle.queue_depth() != 0 {
assert!(
Instant::now() < dequeue_deadline,
"writer drain never dequeued the accepted request"
);
tokio::task::yield_now().await;
}
let scheduling_delay = Duration::from_millis(150);
tokio::time::sleep(scheduling_delay).await;
release_blocker_tx.send(()).unwrap();
blocker.await.unwrap();
reply.await.unwrap().unwrap();
let sample = last_writer_stage_observation(&pool).expect("writer stage sample");
assert!(
sample.total_micros.saturating_sub(sample.queue_wait_micros) >= 100_000,
"the post-dequeue blocking-pool delay must not inflate queue_wait: {sample:?}"
);
});
}
#[tokio::test]
#[serial(tx_registry)]
async fn writer_task_failure_counters_are_acquisition_site_exact() {
{
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("writer_task_failure_counters_rollback.db");
let pool = file_pool(&path);
let handle = spawn(&pool, 8).expect("writer task should spawn");
let before = pool.writer_acquisition_snapshot();
assert_eq!(before.writer_task_request_failures, 0);
assert_eq!(before.writer_task_side_effects_unknown, 0);
let (started_tx, started_rx) = oneshot::channel::<()>();
let (release_tx, release_rx) = std_mpsc::channel::<()>();
let active = tokio::spawn({
let handle = handle.clone();
async move {
handle
.send(move |_conn| -> Result<(), StorageError> {
let _ = started_tx.send(());
release_rx.recv().expect("test must release active op");
panic!("intentional rollback-clean panic for counter test");
})
.await
}
});
tokio::time::timeout(Duration::from_secs(5), started_rx)
.await
.expect("active request did not start")
.expect("active request dropped its start signal");
let queued = handle
.enqueue(|_conn| Ok::<(), StorageError>(()))
.await
.expect("second request must queue behind the active one");
release_tx.send(()).expect("release active op");
let active_result = tokio::time::timeout(Duration::from_secs(5), active)
.await
.expect("active caller hung after panic")
.expect("active caller task join");
assert_writer_task_terminal_state(
active_result,
WriterTaskRequestState::TransactionRolledBack,
);
let queued_result = queued.await.expect("terminal drain must reply");
assert_writer_task_terminal_state(queued_result, WriterTaskRequestState::NotStarted);
let after = pool.writer_acquisition_snapshot();
assert_eq!(
after.writer_task_request_failures, 1,
"only the request that actually reached the seam counts, not the ones \
failed by the queue-close drain"
);
assert_eq!(
after.writer_task_side_effects_unknown, 0,
"a clean rollback must not be counted as an unknown-side-effects outcome"
);
}
{
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("writer_task_failure_counters_unknown.db");
let pool = file_pool(&path);
let handle = spawn(&pool, 8).expect("writer task should spawn");
let before = pool.writer_acquisition_snapshot();
let active_result = handle
.send(|conn| -> Result<(), StorageError> {
conn.authorizer(Some(deny_rollback))
.map_err(|e| StorageError::Pool {
operation: "test_install_authorizer".into(),
message: e.to_string(),
})?;
Err(StorageError::Internal(
"intentional operation failure before denied rollback".into(),
))
})
.await;
assert_writer_task_terminal_state(
active_result,
WriterTaskRequestState::SideEffectsUnknown,
);
let sentinel_result = handle.send(|_conn| Ok::<(), StorageError>(())).await;
assert_writer_task_terminal_state(sentinel_result, WriterTaskRequestState::NotStarted);
let after = pool.writer_acquisition_snapshot();
assert_eq!(
after.writer_task_request_failures - before.writer_task_request_failures,
1
);
assert_eq!(
after.writer_task_side_effects_unknown - before.writer_task_side_effects_unknown,
1
);
}
}
}