1use rusqlite::Connection;
49use std::collections::HashMap;
50use std::panic::{catch_unwind, AssertUnwindSafe};
51use std::path::{Path, PathBuf};
52use std::sync::{Arc, Mutex, OnceLock};
53use std::time::{Duration, Instant};
54use tokio::sync::{mpsc, oneshot};
55
56use khive_storage::error::{StorageError, WriterTaskRequestState};
57
58use crate::error::SqliteError;
59use crate::pool::{ConnectionPool, WriterAcquisitionCounters};
60
61type WriteOp<R> = Box<dyn FnOnce(&Connection) -> Result<R, StorageError> + Send>;
71
72#[derive(Debug, Clone, PartialEq, Eq)]
77pub struct WriterStageObservation {
78 pub queue_wait_micros: u64,
79 pub transaction_acquire_micros: u64,
80 pub body_micros: u64,
81 pub commit_micros: u64,
82 pub total_micros: u64,
83 pub queue_depth_at_entry: u64,
84 pub observed_at_unix_ms: u64,
85}
86
87static WRITER_STAGE_OBSERVATIONS: OnceLock<
88 Mutex<HashMap<Option<PathBuf>, WriterStageObservation>>,
89> = OnceLock::new();
90
91fn writer_stage_observations() -> &'static Mutex<HashMap<Option<PathBuf>, WriterStageObservation>> {
92 WRITER_STAGE_OBSERVATIONS.get_or_init(|| Mutex::new(HashMap::new()))
93}
94
95fn writer_db_key_from_path(path: Option<&Path>) -> Option<PathBuf> {
96 path.map(Path::to_path_buf)
97}
98
99fn writer_db_key(pool: &ConnectionPool) -> Option<PathBuf> {
100 writer_db_key_from_path(pool.canonical_path())
101}
102
103fn duration_micros(duration: Duration) -> u64 {
104 duration.as_micros().min(u128::from(u64::MAX)) as u64
105}
106
107fn observed_at_unix_ms() -> u64 {
108 std::time::SystemTime::now()
109 .duration_since(std::time::UNIX_EPOCH)
110 .map(|duration| duration.as_millis() as u64)
111 .unwrap_or(0)
112}
113
114pub fn last_writer_stage_observation(pool: &ConnectionPool) -> Option<WriterStageObservation> {
117 writer_stage_observations()
118 .lock()
119 .unwrap_or_else(std::sync::PoisonError::into_inner)
120 .get(&writer_db_key(pool))
121 .cloned()
122}
123
124struct WriteTelemetry {
125 backend_key: Option<PathBuf>,
126 db: String,
127 submitted_at: Instant,
128 queue_depth_at_entry: usize,
129 slow_write_threshold: Option<Duration>,
130}
131
132impl WriteTelemetry {
133 fn new(
134 backend_key: Option<PathBuf>,
135 db: String,
136 queue_depth_at_entry: usize,
137 slow_write_threshold: Option<Duration>,
138 ) -> Self {
139 Self {
140 backend_key,
141 db,
142 submitted_at: Instant::now(),
143 queue_depth_at_entry,
144 slow_write_threshold,
145 }
146 }
147
148 fn queue_wait(&self) -> Duration {
149 self.submitted_at.elapsed()
150 }
151
152 fn finish(
153 self,
154 queue_wait: Duration,
155 transaction_acquire: Duration,
156 body: Duration,
157 commit: Duration,
158 ) {
159 let total = self.submitted_at.elapsed();
160 let observation = WriterStageObservation {
161 queue_wait_micros: duration_micros(queue_wait),
162 transaction_acquire_micros: duration_micros(transaction_acquire),
163 body_micros: duration_micros(body),
164 commit_micros: duration_micros(commit),
165 total_micros: duration_micros(total),
166 queue_depth_at_entry: self.queue_depth_at_entry as u64,
167 observed_at_unix_ms: observed_at_unix_ms(),
168 };
169 writer_stage_observations()
170 .lock()
171 .unwrap_or_else(std::sync::PoisonError::into_inner)
172 .insert(self.backend_key, observation.clone());
173
174 if self
175 .slow_write_threshold
176 .is_some_and(|threshold| total >= threshold)
177 {
178 crate::timeout_sink::emit_slow_write(&self.db, &observation);
179 }
180 }
181}
182
183pub struct WriteRequest<R: Send + 'static> {
198 op: WriteOp<R>,
199 reply: oneshot::Sender<Result<R, StorageError>>,
200 top_level: bool,
201 telemetry: WriteTelemetry,
202}
203
204mod sealed {
205 pub trait Sealed {
210 fn execute_and_reply_reporting_terminal(
211 self: Box<Self>,
212 conn: &rusqlite::Connection,
213 tx_span: Option<khive_storage::tx_registry::TxHandle>,
214 queue_wait: std::time::Duration,
215 transaction_acquire: std::time::Duration,
216 ) -> Option<khive_storage::error::WriterTaskRequestState>;
217
218 fn execute_and_reply_top_level_reporting_terminal(
219 self: Box<Self>,
220 conn: &rusqlite::Connection,
221 queue_wait: std::time::Duration,
222 ) -> Option<khive_storage::error::WriterTaskRequestState>;
223
224 fn reply_error_after_begin(
225 self: Box<Self>,
226 err: khive_storage::error::StorageError,
227 queue_wait: std::time::Duration,
228 transaction_acquire: std::time::Duration,
229 );
230 }
231}
232
233pub trait AnyWriteRequest: sealed::Sealed + Send {
239 fn execute_and_reply(self: Box<Self>, conn: &Connection);
252
253 fn execute_and_reply_top_level(self: Box<Self>, conn: &Connection);
263
264 fn reply_error(self: Box<Self>, err: StorageError);
276
277 fn is_top_level(&self) -> bool;
281
282 fn queue_wait(&self) -> Duration;
285}
286
287#[derive(Debug, Clone, Copy, PartialEq, Eq)]
288enum RollbackDisposition {
289 RolledBack,
290 SideEffectsUnknown,
291}
292
293fn rollback_after_failure(conn: &Connection, failure_context: &'static str) -> RollbackDisposition {
296 match conn.execute_batch("ROLLBACK") {
297 Ok(()) if conn.is_autocommit() => RollbackDisposition::RolledBack,
298 Ok(()) => {
299 tracing::error!(
300 failure_context,
301 "writer transaction: ROLLBACK returned success but the connection is still in a \
302 transaction; request side effects are unknown"
303 );
304 RollbackDisposition::SideEffectsUnknown
305 }
306 Err(rollback_error) => {
307 tracing::error!(
308 error = %rollback_error,
309 failure_context,
310 "writer transaction: rollback after request failure failed; request side effects are \
311 unknown"
312 );
313 RollbackDisposition::SideEffectsUnknown
314 }
315 }
316}
317
318pub(crate) fn execute_wrapped_transaction<R, F>(
321 conn: &Connection,
322 commit_operation: &'static str,
323 operation: F,
324) -> (Result<R, StorageError>, Option<WriterTaskRequestState>)
325where
326 F: FnOnce(&Connection) -> Result<R, StorageError>,
327{
328 let profiled = execute_wrapped_transaction_profiled(conn, commit_operation, operation);
329 (profiled.result, profiled.terminal_state)
330}
331
332struct ProfiledWrappedTransaction<R> {
333 result: Result<R, StorageError>,
334 terminal_state: Option<WriterTaskRequestState>,
335 body: Duration,
336 commit: Duration,
337}
338
339fn execute_wrapped_transaction_profiled<R, F>(
340 conn: &Connection,
341 commit_operation: &'static str,
342 operation: F,
343) -> ProfiledWrappedTransaction<R>
344where
345 F: FnOnce(&Connection) -> Result<R, StorageError>,
346{
347 let body_started = Instant::now();
348 let operation_outcome = catch_unwind(AssertUnwindSafe(|| operation(conn)));
349 let body = body_started.elapsed();
350
351 match operation_outcome {
352 Ok(Ok(value)) => {
353 let commit_started = Instant::now();
354 let commit_outcome = conn.execute_batch("COMMIT");
355 let commit = commit_started.elapsed();
356 match commit_outcome {
357 Ok(()) if conn.is_autocommit() => ProfiledWrappedTransaction {
358 result: Ok(value),
359 terminal_state: None,
360 body,
361 commit,
362 },
363 Ok(()) => {
364 tracing::error!(
365 "writer transaction: COMMIT returned success but the connection is still in \
366 a transaction; request side effects are unknown"
367 );
368 let request_state = WriterTaskRequestState::SideEffectsUnknown;
369 ProfiledWrappedTransaction {
370 result: Err(writer_task_terminated(request_state)),
371 terminal_state: Some(request_state),
372 body,
373 commit,
374 }
375 }
376 Err(commit_error) => match rollback_after_failure(conn, "commit failure") {
377 RollbackDisposition::RolledBack => ProfiledWrappedTransaction {
378 result: Err(StorageError::Pool {
379 operation: commit_operation.into(),
380 message: commit_error.to_string(),
381 }),
382 terminal_state: None,
383 body,
384 commit,
385 },
386 RollbackDisposition::SideEffectsUnknown => {
387 let request_state = WriterTaskRequestState::SideEffectsUnknown;
388 ProfiledWrappedTransaction {
389 result: Err(writer_task_terminated(request_state)),
390 terminal_state: Some(request_state),
391 body,
392 commit,
393 }
394 }
395 },
396 }
397 }
398 Ok(Err(operation_error)) => {
399 match rollback_after_failure(conn, "request operation failure") {
400 RollbackDisposition::RolledBack => ProfiledWrappedTransaction {
401 result: Err(operation_error),
402 terminal_state: None,
403 body,
404 commit: Duration::ZERO,
405 },
406 RollbackDisposition::SideEffectsUnknown => {
407 let request_state = WriterTaskRequestState::SideEffectsUnknown;
408 ProfiledWrappedTransaction {
409 result: Err(writer_task_terminated(request_state)),
410 terminal_state: Some(request_state),
411 body,
412 commit: Duration::ZERO,
413 }
414 }
415 }
416 }
417 Err(_panic_payload) => {
418 let request_state = match rollback_after_failure(conn, "request panic") {
419 RollbackDisposition::RolledBack => WriterTaskRequestState::TransactionRolledBack,
420 RollbackDisposition::SideEffectsUnknown => {
421 WriterTaskRequestState::SideEffectsUnknown
422 }
423 };
424 ProfiledWrappedTransaction {
425 result: Err(writer_task_terminated(request_state)),
426 terminal_state: Some(request_state),
427 body,
428 commit: Duration::ZERO,
429 }
430 }
431 }
432}
433
434impl<R: Send + 'static> sealed::Sealed for WriteRequest<R> {
435 fn execute_and_reply_reporting_terminal(
436 self: Box<Self>,
437 conn: &Connection,
438 tx_span: Option<khive_storage::tx_registry::TxHandle>,
439 queue_wait: Duration,
440 transaction_acquire: Duration,
441 ) -> Option<WriterTaskRequestState> {
442 let WriteRequest {
446 op,
447 reply,
448 telemetry,
449 ..
450 } = *self;
451 let profiled = execute_wrapped_transaction_profiled(conn, "writer_task_commit", op);
452 drop(tx_span);
457 telemetry.finish(
458 queue_wait,
459 transaction_acquire,
460 profiled.body,
461 profiled.commit,
462 );
463 let _ = reply.send(profiled.result);
466 profiled.terminal_state
467 }
468
469 fn execute_and_reply_top_level_reporting_terminal(
470 self: Box<Self>,
471 conn: &Connection,
472 queue_wait: Duration,
473 ) -> Option<WriterTaskRequestState> {
474 let WriteRequest {
475 op,
476 reply,
477 telemetry,
478 ..
479 } = *self;
480 let body_started = Instant::now();
481 let outcome = catch_unwind(AssertUnwindSafe(|| op(conn)));
482 let body = body_started.elapsed();
483 telemetry.finish(queue_wait, Duration::ZERO, body, Duration::ZERO);
484 match outcome {
485 Ok(outcome) if conn.is_autocommit() => {
486 let _ = reply.send(outcome);
489 None
490 }
491 Ok(_outcome) => {
492 tracing::error!(
493 "writer task: top-level request returned with an open transaction; request \
494 side effects are unknown"
495 );
496 let request_state = WriterTaskRequestState::SideEffectsUnknown;
497 let _ = reply.send(Err(writer_task_terminated(request_state)));
498 Some(request_state)
499 }
500 Err(_panic_payload) => {
501 let request_state = WriterTaskRequestState::SideEffectsUnknown;
505 let _ = reply.send(Err(writer_task_terminated(request_state)));
506 Some(request_state)
507 }
508 }
509 }
510
511 fn reply_error_after_begin(
512 self: Box<Self>,
513 err: StorageError,
514 queue_wait: Duration,
515 transaction_acquire: Duration,
516 ) {
517 let WriteRequest {
518 reply, telemetry, ..
519 } = *self;
520 telemetry.finish(
521 queue_wait,
522 transaction_acquire,
523 Duration::ZERO,
524 Duration::ZERO,
525 );
526 let _ = reply.send(Err(err));
527 }
528}
529
530impl<R: Send + 'static> AnyWriteRequest for WriteRequest<R> {
531 fn execute_and_reply(self: Box<Self>, conn: &Connection) {
532 let queue_wait = self.queue_wait();
533 let _ = sealed::Sealed::execute_and_reply_reporting_terminal(
534 self,
535 conn,
536 None,
537 queue_wait,
538 Duration::ZERO,
539 );
540 }
541
542 fn execute_and_reply_top_level(self: Box<Self>, conn: &Connection) {
543 let queue_wait = self.queue_wait();
544 let _ =
545 sealed::Sealed::execute_and_reply_top_level_reporting_terminal(self, conn, queue_wait);
546 }
547
548 fn reply_error(self: Box<Self>, err: StorageError) {
549 let queue_wait = self.queue_wait();
550 sealed::Sealed::reply_error_after_begin(self, err, queue_wait, Duration::ZERO);
551 }
552
553 fn is_top_level(&self) -> bool {
554 self.top_level
555 }
556
557 fn queue_wait(&self) -> Duration {
558 self.telemetry.queue_wait()
559 }
560}
561
562fn writer_task_terminated(request_state: WriterTaskRequestState) -> StorageError {
563 StorageError::WriterTaskTerminated { request_state }
564}
565
566fn writer_task_begin_error(error: rusqlite::Error, busy_timeout: Duration) -> StorageError {
567 if crate::timeout_sink::is_busy_or_locked(&error) {
568 StorageError::WriterTaskBusy {
569 timeout_ms: u64::try_from(busy_timeout.as_millis()).unwrap_or(u64::MAX),
570 }
571 } else {
572 StorageError::Pool {
573 operation: "writer_task_begin".into(),
574 message: error.to_string(),
575 }
576 }
577}
578
579#[derive(Clone, Debug)]
583pub struct WriterTaskHandle {
584 tx: mpsc::Sender<Box<dyn AnyWriteRequest + Send>>,
585 backend_key: Option<PathBuf>,
588 db: String,
593 slow_write_threshold: Option<std::time::Duration>,
599 enqueue_timeout: std::time::Duration,
620}
621
622impl WriterTaskHandle {
623 async fn enqueue<R, F>(
637 &self,
638 op: F,
639 ) -> Result<oneshot::Receiver<Result<R, StorageError>>, StorageError>
640 where
641 R: Send + 'static,
642 F: FnOnce(&Connection) -> Result<R, StorageError> + Send + 'static,
643 {
644 self.enqueue_inner(op, false).await
645 }
646
647 async fn enqueue_inner<R, F>(
651 &self,
652 op: F,
653 top_level: bool,
654 ) -> Result<oneshot::Receiver<Result<R, StorageError>>, StorageError>
655 where
656 R: Send + 'static,
657 F: FnOnce(&Connection) -> Result<R, StorageError> + Send + 'static,
658 {
659 let (reply_tx, reply_rx) = oneshot::channel();
660 let telemetry = WriteTelemetry::new(
661 self.backend_key.clone(),
662 self.db.clone(),
663 self.queue_depth(),
664 self.slow_write_threshold,
665 );
666 let request = WriteRequest {
667 op: Box::new(op),
668 reply: reply_tx,
669 top_level,
670 telemetry,
671 };
672
673 self.tx
674 .send(Box::new(request))
675 .await
676 .map_err(|_| writer_task_terminated(WriterTaskRequestState::NotStarted))?;
677
678 Ok(reply_rx)
679 }
680
681 pub async fn send<R, F>(&self, op: F) -> Result<R, StorageError>
688 where
689 R: Send + 'static,
690 F: FnOnce(&Connection) -> Result<R, StorageError> + Send + 'static,
691 {
692 let reply_rx = self.enqueue(op).await?;
693 reply_rx
694 .await
695 .map_err(|_| writer_task_terminated(WriterTaskRequestState::SideEffectsUnknown))?
696 }
697
698 pub async fn send_with_timeout<R, F>(
713 &self,
714 op: F,
715 timeout: std::time::Duration,
716 ) -> Result<R, StorageError>
717 where
718 R: Send + 'static,
719 F: FnOnce(&Connection) -> Result<R, StorageError> + Send + 'static,
720 {
721 let reply_rx = match tokio::time::timeout(timeout, self.enqueue(op)).await {
722 Ok(Ok(reply_rx)) => reply_rx,
723 Ok(Err(e)) => return Err(e),
724 Err(_elapsed) => {
725 let timeout_ms = timeout.as_millis() as u64;
726 crate::timeout_sink::emit_queue_saturation(&self.db, timeout_ms);
727 return Err(StorageError::WriteQueueFull { timeout_ms });
728 }
729 };
730
731 reply_rx
732 .await
733 .map_err(|_| writer_task_terminated(WriterTaskRequestState::SideEffectsUnknown))?
734 }
735
736 pub async fn send_bounded<R, F>(&self, op: F) -> Result<R, StorageError>
747 where
748 R: Send + 'static,
749 F: FnOnce(&Connection) -> Result<R, StorageError> + Send + 'static,
750 {
751 self.send_with_timeout(op, self.enqueue_timeout).await
752 }
753
754 pub async fn send_top_level<R, F>(&self, op: F) -> Result<R, StorageError>
766 where
767 R: Send + 'static,
768 F: FnOnce(&Connection) -> Result<R, StorageError> + Send + 'static,
769 {
770 let reply_rx = self.enqueue_inner(op, true).await?;
771 reply_rx
772 .await
773 .map_err(|_| writer_task_terminated(WriterTaskRequestState::SideEffectsUnknown))?
774 }
775
776 pub async fn send_top_level_bounded<R, F>(&self, op: F) -> Result<R, StorageError>
781 where
782 R: Send + 'static,
783 F: FnOnce(&Connection) -> Result<R, StorageError> + Send + 'static,
784 {
785 let reply_rx =
786 match tokio::time::timeout(self.enqueue_timeout, self.enqueue_inner(op, true)).await {
787 Ok(Ok(reply_rx)) => reply_rx,
788 Ok(Err(e)) => return Err(e),
789 Err(_elapsed) => {
790 let timeout_ms = self.enqueue_timeout.as_millis() as u64;
791 crate::timeout_sink::emit_queue_saturation(&self.db, timeout_ms);
792 return Err(StorageError::WriteQueueFull { timeout_ms });
793 }
794 };
795
796 reply_rx
797 .await
798 .map_err(|_| writer_task_terminated(WriterTaskRequestState::SideEffectsUnknown))?
799 }
800
801 pub fn queue_depth(&self) -> usize {
810 self.tx.max_capacity() - self.tx.capacity()
811 }
812
813 pub fn capacity(&self) -> usize {
816 self.tx.max_capacity()
817 }
818}
819
820pub fn spawn(pool: &ConnectionPool, capacity: usize) -> Result<WriterTaskHandle, SqliteError> {
842 let conn = pool.open_standalone_writer_untracked()?;
846 let acquisition_counters = pool.writer_acquisition_counters();
847 let busy_timeout = pool.config().busy_timeout;
848 let origin = pool.origin();
849 let backend_key = writer_db_key(pool);
850 let db = crate::timeout_sink::db_label(pool);
851 let (tx, rx) = mpsc::channel(capacity.max(1));
852 let join = tokio::spawn(run_writer_task(
853 conn,
854 rx,
855 origin,
856 db.clone(),
857 acquisition_counters,
858 busy_timeout,
859 ));
860 pool.set_writer_task_join(join);
864 Ok(WriterTaskHandle {
865 tx,
866 backend_key,
867 db,
868 slow_write_threshold: crate::timeout_sink::slow_write_threshold(),
869 enqueue_timeout: std::time::Duration::from_millis(
870 pool.config().write_admission_deadline_ms,
871 ),
872 })
873}
874
875async fn close_and_fail_queued_requests(rx: &mut mpsc::Receiver<Box<dyn AnyWriteRequest + Send>>) {
883 rx.close();
884 while let Some(request) = rx.recv().await {
885 request.reply_error(writer_task_terminated(WriterTaskRequestState::NotStarted));
886 }
887}
888
889async fn run_writer_task(
903 mut conn: Connection,
904 mut rx: mpsc::Receiver<Box<dyn AnyWriteRequest + Send>>,
905 origin: khive_storage::tx_registry::TxOrigin,
906 db: String,
907 acquisition_counters: Arc<WriterAcquisitionCounters>,
908 busy_timeout: Duration,
909) {
910 while let Some(request) = rx.recv().await {
911 let queue_wait = request.queue_wait();
915 let origin = origin.clone();
916 let blocking_counters = Arc::clone(&acquisition_counters);
917 let outcome = tokio::task::spawn_blocking(move || {
918 let acquisition_counters = blocking_counters;
919 if !conn.is_autocommit() {
924 tracing::error!(
925 "writer task: connection is not in autocommit mode before request dispatch; \
926 retiring the poisoned writer without running the request"
927 );
928 let request_state = WriterTaskRequestState::NotStarted;
929 request.reply_error(writer_task_terminated(request_state));
930 return (conn, Some(request_state));
931 }
932
933 let terminal_state = if request.is_top_level() {
934 acquisition_counters.record_writer_task_acquisition();
942 sealed::Sealed::execute_and_reply_top_level_reporting_terminal(
943 request, &conn, queue_wait,
944 )
945 } else {
946 let tx_span = khive_storage::tx_registry::register_scoped(
947 Some("writer_task_tx".to_string()),
948 origin,
949 );
950 let transaction_acquire_started = Instant::now();
951 let begin_outcome = conn.execute_batch("BEGIN IMMEDIATE");
952 let transaction_acquire = transaction_acquire_started.elapsed();
953 match begin_outcome {
954 Ok(()) => {
955 acquisition_counters.record_writer_task_acquisition();
956 sealed::Sealed::execute_and_reply_reporting_terminal(
957 request,
958 &conn,
959 Some(tx_span),
960 queue_wait,
961 transaction_acquire,
962 )
963 }
964 Err(e) => {
965 tracing::warn!(
971 error = %e,
972 "writer task: BEGIN IMMEDIATE failed; replying an \
973 error without running the request's operation"
974 );
975 drop(tx_span);
980 let begin_error = writer_task_begin_error(e, busy_timeout);
987 match &begin_error {
988 StorageError::WriterTaskBusy { .. } => {
989 acquisition_counters.record_writer_task_begin_busy()
990 }
991 _ => acquisition_counters.record_writer_task_begin_error(),
992 }
993 sealed::Sealed::reply_error_after_begin(
994 request,
995 begin_error,
996 queue_wait,
997 transaction_acquire,
998 );
999 None
1000 }
1001 }
1002 };
1003 (conn, terminal_state)
1004 })
1005 .await;
1006
1007 match outcome {
1008 Ok((returned_conn, None)) => conn = returned_conn,
1009 Ok((_returned_conn, Some(request_state))) => {
1010 acquisition_counters.record_writer_task_request_failure();
1011 if request_state == WriterTaskRequestState::SideEffectsUnknown {
1012 acquisition_counters.record_writer_task_side_effects_unknown();
1013 }
1014 tracing::error!(
1015 request_state = %request_state,
1016 "writer task reached a terminal request or connection state; closing and \
1017 failing the queue without restarting"
1018 );
1019 crate::timeout_sink::emit_writer_task_retirement(
1020 &db,
1021 &format!("terminal request state: {request_state}"),
1022 );
1023 close_and_fail_queued_requests(&mut rx).await;
1024 return;
1025 }
1026 Err(join_err) => {
1027 acquisition_counters.record_writer_task_request_failure();
1028 tracing::error!(
1029 error = %join_err,
1030 "writer task blocking closure failed outside the request \
1031 panic boundary; closing and failing the queue without restarting"
1032 );
1033 crate::timeout_sink::emit_writer_task_retirement(
1034 &db,
1035 &format!("blocking closure join failure: {join_err}"),
1036 );
1037 close_and_fail_queued_requests(&mut rx).await;
1038 return;
1039 }
1040 }
1041 }
1042}
1043
1044#[cfg(test)]
1045mod tests {
1046 use super::*;
1047 use crate::pool::PoolConfig;
1048 use rusqlite::hooks::{AuthAction, AuthContext, Authorization, TransactionOperation};
1049 use serial_test::serial;
1050
1051 #[test]
1052 fn begin_error_classification_is_code_based_and_narrow() {
1053 for code in [rusqlite::ffi::SQLITE_BUSY, rusqlite::ffi::SQLITE_LOCKED] {
1054 let error = rusqlite::Error::SqliteFailure(
1055 rusqlite::ffi::Error::new(code),
1056 Some("rendered text is irrelevant".to_string()),
1057 );
1058 assert!(matches!(
1059 writer_task_begin_error(error, Duration::from_millis(175)),
1060 StorageError::WriterTaskBusy { timeout_ms: 175 }
1061 ));
1062 }
1063
1064 let structural = rusqlite::Error::SqliteFailure(
1065 rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_CONSTRAINT),
1066 Some("database is locked".to_string()),
1067 );
1068 assert!(matches!(
1069 writer_task_begin_error(structural, Duration::from_millis(175)),
1070 StorageError::Pool { ref operation, .. } if operation == "writer_task_begin"
1071 ));
1072 }
1073 use std::future::Future;
1074 use std::pin::Pin;
1075 use std::sync::atomic::{AtomicBool, Ordering};
1076 use std::sync::mpsc as std_mpsc;
1077 use std::sync::{Arc, Mutex};
1078 use std::task::{Context, Poll, Wake, Waker};
1079 use std::time::Duration;
1080
1081 fn file_pool(path: &std::path::Path) -> ConnectionPool {
1082 let cfg = PoolConfig {
1083 path: Some(path.to_path_buf()),
1084 ..PoolConfig::default()
1085 };
1086 ConnectionPool::new(cfg).expect("pool open")
1087 }
1088
1089 fn deny_commit_and_rollback(ctx: AuthContext<'_>) -> Authorization {
1090 match ctx.action {
1091 AuthAction::Transaction {
1094 operation: TransactionOperation::Unknown | TransactionOperation::Rollback,
1095 } => Authorization::Deny,
1096 _ => Authorization::Allow,
1097 }
1098 }
1099
1100 fn deny_commit(ctx: AuthContext<'_>) -> Authorization {
1101 match ctx.action {
1102 AuthAction::Transaction {
1103 operation: TransactionOperation::Unknown,
1104 } => Authorization::Deny,
1105 _ => Authorization::Allow,
1106 }
1107 }
1108
1109 fn deny_rollback(ctx: AuthContext<'_>) -> Authorization {
1110 match ctx.action {
1111 AuthAction::Transaction {
1112 operation: TransactionOperation::Rollback,
1113 } => Authorization::Deny,
1114 _ => Authorization::Allow,
1115 }
1116 }
1117
1118 fn assert_writer_task_terminal_state<T: std::fmt::Debug>(
1119 result: Result<T, StorageError>,
1120 expected: WriterTaskRequestState,
1121 ) {
1122 match result {
1123 Err(StorageError::WriterTaskTerminated { request_state }) => {
1124 assert_eq!(request_state, expected)
1125 }
1126 other => panic!("expected WriterTaskTerminated({expected:?}), got {other:?}"),
1127 }
1128 }
1129
1130 struct ParkedWake {
1131 entered: std_mpsc::SyncSender<()>,
1132 release: Mutex<std_mpsc::Receiver<()>>,
1133 }
1134
1135 impl Wake for ParkedWake {
1136 fn wake(self: Arc<Self>) {
1137 self.entered
1138 .send(())
1139 .expect("reply sender must rendezvous with the test");
1140 self.release
1141 .lock()
1142 .unwrap_or_else(|poisoned| poisoned.into_inner())
1143 .recv()
1144 .expect("test must release the parked reply sender");
1145 }
1146 }
1147
1148 fn arm_parked_wake<F: Future>(
1149 mut future: Pin<&mut F>,
1150 ) -> (std_mpsc::Receiver<()>, std_mpsc::Sender<()>) {
1151 let (entered_tx, entered_rx) = std_mpsc::sync_channel(0);
1152 let (release_tx, release_rx) = std_mpsc::channel();
1153 let waker = Waker::from(Arc::new(ParkedWake {
1154 entered: entered_tx,
1155 release: Mutex::new(release_rx),
1156 }));
1157 let mut context = Context::from_waker(&waker);
1158 assert!(
1159 matches!(future.as_mut().poll(&mut context), Poll::Pending),
1160 "writer send must remain pending until its operation replies"
1161 );
1162 (entered_rx, release_tx)
1163 }
1164
1165 fn poll_ready<F: Future>(mut future: Pin<&mut F>) -> F::Output {
1166 let mut context = Context::from_waker(Waker::noop());
1167 match future.as_mut().poll(&mut context) {
1168 Poll::Ready(output) => output,
1169 Poll::Pending => panic!("reply wake must make the writer send ready"),
1170 }
1171 }
1172
1173 fn database_tx_view(pool: &ConnectionPool) -> khive_storage::tx_registry::TxOriginFilter {
1174 match pool.origin() {
1175 khive_storage::tx_registry::TxOrigin::Database(identity) => {
1176 khive_storage::tx_registry::TxOriginFilter::Secondary(identity)
1177 }
1178 other => panic!("expected a file-backed database origin, got {other:?}"),
1179 }
1180 }
1181
1182 async fn wait_for_writer_span_to_close(view: &khive_storage::tx_registry::TxOriginFilter) {
1183 tokio::time::timeout(Duration::from_secs(5), async {
1184 while khive_storage::tx_registry::any_open_labeled(view, "writer_task_tx") {
1185 tokio::task::yield_now().await;
1186 }
1187 })
1188 .await
1189 .expect("writer task transaction span must eventually close");
1190 }
1191
1192 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1193 #[serial(tx_registry)]
1194 async fn successful_send_reply_waits_for_writer_tx_deregistration() {
1195 let dir = tempfile::tempdir().unwrap();
1196 let path = dir.path().join("writer_task_success_reply_lifecycle.db");
1197 let pool = file_pool(&path);
1198 let view = database_tx_view(&pool);
1199 let handle = spawn(&pool, 8).expect("writer task spawn");
1200 let (op_started_tx, op_started_rx) = std_mpsc::sync_channel(0);
1201 let (op_release_tx, op_release_rx) = std_mpsc::channel();
1202
1203 let send = handle.send(move |_conn| {
1204 op_started_tx
1205 .send(())
1206 .expect("operation must rendezvous with the test");
1207 op_release_rx
1208 .recv()
1209 .expect("test must release the operation");
1210 Ok::<_, StorageError>(())
1211 });
1212 tokio::pin!(send);
1213 let (reply_entered_rx, reply_release_tx) = arm_parked_wake(send.as_mut());
1214
1215 op_started_rx
1216 .recv_timeout(Duration::from_secs(5))
1217 .expect("writer operation must start");
1218 op_release_tx.send(()).expect("release writer operation");
1219 reply_entered_rx
1220 .recv_timeout(Duration::from_secs(5))
1221 .expect("reply sender must wake the waiting caller");
1222
1223 let reply = poll_ready(send.as_mut());
1224 let span_was_open_at_reply =
1225 khive_storage::tx_registry::any_open_labeled(&view, "writer_task_tx");
1226
1227 reply_release_tx
1228 .send(())
1229 .expect("release parked reply sender");
1230 wait_for_writer_span_to_close(&view).await;
1231
1232 reply.expect("committed operation reply");
1233 assert!(
1234 !span_was_open_at_reply,
1235 "a successful caller reply must not become observable while its committed writer_task_tx span remains registered"
1236 );
1237 }
1238
1239 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1240 #[serial(tx_registry)]
1241 async fn begin_failure_reply_waits_for_writer_tx_deregistration() {
1242 let dir = tempfile::tempdir().unwrap();
1243 let path = dir.path().join("writer_task_begin_reply_lifecycle.db");
1244 let cfg = PoolConfig {
1245 path: Some(path),
1246 busy_timeout: Duration::from_millis(150),
1247 ..PoolConfig::default()
1248 };
1249 let pool = ConnectionPool::new(cfg).unwrap();
1250 let view = database_tx_view(&pool);
1251 let handle = spawn(&pool, 8).expect("writer task spawn");
1252 let lock_holder = pool.try_writer().expect("pool writer");
1253 lock_holder
1254 .conn()
1255 .execute_batch("BEGIN IMMEDIATE")
1256 .expect("hold database write lock");
1257 let op_ran = Arc::new(AtomicBool::new(false));
1258 let op_ran_in_request = Arc::clone(&op_ran);
1259
1260 let send = handle.send(move |_conn| {
1261 op_ran_in_request.store(true, Ordering::SeqCst);
1262 Ok::<_, StorageError>(())
1263 });
1264 tokio::pin!(send);
1265 let (reply_entered_rx, reply_release_tx) = arm_parked_wake(send.as_mut());
1266 reply_entered_rx
1267 .recv_timeout(Duration::from_secs(5))
1268 .expect("BEGIN failure must wake the waiting caller");
1269
1270 let reply = poll_ready(send.as_mut());
1271 let span_was_open_at_reply =
1272 khive_storage::tx_registry::any_open_labeled(&view, "writer_task_tx");
1273
1274 reply_release_tx
1275 .send(())
1276 .expect("release parked reply sender");
1277 wait_for_writer_span_to_close(&view).await;
1278 lock_holder
1279 .conn()
1280 .execute_batch("ROLLBACK")
1281 .expect("release database write lock");
1282
1283 assert!(
1284 matches!(
1285 &reply,
1286 Err(StorageError::WriterTaskBusy { timeout_ms }) if *timeout_ms == 150
1287 ),
1288 "expected typed retryable writer-task contention, got {reply:?}"
1289 );
1290 assert!(!op_ran.load(Ordering::SeqCst));
1291 assert!(
1292 !span_was_open_at_reply,
1293 "a BEGIN-failure caller reply must not become observable while its writer_task_tx span remains registered"
1294 );
1295 }
1296
1297 #[tokio::test]
1304 #[serial(tx_registry)]
1305 async fn begin_immediate_failure_replies_error_without_running_op() {
1306 let dir = tempfile::tempdir().unwrap();
1312 let path = dir.path().join("writer_task_begin_failure.db");
1313 let cfg = PoolConfig {
1314 path: Some(path.clone()),
1315 busy_timeout: Duration::from_millis(150),
1316 ..PoolConfig::default()
1317 };
1318 let pool = ConnectionPool::new(cfg).unwrap();
1319 {
1320 let writer = pool.try_writer().unwrap();
1321 writer
1322 .conn()
1323 .execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)")
1324 .unwrap();
1325 }
1326
1327 let handle = spawn(&pool, 8).expect("writer task should spawn on a file-backed pool");
1328
1329 let lock_holder = pool.try_writer().unwrap();
1330 lock_holder.conn().execute_batch("BEGIN IMMEDIATE").unwrap();
1331
1332 let op_ran = Arc::new(AtomicBool::new(false));
1333 let op_ran_clone = Arc::clone(&op_ran);
1334 let result = handle
1335 .send(move |conn| {
1336 op_ran_clone.store(true, Ordering::SeqCst);
1337 conn.execute("INSERT INTO t (id, v) VALUES (99, 'should-not-land')", [])
1338 .map_err(|e| StorageError::Pool {
1339 operation: "test_insert".into(),
1340 message: e.to_string(),
1341 })
1342 })
1343 .await;
1344
1345 assert!(
1346 matches!(
1347 &result,
1348 Err(StorageError::WriterTaskBusy { timeout_ms }) if *timeout_ms == 150
1349 ),
1350 "expected a typed retryable error on contended BEGIN IMMEDIATE, got {result:?}"
1351 );
1352 assert!(
1353 !op_ran.load(Ordering::SeqCst),
1354 "the request's operation closure must never run when BEGIN \
1355 IMMEDIATE fails — running it would land a partial write in \
1356 autocommit mode for a request the caller is told failed"
1357 );
1358
1359 lock_holder.conn().execute_batch("ROLLBACK").unwrap();
1362 drop(lock_holder);
1363
1364 handle
1365 .send(|conn| {
1366 conn.execute("INSERT INTO t (id, v) VALUES (100, 'next-request')", [])
1367 .map_err(|e| StorageError::Pool {
1368 operation: "test_insert_after_busy".into(),
1369 message: e.to_string(),
1370 })
1371 })
1372 .await
1373 .expect("transient contention must not retire the writer task");
1374
1375 let reader = pool.reader().expect("reader");
1376 let count: i64 = reader
1377 .conn()
1378 .query_row("SELECT COUNT(*) FROM t WHERE id IN (99, 100)", [], |row| {
1379 row.get(0)
1380 })
1381 .unwrap();
1382 assert_eq!(
1383 count, 1,
1384 "the failed request must not land, while the next request commits on the same task"
1385 );
1386 }
1387
1388 #[tokio::test]
1391 #[serial(tx_registry)]
1392 async fn contended_begin_increments_its_own_failure_counter() {
1393 let dir = tempfile::tempdir().unwrap();
1399 let path = dir.path().join("writer_task_begin_busy_counter.db");
1400 let cfg = PoolConfig {
1401 path: Some(path.clone()),
1402 busy_timeout: Duration::from_millis(150),
1403 ..PoolConfig::default()
1404 };
1405 let pool = ConnectionPool::new(cfg).unwrap();
1406 {
1407 let writer = pool.try_writer().unwrap();
1408 writer
1409 .conn()
1410 .execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY)")
1411 .unwrap();
1412 }
1413
1414 let handle = spawn(&pool, 8).expect("writer task should spawn on a file-backed pool");
1415
1416 let before = pool.writer_acquisition_snapshot();
1417 assert_eq!(
1418 before.writer_task_begin_busy, 0,
1419 "baseline: nothing has been refused yet"
1420 );
1421
1422 let lock_holder = pool.try_writer().unwrap();
1423 lock_holder.conn().execute_batch("BEGIN IMMEDIATE").unwrap();
1424
1425 let result = handle
1426 .send(|conn| {
1427 conn.execute("INSERT INTO t (id) VALUES (1)", [])
1428 .map_err(|e| StorageError::Pool {
1429 operation: "test_insert".into(),
1430 message: e.to_string(),
1431 })
1432 })
1433 .await;
1434 assert!(
1435 matches!(&result, Err(StorageError::WriterTaskBusy { .. })),
1436 "precondition: the request must actually be refused busy, got {result:?}"
1437 );
1438
1439 let after = pool.writer_acquisition_snapshot();
1440 assert_eq!(
1441 after.writer_task_begin_busy, 1,
1442 "the refusal the caller was told about must appear in the counters"
1443 );
1444 assert_eq!(
1445 after.writer_task_begin_errors, 0,
1446 "a busy refusal must not be counted as a non-busy BEGIN error"
1447 );
1448 assert_eq!(
1449 after.timeouts, before.timeouts,
1450 "a writer-task BEGIN refusal must not be mislabeled as a pool-mutex \
1451 checkout timeout — separate ADR-135 F6 stages, separate counters"
1452 );
1453
1454 lock_holder.conn().execute_batch("ROLLBACK").unwrap();
1458 drop(lock_holder);
1459 handle
1460 .send(|conn| {
1461 conn.execute("INSERT INTO t (id) VALUES (2)", [])
1462 .map_err(|e| StorageError::Pool {
1463 operation: "test_insert_after_busy".into(),
1464 message: e.to_string(),
1465 })
1466 })
1467 .await
1468 .expect("the writer task survives transient contention");
1469
1470 let settled = pool.writer_acquisition_snapshot();
1471 assert_eq!(
1472 settled.writer_task_begin_busy, 1,
1473 "a successful request must leave the refusal counter untouched"
1474 );
1475 assert!(
1476 settled.writer_task_acquisitions > after.writer_task_acquisitions,
1477 "and it must still register as a success"
1478 );
1479 }
1480
1481 #[tokio::test]
1485 #[serial(tx_registry)]
1486 async fn writer_task_executes_op_and_commits() {
1487 let dir = tempfile::tempdir().unwrap();
1488 let path = dir.path().join("writer_task_commit.db");
1489 let pool = file_pool(&path);
1490 {
1491 let writer = pool.try_writer().unwrap();
1492 writer
1493 .conn()
1494 .execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)")
1495 .unwrap();
1496 }
1497
1498 let handle = spawn(&pool, 8).expect("writer task should spawn on a file-backed pool");
1499
1500 let affected = handle
1501 .send(|conn| {
1502 conn.execute("INSERT INTO t (id, v) VALUES (1, 'hello')", [])
1503 .map_err(|e| StorageError::Pool {
1504 operation: "test_insert".into(),
1505 message: e.to_string(),
1506 })
1507 })
1508 .await
1509 .expect("op should succeed");
1510 assert_eq!(affected, 1);
1511
1512 let reader = pool.reader().expect("reader");
1516 let v: String = reader
1517 .conn()
1518 .query_row("SELECT v FROM t WHERE id = 1", [], |row| row.get(0))
1519 .expect("row must be committed and visible to a reader");
1520 assert_eq!(v, "hello");
1521
1522 let counters = pool.writer_acquisition_snapshot();
1523 assert_eq!(counters.acquisitions, 2);
1524 assert_eq!(counters.pooled_acquisitions, 1);
1525 assert_eq!(counters.standalone_acquisitions, 0);
1526 assert_eq!(counters.writer_task_acquisitions, 1);
1527 assert_eq!(counters.timeouts, 0);
1528 }
1529
1530 #[tokio::test]
1531 async fn writer_task_connection_follows_checkpoint_ownership_claim() {
1532 let dir = tempfile::tempdir().unwrap();
1533 let path = dir.path().join("writer_task_autocheckpoint.db");
1534 let pool = file_pool(&path);
1535 let handle = pool
1539 .writer_task_handle()
1540 .expect("writer task should spawn")
1541 .expect("file-backed pool resolves the write queue on");
1542
1543 let read_pages = |handle: &WriterTaskHandle| {
1544 let handle = handle.clone();
1545 async move {
1546 handle
1547 .send_top_level(|conn| {
1548 conn.pragma_query_value(None, "wal_autocheckpoint", |row| {
1549 row.get::<_, u32>(0)
1550 })
1551 .map_err(|e| StorageError::Pool {
1552 operation: "test_wal_autocheckpoint".into(),
1553 message: e.to_string(),
1554 })
1555 })
1556 .await
1557 .expect("query writer-task connection pragma")
1558 }
1559 };
1560
1561 assert_eq!(
1564 read_pages(&handle).await,
1565 crate::pool::FALLBACK_WAL_AUTOCHECKPOINT_PAGES
1566 );
1567
1568 pool.claim_checkpoint_ownership().expect("claim ownership");
1571 pool.propagate_checkpoint_claim_to_writer_task()
1572 .await
1573 .expect("propagate claim to the running writer task");
1574 assert_eq!(read_pages(&handle).await, 0);
1575 }
1576
1577 #[test]
1578 fn spawn_fails_on_in_memory_pool() {
1579 let cfg = PoolConfig {
1585 path: None,
1586 ..PoolConfig::default()
1587 };
1588 let pool = ConnectionPool::new(cfg).unwrap();
1589 let result = spawn(&pool, 8);
1590 assert!(
1591 result.is_err(),
1592 "in-memory pools must reject spawn, not panic"
1593 );
1594 }
1595
1596 #[tokio::test]
1597 async fn full_channel_applies_backpressure_not_immediate_error() {
1598 let (tx, _rx) = mpsc::channel::<Box<dyn AnyWriteRequest + Send>>(1);
1603 let handle = WriterTaskHandle {
1604 tx,
1605 backend_key: None,
1606 db: "test".to_string(),
1607 slow_write_threshold: None,
1608 enqueue_timeout: Duration::from_secs(5),
1609 };
1610
1611 let first = tokio::spawn({
1614 let handle = handle.clone();
1615 async move {
1616 let _ = handle.send(|_conn| Ok::<(), StorageError>(())).await;
1617 }
1618 });
1619
1620 tokio::time::sleep(Duration::from_millis(20)).await;
1622
1623 let second = tokio::time::timeout(
1626 Duration::from_millis(100),
1627 handle.send(|_conn| Ok::<(), StorageError>(())),
1628 )
1629 .await;
1630
1631 assert!(
1632 second.is_err(),
1633 "a full channel must apply backpressure (send suspends) rather \
1634 than erroring immediately — no try_send escape hatch per ADR-067"
1635 );
1636
1637 first.abort();
1638 }
1639
1640 #[tokio::test]
1641 async fn send_with_timeout_maps_full_channel_to_write_queue_full() {
1642 let (tx, _rx) = mpsc::channel::<Box<dyn AnyWriteRequest + Send>>(1);
1643 let handle = WriterTaskHandle {
1644 tx,
1645 backend_key: None,
1646 db: "test".to_string(),
1647 slow_write_threshold: None,
1648 enqueue_timeout: Duration::from_secs(5),
1649 };
1650
1651 let first = tokio::spawn({
1652 let handle = handle.clone();
1653 async move {
1654 let _ = handle.send(|_conn| Ok::<(), StorageError>(())).await;
1655 }
1656 });
1657 tokio::time::sleep(Duration::from_millis(20)).await;
1658
1659 let result = handle
1660 .send_with_timeout(
1661 |_conn| Ok::<(), StorageError>(()),
1662 Duration::from_millis(50),
1663 )
1664 .await;
1665
1666 match result {
1667 Err(StorageError::WriteQueueFull { timeout_ms }) => assert_eq!(timeout_ms, 50),
1668 other => panic!("expected WriteQueueFull, got {other:?}"),
1669 }
1670
1671 first.abort();
1672 }
1673
1674 #[tokio::test]
1675 async fn configured_enqueue_timeout_rejects_only_unaccepted_request() {
1676 let dir = tempfile::tempdir().unwrap();
1681 let path = dir.path().join("configured_enqueue_timeout.db");
1682 let cfg = PoolConfig {
1683 path: Some(path.clone()),
1684 write_admission_deadline_ms: 100,
1685 ..PoolConfig::default()
1686 };
1687 let pool = ConnectionPool::new(cfg).unwrap();
1688 let handle = spawn(&pool, 1).expect("writer task should spawn on a file-backed pool");
1689
1690 let (started_tx, started_rx) = oneshot::channel::<()>();
1694 let (release_tx, release_rx) = std_mpsc::channel::<()>();
1695 let handle_a = handle.clone();
1696 let a_task = tokio::spawn(async move {
1697 handle_a
1698 .send(move |_conn| {
1699 let _ = started_tx.send(());
1700 release_rx.recv().expect("test must release request A");
1701 Ok::<(), StorageError>(())
1702 })
1703 .await
1704 });
1705 tokio::time::timeout(Duration::from_secs(5), started_rx)
1706 .await
1707 .expect("request A did not start")
1708 .expect("request A dropped its start signal");
1709
1710 let b_reply_rx = tokio::time::timeout(
1714 Duration::from_secs(5),
1715 handle.enqueue(|_conn| Ok::<(), StorageError>(())),
1716 )
1717 .await
1718 .expect("B must be accepted promptly")
1719 .expect("B must be accepted: the one channel slot is free while A drains");
1720
1721 let c_ran = Arc::new(AtomicBool::new(false));
1725 let c_ran_in_op = Arc::clone(&c_ran);
1726 let c_result = handle
1727 .send_bounded(move |_conn| {
1728 c_ran_in_op.store(true, Ordering::SeqCst);
1729 Ok::<(), StorageError>(())
1730 })
1731 .await;
1732 match c_result {
1733 Err(StorageError::WriteQueueFull { .. }) => {}
1734 other => panic!("expected WriteQueueFull, got {other:?}"),
1735 }
1736 assert!(!c_ran.load(Ordering::SeqCst), "C must never run");
1737
1738 release_tx.send(()).expect("release request A");
1740 tokio::time::timeout(Duration::from_secs(5), a_task)
1741 .await
1742 .expect("A did not complete")
1743 .expect("A task join")
1744 .expect("A must complete successfully");
1745 tokio::time::timeout(Duration::from_secs(5), b_reply_rx)
1746 .await
1747 .expect("B did not reply")
1748 .expect("B's reply channel must not be dropped")
1749 .expect("B must complete successfully");
1750 }
1751
1752 #[tokio::test]
1758 #[serial(tx_registry)]
1759 async fn send_with_timeout_returns_op_result_when_op_outlives_the_timeout() {
1760 let dir = tempfile::tempdir().unwrap();
1767 let path = dir.path().join("writer_task_slow_op.db");
1768 let pool = file_pool(&path);
1769 {
1770 let writer = pool.try_writer().unwrap();
1771 writer
1772 .conn()
1773 .execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)")
1774 .unwrap();
1775 }
1776
1777 let handle = spawn(&pool, 8).expect("writer task should spawn on a file-backed pool");
1778
1779 let result = handle
1780 .send_with_timeout(
1781 |conn| {
1782 std::thread::sleep(Duration::from_millis(150));
1785 conn.execute("INSERT INTO t (id, v) VALUES (1, 'slow')", [])
1786 .map_err(|e| StorageError::Pool {
1787 operation: "test_insert".into(),
1788 message: e.to_string(),
1789 })
1790 },
1791 Duration::from_millis(20),
1792 )
1793 .await;
1794
1795 let affected = result.expect(
1796 "an accepted request must return its real result even when the \
1797 op takes longer than the enqueue timeout, not WriteQueueFull",
1798 );
1799 assert_eq!(affected, 1);
1800
1801 let reader = pool.reader().expect("reader");
1804 let v: String = reader
1805 .conn()
1806 .query_row("SELECT v FROM t WHERE id = 1", [], |row| row.get(0))
1807 .expect("the slow op's write must have committed");
1808 assert_eq!(v, "slow");
1809 }
1810
1811 #[tokio::test]
1812 #[serial(tx_registry)]
1813 async fn operation_failure_with_successful_rollback_preserves_error_and_writer_continues() {
1814 let dir = tempfile::tempdir().unwrap();
1815 let path = dir.path().join("writer_task_operation_rollback.db");
1816 let pool = file_pool(&path);
1817 {
1818 let writer = pool.try_writer().unwrap();
1819 writer
1820 .conn()
1821 .execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)")
1822 .unwrap();
1823 }
1824 let handle = spawn(&pool, 8).expect("writer task spawn");
1825
1826 let original_error = handle
1827 .send(|conn| -> Result<(), StorageError> {
1828 conn.execute("INSERT INTO t (id, v) VALUES (1, 'rolled-back')", [])
1829 .map_err(|e| StorageError::Pool {
1830 operation: "test_operation_error_insert".into(),
1831 message: e.to_string(),
1832 })?;
1833 Err(StorageError::Internal(
1834 "intentional operation failure".into(),
1835 ))
1836 })
1837 .await;
1838 assert!(
1839 matches!(
1840 &original_error,
1841 Err(StorageError::Internal(message))
1842 if message == "intentional operation failure"
1843 ),
1844 "a confirmed rollback must preserve the operation error, got {original_error:?}"
1845 );
1846
1847 let affected = handle
1848 .send(|conn| {
1849 conn.execute("INSERT INTO t (id, v) VALUES (2, 'committed')", [])
1850 .map_err(|e| StorageError::Pool {
1851 operation: "test_operation_error_followup_insert".into(),
1852 message: e.to_string(),
1853 })
1854 })
1855 .await
1856 .expect("the writer must continue after a confirmed rollback");
1857 assert_eq!(affected, 1);
1858
1859 let reader = pool.reader().expect("reader");
1860 let rolled_back: i64 = reader
1861 .conn()
1862 .query_row("SELECT COUNT(*) FROM t WHERE id = 1", [], |row| row.get(0))
1863 .unwrap();
1864 let committed: i64 = reader
1865 .conn()
1866 .query_row("SELECT COUNT(*) FROM t WHERE id = 2", [], |row| row.get(0))
1867 .unwrap();
1868 assert_eq!(rolled_back, 0);
1869 assert_eq!(committed, 1);
1870 }
1871
1872 #[tokio::test]
1873 #[serial(tx_registry)]
1874 async fn commit_failure_with_successful_rollback_preserves_error_and_writer_continues() {
1875 let dir = tempfile::tempdir().unwrap();
1876 let path = dir.path().join("writer_task_commit_rollback.db");
1877 let pool = file_pool(&path);
1878 {
1879 let writer = pool.try_writer().unwrap();
1880 writer
1881 .conn()
1882 .execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)")
1883 .unwrap();
1884 }
1885 let handle = spawn(&pool, 8).expect("writer task spawn");
1886
1887 let commit_error = handle
1888 .send(|conn| -> Result<usize, StorageError> {
1889 let affected = conn
1890 .execute("INSERT INTO t (id, v) VALUES (1, 'rolled-back')", [])
1891 .map_err(|e| StorageError::Pool {
1892 operation: "test_commit_error_insert".into(),
1893 message: e.to_string(),
1894 })?;
1895 conn.authorizer(Some(deny_commit))
1896 .map_err(|e| StorageError::Pool {
1897 operation: "test_install_authorizer".into(),
1898 message: e.to_string(),
1899 })?;
1900 Ok(affected)
1901 })
1902 .await;
1903 assert!(
1904 matches!(
1905 &commit_error,
1906 Err(StorageError::Pool { operation, .. })
1907 if operation == "writer_task_commit"
1908 ),
1909 "a confirmed rollback must preserve the commit error, got {commit_error:?}"
1910 );
1911 assert!(
1912 commit_error
1913 .as_ref()
1914 .expect_err("COMMIT must be denied")
1915 .is_retryable(),
1916 "the existing retryable commit-error contract must remain unchanged after a \
1917 confirmed rollback"
1918 );
1919
1920 let affected = handle
1921 .send(|conn| {
1922 conn.authorizer(None::<fn(AuthContext<'_>) -> Authorization>)
1923 .map_err(|e| StorageError::Pool {
1924 operation: "test_remove_authorizer".into(),
1925 message: e.to_string(),
1926 })?;
1927 conn.execute("INSERT INTO t (id, v) VALUES (2, 'committed')", [])
1928 .map_err(|e| StorageError::Pool {
1929 operation: "test_commit_error_followup_insert".into(),
1930 message: e.to_string(),
1931 })
1932 })
1933 .await
1934 .expect("the writer must continue after the failed COMMIT is rolled back");
1935 assert_eq!(affected, 1);
1936
1937 let reader = pool.reader().expect("reader");
1938 let rolled_back: i64 = reader
1939 .conn()
1940 .query_row("SELECT COUNT(*) FROM t WHERE id = 1", [], |row| row.get(0))
1941 .unwrap();
1942 let committed: i64 = reader
1943 .conn()
1944 .query_row("SELECT COUNT(*) FROM t WHERE id = 2", [], |row| row.get(0))
1945 .unwrap();
1946 assert_eq!(rolled_back, 0);
1947 assert_eq!(committed, 1);
1948 }
1949
1950 #[test]
1951 fn top_level_request_returning_with_open_transaction_reports_side_effects_unknown() {
1952 let conn = Connection::open_in_memory().expect("in-memory connection");
1953 conn.execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY)")
1954 .unwrap();
1955 let (reply_tx, mut reply_rx) = oneshot::channel();
1956 let request = WriteRequest {
1957 op: Box::new(|conn| -> Result<usize, StorageError> {
1958 conn.execute_batch("BEGIN IMMEDIATE")
1959 .map_err(|e| StorageError::Pool {
1960 operation: "test_top_level_begin".into(),
1961 message: e.to_string(),
1962 })?;
1963 conn.execute("INSERT INTO t (id) VALUES (1)", [])
1964 .map_err(|e| StorageError::Pool {
1965 operation: "test_top_level_insert".into(),
1966 message: e.to_string(),
1967 })
1968 }),
1969 reply: reply_tx,
1970 top_level: true,
1971 telemetry: WriteTelemetry::new(None, "test".to_string(), 0, None),
1972 };
1973
1974 let terminal_state = sealed::Sealed::execute_and_reply_top_level_reporting_terminal(
1975 Box::new(request),
1976 &conn,
1977 Duration::ZERO,
1978 );
1979 assert_eq!(
1980 terminal_state,
1981 Some(WriterTaskRequestState::SideEffectsUnknown)
1982 );
1983 let reply = reply_rx
1984 .try_recv()
1985 .expect("active request must receive a typed terminal reply");
1986 assert_writer_task_terminal_state(reply, WriterTaskRequestState::SideEffectsUnknown);
1987 assert!(
1988 !conn.is_autocommit(),
1989 "the fixture must prove the post-request autocommit check observed an open transaction"
1990 );
1991 }
1992
1993 #[test]
1994 fn commit_failure_with_failed_rollback_reports_side_effects_unknown() {
1995 let conn = Connection::open_in_memory().expect("in-memory connection");
1996 conn.execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY); BEGIN IMMEDIATE")
1997 .unwrap();
1998 let (reply_tx, mut reply_rx) = oneshot::channel();
1999 let request = WriteRequest {
2000 op: Box::new(|conn| -> Result<usize, StorageError> {
2001 let affected = conn
2002 .execute("INSERT INTO t (id) VALUES (1)", [])
2003 .map_err(|e| StorageError::Pool {
2004 operation: "test_insert_before_commit_failure".into(),
2005 message: e.to_string(),
2006 })?;
2007 conn.authorizer(Some(deny_commit_and_rollback))
2008 .map_err(|e| StorageError::Pool {
2009 operation: "test_install_authorizer".into(),
2010 message: e.to_string(),
2011 })?;
2012 Ok(affected)
2013 }),
2014 reply: reply_tx,
2015 top_level: false,
2016 telemetry: WriteTelemetry::new(None, "test".to_string(), 0, None),
2017 };
2018
2019 let terminal_state = sealed::Sealed::execute_and_reply_reporting_terminal(
2020 Box::new(request),
2021 &conn,
2022 None,
2023 Duration::ZERO,
2024 Duration::ZERO,
2025 );
2026 assert_eq!(
2027 terminal_state,
2028 Some(WriterTaskRequestState::SideEffectsUnknown)
2029 );
2030 let reply = reply_rx
2031 .try_recv()
2032 .expect("active request must receive a typed terminal reply");
2033 assert_writer_task_terminal_state(reply, WriterTaskRequestState::SideEffectsUnknown);
2034 assert!(
2035 !conn.is_autocommit(),
2036 "the denied COMMIT and ROLLBACK must leave the test connection poisoned"
2037 );
2038 }
2039
2040 #[tokio::test]
2041 #[serial(tx_registry)]
2042 async fn poisoned_connection_retires_before_queued_top_level_request() {
2043 let dir = tempfile::tempdir().unwrap();
2044 let path = dir.path().join("writer_task_rollback_poison.db");
2045 let pool = file_pool(&path);
2046 {
2047 let writer = pool.try_writer().unwrap();
2048 writer
2049 .conn()
2050 .execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)")
2051 .unwrap();
2052 }
2053 let handle = spawn(&pool, 8).expect("writer task spawn");
2054 let (started_tx, started_rx) = oneshot::channel::<()>();
2055 let (release_tx, release_rx) = std_mpsc::channel::<()>();
2056
2057 let active = tokio::spawn({
2058 let handle = handle.clone();
2059 async move {
2060 handle
2061 .send(move |conn| -> Result<usize, StorageError> {
2062 let affected = conn
2063 .execute("INSERT INTO t (id, v) VALUES (1, 'active')", [])
2064 .map_err(|e| StorageError::Pool {
2065 operation: "test_active_insert".into(),
2066 message: e.to_string(),
2067 })?;
2068 conn.authorizer(Some(deny_commit_and_rollback))
2069 .map_err(|e| StorageError::Pool {
2070 operation: "test_install_authorizer".into(),
2071 message: e.to_string(),
2072 })?;
2073 let _ = started_tx.send(());
2074 release_rx.recv().expect("test must release active op");
2075 Ok(affected)
2076 })
2077 .await
2078 }
2079 });
2080
2081 tokio::time::timeout(Duration::from_secs(5), started_rx)
2082 .await
2083 .expect("active request did not start")
2084 .expect("active request dropped its start signal");
2085
2086 let queued_ran = Arc::new(AtomicBool::new(false));
2087 let queued_ran_in_op = Arc::clone(&queued_ran);
2088 let queued_top_level = handle
2089 .enqueue_inner(
2090 move |conn| {
2091 queued_ran_in_op.store(true, Ordering::SeqCst);
2092 conn.execute("INSERT INTO t (id, v) VALUES (2, 'queued')", [])
2093 .map_err(|e| StorageError::Pool {
2094 operation: "test_queued_top_level_insert".into(),
2095 message: e.to_string(),
2096 })
2097 },
2098 true,
2099 )
2100 .await
2101 .expect("top-level request must queue behind active request");
2102 release_tx.send(()).expect("release active op");
2103
2104 let active_result = tokio::time::timeout(Duration::from_secs(5), active)
2105 .await
2106 .expect("active caller hung after rollback failure")
2107 .expect("active caller task join");
2108 assert_writer_task_terminal_state(
2109 active_result,
2110 WriterTaskRequestState::SideEffectsUnknown,
2111 );
2112
2113 let queued_result = tokio::time::timeout(Duration::from_secs(5), queued_top_level)
2114 .await
2115 .expect("queued top-level caller hung after terminal failure")
2116 .expect("terminal drain must preserve queued typed reply");
2117 assert_writer_task_terminal_state(queued_result, WriterTaskRequestState::NotStarted);
2118 assert!(
2119 !queued_ran.load(Ordering::SeqCst),
2120 "a top-level request must never run on the poisoned connection"
2121 );
2122
2123 let future_ran = Arc::new(AtomicBool::new(false));
2124 let future_ran_in_op = Arc::clone(&future_ran);
2125 let future_result = handle
2126 .send_top_level(move |_conn| {
2127 future_ran_in_op.store(true, Ordering::SeqCst);
2128 Ok::<(), StorageError>(())
2129 })
2130 .await;
2131 assert_writer_task_terminal_state(future_result, WriterTaskRequestState::NotStarted);
2132 assert!(!future_ran.load(Ordering::SeqCst));
2133 }
2134
2135 #[test]
2136 fn operation_failure_with_failed_rollback_reports_side_effects_unknown() {
2137 let conn = Connection::open_in_memory().expect("in-memory connection");
2138 conn.execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY); BEGIN IMMEDIATE")
2139 .unwrap();
2140 let (reply_tx, mut reply_rx) = oneshot::channel();
2141 let request = WriteRequest {
2142 op: Box::new(|conn| -> Result<(), StorageError> {
2143 conn.authorizer(Some(deny_rollback))
2144 .map_err(|e| StorageError::Pool {
2145 operation: "test_install_authorizer".into(),
2146 message: e.to_string(),
2147 })?;
2148 Err(StorageError::Internal(
2149 "intentional operation failure before denied rollback".into(),
2150 ))
2151 }),
2152 reply: reply_tx,
2153 top_level: false,
2154 telemetry: WriteTelemetry::new(None, "test".to_string(), 0, None),
2155 };
2156
2157 let terminal_state = sealed::Sealed::execute_and_reply_reporting_terminal(
2158 Box::new(request),
2159 &conn,
2160 None,
2161 Duration::ZERO,
2162 Duration::ZERO,
2163 );
2164 assert_eq!(
2165 terminal_state,
2166 Some(WriterTaskRequestState::SideEffectsUnknown)
2167 );
2168 let reply = reply_rx
2169 .try_recv()
2170 .expect("active request must receive a typed terminal reply");
2171 assert_writer_task_terminal_state(reply, WriterTaskRequestState::SideEffectsUnknown);
2172 assert!(
2173 !conn.is_autocommit(),
2174 "the denied ROLLBACK must leave the test connection poisoned"
2175 );
2176 }
2177
2178 #[test]
2179 fn wrapped_panic_with_failed_rollback_reports_side_effects_unknown() {
2180 let conn = Connection::open_in_memory().expect("in-memory connection");
2181 conn.execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY); BEGIN IMMEDIATE")
2182 .unwrap();
2183 let (reply_tx, mut reply_rx) = oneshot::channel();
2184 let request = WriteRequest {
2185 op: Box::new(|conn| -> Result<(), StorageError> {
2186 conn.execute_batch("INSERT INTO t (id) VALUES (1); COMMIT")
2191 .map_err(|e| StorageError::Pool {
2192 operation: "test_force_rollback_failure".into(),
2193 message: e.to_string(),
2194 })?;
2195 panic!("intentional panic after illicit commit");
2196 }),
2197 reply: reply_tx,
2198 top_level: false,
2199 telemetry: WriteTelemetry::new(None, "test".to_string(), 0, None),
2200 };
2201
2202 let terminal_state = sealed::Sealed::execute_and_reply_reporting_terminal(
2203 Box::new(request),
2204 &conn,
2205 None,
2206 Duration::ZERO,
2207 Duration::ZERO,
2208 );
2209 assert_eq!(
2210 terminal_state,
2211 Some(WriterTaskRequestState::SideEffectsUnknown)
2212 );
2213 let reply = reply_rx
2214 .try_recv()
2215 .expect("active request must receive a typed terminal reply");
2216 assert_writer_task_terminal_state(reply, WriterTaskRequestState::SideEffectsUnknown);
2217
2218 let count: i64 = conn
2219 .query_row("SELECT COUNT(*) FROM t", [], |row| row.get(0))
2220 .unwrap();
2221 assert_eq!(
2222 count, 1,
2223 "the fixture's committed side effect proves why the state must be unknown"
2224 );
2225 }
2226
2227 #[tokio::test]
2231 #[serial(tx_registry)]
2232 async fn wrapped_panic_rolls_back_and_terminally_fails_queue() {
2233 let dir = tempfile::tempdir().unwrap();
2234 let path = dir.path().join("writer_task_wrapped_panic.db");
2235 let cfg = PoolConfig {
2236 path: Some(path),
2237 write_queue_enabled: Some(true),
2238 write_queue_capacity: 8,
2239 ..PoolConfig::default()
2240 };
2241 let pool = ConnectionPool::new(cfg).unwrap();
2242 {
2243 let writer = pool.try_writer().unwrap();
2244 writer
2245 .conn()
2246 .execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)")
2247 .unwrap();
2248 }
2249
2250 let handle = pool
2253 .writer_task_handle()
2254 .expect("writer task lookup")
2255 .expect("file-backed queued pool must spawn its writer task");
2256 assert_eq!(pool.writer_task_spawn_count(), 1);
2257
2258 let (started_tx, started_rx) = oneshot::channel::<()>();
2259 let (release_tx, release_rx) = std_mpsc::channel::<()>();
2260 let active = tokio::spawn({
2261 let handle = handle.clone();
2262 async move {
2263 handle
2264 .send(move |conn| -> Result<usize, StorageError> {
2265 conn.execute("INSERT INTO t (id, v) VALUES (1, 'active')", [])
2266 .map_err(|e| StorageError::Pool {
2267 operation: "test_active_insert".into(),
2268 message: e.to_string(),
2269 })?;
2270 let _ = started_tx.send(());
2271 release_rx.recv().expect("test must release active op");
2272 panic!("intentional wrapped writer request panic");
2273 })
2274 .await
2275 }
2276 });
2277
2278 tokio::time::timeout(Duration::from_secs(5), started_rx)
2279 .await
2280 .expect("active request did not start")
2281 .expect("active request dropped its start signal");
2282
2283 let queued_one_ran = Arc::new(AtomicBool::new(false));
2284 let queued_one_ran_in_op = Arc::clone(&queued_one_ran);
2285 let queued_one = handle
2286 .enqueue(move |conn| {
2287 queued_one_ran_in_op.store(true, Ordering::SeqCst);
2288 conn.execute("INSERT INTO t (id, v) VALUES (2, 'queued-one')", [])
2289 .map_err(|e| StorageError::Pool {
2290 operation: "test_queued_one_insert".into(),
2291 message: e.to_string(),
2292 })
2293 })
2294 .await
2295 .expect("first queued request must be accepted");
2296
2297 let queued_two_ran = Arc::new(AtomicBool::new(false));
2298 let queued_two_ran_in_op = Arc::clone(&queued_two_ran);
2299 let queued_two = handle
2300 .enqueue(move |_conn| {
2301 queued_two_ran_in_op.store(true, Ordering::SeqCst);
2302 Ok::<String, StorageError>("queued-two-ran".to_string())
2303 })
2304 .await
2305 .expect("second queued request must be accepted");
2306
2307 assert_eq!(
2308 handle.queue_depth(),
2309 2,
2310 "both heterogeneous requests must be buffered behind the active op"
2311 );
2312 release_tx.send(()).expect("release active op");
2313
2314 let active_result = tokio::time::timeout(Duration::from_secs(5), active)
2315 .await
2316 .expect("active caller hung after panic")
2317 .expect("active caller task join");
2318 assert_writer_task_terminal_state(
2319 active_result,
2320 WriterTaskRequestState::TransactionRolledBack,
2321 );
2322
2323 let queued_one_result = tokio::time::timeout(Duration::from_secs(5), queued_one)
2324 .await
2325 .expect("first queued caller hung after terminal failure")
2326 .expect("terminal drain must preserve first typed reply");
2327 assert_writer_task_terminal_state(queued_one_result, WriterTaskRequestState::NotStarted);
2328
2329 let queued_two_result = tokio::time::timeout(Duration::from_secs(5), queued_two)
2330 .await
2331 .expect("second queued caller hung after terminal failure")
2332 .expect("terminal drain must preserve second typed reply");
2333 assert_writer_task_terminal_state(queued_two_result, WriterTaskRequestState::NotStarted);
2334 assert!(!queued_one_ran.load(Ordering::SeqCst));
2335 assert!(!queued_two_ran.load(Ordering::SeqCst));
2336
2337 let future_ran = Arc::new(AtomicBool::new(false));
2338 let future_ran_in_op = Arc::clone(&future_ran);
2339 let future_result = handle
2340 .send(move |_conn| {
2341 future_ran_in_op.store(true, Ordering::SeqCst);
2342 Ok::<(), StorageError>(())
2343 })
2344 .await;
2345 assert_writer_task_terminal_state(future_result, WriterTaskRequestState::NotStarted);
2346 assert!(!future_ran.load(Ordering::SeqCst));
2347
2348 let cached_after_failure = pool
2349 .writer_task_handle()
2350 .expect("cached writer task lookup")
2351 .expect("pool retains its terminal handle");
2352 assert_eq!(
2353 pool.writer_task_spawn_count(),
2354 1,
2355 "a terminal writer task must not be restarted behind callers' backs"
2356 );
2357 let cached_result = cached_after_failure
2358 .send(|_conn| Ok::<(), StorageError>(()))
2359 .await;
2360 assert_writer_task_terminal_state(cached_result, WriterTaskRequestState::NotStarted);
2361
2362 let reader = pool.reader().expect("reader");
2363 let count: i64 = reader
2364 .conn()
2365 .query_row("SELECT COUNT(*) FROM t", [], |row| row.get(0))
2366 .unwrap();
2367 assert_eq!(
2368 count, 0,
2369 "the active transaction must be rolled back and queued ops must never run"
2370 );
2371 }
2372
2373 #[tokio::test]
2374 async fn top_level_panic_reports_unknown_and_fails_queue_without_running_it() {
2375 let dir = tempfile::tempdir().unwrap();
2376 let path = dir.path().join("writer_task_top_level_panic.db");
2377 let pool = file_pool(&path);
2378 {
2379 let writer = pool.try_writer().unwrap();
2380 writer
2381 .conn()
2382 .execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)")
2383 .unwrap();
2384 }
2385 let handle = spawn(&pool, 8).expect("writer task spawn");
2386
2387 let (started_tx, started_rx) = oneshot::channel::<()>();
2388 let (release_tx, release_rx) = std_mpsc::channel::<()>();
2389 let active = tokio::spawn({
2390 let handle = handle.clone();
2391 async move {
2392 handle
2393 .send_top_level(move |conn| -> Result<usize, StorageError> {
2394 conn.execute("INSERT INTO t (id, v) VALUES (10, 'autocommitted')", [])
2395 .map_err(|e| StorageError::Pool {
2396 operation: "test_top_level_insert".into(),
2397 message: e.to_string(),
2398 })?;
2399 let _ = started_tx.send(());
2400 release_rx.recv().expect("test must release top-level op");
2401 panic!("intentional top-level writer request panic");
2402 })
2403 .await
2404 }
2405 });
2406
2407 tokio::time::timeout(Duration::from_secs(5), started_rx)
2408 .await
2409 .expect("top-level request did not start")
2410 .expect("top-level request dropped its start signal");
2411
2412 let queued_ran = Arc::new(AtomicBool::new(false));
2413 let queued_ran_in_op = Arc::clone(&queued_ran);
2414 let queued = handle
2415 .enqueue(move |conn| {
2416 queued_ran_in_op.store(true, Ordering::SeqCst);
2417 conn.execute("INSERT INTO t (id, v) VALUES (11, 'queued')", [])
2418 .map_err(|e| StorageError::Pool {
2419 operation: "test_top_level_queued_insert".into(),
2420 message: e.to_string(),
2421 })
2422 })
2423 .await
2424 .expect("queued request must be accepted");
2425 assert_eq!(handle.queue_depth(), 1);
2426 release_tx.send(()).expect("release top-level op");
2427
2428 let active_result = tokio::time::timeout(Duration::from_secs(5), active)
2429 .await
2430 .expect("top-level caller hung after panic")
2431 .expect("top-level caller task join");
2432 assert_writer_task_terminal_state(
2433 active_result,
2434 WriterTaskRequestState::SideEffectsUnknown,
2435 );
2436
2437 let queued_result = tokio::time::timeout(Duration::from_secs(5), queued)
2438 .await
2439 .expect("queued caller hung after top-level panic")
2440 .expect("terminal drain must preserve queued typed reply");
2441 assert_writer_task_terminal_state(queued_result, WriterTaskRequestState::NotStarted);
2442 assert!(!queued_ran.load(Ordering::SeqCst));
2443
2444 let reader = pool.reader().expect("reader");
2445 let active_count: i64 = reader
2446 .conn()
2447 .query_row("SELECT COUNT(*) FROM t WHERE id = 10", [], |row| row.get(0))
2448 .unwrap();
2449 let queued_count: i64 = reader
2450 .conn()
2451 .query_row("SELECT COUNT(*) FROM t WHERE id = 11", [], |row| row.get(0))
2452 .unwrap();
2453 assert_eq!(
2454 active_count, 1,
2455 "the completed top-level statement autocommits before the panic"
2456 );
2457 assert_eq!(queued_count, 0, "the queued request must never run");
2458 }
2459
2460 #[tokio::test]
2461 async fn closed_receiver_rejects_all_send_surfaces_as_not_started() {
2462 let (tx, rx) = mpsc::channel::<Box<dyn AnyWriteRequest + Send>>(4);
2466 drop(rx);
2467
2468 let handle = WriterTaskHandle {
2469 tx,
2470 backend_key: None,
2471 db: "test".to_string(),
2472 slow_write_threshold: None,
2473 enqueue_timeout: Duration::from_secs(5),
2474 };
2475 let send_result = handle.send(|_conn| Ok::<(), StorageError>(())).await;
2476 assert_writer_task_terminal_state(send_result, WriterTaskRequestState::NotStarted);
2477
2478 let timed_result = handle
2479 .send_with_timeout(|_conn| Ok::<(), StorageError>(()), Duration::from_secs(1))
2480 .await;
2481 assert_writer_task_terminal_state(timed_result, WriterTaskRequestState::NotStarted);
2482
2483 let top_level_result = handle
2484 .send_top_level(|_conn| Ok::<(), StorageError>(()))
2485 .await;
2486 assert_writer_task_terminal_state(top_level_result, WriterTaskRequestState::NotStarted);
2487 }
2488
2489 #[tokio::test]
2490 async fn accepted_request_lost_reply_is_side_effects_unknown() {
2491 let (tx, mut rx) = mpsc::channel::<Box<dyn AnyWriteRequest + Send>>(1);
2495 let handle = WriterTaskHandle {
2496 tx,
2497 backend_key: None,
2498 db: "test".to_string(),
2499 slow_write_threshold: None,
2500 enqueue_timeout: Duration::from_secs(5),
2501 };
2502 let request_ran = Arc::new(AtomicBool::new(false));
2503 let request_ran_in_op = Arc::clone(&request_ran);
2504
2505 let dropper = tokio::spawn(async move {
2506 let request = rx.recv().await.expect("request must be accepted");
2507 drop(request);
2508 });
2509 let result = tokio::time::timeout(
2510 Duration::from_secs(5),
2511 handle.send(move |_conn| {
2512 request_ran_in_op.store(true, Ordering::SeqCst);
2513 Ok::<(), StorageError>(())
2514 }),
2515 )
2516 .await
2517 .expect("caller hung after accepted request was dropped");
2518 dropper.await.expect("dropper task join");
2519
2520 assert_writer_task_terminal_state(result, WriterTaskRequestState::SideEffectsUnknown);
2521 assert!(!request_ran.load(Ordering::SeqCst));
2522 }
2523
2524 #[cfg(unix)]
2527 #[test]
2528 fn writer_stage_backend_key_preserves_non_utf8_path_bytes() {
2529 use std::ffi::OsString;
2530 use std::os::unix::ffi::OsStringExt;
2531
2532 let path_a =
2533 std::path::PathBuf::from(OsString::from_vec(b"/tmp/khive-writer-\x80.db".to_vec()));
2534 let path_b =
2535 std::path::PathBuf::from(OsString::from_vec(b"/tmp/khive-writer-\x81.db".to_vec()));
2536 assert_eq!(
2537 path_a.display().to_string(),
2538 path_b.display().to_string(),
2539 "fixture must reproduce the lossy display-label collision"
2540 );
2541 assert_ne!(
2542 writer_db_key_from_path(Some(&path_a)),
2543 writer_db_key_from_path(Some(&path_b)),
2544 "backend keys must retain the canonical path's exact OS bytes"
2545 );
2546 }
2547
2548 #[tokio::test]
2552 async fn writer_stage_sample_attributes_a_slow_body() {
2553 let dir = tempfile::tempdir().unwrap();
2554 let path = dir.path().join("writer_stage_sample.db");
2555 let pool = file_pool(&path);
2556 {
2557 let writer = pool.try_writer().unwrap();
2558 writer
2559 .conn()
2560 .execute_batch("CREATE TABLE t (id INTEGER PRIMARY KEY)")
2561 .unwrap();
2562 }
2563 let handle = spawn(&pool, 8).unwrap();
2564
2565 handle
2566 .send(|conn| {
2567 std::thread::sleep(Duration::from_millis(60));
2568 conn.execute("INSERT INTO t VALUES (1)", [])
2569 .map_err(|error| StorageError::Pool {
2570 operation: "writer_stage_sample".into(),
2571 message: error.to_string(),
2572 })
2573 })
2574 .await
2575 .unwrap();
2576
2577 let sample = last_writer_stage_observation(&pool).expect("writer stage sample");
2578 assert!(
2579 sample.body_micros >= 50_000,
2580 "the synthetic delay must land in the body stage: {sample:?}"
2581 );
2582 assert!(
2583 sample.body_micros > sample.queue_wait_micros,
2584 "fast queueing must not receive the body's delay: {sample:?}"
2585 );
2586 assert!(
2587 sample.body_micros > sample.transaction_acquire_micros,
2588 "an uncontended BEGIN must not receive the body's delay: {sample:?}"
2589 );
2590 assert!(
2591 sample.body_micros > sample.commit_micros,
2592 "a fast COMMIT must not receive the body's delay: {sample:?}"
2593 );
2594 assert!(sample.observed_at_unix_ms > 0);
2595 }
2596
2597 #[test]
2602 fn writer_queue_wait_excludes_blocking_pool_scheduling_delay() {
2603 let runtime = tokio::runtime::Builder::new_multi_thread()
2604 .worker_threads(1)
2605 .max_blocking_threads(1)
2606 .enable_all()
2607 .build()
2608 .expect("test runtime");
2609
2610 runtime.block_on(async {
2611 let dir = tempfile::tempdir().unwrap();
2612 let path = dir.path().join("writer_dequeue_boundary.db");
2613 let pool = file_pool(&path);
2614 let handle = spawn(&pool, 8).unwrap();
2615
2616 let (blocker_started_tx, blocker_started_rx) = std_mpsc::sync_channel(0);
2617 let (release_blocker_tx, release_blocker_rx) = std_mpsc::channel();
2618 let blocker = tokio::task::spawn_blocking(move || {
2619 blocker_started_tx.send(()).unwrap();
2620 release_blocker_rx.recv().unwrap();
2621 });
2622 blocker_started_rx
2623 .recv_timeout(Duration::from_secs(1))
2624 .expect("sole blocking worker must be occupied");
2625
2626 let reply = handle
2627 .enqueue(|_conn| Ok::<(), StorageError>(()))
2628 .await
2629 .expect("request must enter the bounded writer channel");
2630 let dequeue_deadline = Instant::now() + Duration::from_secs(1);
2631 while handle.queue_depth() != 0 {
2632 assert!(
2633 Instant::now() < dequeue_deadline,
2634 "writer drain never dequeued the accepted request"
2635 );
2636 tokio::task::yield_now().await;
2637 }
2638
2639 let scheduling_delay = Duration::from_millis(150);
2640 tokio::time::sleep(scheduling_delay).await;
2641 release_blocker_tx.send(()).unwrap();
2642 blocker.await.unwrap();
2643 reply.await.unwrap().unwrap();
2644
2645 let sample = last_writer_stage_observation(&pool).expect("writer stage sample");
2646 assert!(
2647 sample.total_micros.saturating_sub(sample.queue_wait_micros) >= 100_000,
2648 "the post-dequeue blocking-pool delay must not inflate queue_wait: {sample:?}"
2649 );
2650 });
2651 }
2652
2653 #[tokio::test]
2657 #[serial(tx_registry)]
2658 async fn writer_task_failure_counters_are_acquisition_site_exact() {
2659 {
2665 let dir = tempfile::tempdir().unwrap();
2666 let path = dir.path().join("writer_task_failure_counters_rollback.db");
2667 let pool = file_pool(&path);
2668 let handle = spawn(&pool, 8).expect("writer task should spawn");
2669
2670 let before = pool.writer_acquisition_snapshot();
2671 assert_eq!(before.writer_task_request_failures, 0);
2672 assert_eq!(before.writer_task_side_effects_unknown, 0);
2673
2674 let (started_tx, started_rx) = oneshot::channel::<()>();
2675 let (release_tx, release_rx) = std_mpsc::channel::<()>();
2676 let active = tokio::spawn({
2677 let handle = handle.clone();
2678 async move {
2679 handle
2680 .send(move |_conn| -> Result<(), StorageError> {
2681 let _ = started_tx.send(());
2682 release_rx.recv().expect("test must release active op");
2683 panic!("intentional rollback-clean panic for counter test");
2684 })
2685 .await
2686 }
2687 });
2688 tokio::time::timeout(Duration::from_secs(5), started_rx)
2689 .await
2690 .expect("active request did not start")
2691 .expect("active request dropped its start signal");
2692
2693 let queued = handle
2694 .enqueue(|_conn| Ok::<(), StorageError>(()))
2695 .await
2696 .expect("second request must queue behind the active one");
2697
2698 release_tx.send(()).expect("release active op");
2699 let active_result = tokio::time::timeout(Duration::from_secs(5), active)
2700 .await
2701 .expect("active caller hung after panic")
2702 .expect("active caller task join");
2703 assert_writer_task_terminal_state(
2704 active_result,
2705 WriterTaskRequestState::TransactionRolledBack,
2706 );
2707
2708 let queued_result = queued.await.expect("terminal drain must reply");
2709 assert_writer_task_terminal_state(queued_result, WriterTaskRequestState::NotStarted);
2710
2711 let after = pool.writer_acquisition_snapshot();
2712 assert_eq!(
2713 after.writer_task_request_failures, 1,
2714 "only the request that actually reached the seam counts, not the ones \
2715 failed by the queue-close drain"
2716 );
2717 assert_eq!(
2718 after.writer_task_side_effects_unknown, 0,
2719 "a clean rollback must not be counted as an unknown-side-effects outcome"
2720 );
2721 }
2722
2723 {
2727 let dir = tempfile::tempdir().unwrap();
2728 let path = dir.path().join("writer_task_failure_counters_unknown.db");
2729 let pool = file_pool(&path);
2730 let handle = spawn(&pool, 8).expect("writer task should spawn");
2731
2732 let before = pool.writer_acquisition_snapshot();
2733
2734 let active_result = handle
2735 .send(|conn| -> Result<(), StorageError> {
2736 conn.authorizer(Some(deny_rollback))
2737 .map_err(|e| StorageError::Pool {
2738 operation: "test_install_authorizer".into(),
2739 message: e.to_string(),
2740 })?;
2741 Err(StorageError::Internal(
2742 "intentional operation failure before denied rollback".into(),
2743 ))
2744 })
2745 .await;
2746 assert_writer_task_terminal_state(
2747 active_result,
2748 WriterTaskRequestState::SideEffectsUnknown,
2749 );
2750
2751 let sentinel_result = handle.send(|_conn| Ok::<(), StorageError>(())).await;
2758 assert_writer_task_terminal_state(sentinel_result, WriterTaskRequestState::NotStarted);
2759
2760 let after = pool.writer_acquisition_snapshot();
2761 assert_eq!(
2762 after.writer_task_request_failures - before.writer_task_request_failures,
2763 1
2764 );
2765 assert_eq!(
2766 after.writer_task_side_effects_unknown - before.writer_task_side_effects_unknown,
2767 1
2768 );
2769 }
2770 }
2771}