1use std::collections::{BTreeMap, BTreeSet};
2use std::fs;
3use std::path::Path;
4use std::sync::atomic::{AtomicBool, AtomicU64, AtomicU8, Ordering};
5use std::sync::{mpsc, Arc, Mutex, OnceLock};
6use std::thread;
7use std::time::{Duration, SystemTime, UNIX_EPOCH};
8
9use chacha20poly1305::aead::rand_core::{OsRng, RngCore};
10use chio_core::canonical::{canonical_json_bytes, CanonicalBytes};
11use chio_core::capability::{scope::ChioScope, token::CapabilityToken};
12use chio_core::crypto::{sha256_hex, Keypair, Signature};
13use chio_core::receipt::{
14 body::ChioReceipt, crypto_floor::ReceiptCryptoFloor, decision::Decision,
15 economics::FinancialReceiptMetadata, economics::SettlementStatus,
16 governance::GovernedTransactionReceiptMetadata, lineage::ChildRequestReceipt,
17 metadata::ReceiptAttributionMetadata,
18};
19use chio_core::session::{
20 OperationTerminalState, RequestLineageMode, RequestLineageRecord, SessionAnchorReference,
21};
22use chio_kernel::checkpoint::{CheckpointChainFrontier, KernelCheckpoint, KernelCheckpointBody};
23use chio_kernel::cost_attribution::{
24 CostAttributionChainHop, CostAttributionQuery, CostAttributionReceiptRow,
25 CostAttributionReport, CostAttributionSummary, LeafCostAttributionRow, RootCostAttributionRow,
26 MAX_COST_ATTRIBUTION_LIMIT,
27};
28use chio_kernel::dpop::DPOP_SCHEMA;
29use chio_kernel::operator_report::{
30 AuthorizationContextReport, AuthorizationContextRow, AuthorizationContextSenderConstraint,
31 AuthorizationContextSummary, BehavioralFeedGovernedActionSummary,
32 BehavioralFeedMeteredBillingRow, BehavioralFeedMeteredBillingSummary, BehavioralFeedQuery,
33 BehavioralFeedReceiptRow, BehavioralFeedReceiptSelection, BehavioralFeedSettlementSummary,
34 ChioOAuthAuthorizationDiscoveryMetadata, ChioOAuthAuthorizationExampleMapping,
35 ChioOAuthAuthorizationMetadataReport, ChioOAuthAuthorizationProfile,
36 ChioOAuthAuthorizationReviewPack, ChioOAuthAuthorizationReviewPackRecord,
37 ChioOAuthAuthorizationReviewPackSummary, ChioOAuthAuthorizationSupportBoundary,
38 ComplianceReport, EconomicCompletionFlowReport, EconomicCompletionFlowSummary,
39 EconomicReceiptMeteringProjection, EconomicReceiptProjectionReport,
40 EconomicReceiptProjectionRow, EconomicReceiptProjectionSummary,
41 EconomicReceiptSettlementProjection, GovernedAuthorizationCommerceDetail,
42 GovernedAuthorizationDetail, GovernedAuthorizationMeteredBillingDetail,
43 GovernedAuthorizationTransactionContext, MeteredBillingEvidenceRecord,
44 MeteredBillingReconciliationReport, MeteredBillingReconciliationRow,
45 MeteredBillingReconciliationState, MeteredBillingReconciliationSummary, OperatorReportQuery,
46 SettlementReconciliationReport, SettlementReconciliationRow, SettlementReconciliationState,
47 SettlementReconciliationSummary, SharedEvidenceQuery, SharedEvidenceReferenceReport,
48 SharedEvidenceReferenceRow, SharedEvidenceReferenceSummary,
49 CHIO_OAUTH_AUTHORIZATION_COMMERCE_DETAIL_TYPE, CHIO_OAUTH_AUTHORIZATION_CONTEXT_REPORT_SCHEMA,
50 CHIO_OAUTH_AUTHORIZATION_METADATA_SCHEMA, CHIO_OAUTH_AUTHORIZATION_METERED_BILLING_DETAIL_TYPE,
51 CHIO_OAUTH_AUTHORIZATION_REVIEW_PACK_SCHEMA, CHIO_OAUTH_AUTHORIZATION_TOOL_DETAIL_TYPE,
52 CHIO_OAUTH_SENDER_PROOF_CHIO_DPOP, ECONOMIC_COMPLETION_FLOW_SCHEMA,
53};
54use chio_kernel::receipt_analytics::{
55 AgentAnalyticsRow, AnalyticsTimeBucket, ReceiptAnalyticsMetrics, ReceiptAnalyticsQuery,
56 ReceiptAnalyticsResponse, TimeAnalyticsRow, ToolAnalyticsRow, MAX_ANALYTICS_GROUP_LIMIT,
57};
58use chio_kernel::receipt_query::{
59 ReceiptQuery, ReceiptQueryResult, ReceiptReadBoundary, ReceiptReadContext, MAX_QUERY_LIMIT,
60};
61use chio_kernel::receipt_store::{ReceiptLineageStatementLink, ReceiptLineageVerification};
62use chio_kernel::{
63 AtomicReceiptProjection, AuthorizationReceiptConsumption, CapabilitySnapshot,
64 CreditBondDisposition, CreditBondLifecycleState, CreditBondListQuery, CreditBondListReport,
65 CreditBondListSummary, CreditBondRow, CreditFacilityDisposition, CreditFacilityLifecycleState,
66 CreditFacilityListQuery, CreditFacilityListReport, CreditFacilityListSummary,
67 CreditFacilityRow, CreditLossLifecycleEventKind, CreditLossLifecycleListQuery,
68 CreditLossLifecycleListReport, CreditLossLifecycleListSummary, CreditLossLifecycleRow,
69 EvidenceChildReceiptScope, EvidenceExportQuery, ExposureLedgerQuery,
70 FederatedEvidenceShareImport, FederatedEvidenceShareSummary, LiabilityAutoBindDisposition,
71 LiabilityClaimPayoutReconciliationState, LiabilityClaimResponseDisposition,
72 LiabilityClaimSettlementReconciliationState, LiabilityClaimWorkflowQuery,
73 LiabilityClaimWorkflowReport, LiabilityClaimWorkflowRow, LiabilityClaimWorkflowSummary,
74 LiabilityMarketWorkflowQuery, LiabilityMarketWorkflowReport, LiabilityMarketWorkflowRow,
75 LiabilityMarketWorkflowSummary, LiabilityProviderLifecycleState, LiabilityProviderListQuery,
76 LiabilityProviderListReport, LiabilityProviderListSummary, LiabilityProviderResolutionQuery,
77 LiabilityProviderResolutionReport, LiabilityProviderRow, LiabilityQuoteDisposition,
78 PendingSettlementObservation, ReceiptCheckpointCreateReport, ReceiptCheckpointRange,
79 ReceiptCheckpointStatusReport, ReceiptFlushReport, ReceiptStore, ReceiptStoreError,
80 ReceiptStoreHealthReport, ReceiptWalCheckpointReport, ReceiptWriterCounters, RetentionConfig,
81 SignedCreditBond, SignedCreditFacility, SignedCreditLossLifecycle,
82 SignedLiabilityAutoBindDecision, SignedLiabilityBoundCoverage,
83 SignedLiabilityClaimAdjudication, SignedLiabilityClaimDispute, SignedLiabilityClaimPackage,
84 SignedLiabilityClaimPayoutInstruction, SignedLiabilityClaimPayoutReceipt,
85 SignedLiabilityClaimResponse, SignedLiabilityClaimSettlementInstruction,
86 SignedLiabilityClaimSettlementReceipt, SignedLiabilityPlacement,
87 SignedLiabilityPricingAuthority, SignedLiabilityProvider, SignedLiabilityQuoteRequest,
88 SignedLiabilityQuoteResponse, SignedUnderwritingDecision, StoredChildReceipt,
89 StoredToolReceipt, UnderwritingAppealCreateRequest, UnderwritingAppealRecord,
90 UnderwritingAppealResolution, UnderwritingAppealResolveRequest, UnderwritingAppealStatus,
91 UnderwritingDecisionLifecycleState, UnderwritingDecisionListReport,
92 UnderwritingDecisionOutcome, UnderwritingDecisionQuery, UnderwritingDecisionRow,
93 UnderwritingDecisionSummary, CREDIT_BOND_LIST_REPORT_SCHEMA,
94 CREDIT_FACILITY_LIST_REPORT_SCHEMA, CREDIT_LOSS_LIFECYCLE_LIST_REPORT_SCHEMA,
95 LIABILITY_CLAIM_WORKFLOW_REPORT_SCHEMA, LIABILITY_MARKET_WORKFLOW_REPORT_SCHEMA,
96 LIABILITY_PROVIDER_LIST_REPORT_SCHEMA, LIABILITY_PROVIDER_RESOLUTION_REPORT_SCHEMA,
97};
98use chio_supervisor::{
99 HealthFlag, HealthLevel, SupervisedOutcome, SupervisedThread, SupervisorConfig,
100};
101use r2d2::{Pool, PooledConnection};
102use r2d2_sqlite::SqliteConnectionManager;
103use rusqlite::{params, Connection, OptionalExtension};
104
105pub struct SqliteReceiptStore {
106 pub(crate) pool: Pool<SqliteConnectionManager>,
107 receipt_commit_actor: ReceiptCommitActor,
108 settlement_store_binding: Option<chio_settle::SettlementStoreBinding>,
109 pub(crate) strict_tenant_isolation: std::sync::atomic::AtomicBool,
115 pub(crate) incremental_verification: bool,
117}
118
119type FederatedShareSubjectCorpus = (
120 FederatedEvidenceShareSummary,
121 Vec<StoredToolReceipt>,
122 Vec<CapabilitySnapshot>,
123);
124pub(crate) type SqliteStoreConnection = PooledConnection<SqliteConnectionManager>;
125
126const RECEIPT_GROUP_COMMIT_MAX_BATCH: usize = 64;
127const RECEIPT_GROUP_COMMIT_FLUSH_DELAY: Duration = Duration::from_micros(500);
128const RECEIPT_COMMIT_ACTOR_CHANNEL_CAPACITY: usize = RECEIPT_GROUP_COMMIT_MAX_BATCH * 16;
129const RECEIPT_APPEND_TIMEOUT_MARKER: &str = "sqlite receipt commit append timed out";
130const RECEIPT_WRITE_TIMEOUT_MARKER: &str = "sqlite receipt commit write timed out";
131
132fn is_receipt_writer_timeout_marker(message: &str) -> bool {
133 matches!(
134 message,
135 RECEIPT_APPEND_TIMEOUT_MARKER | RECEIPT_WRITE_TIMEOUT_MARKER
136 )
137}
138
139struct ReceiptCommitActor {
140 sender: mpsc::SyncSender<ReceiptCommitCommand>,
141 health: Arc<ReceiptCommitWriterHealth>,
142 worker: Arc<ReceiptCommitWorker>,
146}
147
148struct ReceiptCommitWorker {
149 join: Option<SupervisedReceiptWriter>,
150}
151
152struct SupervisedReceiptWriter {
153 supervisor: Option<SupervisedThread>,
154 health: HealthFlag,
155 thread_id: Arc<OnceLock<thread::ThreadId>>,
156}
157
158impl ReceiptCommitWorker {
159 fn health(&self) -> Option<&HealthFlag> {
160 self.join.as_ref().map(|writer| &writer.health)
161 }
162
163 fn writer_dead_error(&self) -> ReceiptStoreError {
164 let Some(health) = self.health() else {
165 return ReceiptStoreError::WriterDead {
166 restarts: 0,
167 last_error: "receipt commit writer channel disconnected".to_string(),
168 };
169 };
170 let snapshot = health.snapshot();
171 ReceiptStoreError::WriterDead {
172 restarts: snapshot.restart_total,
173 last_error: snapshot
174 .reason
175 .unwrap_or_else(|| "receipt commit writer channel disconnected".to_string()),
176 }
177 }
178}
179
180struct ReceiptCommitWriterHealth {
181 accepted_total: AtomicU64,
182 committed_total: AtomicU64,
183 failed_total: AtomicU64,
184 saturated_total: AtomicU64,
185 inflight: AtomicU64,
186 timed_out_inflight: AtomicU64,
190 timed_out_total: AtomicU64,
191 queue_depth: AtomicU64,
198 last_commit_unix_ms: AtomicU64,
199 first_accept_unix_ms: AtomicU64,
202 backlog_started_unix_ms: AtomicU64,
208 last_error: Mutex<Option<String>>,
209 retention_error: Mutex<Option<String>>,
214 head_checkpoint_seq: AtomicU64,
217 head_checkpointed_entry_seq: AtomicU64,
218 head_claim_log_count: AtomicU64,
219 head_claim_log_max_seq: AtomicU64,
220 head_poisoned: AtomicBool,
227 critical_write_poisoned: AtomicBool,
228}
229
230impl Default for ReceiptCommitWriterHealth {
231 fn default() -> Self {
232 Self {
233 accepted_total: AtomicU64::new(0),
234 committed_total: AtomicU64::new(0),
235 failed_total: AtomicU64::new(0),
236 saturated_total: AtomicU64::new(0),
237 inflight: AtomicU64::new(0),
238 timed_out_inflight: AtomicU64::new(0),
239 timed_out_total: AtomicU64::new(0),
240 queue_depth: AtomicU64::new(0),
241 last_commit_unix_ms: AtomicU64::new(0),
242 first_accept_unix_ms: AtomicU64::new(0),
243 backlog_started_unix_ms: AtomicU64::new(0),
244 last_error: Mutex::new(None),
245 retention_error: Mutex::new(None),
246 head_checkpoint_seq: AtomicU64::new(0),
247 head_checkpointed_entry_seq: AtomicU64::new(0),
248 head_claim_log_count: AtomicU64::new(0),
249 head_claim_log_max_seq: AtomicU64::new(0),
250 head_poisoned: AtomicBool::new(true),
256 critical_write_poisoned: AtomicBool::new(false),
257 }
258 }
259}
260
261impl ReceiptCommitWriterHealth {
262 fn note_accept(&self, previous_inflight: u64) {
271 let now = current_unix_ms();
272 let _ =
273 self.first_accept_unix_ms
274 .compare_exchange(0, now, Ordering::SeqCst, Ordering::SeqCst);
275 if previous_inflight == 0 {
276 self.backlog_started_unix_ms.store(now, Ordering::SeqCst);
277 }
278 }
279
280 fn note_writer_unavailable(&self) {
287 if let Ok(mut last_error) = self.last_error.lock() {
288 *last_error = Some("sqlite receipt commit actor is unavailable".to_string());
289 }
290 }
291
292 fn note_channel_send(&self) {
299 self.queue_depth.fetch_add(1, Ordering::SeqCst);
300 }
301
302 fn note_channel_send_rejected(&self) {
303 atomic_saturating_sub(&self.queue_depth, 1);
304 }
305
306 fn note_channel_dequeue(&self) {
307 atomic_saturating_sub(&self.queue_depth, 1);
308 }
309
310 fn note_timeout(&self, message: &str) {
311 if let Ok(mut last_error) = self.last_error.lock() {
312 if last_error
313 .as_deref()
314 .is_none_or(is_receipt_writer_timeout_marker)
315 {
316 *last_error = Some(message.to_string());
317 }
318 }
319 }
320
321 fn clear_timeout_error_if_drained(&self) {
322 if let Ok(mut last_error) = self.last_error.lock() {
323 if self.timed_out_inflight.load(Ordering::SeqCst) == 0
324 && last_error
325 .as_deref()
326 .is_some_and(is_receipt_writer_timeout_marker)
327 {
328 *last_error = None;
329 }
330 }
331 }
332
333 fn set_head_poisoned(&self, poisoned: bool) {
337 self.head_poisoned.store(poisoned, Ordering::SeqCst);
338 }
339
340 fn store_head_snapshot(&self, head: &VerifiedHead) {
341 self.head_checkpoint_seq
342 .store(head.checkpoint_seq(), Ordering::SeqCst);
343 self.head_checkpointed_entry_seq
344 .store(head.checkpointed_entry_seq(), Ordering::SeqCst);
345 self.head_claim_log_count
346 .store(head.claim_log_count, Ordering::SeqCst);
347 self.head_claim_log_max_seq
348 .store(head.claim_log_max_seq, Ordering::SeqCst);
349 }
350}
351
352const WRITER_COMMAND_PENDING: u8 = 0;
353const WRITER_COMMAND_TIMED_OUT: u8 = 1;
354const WRITER_COMMAND_COMPLETED: u8 = 2;
355
356struct WriterCommandCompletion {
357 state: Arc<AtomicU8>,
358 health: Arc<ReceiptCommitWriterHealth>,
359}
360
361struct WriterCommandTimeout {
362 state: Arc<AtomicU8>,
363 health: Arc<ReceiptCommitWriterHealth>,
364}
365
366fn writer_command_tracker(
367 health: &Arc<ReceiptCommitWriterHealth>,
368) -> (WriterCommandCompletion, WriterCommandTimeout) {
369 let state = Arc::new(AtomicU8::new(WRITER_COMMAND_PENDING));
370 (
371 WriterCommandCompletion {
372 state: Arc::clone(&state),
373 health: Arc::clone(health),
374 },
375 WriterCommandTimeout {
376 state,
377 health: Arc::clone(health),
378 },
379 )
380}
381
382impl WriterCommandCompletion {
383 fn complete(&mut self) {
384 if self.state.swap(WRITER_COMMAND_COMPLETED, Ordering::SeqCst) == WRITER_COMMAND_TIMED_OUT {
385 atomic_saturating_sub(&self.health.timed_out_inflight, 1);
386 self.health.clear_timeout_error_if_drained();
387 }
388 }
389}
390
391impl Drop for WriterCommandCompletion {
392 fn drop(&mut self) {
393 self.complete();
394 }
395}
396
397impl WriterCommandTimeout {
398 fn note_timeout(&self, message: &str) {
402 self.health.timed_out_total.fetch_add(1, Ordering::SeqCst);
403 self.health
404 .timed_out_inflight
405 .fetch_add(1, Ordering::SeqCst);
406 if self
407 .state
408 .compare_exchange(
409 WRITER_COMMAND_PENDING,
410 WRITER_COMMAND_TIMED_OUT,
411 Ordering::SeqCst,
412 Ordering::SeqCst,
413 )
414 .is_err()
415 {
416 atomic_saturating_sub(&self.health.timed_out_inflight, 1);
417 self.health.clear_timeout_error_if_drained();
418 return;
419 }
420
421 self.health.note_timeout(message);
422 if self.state.load(Ordering::SeqCst) == WRITER_COMMAND_COMPLETED {
425 self.health.clear_timeout_error_if_drained();
426 }
427 }
428}
429
430struct ReceiptCommitRequest {
431 receipt: ChioReceipt,
432 raw_json: String,
433 ensure_lineage: bool,
437 response: mpsc::SyncSender<Result<u64, ReceiptStoreError>>,
438}
439
440type WriterResponder = Box<dyn FnOnce(Result<(), ReceiptStoreError>) -> bool + Send + 'static>;
453
454type WriterClosure = Box<
459 dyn FnOnce(Result<&mut SqliteStoreConnection, ReceiptStoreError>) -> WriterResponder
460 + Send
461 + 'static,
462>;
463
464enum ReceiptCommitCommand {
465 Append(Box<ReceiptCommitRequest>),
466 AppendWithTimeout {
467 request: Box<ReceiptCommitRequest>,
468 completion: WriterCommandCompletion,
469 },
470 Flush(mpsc::SyncSender<Result<(), ReceiptStoreError>>),
471 Write {
482 job: WriterClosure,
483 appends_receipts: bool,
484 fail_closed_on_error: bool,
485 completion: WriterCommandCompletion,
486 },
487 ReseedHead(mpsc::SyncSender<Result<(), ReceiptStoreError>>),
490 InstallSigner(BackgroundCheckpointSigner),
493 #[cfg(test)]
494 RestartSupervisor,
495 Rotate {
499 config: Box<RetentionConfig>,
500 response: mpsc::SyncSender<Result<u64, ReceiptStoreError>>,
501 },
502 RetentionRepair {
509 archive_path: String,
510 response: mpsc::SyncSender<Result<u64, ReceiptStoreError>>,
511 },
512}
513
514impl ReceiptCommitCommand {
515 fn into_append(
516 self,
517 ) -> Result<(Box<ReceiptCommitRequest>, Option<WriterCommandCompletion>), Self> {
518 match self {
519 Self::Append(request) => Ok((request, None)),
520 Self::AppendWithTimeout {
521 request,
522 completion,
523 } => Ok((request, Some(completion))),
524 other => Err(other),
525 }
526 }
527}
528
529impl ReceiptCommitActor {
530 fn start(pool: Pool<SqliteConnectionManager>, incremental_verification: bool) -> Self {
531 let (sender, receiver) = receipt_commit_channel();
532 let health = Arc::new(ReceiptCommitWriterHealth::default());
533 let actor_health = Arc::clone(&health);
534 let thread_id = Arc::new(OnceLock::new());
535 let actor_thread_id = Arc::clone(&thread_id);
536 let config = SupervisorConfig {
537 name: "chio-receipt-writer",
538 tcb_critical: true,
541 trip_after: 1,
543 max_restarts: 5,
544 base_backoff: Duration::from_millis(100),
545 max_backoff: Duration::from_secs(30),
546 };
547 let mut checkpoint_signer = None;
551 let supervisor = SupervisedThread::spawn(config, move |_shutdown| {
552 let _ = actor_thread_id.set(thread::current().id());
553 receipt_commit_actor_loop(
554 pool.clone(),
555 &receiver,
556 Arc::clone(&actor_health),
557 incremental_verification,
558 &mut checkpoint_signer,
559 )
560 });
561 let supervisor_health = supervisor.health();
562 Self {
563 sender,
564 health,
565 worker: Arc::new(ReceiptCommitWorker {
566 join: Some(SupervisedReceiptWriter {
567 supervisor: Some(supervisor),
568 health: supervisor_health,
569 thread_id,
570 }),
571 }),
572 }
573 }
574
575 fn writer_dead_error(&self) -> ReceiptStoreError {
579 self.worker.writer_dead_error()
580 }
581
582 fn writer_serving_closed(&self) -> bool {
588 self.worker
589 .health()
590 .is_none_or(HealthFlag::is_serving_closed)
591 || self.health.head_poisoned.load(Ordering::SeqCst)
592 }
593
594 fn writer_health_summary(&self) -> (HealthLevel, u64) {
597 match self.worker.health() {
598 Some(health) => {
599 let snapshot = health.snapshot();
600 (snapshot.level, snapshot.restart_total)
601 }
602 None => (HealthLevel::Failed, 0),
603 }
604 }
605
606 fn append(
607 &self,
608 receipt: ChioReceipt,
609 raw_json: String,
610 ensure_lineage: bool,
611 ) -> Result<u64, ReceiptStoreError> {
612 let (response, result) = mpsc::sync_channel(1);
613 let command = ReceiptCommitCommand::Append(Box::new(ReceiptCommitRequest {
614 receipt,
615 raw_json,
616 ensure_lineage,
617 response,
618 }));
619 let previous_inflight = self.health.inflight.fetch_add(1, Ordering::SeqCst);
629 self.health.note_channel_send();
630 match self.sender.try_send(command) {
631 Ok(()) => {
632 self.health.accepted_total.fetch_add(1, Ordering::SeqCst);
633 self.health.note_accept(previous_inflight);
634 }
635 Err(mpsc::TrySendError::Full(_)) => {
636 atomic_saturating_sub(&self.health.inflight, 1);
637 self.health.note_channel_send_rejected();
638 self.health.saturated_total.fetch_add(1, Ordering::SeqCst);
639 return Err(receipt_actor_saturated_error());
640 }
641 Err(mpsc::TrySendError::Disconnected(_)) => {
642 atomic_saturating_sub(&self.health.inflight, 1);
643 self.health.note_channel_send_rejected();
644 self.health.note_writer_unavailable();
645 return Err(self.writer_dead_error());
646 }
647 }
648 match result.recv() {
649 Ok(result) => result,
650 Err(_) => {
651 atomic_saturating_sub(&self.health.inflight, 1);
652 self.health.failed_total.fetch_add(1, Ordering::SeqCst);
653 self.health.note_writer_unavailable();
654 Err(self.writer_dead_error())
655 }
656 }
657 }
658
659 fn append_with_timeout(
670 &self,
671 receipt: ChioReceipt,
672 raw_json: String,
673 ensure_lineage: bool,
674 timeout: Duration,
675 ) -> Result<u64, ReceiptStoreError> {
676 let (response, result) = mpsc::sync_channel(1);
677 let (completion, timeout_tracker) = writer_command_tracker(&self.health);
678 let command = ReceiptCommitCommand::AppendWithTimeout {
679 request: Box::new(ReceiptCommitRequest {
680 receipt,
681 raw_json,
682 ensure_lineage,
683 response,
684 }),
685 completion,
686 };
687 let previous_inflight = self.health.inflight.fetch_add(1, Ordering::SeqCst);
688 self.health.note_channel_send();
689 match self.sender.try_send(command) {
690 Ok(()) => {
691 self.health.accepted_total.fetch_add(1, Ordering::SeqCst);
692 self.health.note_accept(previous_inflight);
693 }
694 Err(mpsc::TrySendError::Full(_)) => {
695 atomic_saturating_sub(&self.health.inflight, 1);
696 self.health.note_channel_send_rejected();
697 self.health.saturated_total.fetch_add(1, Ordering::SeqCst);
698 return Err(receipt_actor_saturated_error());
699 }
700 Err(mpsc::TrySendError::Disconnected(_)) => {
701 atomic_saturating_sub(&self.health.inflight, 1);
702 self.health.note_channel_send_rejected();
703 self.health.note_writer_unavailable();
704 return Err(receipt_actor_unavailable_error());
705 }
706 }
707 match result.recv_timeout(timeout) {
708 Ok(result) => result,
709 Err(mpsc::RecvTimeoutError::Timeout) => {
710 timeout_tracker.note_timeout(RECEIPT_APPEND_TIMEOUT_MARKER);
711 Err(receipt_actor_append_timeout_error(timeout))
712 }
713 Err(mpsc::RecvTimeoutError::Disconnected) => {
714 atomic_saturating_sub(&self.health.inflight, 1);
715 self.health.failed_total.fetch_add(1, Ordering::SeqCst);
716 if let Ok(mut last_error) = self.health.last_error.lock() {
717 *last_error = Some("sqlite receipt commit actor is unavailable".to_string());
718 }
719 Err(receipt_actor_unavailable_error())
720 }
721 }
722 }
723
724 fn flush(&self) -> Result<(), ReceiptStoreError> {
725 self.flush_with_receiver(|receiver| {
726 receiver.recv().map_err(|_| self.writer_dead_error())?
727 })
728 }
729
730 fn flush_with_timeout(&self, timeout: Duration) -> Result<(), ReceiptStoreError> {
731 self.flush_with_receiver(|receiver| match receiver.recv_timeout(timeout) {
732 Ok(result) => result,
733 Err(mpsc::RecvTimeoutError::Timeout) => Err(receipt_actor_flush_timeout_error(timeout)),
734 Err(mpsc::RecvTimeoutError::Disconnected) => Err(self.writer_dead_error()),
735 })
736 }
737
738 fn flush_with_receiver(
739 &self,
740 receive: impl FnOnce(
741 mpsc::Receiver<Result<(), ReceiptStoreError>>,
742 ) -> Result<(), ReceiptStoreError>,
743 ) -> Result<(), ReceiptStoreError> {
744 let (response, result) = mpsc::sync_channel(1);
745 self.health.note_channel_send();
746 match self.sender.try_send(ReceiptCommitCommand::Flush(response)) {
747 Ok(()) => {}
748 Err(mpsc::TrySendError::Full(_)) => {
749 self.health.note_channel_send_rejected();
750 self.health.saturated_total.fetch_add(1, Ordering::SeqCst);
751 return Err(receipt_actor_saturated_error());
752 }
753 Err(mpsc::TrySendError::Disconnected(_)) => {
754 self.health.note_channel_send_rejected();
755 self.health.note_writer_unavailable();
756 return Err(self.writer_dead_error());
757 }
758 }
759 receive(result)
760 }
761
762 #[cfg(test)]
763 fn reseed_head(&self) -> Result<(), ReceiptStoreError> {
764 let (response, result) = mpsc::sync_channel(1);
765 match self
766 .sender
767 .try_send(ReceiptCommitCommand::ReseedHead(response))
768 {
769 Ok(()) => {}
770 Err(mpsc::TrySendError::Full(_)) => return Err(receipt_actor_saturated_error()),
771 Err(mpsc::TrySendError::Disconnected(_)) => return Err(self.writer_dead_error()),
772 }
773 result.recv().map_err(|_| self.writer_dead_error())?
774 }
775
776 #[cfg(test)]
777 fn install_signer(&self, signer: BackgroundCheckpointSigner) -> Result<(), ReceiptStoreError> {
778 match self
779 .sender
780 .try_send(ReceiptCommitCommand::InstallSigner(signer))
781 {
782 Ok(()) => Ok(()),
783 Err(mpsc::TrySendError::Full(_)) => Err(receipt_actor_saturated_error()),
784 Err(mpsc::TrySendError::Disconnected(_)) => Err(self.writer_dead_error()),
785 }
786 }
787
788 fn backlog_started_unix_ms(&self) -> Option<u64> {
792 match self.health.backlog_started_unix_ms.load(Ordering::SeqCst) {
793 0 => None,
794 value => Some(value),
795 }
796 }
797
798 fn writer_counters(&self) -> ReceiptWriterCounters {
799 let last_commit_unix_ms = match self.health.last_commit_unix_ms.load(Ordering::SeqCst) {
800 0 => None,
801 value => Some(value),
802 };
803 let first_accept_unix_ms = match self.health.first_accept_unix_ms.load(Ordering::SeqCst) {
804 0 => None,
805 value => Some(value),
806 };
807 let last_error = self
808 .health
809 .last_error
810 .lock()
811 .map(|error| error.clone())
812 .unwrap_or_else(|_| Some("receipt commit writer health lock poisoned".to_string()));
813 ReceiptWriterCounters {
814 accepted_total: self.health.accepted_total.load(Ordering::SeqCst),
815 committed_total: self.health.committed_total.load(Ordering::SeqCst),
816 failed_total: self.health.failed_total.load(Ordering::SeqCst),
817 saturated_total: self.health.saturated_total.load(Ordering::SeqCst),
818 inflight: self.health.inflight.load(Ordering::SeqCst),
819 timed_out_total: self.health.timed_out_total.load(Ordering::SeqCst),
820 timed_out_inflight: self.health.timed_out_inflight.load(Ordering::SeqCst),
821 queue_depth: self.health.queue_depth.load(Ordering::SeqCst),
822 last_commit_unix_ms,
823 first_accept_unix_ms,
824 last_error,
825 }
826 }
827}
828
829impl Drop for SupervisedReceiptWriter {
830 fn drop(&mut self) {
831 let Some(supervisor) = self.supervisor.take() else {
832 return;
833 };
834 if self.thread_id.get() == Some(&thread::current().id()) {
835 let _ = thread::Builder::new()
836 .name("chio-receipt-writer-reaper".to_string())
837 .spawn(move || {
838 let _ = supervisor.join();
839 });
840 } else {
841 let _ = supervisor.join();
842 }
843 }
844}
845
846pub(crate) struct WriterHandle {
851 sender: mpsc::SyncSender<ReceiptCommitCommand>,
852 health: Arc<ReceiptCommitWriterHealth>,
853 worker: Arc<ReceiptCommitWorker>,
854 settlement_store_binding: Option<chio_settle::SettlementStoreBinding>,
855}
856
857impl WriterHandle {
858 pub(crate) const fn settlement_store_binding(
859 &self,
860 ) -> Option<chio_settle::SettlementStoreBinding> {
861 self.settlement_store_binding
862 }
863
864 pub(crate) fn run_write<T, F>(&self, job: F) -> Result<T, ReceiptStoreError>
869 where
870 F: FnOnce(&mut SqliteStoreConnection) -> Result<T, ReceiptStoreError> + Send + 'static,
871 T: Send + 'static,
872 {
873 self.run_write_kind(job, false, false)
874 }
875
876 pub(crate) fn run_write_receipt<T, F>(&self, job: F) -> Result<T, ReceiptStoreError>
881 where
882 F: FnOnce(&mut SqliteStoreConnection) -> Result<T, ReceiptStoreError> + Send + 'static,
883 T: Send + 'static,
884 {
885 self.run_write_kind(job, true, false)
886 }
887
888 fn run_critical_receipt_write<T, F>(&self, job: F) -> Result<T, ReceiptStoreError>
889 where
890 F: FnOnce(&mut SqliteStoreConnection) -> Result<T, ReceiptStoreError> + Send + 'static,
891 T: Send + 'static,
892 {
893 self.run_write_kind(job, true, true)
894 }
895
896 pub(crate) fn run_write_receipt_with_timeout<T, F>(
904 &self,
905 job: F,
906 timeout: Duration,
907 ) -> Result<T, ReceiptStoreError>
908 where
909 F: FnOnce(&mut SqliteStoreConnection) -> Result<T, ReceiptStoreError> + Send + 'static,
910 T: Send + 'static,
911 {
912 self.run_write_kind_with_timeout(job, true, false, timeout)
913 }
914
915 fn run_critical_receipt_write_with_timeout<T, F>(
919 &self,
920 job: F,
921 timeout: Duration,
922 ) -> Result<T, ReceiptStoreError>
923 where
924 F: FnOnce(&mut SqliteStoreConnection) -> Result<T, ReceiptStoreError> + Send + 'static,
925 T: Send + 'static,
926 {
927 self.run_write_kind_with_timeout(job, true, true, timeout)
928 }
929
930 pub(crate) fn run_write_with_timeout<T, F>(
936 &self,
937 job: F,
938 timeout: Duration,
939 ) -> Result<T, ReceiptStoreError>
940 where
941 F: FnOnce(&mut SqliteStoreConnection) -> Result<T, ReceiptStoreError> + Send + 'static,
942 T: Send + 'static,
943 {
944 self.run_write_kind_with_timeout(job, false, false, timeout)
945 }
946
947 fn run_write_kind_with_timeout<T, F>(
948 &self,
949 job: F,
950 appends_receipts: bool,
951 fail_closed_on_error: bool,
952 timeout: Duration,
953 ) -> Result<T, ReceiptStoreError>
954 where
955 F: FnOnce(&mut SqliteStoreConnection) -> Result<T, ReceiptStoreError> + Send + 'static,
956 T: Send + 'static,
957 {
958 let (result, timeout_tracker) =
959 self.enqueue_write_job(job, appends_receipts, fail_closed_on_error)?;
960 match result.recv_timeout(timeout) {
961 Ok(outcome) => outcome,
962 Err(mpsc::RecvTimeoutError::Timeout) => {
963 timeout_tracker.note_timeout(RECEIPT_WRITE_TIMEOUT_MARKER);
964 Err(receipt_actor_write_timeout_error(timeout))
965 }
966 Err(mpsc::RecvTimeoutError::Disconnected) => {
967 atomic_saturating_sub(&self.health.inflight, 1);
968 self.health.failed_total.fetch_add(1, Ordering::SeqCst);
969 self.health.note_writer_unavailable();
976 Err(self.worker.writer_dead_error())
977 }
978 }
979 }
980
981 fn run_write_kind<T, F>(
982 &self,
983 job: F,
984 appends_receipts: bool,
985 fail_closed_on_error: bool,
986 ) -> Result<T, ReceiptStoreError>
987 where
988 F: FnOnce(&mut SqliteStoreConnection) -> Result<T, ReceiptStoreError> + Send + 'static,
989 T: Send + 'static,
990 {
991 let (result, _timeout_tracker) =
992 self.enqueue_write_job(job, appends_receipts, fail_closed_on_error)?;
993 match result.recv() {
994 Ok(outcome) => outcome,
995 Err(_) => {
996 atomic_saturating_sub(&self.health.inflight, 1);
1009 self.health.failed_total.fetch_add(1, Ordering::SeqCst);
1010 self.health.note_writer_unavailable();
1013 Err(self.worker.writer_dead_error())
1014 }
1015 }
1016 }
1017
1018 fn enqueue_write_job<T, F>(
1023 &self,
1024 job: F,
1025 appends_receipts: bool,
1026 fail_closed_on_error: bool,
1027 ) -> Result<
1028 (
1029 mpsc::Receiver<Result<T, ReceiptStoreError>>,
1030 WriterCommandTimeout,
1031 ),
1032 ReceiptStoreError,
1033 >
1034 where
1035 F: FnOnce(&mut SqliteStoreConnection) -> Result<T, ReceiptStoreError> + Send + 'static,
1036 T: Send + 'static,
1037 {
1038 let (response, result) = mpsc::sync_channel(1);
1039 let (completion, timeout_tracker) = writer_command_tracker(&self.health);
1040 let writer_health = Arc::clone(&self.health);
1041 let boxed: WriterClosure = Box::new(move |connection| {
1042 let outcome = match connection {
1043 Ok(connection) => {
1053 std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| job(connection)))
1054 .unwrap_or_else(|payload| Err(receipt_writer_job_panic_error(&payload)))
1055 }
1056 Err(error) => Err(error),
1057 };
1058 let responder: WriterResponder =
1063 Box::new(move |resync: Result<(), ReceiptStoreError>| {
1064 let final_outcome = match (outcome, resync) {
1065 (job_outcome, Ok(())) => job_outcome,
1066 (Err(job_error), Err(_)) => Err(job_error),
1067 (Ok(_), Err(resync_error)) => Err(resync_error),
1068 };
1069 if fail_closed_on_error {
1070 if let Err(error) = &final_outcome {
1071 if let Ok(mut last_error) = writer_health.last_error.lock() {
1072 *last_error = Some(error.to_string());
1073 }
1074 writer_health
1075 .critical_write_poisoned
1076 .store(true, Ordering::SeqCst);
1077 writer_health.set_head_poisoned(true);
1078 }
1079 }
1080 let committed = final_outcome.is_ok();
1084 let _ = response.send(final_outcome);
1085 committed
1086 });
1087 responder
1088 });
1089 let previous_inflight = self.health.inflight.fetch_add(1, Ordering::SeqCst);
1094 self.health.note_channel_send();
1095 match self.sender.try_send(ReceiptCommitCommand::Write {
1096 job: boxed,
1097 appends_receipts,
1098 fail_closed_on_error,
1099 completion,
1100 }) {
1101 Ok(()) => {
1102 self.health.accepted_total.fetch_add(1, Ordering::SeqCst);
1112 self.health.note_accept(previous_inflight);
1113 }
1114 Err(mpsc::TrySendError::Full(_)) => {
1115 atomic_saturating_sub(&self.health.inflight, 1);
1116 self.health.note_channel_send_rejected();
1117 self.health.saturated_total.fetch_add(1, Ordering::SeqCst);
1118 return Err(receipt_actor_saturated_error());
1119 }
1120 Err(mpsc::TrySendError::Disconnected(_)) => {
1121 atomic_saturating_sub(&self.health.inflight, 1);
1122 self.health.note_channel_send_rejected();
1123 self.health.note_writer_unavailable();
1124 return Err(self.worker.writer_dead_error());
1125 }
1126 }
1127 Ok((result, timeout_tracker))
1128 }
1129}
1130
1131fn receipt_commit_channel() -> (
1132 mpsc::SyncSender<ReceiptCommitCommand>,
1133 mpsc::Receiver<ReceiptCommitCommand>,
1134) {
1135 mpsc::sync_channel(RECEIPT_COMMIT_ACTOR_CHANNEL_CAPACITY)
1136}
1137
1138fn receipt_actor_unavailable_error() -> ReceiptStoreError {
1139 ReceiptStoreError::Pool("sqlite receipt commit actor is unavailable".to_string())
1140}
1141
1142fn receipt_actor_saturated_error() -> ReceiptStoreError {
1143 ReceiptStoreError::Pool("sqlite receipt commit queue saturated".to_string())
1144}
1145
1146fn receipt_actor_flush_timeout_error(timeout: Duration) -> ReceiptStoreError {
1147 ReceiptStoreError::Timeout {
1148 operation: "sqlite receipt commit flush".to_string(),
1149 timeout_ms: timeout.as_millis().min(u128::from(u64::MAX)) as u64,
1150 }
1151}
1152
1153fn receipt_actor_append_timeout_error(timeout: Duration) -> ReceiptStoreError {
1154 ReceiptStoreError::Timeout {
1155 operation: "sqlite receipt commit append".to_string(),
1156 timeout_ms: timeout.as_millis().min(u128::from(u64::MAX)) as u64,
1157 }
1158}
1159
1160fn receipt_actor_write_timeout_error(timeout: Duration) -> ReceiptStoreError {
1161 ReceiptStoreError::Timeout {
1162 operation: "sqlite receipt commit write".to_string(),
1163 timeout_ms: timeout.as_millis().min(u128::from(u64::MAX)) as u64,
1164 }
1165}
1166
1167enum WriterHeadState {
1170 Verified(Box<VerifiedHead>),
1173 Poisoned(String),
1176}
1177
1178fn poisoned_head_error(message: &str) -> ReceiptStoreError {
1179 ReceiptStoreError::Conflict(format!(
1180 "receipt store verified head is unavailable ({message}); run `chio receipt audit --repair`"
1181 ))
1182}
1183
1184fn poison_head_from_writer_error(
1185 head_state: &mut WriterHeadState,
1186 health: &ReceiptCommitWriterHealth,
1187) {
1188 health.set_head_poisoned(true);
1189 *head_state = WriterHeadState::Poisoned(critical_writer_error_message(health));
1190}
1191
1192fn critical_writer_error_message(health: &ReceiptCommitWriterHealth) -> String {
1193 health
1194 .last_error
1195 .lock()
1196 .map(|error| error.clone())
1197 .unwrap_or_else(|poisoned| poisoned.into_inner().clone())
1198 .unwrap_or_else(|| "critical receipt writer job failed".to_string())
1199}
1200
1201fn receipt_commit_actor_loop(
1202 pool: Pool<SqliteConnectionManager>,
1203 receiver: &mpsc::Receiver<ReceiptCommitCommand>,
1204 health: Arc<ReceiptCommitWriterHealth>,
1205 incremental_verification: bool,
1206 checkpoint_signer: &mut Option<BackgroundCheckpointSigner>,
1207) -> SupervisedOutcome {
1208 let mut head_state = match pool
1209 .get()
1210 .map_err(|error| ReceiptStoreError::Pool(error.to_string()))
1211 .and_then(|connection| {
1212 if incremental_verification {
1213 seed_verified_head(&connection)
1214 } else {
1215 seed_head_snapshot(&connection)
1216 }
1217 }) {
1218 Ok(head) if health.critical_write_poisoned.load(Ordering::SeqCst) => {
1219 health.store_head_snapshot(&head);
1220 health.set_head_poisoned(true);
1221 WriterHeadState::Poisoned(critical_writer_error_message(&health))
1222 }
1223 Ok(head) => {
1224 health.store_head_snapshot(&head);
1225 health.set_head_poisoned(false);
1228 WriterHeadState::Verified(Box::new(head))
1229 }
1230 Err(error) => {
1231 if let Ok(mut last_error) = health.last_error.lock() {
1232 *last_error = Some(error.to_string());
1233 }
1234 health.set_head_poisoned(true);
1235 WriterHeadState::Poisoned(error.to_string())
1236 }
1237 };
1238
1239 let mut pending_flush_error: Option<ReceiptStoreError> = None;
1240 while let Ok(command) = receiver.recv() {
1241 health.note_channel_dequeue();
1245 match command.into_append() {
1246 Ok((request, completion)) => {
1247 let mut requests = vec![*request];
1248 let mut completions: Vec<_> = completion.into_iter().collect();
1249 let mut flushes = Vec::new();
1250 let mut deferred: Option<ReceiptCommitCommand> = None;
1251 while requests.len() < RECEIPT_GROUP_COMMIT_MAX_BATCH {
1252 let next = receiver.recv_timeout(RECEIPT_GROUP_COMMIT_FLUSH_DELAY);
1253 if next.is_ok() {
1254 health.note_channel_dequeue();
1255 }
1256 match next {
1257 Ok(command) => match command.into_append() {
1258 Ok((request, completion)) => {
1259 requests.push(*request);
1260 completions.extend(completion);
1261 }
1262 Err(ReceiptCommitCommand::Flush(response)) => {
1263 flushes.push(response);
1264 break;
1265 }
1266 Err(other) => {
1267 deferred = Some(other);
1271 break;
1272 }
1273 },
1274 Err(mpsc::RecvTimeoutError::Timeout) => break,
1275 Err(mpsc::RecvTimeoutError::Disconnected) => break,
1276 }
1277 }
1278 let request_responses: Vec<_> = requests
1287 .iter()
1288 .map(|request| request.response.clone())
1289 .collect();
1290 pending_flush_error =
1298 match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
1299 commit_receipt_batch_with_completions(
1300 &pool,
1301 &mut head_state,
1302 incremental_verification,
1303 requests,
1304 &health,
1305 completions,
1306 )
1307 })) {
1308 Ok(flush_error) => flush_error,
1309 Err(payload) => {
1310 let panic_error = receipt_writer_job_panic_error(&payload);
1318 health.set_head_poisoned(true);
1319 head_state = WriterHeadState::Poisoned(panic_error.to_string());
1320 Some(fan_out_batch_panic_error(
1321 &health,
1322 request_responses,
1323 panic_error,
1324 ))
1325 }
1326 };
1327 let mut flush_barrier_error = pending_flush_error
1342 .as_ref()
1343 .map(receipt_store_error_snapshot);
1344 if pending_flush_error.is_none() {
1345 if let WriterHeadState::Verified(head) = &mut head_state {
1346 if let Some(error) = build_due_checkpoints_and_record(
1347 &pool,
1348 head,
1349 checkpoint_signer,
1350 &health,
1351 ) {
1352 flush_barrier_error = Some(error);
1353 }
1354 }
1355 }
1356 for response in flushes {
1360 let result = match &flush_barrier_error {
1361 Some(error) => Err(receipt_store_error_snapshot(error)),
1362 None => Ok(()),
1363 };
1364 let _ = response.send(result);
1365 }
1366 if let Some(command) = deferred {
1367 if let Some(outcome) = handle_non_append_command(
1368 &pool,
1369 &mut head_state,
1370 incremental_verification,
1371 &health,
1372 checkpoint_signer,
1373 &mut pending_flush_error,
1374 command,
1375 ) {
1376 return outcome;
1377 }
1378 }
1379 }
1380 Err(ReceiptCommitCommand::Flush(response)) => {
1381 let result = match &pending_flush_error {
1382 Some(error) => Err(receipt_store_error_snapshot(error)),
1383 None => Ok(()),
1384 };
1385 let _ = response.send(result);
1386 }
1387 Err(other) => {
1388 if let Some(outcome) = handle_non_append_command(
1389 &pool,
1390 &mut head_state,
1391 incremental_verification,
1392 &health,
1393 checkpoint_signer,
1394 &mut pending_flush_error,
1395 other,
1396 ) {
1397 return outcome;
1398 }
1399 }
1400 }
1401 }
1402 SupervisedOutcome::Shutdown
1403}
1404
1405fn handle_non_append_command(
1406 pool: &Pool<SqliteConnectionManager>,
1407 head_state: &mut WriterHeadState,
1408 incremental_verification: bool,
1409 health: &ReceiptCommitWriterHealth,
1410 checkpoint_signer: &mut Option<BackgroundCheckpointSigner>,
1411 pending_flush_error: &mut Option<ReceiptStoreError>,
1412 command: ReceiptCommitCommand,
1413) -> Option<SupervisedOutcome> {
1414 match command {
1415 ReceiptCommitCommand::Write {
1416 job,
1417 appends_receipts,
1418 fail_closed_on_error,
1419 mut completion,
1420 } => {
1421 let inflight_guard = WriterInflightGuard::new(&health.inflight);
1439 let mut connection = match pool.get() {
1440 Ok(connection) => connection,
1441 Err(error) => {
1442 let respond = job(Err(ReceiptStoreError::Pool(error.to_string())));
1446 drop(inflight_guard);
1448 let committed = respond(Ok(()));
1449 record_write_job_outcome(health, committed);
1450 completion.complete();
1451 if fail_closed_on_error && !committed {
1452 poison_head_from_writer_error(head_state, health);
1453 }
1454 return None;
1455 }
1456 };
1457 match head_state {
1458 WriterHeadState::Poisoned(message) => {
1459 let respond = job(Err(poisoned_head_error(message)));
1460 drop(inflight_guard);
1462 let committed = respond(Ok(()));
1463 record_write_job_outcome(health, committed);
1464 completion.complete();
1465 }
1466 WriterHeadState::Verified(head) => {
1467 let pre_check = if incremental_verification {
1477 match verify_head_against_latest_checkpoint(&connection, head) {
1484 Ok(()) => validate_writer_adopted_claim_log_baseline(
1485 &connection,
1486 head,
1487 appends_receipts,
1488 ),
1489 Err(error) => Err(error),
1490 }
1491 } else {
1492 verify_latest_checkpoint_integrity(&connection).and_then(|()| {
1493 if appends_receipts {
1494 validate_claim_receipt_log_entries(&connection)
1495 } else {
1496 Ok(())
1497 }
1498 })
1499 };
1500 if let Err(error) = pre_check {
1501 let respond = job(Err(error));
1502 drop(inflight_guard);
1504 let committed = respond(Ok(()));
1505 record_write_job_outcome(health, committed);
1506 completion.complete();
1507 if fail_closed_on_error && !committed {
1508 poison_head_from_writer_error(head_state, health);
1509 }
1510 return None;
1511 }
1512 let pre_checkpoint_seq = head.checkpoint_seq();
1517 let respond = job(Ok(&mut connection));
1523 match resync_head_after_write(&connection, head) {
1528 Ok(()) => {
1529 drop(inflight_guard);
1535 let committed = respond(Ok(()));
1536 record_write_job_outcome(health, committed);
1537 completion.complete();
1538 if fail_closed_on_error && !committed {
1539 poison_head_from_writer_error(head_state, health);
1540 return None;
1541 }
1542 health.store_head_snapshot(head);
1543 if head.checkpoint_seq() > pre_checkpoint_seq {
1557 if let Ok(mut last_error) = health.last_error.lock() {
1558 *last_error = None;
1559 }
1560 }
1561 drop(connection);
1570 if incremental_verification || appends_receipts {
1584 build_due_checkpoints_and_record(
1585 pool,
1586 head,
1587 checkpoint_signer,
1588 health,
1589 );
1590 }
1591 }
1592 Err(error) => {
1593 if let Ok(mut last_error) = health.last_error.lock() {
1594 *last_error = Some(error.to_string());
1595 }
1596 let poison_message = error.to_string();
1597 drop(inflight_guard);
1603 let committed = respond(Err(error));
1604 record_write_job_outcome(health, committed);
1605 completion.complete();
1606 health.set_head_poisoned(true);
1607 *head_state = WriterHeadState::Poisoned(poison_message);
1608 }
1609 }
1610 }
1611 }
1612 }
1613 ReceiptCommitCommand::Rotate { config, response } => {
1614 atomic_saturating_sub(&health.inflight, 1);
1621 if let WriterHeadState::Poisoned(message) = head_state {
1625 let _ = response.send(Err(poisoned_head_error(message)));
1626 return None;
1627 }
1628 let mut connection = match pool.get() {
1629 Ok(connection) => connection,
1630 Err(error) => {
1631 let _ = response.send(Err(ReceiptStoreError::Pool(error.to_string())));
1632 return None;
1633 }
1634 };
1635 let verified_latest_checkpoint = match verify_checkpoint_chain_integrity(&connection) {
1651 Ok(latest) => latest,
1652 Err(error) => {
1653 let _ = response.send(Err(error));
1654 return None;
1655 }
1656 };
1657 if let Err(error) = validate_claim_receipt_log_entries(&connection) {
1668 let _ = response.send(Err(error));
1669 return None;
1670 }
1671 let verified_ceiling = if incremental_verification {
1687 Some(
1688 verified_latest_checkpoint
1689 .as_ref()
1690 .map_or(0, |checkpoint| checkpoint.body.batch_end_seq),
1691 )
1692 } else {
1693 None
1694 };
1695 let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
1700 evidence_retention::rotate_on_writer_connection(
1701 &mut connection,
1702 &config,
1703 verified_ceiling,
1704 )
1705 }))
1706 .unwrap_or_else(|payload| Err(receipt_writer_job_panic_error(&payload)));
1707 if outcome.is_ok() {
1714 if let WriterHeadState::Verified(head) = head_state {
1715 if let Ok((count, max_seq)) = claim_log_delta_count_and_max_seq(&connection, 0)
1716 {
1717 head.claim_log_count = count;
1718 head.claim_log_max_seq = max_seq;
1719 }
1720 health.store_head_snapshot(head);
1721 }
1722 }
1723 let _ = response.send(outcome);
1724 }
1725 ReceiptCommitCommand::InstallSigner(signer) => {
1726 *checkpoint_signer = Some(signer);
1727 if incremental_verification {
1754 if let WriterHeadState::Verified(head) = head_state {
1755 build_due_checkpoints_and_record(pool, head, checkpoint_signer, health);
1756 }
1757 }
1758 }
1759 ReceiptCommitCommand::ReseedHead(response) => {
1760 let outcome = if health.critical_write_poisoned.load(Ordering::SeqCst) {
1761 Err(ReceiptStoreError::Conflict(format!(
1762 "{}; repair the critical receipt projection and reopen the receipt store",
1763 critical_writer_error_message(health)
1764 )))
1765 } else {
1766 pool.get()
1767 .map_err(|error| ReceiptStoreError::Pool(error.to_string()))
1768 .and_then(|connection| {
1769 support::audit_receipt_cost_projection(&connection)?;
1770 seed_verified_head(&connection)
1787 })
1788 };
1789 let result = match outcome {
1790 Ok(head) => {
1791 health.store_head_snapshot(&head);
1792 if let Ok(mut last_error) = health.last_error.lock() {
1793 *last_error = None;
1794 }
1795 *pending_flush_error = None;
1804 health.set_head_poisoned(false);
1805 *head_state = WriterHeadState::Verified(Box::new(head));
1806 if let WriterHeadState::Verified(head) = head_state {
1822 build_due_checkpoints_and_record(pool, head, checkpoint_signer, health);
1823 }
1824 Ok(())
1825 }
1826 Err(error) => {
1827 if let Ok(mut last_error) = health.last_error.lock() {
1828 *last_error = Some(error.to_string());
1829 }
1830 health.set_head_poisoned(true);
1831 *head_state = WriterHeadState::Poisoned(error.to_string());
1832 Err(error)
1833 }
1834 };
1835 let _ = response.send(result);
1836 }
1837 ReceiptCommitCommand::RetentionRepair {
1838 archive_path,
1839 response,
1840 } => {
1841 atomic_saturating_sub(&health.inflight, 1);
1845 let outcome = pool
1851 .get()
1852 .map_err(|error| ReceiptStoreError::Pool(error.to_string()))
1853 .and_then(|mut connection| {
1854 evidence_retention::retention_repair_on_writer(&mut connection, &archive_path)
1855 });
1856 if outcome.is_ok() {
1857 let reseed = pool
1866 .get()
1867 .map_err(|error| ReceiptStoreError::Pool(error.to_string()))
1868 .and_then(|connection| {
1869 if incremental_verification {
1870 seed_verified_head(&connection)
1871 } else {
1872 seed_head_snapshot(&connection)
1873 }
1874 });
1875 match reseed {
1876 Ok(head) => {
1877 health.store_head_snapshot(&head);
1878 if let Ok(mut last_error) = health.last_error.lock() {
1879 *last_error = None;
1880 }
1881 *pending_flush_error = None;
1889 *head_state = WriterHeadState::Verified(Box::new(head));
1890 }
1891 Err(error) => {
1892 if let Ok(mut last_error) = health.last_error.lock() {
1893 *last_error = Some(error.to_string());
1894 }
1895 *head_state = WriterHeadState::Poisoned(error.to_string());
1896 }
1897 }
1898 }
1899 let _ = response.send(outcome);
1900 }
1901 ReceiptCommitCommand::Append(request) => {
1904 let _ = request
1905 .response
1906 .send(Err(receipt_actor_unavailable_error()));
1907 }
1908 ReceiptCommitCommand::AppendWithTimeout {
1909 request,
1910 mut completion,
1911 } => {
1912 completion.complete();
1913 let _ = request
1914 .response
1915 .send(Err(receipt_actor_unavailable_error()));
1916 }
1917 ReceiptCommitCommand::Flush(response) => {
1918 let _ = response.send(Err(receipt_actor_unavailable_error()));
1919 }
1920 #[cfg(test)]
1921 ReceiptCommitCommand::RestartSupervisor => return Some(SupervisedOutcome::Restart),
1922 }
1923 None
1924}
1925
1926fn build_due_checkpoints_and_record(
1934 pool: &Pool<SqliteConnectionManager>,
1935 head: &mut VerifiedHead,
1936 checkpoint_signer: &Option<BackgroundCheckpointSigner>,
1937 health: &ReceiptCommitWriterHealth,
1938) -> Option<ReceiptStoreError> {
1939 let signer = checkpoint_signer.as_ref()?;
1940 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
1945 build_due_checkpoints(pool, head, signer)
1946 }))
1947 .unwrap_or_else(|payload| Err(receipt_writer_job_panic_error(&payload)));
1948 match result {
1949 Ok(advanced) => {
1950 health.store_head_snapshot(head);
1951 if advanced {
1955 if let Ok(mut last_error) = health.last_error.lock() {
1956 *last_error = None;
1957 }
1958 }
1959 None
1960 }
1961 Err(error) => {
1962 if let Ok(mut last_error) = health.last_error.lock() {
1963 *last_error = Some(error.to_string());
1964 }
1965 Some(error)
1966 }
1967 }
1968}
1969
1970fn build_due_checkpoints(
1971 pool: &Pool<SqliteConnectionManager>,
1972 head: &mut VerifiedHead,
1973 signer: &BackgroundCheckpointSigner,
1974) -> Result<bool, ReceiptStoreError> {
1975 if signer.max_batch == 0 {
1976 return Ok(false); }
1978 let mut connection = pool
1979 .get()
1980 .map_err(|error| ReceiptStoreError::Pool(error.to_string()))?;
1981 let checkpoint_seq_before_refresh = head.checkpoint_seq();
1982 verify_head_against_latest_checkpoint(&connection, head)?;
1995 let refreshed = head.checkpoint_seq() > checkpoint_seq_before_refresh;
1996 maybe_build_checkpoint(&mut connection, head, signer).map(|advanced| refreshed || advanced)
1997}
1998
1999fn maybe_build_checkpoint(
2006 connection: &mut SqliteStoreConnection,
2007 head: &mut VerifiedHead,
2008 signer: &BackgroundCheckpointSigner,
2009) -> Result<bool, ReceiptStoreError> {
2010 if signer.max_batch == 0 {
2011 return Ok(false);
2012 }
2013 if head
2014 .claim_log_max_seq
2015 .saturating_sub(head.checkpointed_entry_seq())
2016 < signer.max_batch
2017 {
2018 return Ok(false);
2019 }
2020 let cached = head
2028 .chain_frontier
2029 .as_ref()
2030 .filter(|frontier| frontier.leaf_count() == head.checkpoint_seq())
2031 .cloned();
2032 let mut advanced = false;
2033 let mut chain_frontier = match cached {
2034 Some(frontier) => frontier,
2035 None => {
2036 let (frontier, cache_advanced) =
2037 build_checkpoint_after_frontier_cache_miss(connection, head, signer)?;
2038 advanced = cache_advanced;
2039 frontier
2040 }
2041 };
2042 if chain_frontier.leaf_count() != head.checkpoint_seq() {
2043 return Err(ReceiptStoreError::Conflict(format!(
2044 "persisted chain covers {} checkpoints but the head is at {}",
2045 chain_frontier.leaf_count(),
2046 head.checkpoint_seq()
2047 )));
2048 }
2049 while head
2050 .claim_log_max_seq
2051 .saturating_sub(head.checkpointed_entry_seq())
2052 >= signer.max_batch
2053 {
2054 let start_seq = head.checkpointed_entry_seq().saturating_add(1);
2055 let end_seq = start_seq.saturating_add(signer.max_batch - 1);
2056 ensure_claim_log_range_contiguous(connection, start_seq, end_seq, "checkpoint range")?;
2057 let receipt_bytes = load_claim_tree_canonical_bytes_range(connection, start_seq, end_seq)?
2058 .into_iter()
2059 .map(|(_, bytes)| bytes)
2060 .collect::<Vec<_>>();
2061 let checkpoint_seq = head
2062 .checkpoint_seq()
2063 .checked_add(1)
2064 .ok_or_else(|| ReceiptStoreError::Conflict("checkpoint_seq overflow".to_string()))?;
2065 let checkpoint = chio_kernel::checkpoint::build_checkpoint_with_chain_frontier(
2068 checkpoint_seq,
2069 start_seq,
2070 end_seq,
2071 &receipt_bytes,
2072 &signer.keypair,
2073 head.latest_checkpoint.as_ref(),
2074 &chain_frontier,
2075 )
2076 .map_err(checkpoint_error_to_receipt_store)?;
2077 #[cfg(test)]
2078 if test_hooks::panic_during_checkpoint_build(signer.max_batch) {
2079 panic!("injected test panic during background checkpoint build");
2080 }
2081 #[cfg(test)]
2082 if test_hooks::fail_checkpoint_build(signer.max_batch) {
2083 return Err(ReceiptStoreError::Conflict(
2084 "injected test checkpoint build failure".to_string(),
2085 ));
2086 }
2087 let (adopted, adopted_frontier) = insert_background_checkpoint_guarded(
2093 connection,
2094 head.latest_checkpoint.as_ref(),
2095 &chain_frontier,
2096 &checkpoint,
2097 )?;
2098 chain_frontier = adopted_frontier;
2099 head.latest_checkpoint = Some(adopted);
2100 advanced = true;
2101 }
2102 head.chain_frontier = Some(chain_frontier);
2103 Ok(advanced)
2104}
2105
2106fn resync_head_after_write(
2109 connection: &Connection,
2110 head: &mut VerifiedHead,
2111) -> Result<(), ReceiptStoreError> {
2112 let pre_resync_max = head.claim_log_max_seq;
2113 let (delta_count, post_max) = claim_log_delta_count_and_max_seq(connection, pre_resync_max)?;
2114 if delta_count > 0 {
2127 validate_adopted_claim_log_delta(connection, pre_resync_max, post_max)?;
2128 }
2129 head.claim_log_count = head.claim_log_count.saturating_add(delta_count);
2130 head.claim_log_max_seq = post_max;
2131 verify_head_against_latest_checkpoint(connection, head)
2132}
2133
2134#[cfg(test)]
2135fn commit_receipt_batch(
2136 pool: &Pool<SqliteConnectionManager>,
2137 head_state: &mut WriterHeadState,
2138 incremental_verification: bool,
2139 requests: Vec<ReceiptCommitRequest>,
2140 health: &ReceiptCommitWriterHealth,
2141) -> Option<ReceiptStoreError> {
2142 commit_receipt_batch_with_completions(
2143 pool,
2144 head_state,
2145 incremental_verification,
2146 requests,
2147 health,
2148 Vec::new(),
2149 )
2150}
2151
2152fn commit_receipt_batch_with_completions(
2153 pool: &Pool<SqliteConnectionManager>,
2154 head_state: &mut WriterHeadState,
2155 incremental_verification: bool,
2156 requests: Vec<ReceiptCommitRequest>,
2157 health: &ReceiptCommitWriterHealth,
2158 mut completions: Vec<WriterCommandCompletion>,
2159) -> Option<ReceiptStoreError> {
2160 let batch_outcome = match head_state {
2161 WriterHeadState::Verified(head) => {
2162 match append_receipt_batch(pool, head, incremental_verification, &requests) {
2163 Ok(results) => {
2164 health.store_head_snapshot(head);
2165 Ok(results)
2166 }
2167 Err(error) => Err(error),
2168 }
2169 }
2170 WriterHeadState::Poisoned(message) => Ok(receipt_batch_error_results(
2171 requests.len(),
2172 poisoned_head_error(message),
2173 )),
2174 };
2175 let results = match batch_outcome {
2176 Ok(results) => results,
2177 Err(error) => {
2178 let message = error.to_string();
2185 let results = receipt_batch_error_results(requests.len(), error);
2186 health.set_head_poisoned(true);
2187 *head_state = WriterHeadState::Poisoned(message);
2188 results
2189 }
2190 };
2191 let flush_error = results
2192 .iter()
2193 .find_map(|result| result.as_ref().err().map(receipt_store_error_snapshot));
2194 let committed = results.iter().filter(|result| result.is_ok()).count() as u64;
2195 let failed = results.iter().filter(|result| result.is_err()).count() as u64;
2196 if committed > 0 {
2197 health
2198 .committed_total
2199 .fetch_add(committed, Ordering::SeqCst);
2200 health
2201 .last_commit_unix_ms
2202 .store(current_unix_ms(), Ordering::SeqCst);
2203 }
2204 if failed > 0 {
2205 health.failed_total.fetch_add(failed, Ordering::SeqCst);
2206 }
2207 atomic_saturating_sub(&health.inflight, results.len() as u64);
2208 for completion in &mut completions {
2209 completion.complete();
2210 }
2211 if let Some(error) = &flush_error {
2212 if let Ok(mut last_error) = health.last_error.lock() {
2213 *last_error = Some(error.to_string());
2214 }
2215 } else {
2216 health.clear_timeout_error_if_drained();
2217 }
2218 for (request, result) in requests.into_iter().zip(results) {
2223 let _ = request.response.send(result);
2224 }
2225 flush_error
2226}
2227
2228fn current_unix_ms() -> u64 {
2229 SystemTime::now()
2230 .duration_since(UNIX_EPOCH)
2231 .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
2232 .unwrap_or(0)
2233}
2234
2235fn atomic_saturating_sub(value: &AtomicU64, amount: u64) {
2236 let mut current = value.load(Ordering::SeqCst);
2237 loop {
2238 let next = current.saturating_sub(amount);
2239 match value.compare_exchange(current, next, Ordering::SeqCst, Ordering::SeqCst) {
2240 Ok(_) => return,
2241 Err(observed) => current = observed,
2242 }
2243 }
2244}
2245
2246fn classify_writer_liveness(
2268 counters: &ReceiptWriterCounters,
2269 stall_threshold_ms: u64,
2270 channel_capacity: u64,
2271 backlog_started_unix_ms: Option<u64>,
2272 now_unix_ms: u64,
2273) -> chio_kernel::ReceiptWriterLiveness {
2274 use chio_kernel::ReceiptWriterLiveness as Liveness;
2275 if counters
2276 .last_error
2277 .as_deref()
2278 .is_some_and(|error| error.contains("unavailable"))
2279 {
2280 return Liveness::Dead;
2281 }
2282 if counters.timed_out_inflight > 0 {
2283 return Liveness::Wedged;
2284 }
2285 let backlogged = counters.inflight > 0
2286 || counters.accepted_total
2287 > counters
2288 .committed_total
2289 .saturating_add(counters.failed_total);
2290 let progress_reference = counters
2295 .last_commit_unix_ms
2296 .into_iter()
2297 .chain(backlog_started_unix_ms)
2298 .max();
2299 let stalled = match progress_reference {
2300 Some(reference) => now_unix_ms.saturating_sub(reference) >= stall_threshold_ms,
2301 None => false,
2302 };
2303 if backlogged && stalled {
2304 return Liveness::Wedged;
2305 }
2306 if counters.queue_depth >= channel_capacity {
2307 return Liveness::Saturated;
2308 }
2309 Liveness::Healthy
2310}
2311
2312fn receipt_store_healthy(
2320 checkpoint_healthy: bool,
2321 writer_last_error: Option<&str>,
2322 writer_liveness: chio_kernel::ReceiptWriterLiveness,
2323) -> bool {
2324 use chio_kernel::ReceiptWriterLiveness as Liveness;
2325 checkpoint_healthy
2326 && writer_last_error.is_none()
2327 && !matches!(
2328 writer_liveness,
2329 Liveness::Wedged | Liveness::Saturated | Liveness::Dead
2330 )
2331}
2332
2333#[cfg(test)]
2334mod writer_liveness_classifier_tests {
2335 use super::*;
2336 use chio_kernel::ReceiptWriterLiveness as Liveness;
2337
2338 const CAPACITY: u64 = RECEIPT_COMMIT_ACTOR_CHANNEL_CAPACITY as u64;
2339 const NOW: u64 = 1_000_000;
2340 const STALL_MS: u64 = 10_000;
2341
2342 #[test]
2343 fn timed_out_inflight_append_reports_wedged() {
2344 let counters = ReceiptWriterCounters {
2347 accepted_total: 1,
2348 inflight: 1,
2349 ..ReceiptWriterCounters::default()
2350 };
2351 assert_eq!(
2352 classify_writer_liveness(&counters, STALL_MS, CAPACITY, Some(NOW - 20_000), NOW),
2353 Liveness::Wedged
2354 );
2355 }
2356
2357 #[test]
2358 fn outstanding_timeout_reports_wedged_before_the_stall_threshold() {
2359 let counters = ReceiptWriterCounters {
2360 accepted_total: 1,
2361 inflight: 1,
2362 timed_out_total: 1,
2363 timed_out_inflight: 1,
2364 last_commit_unix_ms: None,
2365 last_error: Some("sqlite receipt commit append timed out".to_string()),
2366 ..ReceiptWriterCounters::default()
2367 };
2368 assert_eq!(
2369 classify_writer_liveness(&counters, STALL_MS, CAPACITY, Some(NOW - 6_000), NOW),
2370 Liveness::Wedged
2371 );
2372 }
2373
2374 #[test]
2375 fn never_committed_backlog_reports_wedged() {
2376 let counters = ReceiptWriterCounters {
2379 accepted_total: 1,
2380 inflight: 1,
2381 last_commit_unix_ms: None,
2382 ..ReceiptWriterCounters::default()
2383 };
2384 assert_eq!(
2385 classify_writer_liveness(&counters, STALL_MS, CAPACITY, Some(NOW - 20_000), NOW),
2386 Liveness::Wedged
2387 );
2388 }
2389
2390 #[test]
2391 fn honors_configured_stall_threshold() {
2392 let counters = ReceiptWriterCounters {
2396 accepted_total: 2,
2397 committed_total: 1,
2398 inflight: 1,
2399 last_commit_unix_ms: Some(NOW - 600),
2400 ..ReceiptWriterCounters::default()
2401 };
2402 assert_eq!(
2403 classify_writer_liveness(&counters, 500, CAPACITY, None, NOW),
2404 Liveness::Wedged
2405 );
2406 assert_eq!(
2407 classify_writer_liveness(&counters, 10_000, CAPACITY, None, NOW),
2408 Liveness::Healthy
2409 );
2410 }
2411
2412 #[test]
2413 fn full_commit_channel_reports_saturated() {
2414 let counters = ReceiptWriterCounters {
2418 accepted_total: CAPACITY + 5,
2419 committed_total: 4,
2420 inflight: CAPACITY,
2421 queue_depth: CAPACITY,
2422 last_commit_unix_ms: Some(NOW - 100),
2423 ..ReceiptWriterCounters::default()
2424 };
2425 assert_eq!(
2426 classify_writer_liveness(&counters, STALL_MS, CAPACITY, None, NOW),
2427 Liveness::Saturated
2428 );
2429 assert!(!Liveness::Saturated.healthy());
2430 }
2431
2432 #[test]
2433 fn a_drained_but_committing_batch_is_not_reported_saturated() {
2434 let counters = ReceiptWriterCounters {
2441 accepted_total: CAPACITY + RECEIPT_GROUP_COMMIT_MAX_BATCH as u64,
2442 committed_total: 0,
2443 inflight: CAPACITY,
2444 queue_depth: CAPACITY - RECEIPT_GROUP_COMMIT_MAX_BATCH as u64,
2445 last_commit_unix_ms: Some(NOW - 100),
2446 ..ReceiptWriterCounters::default()
2447 };
2448 assert_eq!(
2449 classify_writer_liveness(&counters, STALL_MS, CAPACITY, Some(NOW - 100), NOW),
2450 Liveness::Healthy
2451 );
2452 }
2453
2454 #[test]
2455 fn idle_writer_with_fresh_backlog_is_not_wedged() {
2456 let counters = ReceiptWriterCounters {
2462 accepted_total: 6,
2463 committed_total: 5,
2464 inflight: 1,
2465 last_commit_unix_ms: Some(NOW - 60_000),
2466 ..ReceiptWriterCounters::default()
2467 };
2468 assert_eq!(
2469 classify_writer_liveness(&counters, STALL_MS, CAPACITY, Some(NOW - 100), NOW),
2470 Liveness::Healthy,
2471 "fresh work after idle must not be judged wedged by the stale last commit"
2472 );
2473 assert_eq!(
2476 classify_writer_liveness(&counters, STALL_MS, CAPACITY, Some(NOW - 20_000), NOW),
2477 Liveness::Wedged,
2478 "a backlog stalled past the threshold must still report wedged"
2479 );
2480 }
2481
2482 #[test]
2483 fn unavailable_writer_reports_dead() {
2484 let counters = ReceiptWriterCounters {
2485 last_error: Some("sqlite receipt commit actor is unavailable".to_string()),
2486 ..ReceiptWriterCounters::default()
2487 };
2488 assert_eq!(
2489 classify_writer_liveness(&counters, STALL_MS, CAPACITY, None, NOW),
2490 Liveness::Dead
2491 );
2492 }
2493
2494 #[test]
2495 fn drained_writer_reports_healthy() {
2496 let counters = ReceiptWriterCounters {
2497 accepted_total: 10,
2498 committed_total: 10,
2499 inflight: 0,
2500 last_commit_unix_ms: Some(NOW - 50),
2501 ..ReceiptWriterCounters::default()
2502 };
2503 assert_eq!(
2504 classify_writer_liveness(&counters, STALL_MS, CAPACITY, None, NOW),
2505 Liveness::Healthy
2506 );
2507 }
2508
2509 #[test]
2510 fn a_non_healthy_writer_makes_the_store_unhealthy() {
2511 assert!(!receipt_store_healthy(true, None, Liveness::Wedged));
2515 assert!(!receipt_store_healthy(true, None, Liveness::Saturated));
2516 assert!(!receipt_store_healthy(true, None, Liveness::Dead));
2517 }
2518
2519 #[test]
2520 fn healthy_and_unknown_writers_do_not_downgrade_store_health() {
2521 assert!(receipt_store_healthy(true, None, Liveness::Healthy));
2522 assert!(receipt_store_healthy(true, None, Liveness::Unknown));
2525 assert!(!receipt_store_healthy(false, None, Liveness::Healthy));
2528 assert!(!receipt_store_healthy(
2529 true,
2530 Some("checkpoint build failed"),
2531 Liveness::Healthy
2532 ));
2533 }
2534}
2535
2536struct WriterInflightGuard<'a> {
2548 inflight: &'a AtomicU64,
2549}
2550
2551impl<'a> WriterInflightGuard<'a> {
2552 fn new(inflight: &'a AtomicU64) -> Self {
2553 Self { inflight }
2554 }
2555}
2556
2557impl Drop for WriterInflightGuard<'_> {
2558 fn drop(&mut self) {
2559 atomic_saturating_sub(self.inflight, 1);
2560 }
2561}
2562
2563fn record_write_job_outcome(health: &ReceiptCommitWriterHealth, committed: bool) {
2574 if committed {
2575 health.committed_total.fetch_add(1, Ordering::SeqCst);
2576 health
2577 .last_commit_unix_ms
2578 .store(current_unix_ms(), Ordering::SeqCst);
2579 health.clear_timeout_error_if_drained();
2583 } else {
2584 health.failed_total.fetch_add(1, Ordering::SeqCst);
2585 }
2586}
2587
2588#[derive(Clone)]
2592pub struct BackgroundCheckpointSigner {
2593 pub keypair: Arc<Keypair>,
2594 pub max_batch: u64,
2595}
2596
2597#[derive(Clone, Debug, Default)]
2600pub(crate) struct VerifiedHead {
2601 latest_checkpoint: Option<KernelCheckpoint>,
2604 chain_frontier: Option<CheckpointChainFrontier>,
2610 claim_log_count: u64,
2612 claim_log_max_seq: u64,
2614}
2615
2616impl VerifiedHead {
2617 pub(crate) fn checkpoint_seq(&self) -> u64 {
2618 self.latest_checkpoint
2619 .as_ref()
2620 .map_or(0, |checkpoint| checkpoint.body.checkpoint_seq)
2621 }
2622
2623 pub(crate) fn checkpointed_entry_seq(&self) -> u64 {
2624 self.latest_checkpoint
2625 .as_ref()
2626 .map_or(0, |checkpoint| checkpoint.body.batch_end_seq)
2627 }
2628}
2629
2630pub(crate) struct WriterHeadSnapshot {
2634 pub(crate) checkpoint_seq: u64,
2635 pub(crate) checkpointed_entry_seq: u64,
2636 #[cfg_attr(not(test), allow(dead_code))]
2641 pub(crate) claim_log_count: u64,
2642 #[cfg_attr(not(test), allow(dead_code))]
2643 pub(crate) claim_log_max_seq: u64,
2644}
2645
2646fn seed_verified_head(connection: &Connection) -> Result<VerifiedHead, ReceiptStoreError> {
2649 validate_claim_receipt_log_entries(connection)?;
2650 let (latest_checkpoint, chain_frontier) =
2651 verify_checkpoint_chain_integrity_with_frontier(connection)?;
2652 let (claim_log_count, claim_log_max_seq) = claim_log_delta_count_and_max_seq(connection, 0)?;
2653 Ok(VerifiedHead {
2654 latest_checkpoint,
2655 chain_frontier: Some(chain_frontier),
2656 claim_log_count,
2657 claim_log_max_seq,
2658 })
2659}
2660
2661fn seed_head_snapshot(connection: &Connection) -> Result<VerifiedHead, ReceiptStoreError> {
2666 let latest_checkpoint = load_latest_persisted_checkpoint_row(connection)?
2667 .map(parse_persisted_checkpoint_row)
2668 .transpose()?;
2669 let (claim_log_count, claim_log_max_seq) = claim_log_delta_count_and_max_seq(connection, 0)?;
2670 Ok(VerifiedHead {
2671 latest_checkpoint,
2672 chain_frontier: None,
2673 claim_log_count,
2674 claim_log_max_seq,
2675 })
2676}
2677
2678fn claim_log_delta_count_and_max_seq(
2683 connection: &Connection,
2684 floor_entry_seq: u64,
2685) -> Result<(u64, u64), ReceiptStoreError> {
2686 let floor = sqlite_i64(floor_entry_seq, "claim log delta floor entry_seq")?;
2687 let (count, max_seq) = connection.query_row(
2688 "SELECT COUNT(*), COALESCE(MAX(entry_seq), ?1) FROM claim_receipt_log_entries WHERE entry_seq > ?1",
2689 params![floor],
2690 |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?)),
2691 )?;
2692 Ok((
2693 sqlite_u64(count, "claim log delta count")?,
2694 sqlite_u64(max_seq, "claim log delta max entry_seq")?,
2695 ))
2696}
2697
2698fn validate_writer_adopted_claim_log_baseline(
2714 connection: &Connection,
2715 head: &VerifiedHead,
2716 appends_receipts: bool,
2717) -> Result<(), ReceiptStoreError> {
2718 if !appends_receipts {
2719 return Ok(());
2720 }
2721 let (pre_delta, baseline_max) =
2722 claim_log_delta_count_and_max_seq(connection, head.claim_log_max_seq)?;
2723 if pre_delta > 0 {
2724 validate_adopted_claim_log_delta(connection, head.claim_log_max_seq, baseline_max)?;
2725 }
2726 Ok(())
2727}
2728
2729fn verify_head_against_latest_checkpoint(
2735 connection: &Connection,
2736 head: &mut VerifiedHead,
2737) -> Result<(), ReceiptStoreError> {
2738 let persisted = load_latest_persisted_checkpoint_row(connection)?;
2739 let cached_seq = head.checkpoint_seq();
2740 match persisted {
2741 None if head.latest_checkpoint.is_none() => Ok(()),
2742 None => Err(ReceiptStoreError::Conflict(
2743 "latest checkpoint disappeared behind the verified head; run `chio receipt audit`"
2744 .to_string(),
2745 )),
2746 Some(row) if row.checkpoint_seq < cached_seq => Err(ReceiptStoreError::Conflict(format!(
2747 "checkpoint chain regressed from verified head {cached_seq} to {}; run `chio receipt audit`",
2748 row.checkpoint_seq
2749 ))),
2750 Some(row) if row.checkpoint_seq == cached_seq => {
2751 let Some(cached) = head.latest_checkpoint.as_ref() else {
2752 return Err(ReceiptStoreError::Conflict(
2753 "checkpoint presence diverged from verified head; run `chio receipt audit`"
2754 .to_string(),
2755 ));
2756 };
2757 let persisted_body: KernelCheckpointBody = serde_json::from_str(&row.statement_json)?;
2762 let persisted_digest = chio_kernel::checkpoint::checkpoint_body_sha256(&persisted_body)
2763 .map_err(checkpoint_error_to_receipt_store)?;
2764 let cached_digest = chio_kernel::checkpoint::checkpoint_body_sha256(&cached.body)
2765 .map_err(checkpoint_error_to_receipt_store)?;
2766 if persisted_digest != cached_digest {
2767 return Err(ReceiptStoreError::Conflict(
2768 "latest checkpoint diverged from verified head; run `chio receipt audit`"
2769 .to_string(),
2770 ));
2771 }
2772 ensure_checkpoint_columns_match_body(&row, &persisted_body)?;
2783 if row.signature_hex != cached.signature.to_hex() {
2788 return Err(ReceiptStoreError::Conflict(
2789 "latest checkpoint signature column diverged from verified head; run `chio receipt audit`"
2790 .to_string(),
2791 ));
2792 }
2793 validate_checkpoint_projection_rows(connection, &row, cached)?;
2809 Ok(())
2810 }
2811 Some(row) => catch_up_verified_head_to(connection, head, row.checkpoint_seq),
2812 }
2813}
2814
2815fn catch_up_verified_head_to(
2825 connection: &Connection,
2826 head: &mut VerifiedHead,
2827 latest_seq: u64,
2828) -> Result<(), ReceiptStoreError> {
2829 let mut cursor = head.checkpoint_seq();
2830 let watermark = trusted_retention_watermark(connection)?;
2838 while cursor < latest_seq {
2839 let next_seq = cursor.saturating_add(1);
2840 let Some(row) = load_persisted_checkpoint_row(connection, next_seq)? else {
2841 return Err(ReceiptStoreError::Conflict(format!(
2842 "checkpoint chain gap at {next_seq} behind latest {latest_seq}; run `chio receipt audit`"
2843 )));
2844 };
2845 let checkpoint = parse_persisted_checkpoint_row(row.clone())?;
2846 match head.latest_checkpoint.as_ref() {
2847 Some(predecessor) => {
2848 chio_kernel::checkpoint::validate_checkpoint_predecessor(predecessor, &checkpoint)
2849 .map_err(checkpoint_error_to_receipt_store)?;
2850 }
2851 None => validate_checkpoint_base(&checkpoint)?,
2852 }
2853 if checkpoint.body.batch_end_seq > watermark {
2854 validate_checkpoint_against_claim_log(connection, &checkpoint)?;
2855 }
2856 validate_checkpoint_projection_rows(connection, &row, &checkpoint)?;
2865 head.chain_frontier = Some(advance_verified_checkpoint_chain_frontier(
2868 connection,
2869 head.chain_frontier.as_ref(),
2870 head.latest_checkpoint.as_ref(),
2871 &checkpoint,
2872 )?);
2873 head.latest_checkpoint = Some(checkpoint);
2874 cursor = next_seq;
2875 }
2876 Ok(())
2877}
2878
2879fn append_single_receipt_record(
2887 tx: &rusqlite::Transaction<'_>,
2888 request: &ReceiptCommitRequest,
2889) -> Result<u64, ReceiptStoreError> {
2890 let seq = append_chio_receipt_tx(tx, &request.receipt, &request.raw_json)?;
2891 if request.ensure_lineage {
2892 #[cfg(test)]
2893 if test_hooks::fail_between_receipt_and_lineage() {
2894 return Err(ReceiptStoreError::Conflict(
2895 "injected failure between receipt insert and lineage insert".to_string(),
2896 ));
2897 }
2898 ensure_receipt_lineage_statement_for_receipt_id_tx(tx, &request.receipt.id)?;
2899 }
2900 Ok(seq)
2901}
2902
2903fn append_receipt_batch(
2909 pool: &Pool<SqliteConnectionManager>,
2910 head: &mut VerifiedHead,
2911 incremental_verification: bool,
2912 requests: &[ReceiptCommitRequest],
2913) -> Result<Vec<Result<u64, ReceiptStoreError>>, ReceiptStoreError> {
2914 let mut connection = pool
2915 .get()
2916 .map_err(|error| ReceiptStoreError::Pool(error.to_string()))?;
2917 ensure_checkpoint_transparency_guards(&connection)?;
2918 if incremental_verification {
2919 verify_head_against_latest_checkpoint(&connection, head)?;
2921 } else {
2922 validate_claim_receipt_log_entries(&connection)?;
2923 }
2924 let tx = connection
2925 .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)
2926 .map_err(ReceiptStoreError::Sqlite)?;
2927 if !incremental_verification {
2928 verify_latest_checkpoint_integrity(&tx)?;
2929 }
2930 let (pre_delta, baseline_max) = claim_log_delta_count_and_max_seq(&tx, head.claim_log_max_seq)?;
2934 if pre_delta > 0 {
2944 validate_adopted_claim_log_delta(&tx, head.claim_log_max_seq, baseline_max)?;
2945 }
2946 let mut results = Vec::with_capacity(requests.len());
2947 for request in requests {
2948 #[cfg(test)]
2949 if test_hooks::panic_during_append_batch(&request.receipt.content_hash) {
2950 panic!("injected test panic during append batch");
2951 }
2952 tx.execute_batch("SAVEPOINT chio_append_record")
2964 .map_err(ReceiptStoreError::Sqlite)?;
2965 match append_single_receipt_record(&tx, request) {
2966 Ok(seq) => {
2967 tx.execute_batch("RELEASE chio_append_record")
2968 .map_err(ReceiptStoreError::Sqlite)?;
2969 results.push(Ok(seq));
2970 }
2971 Err(error) => {
2972 tx.execute_batch("ROLLBACK TO chio_append_record; RELEASE chio_append_record")
2977 .map_err(ReceiptStoreError::Sqlite)?;
2978 results.push(Err(error));
2979 }
2980 }
2981 }
2982 let inserted = results
2993 .iter()
2994 .filter_map(|result| result.as_ref().ok())
2995 .filter(|seq| **seq > baseline_max)
2996 .copied()
2997 .collect::<std::collections::BTreeSet<u64>>()
2998 .len() as u64;
2999 #[cfg(feature = "chaos-test-hooks")]
3000 chaos_test_hooks::pause_after_receipt_write_before_commit(inserted > 0)?;
3001 let (delta_count, post_max) = claim_log_delta_count_and_max_seq(&tx, baseline_max)?;
3005 if delta_count != inserted || post_max < baseline_max {
3006 return Err(ReceiptStoreError::Conflict(
3007 "claim receipt log projection drift on append; run `chio receipt audit`".to_string(),
3008 ));
3009 }
3010 if delta_count > 0 {
3026 validate_adopted_claim_log_delta(&tx, baseline_max, post_max)?;
3027 }
3028 tx.commit().map_err(ReceiptStoreError::Sqlite)?;
3029 head.claim_log_count = head
3030 .claim_log_count
3031 .saturating_add(pre_delta)
3032 .saturating_add(delta_count);
3033 head.claim_log_max_seq = post_max.max(baseline_max);
3034 Ok(results)
3035}
3036
3037fn receipt_batch_error_results(
3038 count: usize,
3039 error: ReceiptStoreError,
3040) -> Vec<Result<u64, ReceiptStoreError>> {
3041 let snapshot = receipt_store_error_snapshot(&error);
3042 let mut original = Some(error);
3043 (0..count)
3044 .map(|_| {
3045 Err(original
3046 .take()
3047 .unwrap_or_else(|| receipt_store_error_snapshot(&snapshot)))
3048 })
3049 .collect()
3050}
3051
3052fn receipt_store_error_snapshot(error: &ReceiptStoreError) -> ReceiptStoreError {
3053 match error {
3054 ReceiptStoreError::Sqlite(error) => {
3055 ReceiptStoreError::Sqlite(rusqlite::Error::ToSqlConversionFailure(Box::new(
3056 std::io::Error::other(error.to_string()),
3057 )))
3058 }
3059 ReceiptStoreError::Pool(message) => ReceiptStoreError::Pool(message.clone()),
3060 ReceiptStoreError::Timeout {
3061 operation,
3062 timeout_ms,
3063 } => ReceiptStoreError::Timeout {
3064 operation: operation.clone(),
3065 timeout_ms: *timeout_ms,
3066 },
3067 ReceiptStoreError::Json(error) => ReceiptStoreError::Json(serde_json::Error::io(
3068 std::io::Error::other(error.to_string()),
3069 )),
3070 ReceiptStoreError::Io(error) => {
3071 ReceiptStoreError::Io(std::io::Error::new(error.kind(), error.to_string()))
3072 }
3073 ReceiptStoreError::CryptoDecode(message) => {
3074 ReceiptStoreError::CryptoDecode(message.clone())
3075 }
3076 ReceiptStoreError::Canonical(message) => ReceiptStoreError::Canonical(message.clone()),
3077 ReceiptStoreError::InvalidOutcome(message) => {
3078 ReceiptStoreError::InvalidOutcome(message.clone())
3079 }
3080 ReceiptStoreError::ReadBoundary(message) => {
3081 ReceiptStoreError::ReadBoundary(message.clone())
3082 }
3083 ReceiptStoreError::Conflict(message) => ReceiptStoreError::Conflict(message.clone()),
3084 ReceiptStoreError::NotFound(message) => ReceiptStoreError::NotFound(message.clone()),
3085 ReceiptStoreError::Unsupported(message) => ReceiptStoreError::Unsupported(message.clone()),
3086 ReceiptStoreError::Fenced => ReceiptStoreError::Fenced,
3087 ReceiptStoreError::OutcomeUnknown(message) => {
3088 ReceiptStoreError::OutcomeUnknown(message.clone())
3089 }
3090 ReceiptStoreError::RetentionArchiveIncomplete {
3091 table,
3092 live,
3093 archived,
3094 } => ReceiptStoreError::RetentionArchiveIncomplete {
3095 table,
3096 live: *live,
3097 archived: *archived,
3098 },
3099 ReceiptStoreError::RetentionWatermarkRegression { attempted, current } => {
3100 ReceiptStoreError::RetentionWatermarkRegression {
3101 attempted: *attempted,
3102 current: *current,
3103 }
3104 }
3105 ReceiptStoreError::ArchivedRangeProjection { watermark } => {
3106 ReceiptStoreError::ArchivedRangeProjection {
3107 watermark: *watermark,
3108 }
3109 }
3110 ReceiptStoreError::RetentionTenantScopeUnsupported => {
3111 ReceiptStoreError::RetentionTenantScopeUnsupported
3112 }
3113 ReceiptStoreError::WriterDead {
3114 restarts,
3115 last_error,
3116 } => ReceiptStoreError::WriterDead {
3117 restarts: *restarts,
3118 last_error: last_error.clone(),
3119 },
3120 }
3121}
3122
3123fn receipt_writer_job_panic_error(payload: &(dyn std::any::Any + Send)) -> ReceiptStoreError {
3129 let message = payload
3130 .downcast_ref::<&str>()
3131 .map(|message| (*message).to_string())
3132 .or_else(|| payload.downcast_ref::<String>().cloned())
3133 .unwrap_or_else(|| "non-string panic payload".to_string());
3134 ReceiptStoreError::Canonical(format!("receipt writer job panicked: {message}"))
3135}
3136
3137fn fan_out_batch_panic_error(
3149 health: &ReceiptCommitWriterHealth,
3150 request_responses: Vec<mpsc::SyncSender<Result<u64, ReceiptStoreError>>>,
3151 error: ReceiptStoreError,
3152) -> ReceiptStoreError {
3153 let batch_len = request_responses.len() as u64;
3154 health.failed_total.fetch_add(batch_len, Ordering::SeqCst);
3155 atomic_saturating_sub(&health.inflight, batch_len);
3156 if let Ok(mut last_error) = health.last_error.lock() {
3157 *last_error = Some(error.to_string());
3158 }
3159 for response in request_responses {
3160 let _ = response.send(Err(receipt_store_error_snapshot(&error)));
3161 }
3162 error
3163}
3164
3165#[cfg(test)]
3166pub(crate) mod test_hooks {
3167 use std::sync::atomic::{AtomicBool, Ordering};
3168
3169 pub(crate) static FAIL_BETWEEN_RECEIPT_AND_LINEAGE: AtomicBool = AtomicBool::new(false);
3172
3173 pub(crate) fn fail_between_receipt_and_lineage() -> bool {
3174 FAIL_BETWEEN_RECEIPT_AND_LINEAGE.load(Ordering::SeqCst)
3175 }
3176
3177 pub(crate) static PANIC_DURING_CHECKPOINT_BUILD: AtomicBool = AtomicBool::new(false);
3187
3188 pub(crate) const PANIC_DURING_CHECKPOINT_BUILD_MARKER_MAX_BATCH: u64 = 5;
3189
3190 pub(crate) fn panic_during_checkpoint_build(max_batch: u64) -> bool {
3191 max_batch == PANIC_DURING_CHECKPOINT_BUILD_MARKER_MAX_BATCH
3192 && PANIC_DURING_CHECKPOINT_BUILD.load(Ordering::SeqCst)
3193 }
3194
3195 pub(crate) static FAIL_CHECKPOINT_BUILD: AtomicBool = AtomicBool::new(false);
3203
3204 pub(crate) const FAIL_CHECKPOINT_BUILD_MARKER_MAX_BATCH: u64 = 7;
3205
3206 pub(crate) fn fail_checkpoint_build(max_batch: u64) -> bool {
3207 max_batch == FAIL_CHECKPOINT_BUILD_MARKER_MAX_BATCH
3208 && FAIL_CHECKPOINT_BUILD.load(Ordering::SeqCst)
3209 }
3210
3211 pub(crate) static PANIC_DURING_APPEND_BATCH: AtomicBool = AtomicBool::new(false);
3223
3224 pub(crate) const PANIC_DURING_APPEND_BATCH_MARKER_RECEIPT_ID: &str =
3225 "rcpt-test-hook-panic-during-append-batch";
3226
3227 pub(crate) const PANIC_DURING_APPEND_BATCH_MARKER_CONTENT_HASH: &str =
3230 "content-rcpt-test-hook-panic-during-append-batch";
3231
3232 pub(crate) fn panic_during_append_batch(content_hash: &str) -> bool {
3233 content_hash == PANIC_DURING_APPEND_BATCH_MARKER_CONTENT_HASH
3234 && PANIC_DURING_APPEND_BATCH.load(Ordering::SeqCst)
3235 }
3236}
3237
3238#[path = "receipt_store/bootstrap.rs"]
3239mod bootstrap;
3240mod chaos_test_hooks;
3241#[path = "receipt_store/evidence_retention.rs"]
3242mod evidence_retention;
3243#[path = "receipt_store/liability_claims.rs"]
3244mod liability_claims;
3245#[path = "receipt_store/liability_market.rs"]
3246mod liability_market;
3247#[path = "receipt_store/reports.rs"]
3248mod reports;
3249#[path = "receipt_store/support.rs"]
3250pub(crate) mod support;
3251#[cfg(test)]
3252#[path = "receipt_store/tests.rs"]
3253mod tests;
3254#[path = "receipt_store/underwriting_credit.rs"]
3255mod underwriting_credit;
3256
3257use support::*;
3258pub(crate) use support::{decode_verified_child_receipt, decode_verified_chio_receipt, sqlite_u64};
3259
3260impl SqliteReceiptStore {
3261 pub(crate) fn connection(&self) -> Result<SqliteStoreConnection, ReceiptStoreError> {
3266 self.pool
3267 .get()
3268 .map_err(|error| ReceiptStoreError::Pool(error.to_string()))
3269 }
3270
3271 #[cfg(test)]
3272 pub(crate) fn reader_connection_for_test(
3273 &self,
3274 ) -> Result<SqliteStoreConnection, ReceiptStoreError> {
3275 self.connection()
3276 }
3277
3278 pub(crate) fn writer_handle(&self) -> WriterHandle {
3279 WriterHandle {
3280 sender: self.receipt_commit_actor.sender.clone(),
3281 health: Arc::clone(&self.receipt_commit_actor.health),
3282 worker: Arc::clone(&self.receipt_commit_actor.worker),
3283 settlement_store_binding: self.settlement_store_binding,
3284 }
3285 }
3286
3287 pub fn max_tool_receipt_seq(&self) -> Result<u64, ReceiptStoreError> {
3290 let connection = self.connection()?;
3291 let seq: i64 = connection.query_row(
3292 "SELECT COALESCE(MAX(seq), 0) FROM chio_tool_receipts",
3293 [],
3294 |row| row.get(0),
3295 )?;
3296 Ok(seq.max(0) as u64)
3297 }
3298
3299 pub fn max_child_receipt_seq(&self) -> Result<u64, ReceiptStoreError> {
3301 let connection = self.connection()?;
3302 let seq: i64 = connection.query_row(
3303 "SELECT COALESCE(MAX(seq), 0) FROM chio_child_receipts",
3304 [],
3305 |row| row.get(0),
3306 )?;
3307 Ok(seq.max(0) as u64)
3308 }
3309
3310 pub fn with_strict_tenant_isolation(&self, strict: bool) {
3325 self.strict_tenant_isolation
3326 .store(strict, std::sync::atomic::Ordering::SeqCst);
3327 }
3328
3329 #[must_use]
3331 pub fn strict_tenant_isolation_enabled(&self) -> bool {
3332 self.strict_tenant_isolation
3333 .load(std::sync::atomic::Ordering::SeqCst)
3334 }
3335
3336 #[must_use]
3338 pub fn incremental_verification_enabled(&self) -> bool {
3339 self.incremental_verification
3340 }
3341
3342 pub(crate) fn writer_head_snapshot(&self) -> WriterHeadSnapshot {
3343 let health = &self.receipt_commit_actor.health;
3344 WriterHeadSnapshot {
3345 checkpoint_seq: health.head_checkpoint_seq.load(Ordering::SeqCst),
3346 checkpointed_entry_seq: health.head_checkpointed_entry_seq.load(Ordering::SeqCst),
3347 claim_log_count: health.head_claim_log_count.load(Ordering::SeqCst),
3348 claim_log_max_seq: health.head_claim_log_max_seq.load(Ordering::SeqCst),
3349 }
3350 }
3351
3352 pub fn append_chio_receipt_canonical(
3353 &self,
3354 canonical: Arc<CanonicalBytes>,
3355 ) -> Result<(), ReceiptStoreError> {
3356 self.append_chio_receipt_canonical_returning_seq(canonical)
3357 .map(|_| ())
3358 }
3359
3360 pub fn append_chio_receipt_canonical_bytes(
3361 &self,
3362 canonical: Arc<CanonicalBytes>,
3363 ) -> Result<(), ReceiptStoreError> {
3364 self.append_chio_receipt_canonical(canonical)
3365 }
3366
3367 pub fn append_chio_receipt_canonical_returning_seq(
3368 &self,
3369 canonical: Arc<CanonicalBytes>,
3370 ) -> Result<u64, ReceiptStoreError> {
3371 let receipt = decode_canonical_chio_receipt(canonical.as_ref())?;
3372 let raw_json = canonical_receipt_json(canonical.as_ref())?;
3373 self.append_verified_chio_receipt_record(&receipt, raw_json, false)
3374 }
3375
3376 pub fn append_chio_receipt_canonical_bytes_returning_seq(
3377 &self,
3378 canonical: Arc<CanonicalBytes>,
3379 ) -> Result<u64, ReceiptStoreError> {
3380 self.append_chio_receipt_canonical_returning_seq(canonical)
3381 }
3382
3383 fn append_verified_chio_receipt_record(
3384 &self,
3385 receipt: &ChioReceipt,
3386 raw_json: &str,
3387 ensure_lineage: bool,
3388 ) -> Result<u64, ReceiptStoreError> {
3389 ensure_chio_receipt_verified(receipt)?;
3390 sqlite_i64(receipt.timestamp, "receipt timestamp")?;
3391 self.receipt_commit_actor
3392 .append(receipt.clone(), raw_json.to_string(), ensure_lineage)
3393 }
3394
3395 fn append_verified_chio_receipt_record_with_timeout(
3396 &self,
3397 receipt: &ChioReceipt,
3398 raw_json: &str,
3399 ensure_lineage: bool,
3400 timeout: Duration,
3401 ) -> Result<u64, ReceiptStoreError> {
3402 ensure_chio_receipt_verified(receipt)?;
3403 sqlite_i64(receipt.timestamp, "receipt timestamp")?;
3404 self.receipt_commit_actor.append_with_timeout(
3405 receipt.clone(),
3406 raw_json.to_string(),
3407 ensure_lineage,
3408 timeout,
3409 )
3410 }
3411
3412 pub fn writer_liveness(&self, stall_threshold: Duration) -> chio_kernel::ReceiptWriterLiveness {
3416 classify_writer_liveness(
3417 &self.receipt_commit_actor.writer_counters(),
3418 u64::try_from(stall_threshold.as_millis()).unwrap_or(u64::MAX),
3419 RECEIPT_COMMIT_ACTOR_CHANNEL_CAPACITY as u64,
3420 self.receipt_commit_actor.backlog_started_unix_ms(),
3421 current_unix_ms(),
3422 )
3423 }
3424
3425 pub fn append_chio_receipt_consuming_authorization(
3426 &self,
3427 receipt: &ChioReceipt,
3428 consumption: &AuthorizationReceiptConsumption,
3429 ) -> Result<(), ReceiptStoreError> {
3430 ensure_chio_receipt_verified(receipt)?;
3431 if receipt.id != consumption.consumer_receipt_id {
3432 return Err(ReceiptStoreError::Conflict(
3433 "authorization consumption consumer receipt id does not match appended receipt"
3434 .to_string(),
3435 ));
3436 }
3437 if receipt.tenant_id.as_deref() != consumption.tenant_id.as_deref() {
3438 return Err(ReceiptStoreError::Conflict(
3439 "authorization consumption tenant id does not match appended receipt".to_string(),
3440 ));
3441 }
3442 sqlite_i64(receipt.timestamp, "receipt timestamp")?;
3443 let raw_json = canonical_json_bytes(receipt)
3444 .map_err(|error| ReceiptStoreError::Canonical(error.to_string()))?;
3445 let raw_json = std::str::from_utf8(raw_json.as_slice()).map_err(|error| {
3446 ReceiptStoreError::Canonical(format!("canonical receipt bytes are not UTF-8: {error}"))
3447 })?;
3448 let raw_json = raw_json.to_string();
3449 let receipt = receipt.clone();
3450 let consumption = consumption.clone();
3451 self.writer_handle().run_write_receipt(move |connection| {
3452 ensure_checkpoint_transparency_guards(connection)?;
3453 let tx =
3454 connection.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
3455 consume_authorization_receipt_tx(&tx, &consumption)?;
3456 append_chio_receipt_tx(&tx, &receipt, &raw_json)?;
3457 ensure_receipt_lineage_statement_for_receipt_id_tx(&tx, &receipt.id)?;
3458 tx.commit()?;
3459 Ok(())
3460 })
3461 }
3462
3463 pub fn flush_receipt_writes(&self) -> Result<ReceiptFlushReport, ReceiptStoreError> {
3464 self.receipt_commit_actor.flush()?;
3465 let wal_checkpoint = Some(self.wal_checkpoint_passive()?);
3466 self.flush_report(wal_checkpoint)
3467 }
3468
3469 pub fn retention_repair(&self, archive_path: &str) -> Result<u64, ReceiptStoreError> {
3487 let (response, result) = mpsc::sync_channel(1);
3488 let health = &self.receipt_commit_actor.health;
3489 health.inflight.fetch_add(1, Ordering::SeqCst);
3497 if let Err(error) =
3498 self.receipt_commit_actor
3499 .sender
3500 .try_send(ReceiptCommitCommand::RetentionRepair {
3501 archive_path: archive_path.to_string(),
3502 response,
3503 })
3504 {
3505 atomic_saturating_sub(&health.inflight, 1);
3506 return Err(match error {
3507 mpsc::TrySendError::Full(_) => receipt_actor_saturated_error(),
3508 mpsc::TrySendError::Disconnected(_) => receipt_actor_unavailable_error(),
3509 });
3510 }
3511 match result.recv() {
3512 Ok(outcome) => outcome,
3513 Err(_) => {
3514 atomic_saturating_sub(&health.inflight, 1);
3515 Err(receipt_actor_unavailable_error())
3516 }
3517 }
3518 }
3519
3520 pub fn audit_receipt_cost_projection(&self) -> Result<(), ReceiptStoreError> {
3521 let connection = self.connection()?;
3522 support::audit_receipt_cost_projection(&connection)
3523 }
3524
3525 pub fn reseed_verified_head(&self) -> Result<(), ReceiptStoreError> {
3529 let (response, result) = mpsc::sync_channel(1);
3530 self.receipt_commit_actor.health.note_channel_send();
3531 match self
3532 .receipt_commit_actor
3533 .sender
3534 .try_send(ReceiptCommitCommand::ReseedHead(response))
3535 {
3536 Ok(()) => {}
3537 Err(mpsc::TrySendError::Full(_)) => {
3538 self.receipt_commit_actor
3539 .health
3540 .note_channel_send_rejected();
3541 return Err(receipt_actor_saturated_error());
3542 }
3543 Err(mpsc::TrySendError::Disconnected(_)) => {
3544 self.receipt_commit_actor
3545 .health
3546 .note_channel_send_rejected();
3547 self.receipt_commit_actor.health.note_writer_unavailable();
3552 return Err(receipt_actor_unavailable_error());
3553 }
3554 }
3555 result
3556 .recv()
3557 .map_err(|_| receipt_actor_unavailable_error())?
3558 }
3559
3560 pub fn enable_background_checkpoints(
3564 &self,
3565 signer: BackgroundCheckpointSigner,
3566 ) -> Result<(), ReceiptStoreError> {
3567 self.receipt_commit_actor.health.note_channel_send();
3568 match self
3569 .receipt_commit_actor
3570 .sender
3571 .try_send(ReceiptCommitCommand::InstallSigner(signer))
3572 {
3573 Ok(()) => Ok(()),
3574 Err(mpsc::TrySendError::Full(_)) => {
3575 self.receipt_commit_actor
3576 .health
3577 .note_channel_send_rejected();
3578 Err(receipt_actor_saturated_error())
3579 }
3580 Err(mpsc::TrySendError::Disconnected(_)) => {
3581 self.receipt_commit_actor
3582 .health
3583 .note_channel_send_rejected();
3584 self.receipt_commit_actor.health.note_writer_unavailable();
3587 Err(receipt_actor_unavailable_error())
3588 }
3589 }
3590 }
3591
3592 pub fn flush_receipt_writes_with_timeout(
3593 &self,
3594 timeout: Duration,
3595 ) -> Result<ReceiptFlushReport, ReceiptStoreError> {
3596 self.receipt_commit_actor.flush_with_timeout(timeout)?;
3597 let wal_checkpoint = Some(self.wal_checkpoint_passive()?);
3598 self.flush_report(wal_checkpoint)
3599 }
3600
3601 pub fn record_retention_rotation_outcome(&self, failure: Option<&str>) {
3610 if let Ok(mut retention_error) = self.receipt_commit_actor.health.retention_error.lock() {
3611 *retention_error = failure.map(ToString::to_string);
3612 }
3613 }
3614
3615 pub fn receipt_store_health(&self) -> Result<ReceiptStoreHealthReport, ReceiptStoreError> {
3616 self.validate_claim_receipt_log_projection_current()?;
3617 let status = self.receipt_checkpoint_status(Some(1))?;
3618 if status.checkpoint_error.is_none()
3627 && status.latest_committed_entry_seq > status.latest_checkpointed_entry_seq
3628 {
3629 let connection = self.connection()?;
3630 let start_seq = status.latest_checkpointed_entry_seq + 1;
3631 load_claim_tree_canonical_bytes_range(
3632 &connection,
3633 start_seq,
3634 status.latest_committed_entry_seq,
3635 )?;
3636 }
3637 let writer_counters = self.receipt_commit_actor.writer_counters();
3638 let (writer_level, writer_restart_total) =
3639 self.receipt_commit_actor.writer_health_summary();
3640 let writer_liveness = self.writer_liveness(std::time::Duration::from_millis(
3644 chio_kernel::DEFAULT_RECEIPT_WRITER_STALL_MS,
3645 ));
3646 let retention_error = self
3647 .receipt_commit_actor
3648 .health
3649 .retention_error
3650 .lock()
3651 .ok()
3652 .and_then(|guard| guard.clone());
3653 let healthy = receipt_store_healthy(
3662 status.healthy,
3663 writer_counters.last_error.as_deref(),
3664 writer_liveness,
3665 ) && !self.receipt_commit_actor.writer_serving_closed()
3666 && matches!(writer_level, HealthLevel::Healthy)
3667 && retention_error.is_none();
3668 let (uncheckpointed_start_seq, uncheckpointed_end_seq) = uncheckpointed_range(
3669 status.latest_checkpointed_entry_seq,
3670 status.latest_committed_entry_seq,
3671 );
3672 Ok(ReceiptStoreHealthReport {
3673 healthy,
3674 writer: writer_counters,
3675 writer_liveness: writer_liveness.as_label().to_string(),
3676 latest_committed_entry_seq: status.latest_committed_entry_seq,
3677 latest_checkpoint_seq: status.latest_checkpoint_seq,
3678 latest_checkpointed_entry_seq: status.latest_checkpointed_entry_seq,
3679 uncheckpointed_start_seq,
3680 uncheckpointed_end_seq,
3681 checkpoint_error: status.checkpoint_error,
3682 db_size_bytes: self.db_size_bytes().ok(),
3683 retention_watermark_entry_seq: status.retention_watermark_entry_seq,
3684 retention_error,
3685 writer_level,
3686 writer_restart_total,
3687 })
3688 }
3689
3690 pub fn writer_serving_closed(&self) -> bool {
3696 self.receipt_commit_actor.writer_serving_closed()
3697 }
3698
3699 pub fn receipt_store_health_read_only(
3715 path: &Path,
3716 ) -> Result<ReceiptStoreHealthReport, ReceiptStoreError> {
3717 let connection = Connection::open_with_flags(
3718 path,
3719 rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX,
3720 )
3721 .map_err(|error| {
3722 if error.sqlite_error_code() == Some(rusqlite::ErrorCode::CannotOpen) {
3723 ReceiptStoreError::NotFound(format!(
3724 "receipt database {} does not exist",
3725 path.display()
3726 ))
3727 } else {
3728 ReceiptStoreError::Sqlite(error)
3729 }
3730 })?;
3731 let live_committed_entry_seq = latest_claim_log_entry_seq(&connection)?;
3732 let retention_watermark_entry_seq = support::retention_watermark(&connection)?;
3733 let latest_committed_entry_seq =
3740 live_committed_entry_seq.max(retention_watermark_entry_seq.unwrap_or(0));
3741 match verify_checkpoint_chain_integrity(&connection) {
3751 Ok(latest) => {
3752 let latest_checkpoint_seq = latest
3753 .as_ref()
3754 .map(|checkpoint| checkpoint.body.checkpoint_seq);
3755 let latest_checkpointed_entry_seq = latest
3756 .as_ref()
3757 .map_or(0, |checkpoint| checkpoint.body.batch_end_seq);
3758 let (uncheckpointed_start_seq, uncheckpointed_end_seq) =
3759 uncheckpointed_range(latest_checkpointed_entry_seq, latest_committed_entry_seq);
3760 Ok(ReceiptStoreHealthReport {
3761 healthy: latest_committed_entry_seq >= latest_checkpointed_entry_seq,
3762 writer: ReceiptWriterCounters::default(),
3763 writer_liveness: chio_kernel::ReceiptWriterLiveness::Unknown
3766 .as_label()
3767 .to_string(),
3768 latest_committed_entry_seq,
3769 latest_checkpoint_seq,
3770 latest_checkpointed_entry_seq,
3771 uncheckpointed_start_seq,
3772 uncheckpointed_end_seq,
3773 checkpoint_error: None,
3774 db_size_bytes: None,
3775 retention_watermark_entry_seq,
3776 retention_error: None,
3780 writer_level: HealthLevel::default(),
3782 writer_restart_total: 0,
3783 })
3784 }
3785 Err(error) => {
3786 let (uncheckpointed_start_seq, uncheckpointed_end_seq) =
3787 uncheckpointed_range(0, latest_committed_entry_seq);
3788 Ok(ReceiptStoreHealthReport {
3789 healthy: false,
3790 writer: ReceiptWriterCounters::default(),
3791 writer_liveness: chio_kernel::ReceiptWriterLiveness::Unknown
3792 .as_label()
3793 .to_string(),
3794 latest_committed_entry_seq,
3795 latest_checkpoint_seq: None,
3796 latest_checkpointed_entry_seq: 0,
3797 uncheckpointed_start_seq,
3798 uncheckpointed_end_seq,
3799 checkpoint_error: Some(error.to_string()),
3800 db_size_bytes: None,
3801 retention_watermark_entry_seq,
3802 retention_error: None,
3803 writer_level: HealthLevel::default(),
3804 writer_restart_total: 0,
3805 })
3806 }
3807 }
3808 }
3809
3810 pub fn latest_committed_entry_seq(&self) -> Result<u64, ReceiptStoreError> {
3811 let connection = self.connection()?;
3812 let live = latest_claim_log_entry_seq(&connection)?;
3820 let watermark = support::retention_watermark(&connection)?.unwrap_or(0);
3821 Ok(live.max(watermark))
3822 }
3823
3824 pub fn latest_checkpointed_entry_seq(&self) -> Result<u64, ReceiptStoreError> {
3825 let connection = self.connection()?;
3826 latest_checkpointed_entry_seq(&connection)
3827 }
3828
3829 pub fn next_checkpoint_range(
3830 &self,
3831 max_batch: u64,
3832 ) -> Result<Option<ReceiptCheckpointRange>, ReceiptStoreError> {
3833 let connection = self.connection()?;
3834 next_checkpoint_range_for_connection(&connection, max_batch)
3835 }
3836
3837 pub fn receipt_checkpoint_status(
3838 &self,
3839 max_batch: Option<u64>,
3840 ) -> Result<ReceiptCheckpointStatusReport, ReceiptStoreError> {
3841 self.validate_claim_receipt_log_projection_current()?;
3842 let connection = self.connection()?;
3843 let retention_watermark_entry_seq = support::retention_watermark(&connection)?;
3847 let latest_committed_entry_seq = latest_claim_log_entry_seq(&connection)?
3855 .max(retention_watermark_entry_seq.unwrap_or(0));
3856 match verify_checkpoint_chain_integrity(&connection) {
3857 Ok(latest) => {
3858 let latest_checkpoint_seq = latest
3859 .as_ref()
3860 .map(|checkpoint| checkpoint.body.checkpoint_seq);
3861 let latest_checkpointed_entry_seq = latest
3862 .as_ref()
3863 .map_or(0, |checkpoint| checkpoint.body.batch_end_seq);
3864 if latest_committed_entry_seq > latest_checkpointed_entry_seq {
3865 let start_seq = latest_checkpointed_entry_seq + 1;
3866 if let Err(error) = ensure_claim_log_range_contiguous(
3867 &connection,
3868 start_seq,
3869 latest_committed_entry_seq,
3870 "uncheckpointed range",
3871 ) {
3872 return Ok(ReceiptCheckpointStatusReport {
3873 healthy: false,
3874 latest_committed_entry_seq,
3875 latest_checkpoint_seq,
3876 latest_checkpointed_entry_seq,
3877 next_range: None,
3878 checkpoint_error: Some(error.to_string()),
3879 retention_watermark_entry_seq,
3880 });
3881 }
3882 }
3883 let next_range = match max_batch {
3884 Some(max_batch) => {
3885 next_checkpoint_range_for_connection(&connection, max_batch)?
3886 }
3887 None => None,
3888 };
3889 Ok(ReceiptCheckpointStatusReport {
3890 healthy: true,
3891 latest_committed_entry_seq,
3892 latest_checkpoint_seq,
3893 latest_checkpointed_entry_seq,
3894 next_range,
3895 checkpoint_error: None,
3896 retention_watermark_entry_seq,
3897 })
3898 }
3899 Err(error) => Ok(ReceiptCheckpointStatusReport {
3900 healthy: false,
3901 latest_committed_entry_seq,
3902 latest_checkpoint_seq: None,
3903 latest_checkpointed_entry_seq: 0,
3904 next_range: None,
3905 checkpoint_error: Some(error.to_string()),
3906 retention_watermark_entry_seq,
3907 }),
3908 }
3909 }
3910
3911 pub fn create_next_receipt_checkpoint(
3912 &self,
3913 max_batch: u64,
3914 keypair: &Keypair,
3915 ) -> Result<ReceiptCheckpointCreateReport, ReceiptStoreError> {
3916 let keypair = keypair.clone();
3917 self.writer_handle().run_write(move |connection| {
3918 validate_claim_receipt_log_entries(connection)?;
3919 create_next_receipt_checkpoint_atomic(connection, max_batch, &keypair)
3920 })
3921 }
3922
3923 fn flush_report(
3924 &self,
3925 wal_checkpoint: Option<ReceiptWalCheckpointReport>,
3926 ) -> Result<ReceiptFlushReport, ReceiptStoreError> {
3927 let head = self.writer_head_snapshot();
3928 let connection = self.connection()?;
3929 let latest_committed_entry_seq = latest_claim_log_entry_seq(&connection)?
3938 .max(support::retention_watermark(&connection)?.unwrap_or(0));
3939 let trusted_watermark = trusted_retention_watermark(&connection)?;
3994 let verified_persisted = load_latest_persisted_checkpoint_row(&connection)?
3995 .and_then(|row| parse_persisted_checkpoint_row(row).ok())
3996 .filter(|checkpoint| {
3997 latest_checkpoint_is_chain_connected(&connection, checkpoint).is_ok()
3998 && (checkpoint.body.batch_end_seq <= trusted_watermark
3999 || validate_checkpoint_against_claim_log(&connection, checkpoint).is_ok())
4000 });
4001 let persisted_checkpoint_seq = verified_persisted
4002 .as_ref()
4003 .map_or(0, |checkpoint| checkpoint.body.checkpoint_seq);
4004 let persisted_checkpointed_entry_seq = verified_persisted
4005 .as_ref()
4006 .map_or(0, |checkpoint| checkpoint.body.batch_end_seq);
4007 let checkpoint_seq = head.checkpoint_seq.max(persisted_checkpoint_seq);
4008 let latest_checkpointed_entry_seq = head
4009 .checkpointed_entry_seq
4010 .max(persisted_checkpointed_entry_seq);
4011 let latest_checkpoint_seq = (checkpoint_seq > 0).then_some(checkpoint_seq);
4012 let (uncheckpointed_start_seq, uncheckpointed_end_seq) =
4013 uncheckpointed_range(latest_checkpointed_entry_seq, latest_committed_entry_seq);
4014 Ok(ReceiptFlushReport {
4015 writer: self.receipt_commit_actor.writer_counters(),
4016 latest_committed_entry_seq,
4017 latest_checkpoint_seq,
4018 latest_checkpointed_entry_seq,
4019 uncheckpointed_start_seq,
4020 uncheckpointed_end_seq,
4021 wal_checkpoint,
4022 db_size_bytes: self.db_size_bytes().ok(),
4023 })
4024 }
4025
4026 fn validate_claim_receipt_log_projection_current(&self) -> Result<(), ReceiptStoreError> {
4027 let connection = self.connection()?;
4028 validate_claim_receipt_log_entries(&connection)
4029 }
4030
4031 fn wal_checkpoint_passive(&self) -> Result<ReceiptWalCheckpointReport, ReceiptStoreError> {
4032 let connection = self.connection()?;
4033 let (busy, log_frames, checkpointed_frames) =
4034 connection.query_row("PRAGMA wal_checkpoint(PASSIVE)", [], |row| {
4035 Ok((
4036 row.get::<_, i64>(0)?,
4037 row.get::<_, i64>(1)?,
4038 row.get::<_, i64>(2)?,
4039 ))
4040 })?;
4041 Ok(ReceiptWalCheckpointReport {
4042 busy: sqlite_u64(busy, "wal checkpoint busy")?,
4043 log_frames: wal_checkpoint_frame_count(log_frames, "wal checkpoint log frames")?,
4044 checkpointed_frames: wal_checkpoint_frame_count(
4045 checkpointed_frames,
4046 "wal checkpointed frames",
4047 )?,
4048 })
4049 }
4050}
4051
4052fn wal_checkpoint_frame_count(value: i64, field: &str) -> Result<u64, ReceiptStoreError> {
4060 if value == -1 {
4061 return Ok(0);
4062 }
4063 sqlite_u64(value, field)
4064}
4065
4066fn uncheckpointed_range(checkpointed: u64, committed: u64) -> (Option<u64>, Option<u64>) {
4067 if committed > checkpointed {
4068 (Some(checkpointed + 1), Some(committed))
4069 } else {
4070 (None, None)
4071 }
4072}
4073
4074fn latest_claim_log_entry_seq(connection: &Connection) -> Result<u64, ReceiptStoreError> {
4075 connection
4076 .query_row(
4077 "SELECT COALESCE(MAX(entry_seq), 0) FROM claim_receipt_log_entries",
4078 [],
4079 |row| row.get::<_, i64>(0),
4080 )
4081 .map_err(ReceiptStoreError::from)
4082 .and_then(|value| sqlite_u64(value, "latest claim receipt log entry_seq"))
4083}
4084
4085fn latest_checkpointed_entry_seq(connection: &Connection) -> Result<u64, ReceiptStoreError> {
4086 verify_checkpoint_chain_integrity(connection)
4087 .map(|latest| latest.map_or(0, |checkpoint| checkpoint.body.batch_end_seq))
4088}
4089
4090fn next_checkpoint_range_for_connection(
4091 connection: &Connection,
4092 max_batch: u64,
4093) -> Result<Option<ReceiptCheckpointRange>, ReceiptStoreError> {
4094 if max_batch == 0 {
4095 return Err(ReceiptStoreError::Conflict(
4096 "checkpoint max_batch must be greater than zero".to_string(),
4097 ));
4098 }
4099 let latest_committed = latest_claim_log_entry_seq(connection)?;
4100 let latest_checkpointed = latest_checkpointed_entry_seq(connection)?;
4101 if latest_committed <= latest_checkpointed {
4102 return Ok(None);
4103 }
4104 let start_seq = latest_checkpointed + 1;
4105 let end_seq = latest_committed.min(start_seq.saturating_add(max_batch - 1));
4106 ensure_claim_log_range_contiguous(connection, start_seq, end_seq, "checkpoint range")?;
4107 Ok(Some(ReceiptCheckpointRange { start_seq, end_seq }))
4108}
4109
4110fn ensure_claim_log_range_contiguous(
4111 connection: &Connection,
4112 start_seq: u64,
4113 end_seq: u64,
4114 context: &str,
4115) -> Result<(), ReceiptStoreError> {
4116 if end_seq < start_seq {
4117 return Err(ReceiptStoreError::Conflict(format!(
4118 "claim receipt log {context} end {end_seq} is before start {start_seq}"
4119 )));
4120 }
4121 let (count, min_seq, max_seq) = connection.query_row(
4122 r#"
4123 SELECT COUNT(*), MIN(entry_seq), MAX(entry_seq)
4124 FROM claim_receipt_log_entries
4125 WHERE entry_seq >= ?1 AND entry_seq <= ?2
4126 "#,
4127 params![
4128 sqlite_i64(start_seq, "claim log range start_seq")?,
4129 sqlite_i64(end_seq, "claim log range end_seq")?,
4130 ],
4131 |row| {
4132 Ok((
4133 row.get::<_, i64>(0)?,
4134 row.get::<_, Option<i64>>(1)?,
4135 row.get::<_, Option<i64>>(2)?,
4136 ))
4137 },
4138 )?;
4139 let expected = end_seq - start_seq + 1;
4140 let count = sqlite_u64(count, "claim receipt log range count")?;
4141 let min_seq = min_seq
4142 .map(|value| sqlite_u64(value, "claim receipt log range min_seq"))
4143 .transpose()?;
4144 let max_seq = max_seq
4145 .map(|value| sqlite_u64(value, "claim receipt log range max_seq"))
4146 .transpose()?;
4147 if count != expected || min_seq != Some(start_seq) || max_seq != Some(end_seq) {
4148 return Err(ReceiptStoreError::Conflict(format!(
4149 "claim receipt log has a gap in {context} {start_seq}..={end_seq}"
4150 )));
4151 }
4152 Ok(())
4153}
4154
4155fn claim_log_entry_seq_for_source_tx(
4156 tx: &rusqlite::Transaction<'_>,
4157 receipt_kind: &str,
4158 source_seq: u64,
4159) -> Result<u64, ReceiptStoreError> {
4160 let source_seq_i64 = sqlite_i64(source_seq, "claim receipt source_seq")?;
4161 let (entry_seq, log_receipt_id, log_raw_json, source_receipt_id, source_raw_json) =
4162 match receipt_kind {
4163 "tool_receipt" => tx.query_row(
4164 r#"
4165 SELECT l.entry_seq, l.receipt_id, l.raw_json, r.receipt_id, r.raw_json
4166 FROM claim_receipt_log_entries l
4167 JOIN chio_tool_receipts r ON r.seq = l.source_seq
4168 WHERE l.receipt_kind = ?1 AND l.source_seq = ?2
4169 "#,
4170 params![receipt_kind, source_seq_i64],
4171 |row| {
4172 Ok((
4173 row.get::<_, i64>(0)?,
4174 row.get::<_, String>(1)?,
4175 row.get::<_, String>(2)?,
4176 row.get::<_, String>(3)?,
4177 row.get::<_, String>(4)?,
4178 ))
4179 },
4180 ),
4181 "child_receipt" => tx.query_row(
4182 r#"
4183 SELECT l.entry_seq, l.receipt_id, l.raw_json, r.receipt_id, r.raw_json
4184 FROM claim_receipt_log_entries l
4185 JOIN chio_child_receipts r ON r.seq = l.source_seq
4186 WHERE l.receipt_kind = ?1 AND l.source_seq = ?2
4187 "#,
4188 params![receipt_kind, source_seq_i64],
4189 |row| {
4190 Ok((
4191 row.get::<_, i64>(0)?,
4192 row.get::<_, String>(1)?,
4193 row.get::<_, String>(2)?,
4194 row.get::<_, String>(3)?,
4195 row.get::<_, String>(4)?,
4196 ))
4197 },
4198 ),
4199 other => {
4200 return Err(ReceiptStoreError::Conflict(format!(
4201 "unsupported claim receipt log kind `{other}`"
4202 )));
4203 }
4204 }
4205 .optional()?
4206 .ok_or_else(|| {
4207 ReceiptStoreError::Conflict(format!(
4208 "claim receipt log entry missing for {receipt_kind} source seq {source_seq}"
4209 ))
4210 })?;
4211 if log_receipt_id != source_receipt_id || log_raw_json != source_raw_json {
4212 return Err(ReceiptStoreError::Conflict(format!(
4213 "claim receipt log entry for {receipt_kind} source seq {source_seq} diverges from source row"
4214 )));
4215 }
4216 sqlite_positive_u64(entry_seq, "claim receipt log entry_seq")
4217}
4218
4219fn append_chio_receipt_tx(
4220 tx: &rusqlite::Transaction<'_>,
4221 receipt: &ChioReceipt,
4222 raw_json: &str,
4223) -> Result<u64, ReceiptStoreError> {
4224 append_chio_receipt_tx_with_insert_status(tx, receipt, raw_json).map(|(seq, _)| seq)
4225}
4226
4227fn append_chio_receipt_tx_with_insert_status(
4228 tx: &rusqlite::Transaction<'_>,
4229 receipt: &ChioReceipt,
4230 raw_json: &str,
4231) -> Result<(u64, bool), ReceiptStoreError> {
4232 let (cost_currency, cost_charged_be) = receipt_cost_projection(receipt)?;
4233 let attribution = extract_receipt_attribution(receipt);
4234 let mut subject_key = attribution.subject_key;
4235 let mut issuer_key = attribution.issuer_key;
4236 if subject_key.is_none() || issuer_key.is_none() {
4237 if let Some((lineage_subject_key, lineage_issuer_key)) = tx
4238 .query_row(
4239 "SELECT subject_key, issuer_key FROM capability_lineage WHERE capability_id = ?1",
4240 params![receipt.capability_id.as_str()],
4241 |row| {
4242 Ok((
4243 row.get::<_, Option<String>>(0)?,
4244 row.get::<_, Option<String>>(1)?,
4245 ))
4246 },
4247 )
4248 .optional()?
4249 {
4250 if subject_key.is_none() {
4251 subject_key = lineage_subject_key;
4252 }
4253 if issuer_key.is_none() {
4254 issuer_key = lineage_issuer_key;
4255 }
4256 }
4257 }
4258 let source_seq = tx
4259 .query_row(
4260 r#"
4261 INSERT INTO chio_tool_receipts (receipt_id, timestamp, capability_id, subject_key, issuer_key, grant_index, tool_server, tool_name, decision_kind, policy_hash, content_hash, tenant_id, raw_json, cost_currency, cost_charged_be) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(receipt_id) DO NOTHING RETURNING seq
4262 "#,
4263 params![
4264 receipt.id.as_str(),
4265 sqlite_i64(receipt.timestamp, "receipt timestamp")?,
4266 receipt.capability_id.as_str(),
4267 subject_key,
4268 issuer_key,
4269 attribution.grant_index.map(i64::from),
4270 receipt.tool_server.as_str(),
4271 receipt.tool_name.as_str(),
4272 receipt_decision_kind(receipt),
4273 receipt.policy_hash.as_str(),
4274 receipt.content_hash.as_str(),
4275 receipt.tenant_id.as_deref(),
4276 raw_json,
4277 cost_currency.as_deref(),
4278 cost_charged_be.as_deref(),
4279 ],
4280 |row| row.get::<_, i64>(0),
4281 )
4282 .optional()?;
4283 let Some(source_seq) = source_seq else {
4284 let (existing_source_seq, existing_raw_json, existing_currency, existing_key) = tx.query_row(
4285 "SELECT seq, raw_json, cost_currency, cost_charged_be FROM chio_tool_receipts WHERE receipt_id = ?1",
4286 params![receipt.id.as_str()],
4287 |row| {
4288 Ok((
4289 row.get::<_, i64>(0)?,
4290 row.get::<_, String>(1)?,
4291 row.get::<_, Option<String>>(2)?,
4292 row.get::<_, Option<Vec<u8>>>(3)?,
4293 ))
4294 },
4295 )?;
4296 let existing_source_seq =
4297 sqlite_positive_u64(existing_source_seq, "tool receipt source_seq")?;
4298 if existing_raw_json != raw_json {
4299 return Err(ReceiptStoreError::Conflict(format!(
4300 "tool receipt `{}` already exists with different content",
4301 receipt.id
4302 )));
4303 }
4304 if existing_currency != cost_currency || existing_key != cost_charged_be {
4305 return Err(ReceiptStoreError::Conflict(format!(
4306 "tool receipt `{}` already exists with different cost projection",
4307 receipt.id
4308 )));
4309 }
4310 decode_verified_chio_receipt(
4311 &existing_raw_json,
4312 "persisted duplicate tool receipt",
4313 Some(existing_source_seq),
4314 )?;
4315 return claim_log_entry_seq_for_source_tx(tx, "tool_receipt", existing_source_seq)
4316 .map(|seq| (seq, false));
4317 };
4318 let source_seq = sqlite_positive_u64(source_seq, "tool receipt source_seq")?;
4319 claim_log_entry_seq_for_source_tx(tx, "tool_receipt", source_seq).map(|seq| (seq, true))
4320}
4321
4322fn consume_authorization_receipt_tx(
4323 tx: &rusqlite::Transaction<'_>,
4324 consumption: &AuthorizationReceiptConsumption,
4325) -> Result<(), ReceiptStoreError> {
4326 if consumption.authorization_receipt_id.trim().is_empty()
4327 || consumption.consumer_receipt_id.trim().is_empty()
4328 || consumption.request_id.trim().is_empty()
4329 || consumption.session_id.trim().is_empty()
4330 || consumption.tool_call_id.trim().is_empty()
4331 || consumption.parameter_hash.trim().is_empty()
4332 {
4333 return Err(ReceiptStoreError::Conflict(
4334 "authorization receipt consumption requires non-empty binding fields".to_string(),
4335 ));
4336 }
4337 if matches!(&consumption.tenant_id, Some(tenant) if tenant.trim().is_empty()) {
4340 return Err(ReceiptStoreError::Conflict(
4341 "authorization receipt consumption tenant id must not be empty when present"
4342 .to_string(),
4343 ));
4344 }
4345 let consumed_at = sqlite_i64(
4346 consumption.consumed_at_unix_ms,
4347 "authorization receipt consumed_at_unix_ms",
4348 )?;
4349 let authorization_tenant = tx
4350 .query_row(
4351 "SELECT tenant_id FROM chio_tool_receipts WHERE receipt_id = ?1",
4352 params![consumption.authorization_receipt_id.as_str()],
4353 |row| row.get::<_, Option<String>>(0),
4354 )
4355 .optional()?
4356 .ok_or_else(|| {
4357 ReceiptStoreError::NotFound(format!(
4358 "authorization receipt {} was not found",
4359 consumption.authorization_receipt_id
4360 ))
4361 })?;
4362 if authorization_tenant.as_deref() != consumption.tenant_id.as_deref() {
4363 return Err(ReceiptStoreError::Conflict(
4364 "authorization receipt tenant id does not match consumption tenant".to_string(),
4365 ));
4366 }
4367 match tx.execute(
4368 r#"
4369 INSERT INTO chio_authorization_receipt_consumptions (
4370 authorization_receipt_id,
4371 consumer_receipt_id,
4372 request_id,
4373 session_id,
4374 tool_call_id,
4375 tenant_id,
4376 parameter_hash,
4377 consumed_at_unix_ms
4378 ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)
4379 "#,
4380 params![
4381 consumption.authorization_receipt_id.as_str(),
4382 consumption.consumer_receipt_id.as_str(),
4383 consumption.request_id.as_str(),
4384 consumption.session_id.as_str(),
4385 consumption.tool_call_id.as_str(),
4386 consumption.tenant_id.as_deref(),
4387 consumption.parameter_hash.as_str(),
4388 consumed_at,
4389 ],
4390 ) {
4391 Ok(_) => Ok(()),
4392 Err(error)
4393 if matches!(
4394 error.sqlite_error_code(),
4395 Some(rusqlite::ErrorCode::ConstraintViolation)
4396 ) =>
4397 {
4398 Err(ReceiptStoreError::Conflict(
4399 "authorization receipt already consumed".to_string(),
4400 ))
4401 }
4402 Err(error) => Err(ReceiptStoreError::Sqlite(error)),
4403 }
4404}
4405
4406fn decode_canonical_chio_receipt(
4407 canonical: &CanonicalBytes,
4408) -> Result<ChioReceipt, ReceiptStoreError> {
4409 let receipt: ChioReceipt =
4410 serde_json::from_slice(canonical.as_bytes()).map_err(ReceiptStoreError::from)?;
4411 let expected = canonical_json_bytes(&receipt)
4412 .map_err(|error| ReceiptStoreError::Canonical(error.to_string()))?;
4413 if expected.as_slice() != canonical.as_bytes() {
4414 return Err(ReceiptStoreError::Canonical(
4415 "canonical receipt bytes do not match ChioReceipt serialization".to_string(),
4416 ));
4417 }
4418 Ok(receipt)
4419}
4420
4421fn canonical_receipt_json(canonical: &CanonicalBytes) -> Result<&str, ReceiptStoreError> {
4422 std::str::from_utf8(canonical.as_bytes()).map_err(|error| {
4423 ReceiptStoreError::Canonical(format!("canonical receipt bytes are not UTF-8: {error}"))
4424 })
4425}
4426#[cfg(test)]
4427mod receipt_commit_actor_tests {
4428 use super::*;
4429
4430 #[test]
4431 fn writer_health_starts_with_a_poisoned_head_until_seeding_clears_it() {
4432 let health = ReceiptCommitWriterHealth::default();
4440 assert!(
4441 health.head_poisoned.load(Ordering::SeqCst),
4442 "writer health must start head-poisoned (serving closed) until a seeded head clears it"
4443 );
4444 }
4445
4446 fn idle_worker() -> Arc<ReceiptCommitWorker> {
4447 Arc::new(ReceiptCommitWorker { join: None })
4448 }
4449
4450 fn actor_test_receipt() -> Result<ChioReceipt, ReceiptStoreError> {
4451 let keypair = chio_core::crypto::Keypair::generate();
4452 ChioReceipt::sign(
4453 chio_core::receipt::body::ChioReceiptBody {
4454 id: "rcpt-actor-test".to_string(),
4455 timestamp: 1,
4456 capability_id: "cap-actor".to_string(),
4457 tool_server: "shell".to_string(),
4458 tool_name: "bash".to_string(),
4459 action: chio_core::receipt::decision::ToolCallAction::from_parameters(
4460 serde_json::json!({}),
4461 )
4462 .map_err(|error| ReceiptStoreError::Canonical(error.to_string()))?,
4463 decision: Some(Decision::Allow),
4464 receipt_kind: Default::default(),
4465 boundary_class: Default::default(),
4466 observation_outcome: None,
4467 tool_origin: Default::default(),
4468 redaction_mode: Default::default(),
4469 actor_chain: Vec::new(),
4470 content_hash: "content".to_string(),
4471 policy_hash: "policy".to_string(),
4472 evidence: Vec::new(),
4473 metadata: None,
4474 trust_level: chio_core::receipt::kinds::TrustLevel::default(),
4475 tenant_id: None,
4476 kernel_key: keypair.public_key(),
4477 bbs_projection_version: None,
4478 },
4479 &keypair,
4480 )
4481 .map_err(|error| ReceiptStoreError::CryptoDecode(error.to_string()))
4482 }
4483
4484 #[test]
4485 fn receipt_commit_actor_channel_has_fixed_capacity() -> Result<(), Box<dyn std::error::Error>> {
4486 let (sender, _receiver) = receipt_commit_channel();
4487 for _ in 0..RECEIPT_COMMIT_ACTOR_CHANNEL_CAPACITY {
4488 let (response, _result) = mpsc::sync_channel(1);
4489 sender.try_send(ReceiptCommitCommand::Flush(response))?;
4490 }
4491
4492 let (response, _result) = mpsc::sync_channel(1);
4493 match sender.try_send(ReceiptCommitCommand::Flush(response)) {
4494 Err(mpsc::TrySendError::Full(_)) => Ok(()),
4495 Err(mpsc::TrySendError::Disconnected(_)) => {
4496 Err("commit actor channel disconnected unexpectedly".into())
4497 }
4498 Ok(()) => Err("commit actor channel accepted beyond fixed capacity".into()),
4499 }
4500 }
4501
4502 #[test]
4503 fn receipt_commit_actor_append_fails_closed_when_queue_is_full(
4504 ) -> Result<(), Box<dyn std::error::Error>> {
4505 let (sender, _receiver) = receipt_commit_channel();
4506 let health = Arc::new(ReceiptCommitWriterHealth::default());
4507 for _ in 0..RECEIPT_COMMIT_ACTOR_CHANNEL_CAPACITY {
4508 let (response, _result) = mpsc::sync_channel(1);
4509 sender.try_send(ReceiptCommitCommand::Flush(response))?;
4510 }
4511 let actor = ReceiptCommitActor {
4512 sender,
4513 health,
4514 worker: idle_worker(),
4515 };
4516
4517 let error = actor.append(actor_test_receipt()?, "{}".to_string(), false);
4518
4519 assert!(error
4520 .err()
4521 .ok_or("expected queue saturation error")?
4522 .to_string()
4523 .contains("sqlite receipt commit queue saturated"));
4524 Ok(())
4525 }
4526
4527 #[test]
4528 fn receipt_commit_actor_flush_honors_timeout() -> Result<(), Box<dyn std::error::Error>> {
4529 let (sender, _receiver) = receipt_commit_channel();
4530 let health = Arc::new(ReceiptCommitWriterHealth::default());
4531 let actor = ReceiptCommitActor {
4532 sender,
4533 health,
4534 worker: idle_worker(),
4535 };
4536
4537 let error = actor.flush_with_timeout(Duration::from_millis(1));
4538
4539 match error.err().ok_or("expected flush timeout error")? {
4540 ReceiptStoreError::Timeout {
4541 operation,
4542 timeout_ms,
4543 } => {
4544 assert_eq!(operation, "sqlite receipt commit flush");
4545 assert_eq!(timeout_ms, 1);
4546 }
4547 other => {
4548 return Err(
4549 std::io::Error::other(format!("expected timeout error, got {other}")).into(),
4550 );
4551 }
4552 }
4553 Ok(())
4554 }
4555
4556 #[test]
4557 fn append_with_timeout_maps_to_timeout_and_keeps_inflight_elevated(
4558 ) -> Result<(), Box<dyn std::error::Error>> {
4559 let (sender, _receiver) = receipt_commit_channel();
4562 let health = Arc::new(ReceiptCommitWriterHealth::default());
4563 let actor = ReceiptCommitActor {
4564 sender,
4565 health,
4566 worker: idle_worker(),
4567 };
4568 let inflight_before = actor.health.inflight.load(Ordering::SeqCst);
4569
4570 let start = std::time::Instant::now();
4571 let error = actor.append_with_timeout(
4572 actor_test_receipt()?,
4573 "{}".to_string(),
4574 false,
4575 Duration::from_millis(250),
4576 );
4577 assert!(start.elapsed() < Duration::from_secs(2));
4578
4579 match error.err().ok_or("expected append timeout error")? {
4580 ReceiptStoreError::Timeout { operation, .. } => {
4581 assert_eq!(operation, "sqlite receipt commit append");
4582 }
4583 other => {
4584 return Err(
4585 std::io::Error::other(format!("expected timeout error, got {other}")).into(),
4586 );
4587 }
4588 }
4589 assert_eq!(
4592 actor.health.inflight.load(Ordering::SeqCst),
4593 inflight_before + 1
4594 );
4595 assert_eq!(actor.health.failed_total.load(Ordering::SeqCst), 0);
4596 assert_eq!(actor.health.timed_out_inflight.load(Ordering::SeqCst), 1);
4597 assert_eq!(actor.health.timed_out_total.load(Ordering::SeqCst), 1);
4598 Ok(())
4599 }
4600
4601 #[test]
4602 fn enqueue_on_a_disconnected_actor_records_writer_dead(
4603 ) -> Result<(), Box<dyn std::error::Error>> {
4604 let (sender, receiver) = receipt_commit_channel();
4610 drop(receiver);
4611 let health = Arc::new(ReceiptCommitWriterHealth::default());
4612 let actor = ReceiptCommitActor {
4613 sender,
4614 health,
4615 worker: idle_worker(),
4616 };
4617
4618 let error = actor.append_with_timeout(
4619 actor_test_receipt()?,
4620 "{}".to_string(),
4621 false,
4622 Duration::from_millis(250),
4623 );
4624 assert!(error
4625 .err()
4626 .ok_or("expected writer-unavailable error")?
4627 .to_string()
4628 .contains("unavailable"));
4629
4630 let counters = actor.writer_counters();
4631 assert!(
4632 counters
4633 .last_error
4634 .as_deref()
4635 .is_some_and(|error| error.contains("unavailable")),
4636 "the disconnected enqueue must record the writer death"
4637 );
4638 assert_eq!(
4639 classify_writer_liveness(
4640 &counters,
4641 10_000,
4642 RECEIPT_COMMIT_ACTOR_CHANNEL_CAPACITY as u64,
4643 None,
4644 1_000_000,
4645 ),
4646 chio_kernel::ReceiptWriterLiveness::Dead,
4647 "a disconnected writer must classify as Dead so admission stops"
4648 );
4649 Ok(())
4650 }
4651
4652 #[test]
4653 fn note_accept_restamps_backlog_start_only_on_a_fresh_backlog() {
4654 let health = ReceiptCommitWriterHealth::default();
4655
4656 health.note_accept(0);
4658 assert_ne!(
4659 health.backlog_started_unix_ms.load(Ordering::SeqCst),
4660 0,
4661 "the first enqueue of a backlog must stamp its start"
4662 );
4663
4664 health.backlog_started_unix_ms.store(1, Ordering::SeqCst);
4666 health.note_accept(1);
4667 assert_eq!(
4668 health.backlog_started_unix_ms.load(Ordering::SeqCst),
4669 1,
4670 "a growing backlog must keep its original start"
4671 );
4672
4673 health.backlog_started_unix_ms.store(1, Ordering::SeqCst);
4675 health.note_accept(0);
4676 assert_ne!(
4677 health.backlog_started_unix_ms.load(Ordering::SeqCst),
4678 1,
4679 "a fresh backlog after draining must restamp the start"
4680 );
4681 }
4682
4683 #[test]
4684 fn timeout_tracker_attributes_timeout_until_its_actor_token_completes() {
4685 let health = Arc::new(ReceiptCommitWriterHealth::default());
4686 let (mut completion, timeout) = writer_command_tracker(&health);
4687
4688 timeout.note_timeout("sqlite receipt commit write timed out");
4689 assert_eq!(health.timed_out_inflight.load(Ordering::SeqCst), 1);
4690 assert_eq!(health.timed_out_total.load(Ordering::SeqCst), 1);
4691 assert!(health.last_error.lock().is_ok_and(|error| error
4692 .as_deref()
4693 .is_some_and(|message| message.contains("timed out"))));
4694
4695 completion.complete();
4696 assert_eq!(health.timed_out_inflight.load(Ordering::SeqCst), 0);
4697 assert_eq!(health.timed_out_total.load(Ordering::SeqCst), 1);
4698 assert!(health.last_error.lock().is_ok_and(|error| error.is_none()));
4699 }
4700
4701 #[test]
4702 fn timeout_tracker_undoes_registration_when_actor_completed_first() {
4703 let health = Arc::new(ReceiptCommitWriterHealth::default());
4704 let (mut completion, timeout) = writer_command_tracker(&health);
4705
4706 completion.complete();
4707 timeout.note_timeout("sqlite receipt commit write timed out");
4708
4709 assert_eq!(health.timed_out_inflight.load(Ordering::SeqCst), 0);
4710 assert_eq!(health.timed_out_total.load(Ordering::SeqCst), 1);
4711 assert!(health.last_error.lock().is_ok_and(|error| error.is_none()));
4712 }
4713
4714 #[test]
4715 fn timeout_tracker_preserves_a_genuine_error_containing_timeout_words() {
4716 let health = Arc::new(ReceiptCommitWriterHealth::default());
4717 if let Ok(mut last_error) = health.last_error.lock() {
4718 *last_error = Some("database lock timed out".to_string());
4719 }
4720 let (mut completion, timeout) = writer_command_tracker(&health);
4721
4722 timeout.note_timeout(RECEIPT_WRITE_TIMEOUT_MARKER);
4723 completion.complete();
4724
4725 assert_eq!(health.timed_out_total.load(Ordering::SeqCst), 1);
4726 assert_eq!(health.timed_out_inflight.load(Ordering::SeqCst), 0);
4727 assert!(health
4728 .last_error
4729 .lock()
4730 .is_ok_and(|error| { error.as_deref() == Some("database lock timed out") }));
4731 }
4732
4733 #[test]
4734 fn unrelated_commit_preserves_an_outstanding_timeout_marker() {
4735 let health = ReceiptCommitWriterHealth::default();
4736 health.timed_out_inflight.store(1, Ordering::SeqCst);
4737 if let Ok(mut last_error) = health.last_error.lock() {
4738 *last_error = Some("sqlite receipt commit write timed out".to_string());
4739 }
4740 record_write_job_outcome(&health, true);
4743 let preserved = match health.last_error.lock() {
4744 Ok(guard) => guard
4745 .as_deref()
4746 .is_some_and(|error| error.contains("timed out")),
4747 Err(_) => false,
4748 };
4749 assert!(
4750 preserved,
4751 "an unrelated earlier commit must not clear a queued command's timeout marker"
4752 );
4753 assert_eq!(health.committed_total.load(Ordering::SeqCst), 1);
4754
4755 atomic_saturating_sub(&health.timed_out_inflight, 1);
4756 health.clear_timeout_error_if_drained();
4757 assert!(health.last_error.lock().is_ok_and(|error| error.is_none()));
4758 }
4759
4760 #[test]
4761 fn committed_write_preserves_a_genuine_writer_error() {
4762 let health = ReceiptCommitWriterHealth::default();
4763 if let Ok(mut last_error) = health.last_error.lock() {
4766 *last_error = Some("receipt store verified head is unavailable".to_string());
4767 }
4768 record_write_job_outcome(&health, true);
4769 let preserved = match health.last_error.lock() {
4770 Ok(guard) => guard.as_deref() == Some("receipt store verified head is unavailable"),
4771 Err(_) => false,
4772 };
4773 assert!(
4774 preserved,
4775 "a committed write must not clear an unrelated writer error"
4776 );
4777 }
4778
4779 #[test]
4780 fn run_write_receipt_with_timeout_fails_closed_when_writer_never_drains(
4781 ) -> Result<(), Box<dyn std::error::Error>> {
4782 let (sender, _receiver) = receipt_commit_channel();
4786 let health = Arc::new(ReceiptCommitWriterHealth::default());
4787 let handle = WriterHandle {
4788 sender,
4789 health: Arc::clone(&health),
4790 worker: idle_worker(),
4791 settlement_store_binding: None,
4792 };
4793 let inflight_before = health.inflight.load(Ordering::SeqCst);
4794
4795 let start = std::time::Instant::now();
4796 let error =
4797 handle.run_write_receipt_with_timeout(|_connection| Ok(()), Duration::from_millis(250));
4798 assert!(start.elapsed() < Duration::from_secs(2));
4799
4800 match error.err().ok_or("expected write timeout error")? {
4801 ReceiptStoreError::Timeout { operation, .. } => {
4802 assert_eq!(operation, "sqlite receipt commit write");
4803 }
4804 other => {
4805 return Err(
4806 std::io::Error::other(format!("expected timeout error, got {other}")).into(),
4807 );
4808 }
4809 }
4810 assert_eq!(health.inflight.load(Ordering::SeqCst), inflight_before + 1);
4813 assert_eq!(health.failed_total.load(Ordering::SeqCst), 0);
4814 assert_eq!(health.timed_out_inflight.load(Ordering::SeqCst), 1);
4815 assert_eq!(health.timed_out_total.load(Ordering::SeqCst), 1);
4816 Ok(())
4817 }
4818
4819 #[test]
4820 fn earlier_success_does_not_clear_a_later_queued_timeout(
4821 ) -> Result<(), Box<dyn std::error::Error>> {
4822 let path = std::env::temp_dir().join(format!(
4823 "chio-write-timeout-ordering-{}-{}.sqlite3",
4824 std::process::id(),
4825 SystemTime::now()
4826 .duration_since(UNIX_EPOCH)
4827 .map(|duration| duration.as_nanos())
4828 .unwrap_or(0)
4829 ));
4830 let store = SqliteReceiptStore::open(&path)?;
4831 let baseline = store.receipt_commit_actor.writer_counters();
4832 let writer_a = store.writer_handle();
4833 let writer_b = store.writer_handle();
4834 let (a_started_tx, a_started_rx) = mpsc::sync_channel(1);
4835 let (a_release_tx, a_release_rx) = mpsc::sync_channel(1);
4836 let a = std::thread::spawn(move || {
4837 writer_a.run_write(move |_connection| {
4838 let _ = a_started_tx.send(());
4839 let _ = a_release_rx.recv();
4840 Ok(())
4841 })
4842 });
4843 a_started_rx.recv()?;
4844
4845 let (b_started_tx, b_started_rx) = mpsc::sync_channel(1);
4846 let (b_release_tx, b_release_rx) = mpsc::sync_channel(1);
4847 let b_error = writer_b
4848 .run_write_with_timeout(
4849 move |_connection| {
4850 let _ = b_started_tx.send(());
4851 let _ = b_release_rx.recv();
4852 Ok(())
4853 },
4854 Duration::from_millis(25),
4855 )
4856 .err()
4857 .ok_or("B must time out while queued behind A")?;
4858 assert!(matches!(b_error, ReceiptStoreError::Timeout { .. }));
4859 assert_eq!(
4860 store
4861 .receipt_commit_actor
4862 .health
4863 .timed_out_inflight
4864 .load(Ordering::SeqCst),
4865 1
4866 );
4867 assert_eq!(
4868 store.writer_liveness(Duration::from_secs(60)),
4869 chio_kernel::ReceiptWriterLiveness::Wedged
4870 );
4871
4872 a_release_tx.send(())?;
4873 a.join().map_err(|_| "A writer thread panicked")??;
4874 b_started_rx.recv()?;
4875 let after_a = store.receipt_commit_actor.writer_counters();
4876 assert_eq!(after_a.accepted_total, baseline.accepted_total + 2);
4877 assert_eq!(after_a.committed_total, baseline.committed_total + 1);
4878 assert_eq!(after_a.failed_total, baseline.failed_total);
4879 assert_eq!(after_a.timed_out_total, baseline.timed_out_total + 1);
4880 assert_eq!(after_a.timed_out_inflight, 1);
4881 assert_eq!(
4882 store
4883 .receipt_commit_actor
4884 .health
4885 .timed_out_inflight
4886 .load(Ordering::SeqCst),
4887 1,
4888 "A's success must not clear B's outstanding timeout"
4889 );
4890 assert!(after_a
4891 .last_error
4892 .as_deref()
4893 .is_some_and(|error| error.contains("timed out")));
4894 assert_eq!(
4895 store.writer_liveness(Duration::from_secs(60)),
4896 chio_kernel::ReceiptWriterLiveness::Wedged
4897 );
4898
4899 b_release_tx.send(())?;
4900 assert!(wait_until(|| {
4901 store
4902 .receipt_commit_actor
4903 .health
4904 .timed_out_inflight
4905 .load(Ordering::SeqCst)
4906 == 0
4907 && store
4908 .receipt_commit_actor
4909 .health
4910 .committed_total
4911 .load(Ordering::SeqCst)
4912 == baseline.committed_total + 2
4913 }));
4914 let drained = store.receipt_commit_actor.writer_counters();
4915 assert_eq!(drained.failed_total, baseline.failed_total);
4916 assert_eq!(drained.timed_out_total, baseline.timed_out_total + 1);
4917 assert_eq!(drained.timed_out_inflight, 0);
4918 assert_eq!(
4919 drained.accepted_total,
4920 drained.committed_total + drained.failed_total
4921 );
4922 assert!(drained.last_error.is_none());
4923 assert_eq!(
4924 store.writer_liveness(Duration::from_secs(60)),
4925 chio_kernel::ReceiptWriterLiveness::Healthy
4926 );
4927
4928 drop(store);
4929 let _ = fs::remove_file(path);
4930 Ok(())
4931 }
4932
4933 #[test]
4934 fn critical_write_timeout_keeps_inflight_and_late_failure_poisoning(
4935 ) -> Result<(), Box<dyn std::error::Error>> {
4936 let (sender, receiver) = receipt_commit_channel();
4937 let health = Arc::new(ReceiptCommitWriterHealth::default());
4938 let handle = WriterHandle {
4939 sender,
4940 health: Arc::clone(&health),
4941 worker: idle_worker(),
4942 settlement_store_binding: None,
4943 };
4944 let inflight_before = health.inflight.load(Ordering::SeqCst);
4945
4946 let error = handle
4947 .run_critical_receipt_write_with_timeout(
4948 |_connection| Ok(()),
4949 Duration::from_millis(10),
4950 )
4951 .err()
4952 .ok_or("expected critical write timeout")?;
4953 assert!(matches!(error, ReceiptStoreError::Timeout { .. }));
4954 assert_eq!(health.inflight.load(Ordering::SeqCst), inflight_before + 1);
4955 assert_eq!(health.failed_total.load(Ordering::SeqCst), 0);
4956 assert_eq!(health.timed_out_inflight.load(Ordering::SeqCst), 1);
4957 assert_eq!(health.timed_out_total.load(Ordering::SeqCst), 1);
4958 assert!(!health.critical_write_poisoned.load(Ordering::SeqCst));
4959
4960 match receiver.recv()? {
4961 ReceiptCommitCommand::Write {
4962 job,
4963 appends_receipts,
4964 fail_closed_on_error,
4965 mut completion,
4966 } => {
4967 assert!(appends_receipts);
4968 assert!(fail_closed_on_error);
4969 health.note_channel_dequeue();
4970 let respond = job(Err(ReceiptStoreError::Conflict(
4971 "late critical write failure".to_string(),
4972 )));
4973 atomic_saturating_sub(&health.inflight, 1);
4974 let committed = respond(Ok(()));
4975 assert!(!committed);
4976 record_write_job_outcome(&health, committed);
4977 completion.complete();
4978 }
4979 _ => return Err("expected queued critical Write command".into()),
4980 }
4981 assert!(health.critical_write_poisoned.load(Ordering::SeqCst));
4982 assert!(health.head_poisoned.load(Ordering::SeqCst));
4983 assert_eq!(health.timed_out_inflight.load(Ordering::SeqCst), 0);
4984 assert_eq!(health.timed_out_total.load(Ordering::SeqCst), 1);
4985 assert_eq!(health.accepted_total.load(Ordering::SeqCst), 1);
4986 assert_eq!(health.committed_total.load(Ordering::SeqCst), 0);
4987 assert_eq!(health.failed_total.load(Ordering::SeqCst), 1);
4988 Ok(())
4989 }
4990
4991 #[test]
4992 fn run_write_with_timeout_fails_closed_when_writer_never_drains(
4993 ) -> Result<(), Box<dyn std::error::Error>> {
4994 let (sender, _receiver) = receipt_commit_channel();
4998 let health = Arc::new(ReceiptCommitWriterHealth::default());
4999 let handle = WriterHandle {
5000 sender,
5001 health: Arc::clone(&health),
5002 worker: idle_worker(),
5003 settlement_store_binding: None,
5004 };
5005 let inflight_before = health.inflight.load(Ordering::SeqCst);
5006
5007 let start = std::time::Instant::now();
5008 let error = handle.run_write_with_timeout(|_connection| Ok(()), Duration::from_millis(250));
5009 assert!(start.elapsed() < Duration::from_secs(2));
5010
5011 match error.err().ok_or("expected write timeout error")? {
5012 ReceiptStoreError::Timeout { operation, .. } => {
5013 assert_eq!(operation, "sqlite receipt commit write");
5014 }
5015 other => {
5016 return Err(
5017 std::io::Error::other(format!("expected timeout error, got {other}")).into(),
5018 );
5019 }
5020 }
5021 assert_eq!(health.inflight.load(Ordering::SeqCst), inflight_before + 1);
5024 assert_eq!(health.failed_total.load(Ordering::SeqCst), 0);
5025 assert_eq!(health.timed_out_inflight.load(Ordering::SeqCst), 1);
5026 assert_eq!(health.timed_out_total.load(Ordering::SeqCst), 1);
5027 Ok(())
5028 }
5029
5030 #[test]
5031 fn disconnected_bounded_write_records_writer_death_for_liveness(
5032 ) -> Result<(), Box<dyn std::error::Error>> {
5033 let (sender, receiver) = receipt_commit_channel();
5040 let health = Arc::new(ReceiptCommitWriterHealth::default());
5041 let handle = WriterHandle {
5042 sender,
5043 health: Arc::clone(&health),
5044 worker: idle_worker(),
5045 settlement_store_binding: None,
5046 };
5047 let actor = std::thread::spawn(move || {
5050 if let Ok(command) = receiver.recv() {
5051 drop(command);
5052 }
5053 drop(receiver);
5054 });
5055
5056 let error =
5057 handle.run_write_receipt_with_timeout(|_connection| Ok(()), Duration::from_secs(5));
5058 actor.join().map_err(|_| "actor thread panicked")?;
5059 assert!(error.is_err(), "a disconnected writer must fail closed");
5060
5061 let counters = ReceiptCommitActor {
5062 sender: receipt_commit_channel().0,
5063 health: Arc::clone(&health),
5064 worker: idle_worker(),
5065 }
5066 .writer_counters();
5067 assert!(
5068 counters
5069 .last_error
5070 .as_deref()
5071 .is_some_and(|reason| reason.contains("unavailable")),
5072 "writer death must be recorded for the liveness probe"
5073 );
5074 assert_eq!(
5075 classify_writer_liveness(
5076 &counters,
5077 10_000,
5078 RECEIPT_COMMIT_ACTOR_CHANNEL_CAPACITY as u64,
5079 None,
5080 current_unix_ms(),
5081 ),
5082 chio_kernel::ReceiptWriterLiveness::Dead
5083 );
5084 Ok(())
5085 }
5086
5087 #[test]
5088 fn disconnected_reseed_flips_writer_liveness_dead_immediately(
5089 ) -> Result<(), Box<dyn std::error::Error>> {
5090 let path = std::env::temp_dir().join(format!(
5095 "chio-reseed-dead-{}-{}.sqlite3",
5096 std::process::id(),
5097 SystemTime::now()
5098 .duration_since(UNIX_EPOCH)
5099 .map(|d| d.as_nanos())
5100 .unwrap_or(0)
5101 ));
5102 let mut store = SqliteReceiptStore::open(&path)?;
5103
5104 let (sender, receiver) = receipt_commit_channel();
5108 drop(receiver);
5109 store.receipt_commit_actor = ReceiptCommitActor {
5110 sender,
5111 health: Arc::new(ReceiptCommitWriterHealth::default()),
5112 worker: idle_worker(),
5113 };
5114
5115 let error = match store.reseed_verified_head() {
5116 Ok(()) => return Err("reseed against a dead actor must fail closed".into()),
5117 Err(error) => error,
5118 };
5119 assert!(error.to_string().contains("unavailable"));
5120
5121 assert_eq!(
5122 store.writer_liveness(Duration::from_secs(60)),
5123 chio_kernel::ReceiptWriterLiveness::Dead,
5124 "a disconnected reseed must flip writer liveness to Dead immediately"
5125 );
5126
5127 let _ = std::fs::remove_file(&path);
5128 Ok(())
5129 }
5130
5131 #[test]
5132 fn run_write_executes_jobs_serially_on_the_writer_thread(
5133 ) -> Result<(), Box<dyn std::error::Error>> {
5134 let path = std::env::temp_dir().join(format!(
5135 "chio-run-write-{}-{}.sqlite3",
5136 std::process::id(),
5137 SystemTime::now()
5138 .duration_since(UNIX_EPOCH)
5139 .map(|d| d.as_nanos())
5140 .unwrap_or(0)
5141 ));
5142 let store = SqliteReceiptStore::open(&path)?;
5143 let writer = store.writer_handle();
5144
5145 let first_thread = writer.run_write(|_connection| Ok(std::thread::current().id()))?;
5146 let second_thread = writer.run_write(|_connection| Ok(std::thread::current().id()))?;
5147
5148 assert_eq!(
5149 first_thread, second_thread,
5150 "all write jobs must run on the single writer thread"
5151 );
5152 assert_ne!(
5153 first_thread,
5154 std::thread::current().id(),
5155 "write jobs must not run on the caller thread"
5156 );
5157
5158 let journal_mode = writer.run_write(|connection| {
5160 connection
5161 .query_row("PRAGMA journal_mode", [], |row| row.get::<_, String>(0))
5162 .map_err(ReceiptStoreError::from)
5163 })?;
5164 assert!(journal_mode.eq_ignore_ascii_case("wal"));
5165
5166 assert_eq!(
5168 store
5169 .receipt_commit_actor
5170 .health
5171 .inflight
5172 .load(Ordering::SeqCst),
5173 0
5174 );
5175
5176 let _ = fs::remove_file(path);
5177 Ok(())
5178 }
5179
5180 #[test]
5181 fn run_write_fails_closed_when_queue_is_full() -> Result<(), Box<dyn std::error::Error>> {
5182 let (sender, _receiver) = receipt_commit_channel();
5183 let health = Arc::new(ReceiptCommitWriterHealth::default());
5184 for _ in 0..RECEIPT_COMMIT_ACTOR_CHANNEL_CAPACITY {
5185 let (response, _result) = mpsc::sync_channel(1);
5186 sender.try_send(ReceiptCommitCommand::Flush(response))?;
5187 }
5188 let handle = WriterHandle {
5189 sender,
5190 health: Arc::clone(&health),
5191 worker: idle_worker(),
5192 settlement_store_binding: None,
5193 };
5194
5195 let error = handle.run_write(|_connection| Ok(()));
5196
5197 assert!(error
5198 .err()
5199 .ok_or("expected queue saturation error")?
5200 .to_string()
5201 .contains("sqlite receipt commit queue saturated"));
5202 assert_eq!(
5203 health.inflight.load(Ordering::SeqCst),
5204 0,
5205 "speculative inflight increment must be undone on saturation"
5206 );
5207 assert_eq!(health.saturated_total.load(Ordering::SeqCst), 1);
5208 Ok(())
5209 }
5210
5211 #[test]
5217 fn write_job_holds_inflight_for_its_duration() -> Result<(), Box<dyn std::error::Error>> {
5218 let path = std::env::temp_dir().join(format!(
5219 "chio-write-inflight-{}-{}.sqlite3",
5220 std::process::id(),
5221 SystemTime::now()
5222 .duration_since(UNIX_EPOCH)
5223 .map(|d| d.as_nanos())
5224 .unwrap_or(0)
5225 ));
5226 let store = SqliteReceiptStore::open(&path)?;
5227 let writer = store.writer_handle();
5228
5229 let drained_baseline = wait_until(|| {
5232 store
5233 .receipt_commit_actor
5234 .health
5235 .inflight
5236 .load(Ordering::SeqCst)
5237 == 0
5238 });
5239 assert!(drained_baseline, "writer failed to drain to baseline");
5240
5241 let (started_tx, started_rx) = mpsc::sync_channel::<()>(1);
5243 let (release_tx, release_rx) = mpsc::sync_channel::<()>(1);
5244 let worker = std::thread::spawn(move || {
5245 writer.run_write(move |_connection| {
5246 let _ = started_tx.send(());
5249 let _ = release_rx.recv();
5250 Ok(())
5251 })
5252 });
5253
5254 started_rx.recv().map_err(|_| "write job never started")?;
5257 assert_eq!(
5258 store
5259 .receipt_commit_actor
5260 .health
5261 .inflight
5262 .load(Ordering::SeqCst),
5263 1,
5264 "a running Write job must report inflight > 0"
5265 );
5266
5267 release_tx.send(())?;
5272 worker
5273 .join()
5274 .map_err(|_| "write worker thread panicked")??;
5275 let drained = wait_until(|| {
5276 store
5277 .receipt_commit_actor
5278 .health
5279 .inflight
5280 .load(Ordering::SeqCst)
5281 == 0
5282 });
5283 assert!(
5284 drained,
5285 "inflight must return to baseline after the Write completes"
5286 );
5287
5288 let _ = fs::remove_file(path);
5289 Ok(())
5290 }
5291
5292 #[test]
5304 fn write_decrements_inflight_before_returning_to_caller(
5305 ) -> Result<(), Box<dyn std::error::Error>> {
5306 let path = std::env::temp_dir().join(format!(
5307 "chio-write-inflight-order-{}-{}.sqlite3",
5308 std::process::id(),
5309 SystemTime::now()
5310 .duration_since(UNIX_EPOCH)
5311 .map(|d| d.as_nanos())
5312 .unwrap_or(0)
5313 ));
5314 let store = SqliteReceiptStore::open(&path)?;
5315 let writer = store.writer_handle();
5316
5317 let drained_baseline = wait_until(|| {
5319 store
5320 .receipt_commit_actor
5321 .health
5322 .inflight
5323 .load(Ordering::SeqCst)
5324 == 0
5325 });
5326 assert!(drained_baseline, "writer failed to drain to baseline");
5327
5328 for iteration in 0..512 {
5335 writer.run_write(|_connection| Ok(()))?;
5336 let observed = store
5337 .receipt_commit_actor
5338 .health
5339 .inflight
5340 .load(Ordering::SeqCst);
5341 assert_eq!(
5342 observed, 0,
5343 "caller returned from run_write with inflight still counted \
5344 (iteration {iteration}); the decrement must precede the response"
5345 );
5346 }
5347
5348 let _ = fs::remove_file(path);
5349 Ok(())
5350 }
5351
5352 fn wait_until(predicate: impl Fn() -> bool) -> bool {
5354 for _ in 0..1_000 {
5355 if predicate() {
5356 return true;
5357 }
5358 std::thread::sleep(std::time::Duration::from_millis(1));
5359 }
5360 predicate()
5361 }
5362}