Skip to main content

chio_store_sqlite/
receipt_store.rs

1use std::collections::{BTreeMap, BTreeSet};
2use std::fs;
3use std::path::Path;
4use std::sync::atomic::{AtomicBool, AtomicU64, AtomicU8, Ordering};
5use std::sync::{mpsc, Arc, Mutex, OnceLock};
6use std::thread;
7use std::time::{Duration, SystemTime, UNIX_EPOCH};
8
9use chacha20poly1305::aead::rand_core::{OsRng, RngCore};
10use chio_core::canonical::{canonical_json_bytes, CanonicalBytes};
11use chio_core::capability::{scope::ChioScope, token::CapabilityToken};
12use chio_core::crypto::{sha256_hex, Keypair, Signature};
13use chio_core::receipt::{
14    body::ChioReceipt, crypto_floor::ReceiptCryptoFloor, decision::Decision,
15    economics::FinancialReceiptMetadata, economics::SettlementStatus,
16    governance::GovernedTransactionReceiptMetadata, lineage::ChildRequestReceipt,
17    metadata::ReceiptAttributionMetadata,
18};
19use chio_core::session::{
20    OperationTerminalState, RequestLineageMode, RequestLineageRecord, SessionAnchorReference,
21};
22use chio_kernel::checkpoint::{CheckpointChainFrontier, KernelCheckpoint, KernelCheckpointBody};
23use chio_kernel::cost_attribution::{
24    CostAttributionChainHop, CostAttributionQuery, CostAttributionReceiptRow,
25    CostAttributionReport, CostAttributionSummary, LeafCostAttributionRow, RootCostAttributionRow,
26    MAX_COST_ATTRIBUTION_LIMIT,
27};
28use chio_kernel::dpop::DPOP_SCHEMA;
29use chio_kernel::operator_report::{
30    AuthorizationContextReport, AuthorizationContextRow, AuthorizationContextSenderConstraint,
31    AuthorizationContextSummary, BehavioralFeedGovernedActionSummary,
32    BehavioralFeedMeteredBillingRow, BehavioralFeedMeteredBillingSummary, BehavioralFeedQuery,
33    BehavioralFeedReceiptRow, BehavioralFeedReceiptSelection, BehavioralFeedSettlementSummary,
34    ChioOAuthAuthorizationDiscoveryMetadata, ChioOAuthAuthorizationExampleMapping,
35    ChioOAuthAuthorizationMetadataReport, ChioOAuthAuthorizationProfile,
36    ChioOAuthAuthorizationReviewPack, ChioOAuthAuthorizationReviewPackRecord,
37    ChioOAuthAuthorizationReviewPackSummary, ChioOAuthAuthorizationSupportBoundary,
38    ComplianceReport, EconomicCompletionFlowReport, EconomicCompletionFlowSummary,
39    EconomicReceiptMeteringProjection, EconomicReceiptProjectionReport,
40    EconomicReceiptProjectionRow, EconomicReceiptProjectionSummary,
41    EconomicReceiptSettlementProjection, GovernedAuthorizationCommerceDetail,
42    GovernedAuthorizationDetail, GovernedAuthorizationMeteredBillingDetail,
43    GovernedAuthorizationTransactionContext, MeteredBillingEvidenceRecord,
44    MeteredBillingReconciliationReport, MeteredBillingReconciliationRow,
45    MeteredBillingReconciliationState, MeteredBillingReconciliationSummary, OperatorReportQuery,
46    SettlementReconciliationReport, SettlementReconciliationRow, SettlementReconciliationState,
47    SettlementReconciliationSummary, SharedEvidenceQuery, SharedEvidenceReferenceReport,
48    SharedEvidenceReferenceRow, SharedEvidenceReferenceSummary,
49    CHIO_OAUTH_AUTHORIZATION_COMMERCE_DETAIL_TYPE, CHIO_OAUTH_AUTHORIZATION_CONTEXT_REPORT_SCHEMA,
50    CHIO_OAUTH_AUTHORIZATION_METADATA_SCHEMA, CHIO_OAUTH_AUTHORIZATION_METERED_BILLING_DETAIL_TYPE,
51    CHIO_OAUTH_AUTHORIZATION_REVIEW_PACK_SCHEMA, CHIO_OAUTH_AUTHORIZATION_TOOL_DETAIL_TYPE,
52    CHIO_OAUTH_SENDER_PROOF_CHIO_DPOP, ECONOMIC_COMPLETION_FLOW_SCHEMA,
53};
54use chio_kernel::receipt_analytics::{
55    AgentAnalyticsRow, AnalyticsTimeBucket, ReceiptAnalyticsMetrics, ReceiptAnalyticsQuery,
56    ReceiptAnalyticsResponse, TimeAnalyticsRow, ToolAnalyticsRow, MAX_ANALYTICS_GROUP_LIMIT,
57};
58use chio_kernel::receipt_query::{
59    ReceiptQuery, ReceiptQueryResult, ReceiptReadBoundary, ReceiptReadContext, MAX_QUERY_LIMIT,
60};
61use chio_kernel::receipt_store::{ReceiptLineageStatementLink, ReceiptLineageVerification};
62use chio_kernel::{
63    AtomicReceiptProjection, AuthorizationReceiptConsumption, CapabilitySnapshot,
64    CreditBondDisposition, CreditBondLifecycleState, CreditBondListQuery, CreditBondListReport,
65    CreditBondListSummary, CreditBondRow, CreditFacilityDisposition, CreditFacilityLifecycleState,
66    CreditFacilityListQuery, CreditFacilityListReport, CreditFacilityListSummary,
67    CreditFacilityRow, CreditLossLifecycleEventKind, CreditLossLifecycleListQuery,
68    CreditLossLifecycleListReport, CreditLossLifecycleListSummary, CreditLossLifecycleRow,
69    EvidenceChildReceiptScope, EvidenceExportQuery, ExposureLedgerQuery,
70    FederatedEvidenceShareImport, FederatedEvidenceShareSummary, LiabilityAutoBindDisposition,
71    LiabilityClaimPayoutReconciliationState, LiabilityClaimResponseDisposition,
72    LiabilityClaimSettlementReconciliationState, LiabilityClaimWorkflowQuery,
73    LiabilityClaimWorkflowReport, LiabilityClaimWorkflowRow, LiabilityClaimWorkflowSummary,
74    LiabilityMarketWorkflowQuery, LiabilityMarketWorkflowReport, LiabilityMarketWorkflowRow,
75    LiabilityMarketWorkflowSummary, LiabilityProviderLifecycleState, LiabilityProviderListQuery,
76    LiabilityProviderListReport, LiabilityProviderListSummary, LiabilityProviderResolutionQuery,
77    LiabilityProviderResolutionReport, LiabilityProviderRow, LiabilityQuoteDisposition,
78    PendingSettlementObservation, ReceiptCheckpointCreateReport, ReceiptCheckpointRange,
79    ReceiptCheckpointStatusReport, ReceiptFlushReport, ReceiptStore, ReceiptStoreError,
80    ReceiptStoreHealthReport, ReceiptWalCheckpointReport, ReceiptWriterCounters, RetentionConfig,
81    SignedCreditBond, SignedCreditFacility, SignedCreditLossLifecycle,
82    SignedLiabilityAutoBindDecision, SignedLiabilityBoundCoverage,
83    SignedLiabilityClaimAdjudication, SignedLiabilityClaimDispute, SignedLiabilityClaimPackage,
84    SignedLiabilityClaimPayoutInstruction, SignedLiabilityClaimPayoutReceipt,
85    SignedLiabilityClaimResponse, SignedLiabilityClaimSettlementInstruction,
86    SignedLiabilityClaimSettlementReceipt, SignedLiabilityPlacement,
87    SignedLiabilityPricingAuthority, SignedLiabilityProvider, SignedLiabilityQuoteRequest,
88    SignedLiabilityQuoteResponse, SignedUnderwritingDecision, StoredChildReceipt,
89    StoredToolReceipt, UnderwritingAppealCreateRequest, UnderwritingAppealRecord,
90    UnderwritingAppealResolution, UnderwritingAppealResolveRequest, UnderwritingAppealStatus,
91    UnderwritingDecisionLifecycleState, UnderwritingDecisionListReport,
92    UnderwritingDecisionOutcome, UnderwritingDecisionQuery, UnderwritingDecisionRow,
93    UnderwritingDecisionSummary, CREDIT_BOND_LIST_REPORT_SCHEMA,
94    CREDIT_FACILITY_LIST_REPORT_SCHEMA, CREDIT_LOSS_LIFECYCLE_LIST_REPORT_SCHEMA,
95    LIABILITY_CLAIM_WORKFLOW_REPORT_SCHEMA, LIABILITY_MARKET_WORKFLOW_REPORT_SCHEMA,
96    LIABILITY_PROVIDER_LIST_REPORT_SCHEMA, LIABILITY_PROVIDER_RESOLUTION_REPORT_SCHEMA,
97};
98use chio_supervisor::{
99    HealthFlag, HealthLevel, SupervisedOutcome, SupervisedThread, SupervisorConfig,
100};
101use r2d2::{Pool, PooledConnection};
102use r2d2_sqlite::SqliteConnectionManager;
103use rusqlite::{params, Connection, OptionalExtension};
104
105pub struct SqliteReceiptStore {
106    pub(crate) pool: Pool<SqliteConnectionManager>,
107    receipt_commit_actor: ReceiptCommitActor,
108    settlement_store_binding: Option<chio_settle::SettlementStoreBinding>,
109    /// Multi-tenant receipt isolation: when true, tenant-
110    /// scoped queries exclude the pre-multitenant NULL-tagged set. When
111    /// false, queries with `tenant_filter = Some(id)` return rows where
112    /// `tenant_id = id OR tenant_id IS NULL`, which keeps pre-multitenant
113    /// (NULL-tagged) receipts visible during explicit compatibility mode.
114    pub(crate) strict_tenant_isolation: std::sync::atomic::AtomicBool,
115    /// Staged-rollout flag: read-only after open.
116    pub(crate) incremental_verification: bool,
117}
118
119type FederatedShareSubjectCorpus = (
120    FederatedEvidenceShareSummary,
121    Vec<StoredToolReceipt>,
122    Vec<CapabilitySnapshot>,
123);
124pub(crate) type SqliteStoreConnection = PooledConnection<SqliteConnectionManager>;
125
126const RECEIPT_GROUP_COMMIT_MAX_BATCH: usize = 64;
127const RECEIPT_GROUP_COMMIT_FLUSH_DELAY: Duration = Duration::from_micros(500);
128const RECEIPT_COMMIT_ACTOR_CHANNEL_CAPACITY: usize = RECEIPT_GROUP_COMMIT_MAX_BATCH * 16;
129const RECEIPT_APPEND_TIMEOUT_MARKER: &str = "sqlite receipt commit append timed out";
130const RECEIPT_WRITE_TIMEOUT_MARKER: &str = "sqlite receipt commit write timed out";
131
132fn is_receipt_writer_timeout_marker(message: &str) -> bool {
133    matches!(
134        message,
135        RECEIPT_APPEND_TIMEOUT_MARKER | RECEIPT_WRITE_TIMEOUT_MARKER
136    )
137}
138
139struct ReceiptCommitActor {
140    sender: mpsc::SyncSender<ReceiptCommitCommand>,
141    health: Arc<ReceiptCommitWriterHealth>,
142    /// Retains the supervised writer until the last store or writer handle drops.
143    /// The sender precedes this field so channel disconnect drains queued work
144    /// before the final owner joins the writer.
145    worker: Arc<ReceiptCommitWorker>,
146}
147
148struct ReceiptCommitWorker {
149    join: Option<SupervisedReceiptWriter>,
150}
151
152struct SupervisedReceiptWriter {
153    supervisor: Option<SupervisedThread>,
154    health: HealthFlag,
155    thread_id: Arc<OnceLock<thread::ThreadId>>,
156}
157
158impl ReceiptCommitWorker {
159    fn health(&self) -> Option<&HealthFlag> {
160        self.join.as_ref().map(|writer| &writer.health)
161    }
162
163    fn writer_dead_error(&self) -> ReceiptStoreError {
164        let Some(health) = self.health() else {
165            return ReceiptStoreError::WriterDead {
166                restarts: 0,
167                last_error: "receipt commit writer channel disconnected".to_string(),
168            };
169        };
170        let snapshot = health.snapshot();
171        ReceiptStoreError::WriterDead {
172            restarts: snapshot.restart_total,
173            last_error: snapshot
174                .reason
175                .unwrap_or_else(|| "receipt commit writer channel disconnected".to_string()),
176        }
177    }
178}
179
180struct ReceiptCommitWriterHealth {
181    accepted_total: AtomicU64,
182    committed_total: AtomicU64,
183    failed_total: AtomicU64,
184    saturated_total: AtomicU64,
185    inflight: AtomicU64,
186    /// Caller-timed-out commands that are still owned by the actor. This is
187    /// command-specific liveness state; terminal outcome counters remain owned
188    /// by the actor and are updated only when each command actually drains.
189    timed_out_inflight: AtomicU64,
190    timed_out_total: AtomicU64,
191    /// Commands currently sitting in the commit-actor channel, not yet pulled
192    /// for processing. Incremented before every send and decremented when the
193    /// actor pulls a command, so it tracks true channel occupancy rather than
194    /// `inflight`, which stays elevated through the commit of a drained batch.
195    /// The saturation gate reads this: a drained but still-committing batch must
196    /// not read as a full channel when the next send would in fact succeed.
197    queue_depth: AtomicU64,
198    last_commit_unix_ms: AtomicU64,
199    /// Wall-clock (unix-ms) of the first accepted append, set once (0 = unset).
200    /// Retained for operator display of when this writer first did work.
201    first_accept_unix_ms: AtomicU64,
202    /// Wall-clock (unix-ms) at which the CURRENT unserviced backlog began, i.e.
203    /// the enqueue that took `inflight` from 0 to 1 (0 = no backlog yet). The
204    /// wedged-writer stall clock anchors here so a writer that was merely idle
205    /// (an old last commit) is not judged wedged the instant fresh work arrives;
206    /// it re-stamps on each new backlog and a growing backlog keeps its start.
207    backlog_started_unix_ms: AtomicU64,
208    last_error: Mutex<Option<String>>,
209    // Last background-retention rotation failure, set by the kernel maintenance
210    // worker via `record_retention_rotation_outcome` and cleared on the next
211    // successful rotation. Surfaced by `receipt_store_health` so a silently
212    // failing background retention task is observable rather than healthy.
213    retention_error: Mutex<Option<String>>,
214    // Verified-head snapshot, written only by the actor thread; read by
215    // flush_report / receipt_store_health / kernel counters.
216    head_checkpoint_seq: AtomicU64,
217    head_checkpointed_entry_seq: AtomicU64,
218    head_claim_log_count: AtomicU64,
219    head_claim_log_max_seq: AtomicU64,
220    // Mirror of the actor thread's `WriterHeadState` poison bit. The head lives
221    // only on the actor thread, so a poisoned head (every append rejected with a
222    // Conflict) is invisible to the supervised thread flag: the writer thread is
223    // still alive. Publishing it here lets `writer_serving_closed` deny at the
224    // pre-dispatch gate, so a tool is never executed against a store that cannot
225    // persist its receipt.
226    head_poisoned: AtomicBool,
227    critical_write_poisoned: AtomicBool,
228}
229
230impl Default for ReceiptCommitWriterHealth {
231    fn default() -> Self {
232        Self {
233            accepted_total: AtomicU64::new(0),
234            committed_total: AtomicU64::new(0),
235            failed_total: AtomicU64::new(0),
236            saturated_total: AtomicU64::new(0),
237            inflight: AtomicU64::new(0),
238            timed_out_inflight: AtomicU64::new(0),
239            timed_out_total: AtomicU64::new(0),
240            queue_depth: AtomicU64::new(0),
241            last_commit_unix_ms: AtomicU64::new(0),
242            first_accept_unix_ms: AtomicU64::new(0),
243            backlog_started_unix_ms: AtomicU64::new(0),
244            last_error: Mutex::new(None),
245            retention_error: Mutex::new(None),
246            head_checkpoint_seq: AtomicU64::new(0),
247            head_checkpointed_entry_seq: AtomicU64::new(0),
248            head_claim_log_count: AtomicU64::new(0),
249            head_claim_log_max_seq: AtomicU64::new(0),
250            // Fail closed until the actor thread seeds a verified head. The head
251            // is seeded asynchronously after construction, so starting open would
252            // let a corrupt or still-attaching store pass the pre-dispatch gate
253            // and run a tool before the first append could reject. The seed path
254            // clears this the moment it succeeds.
255            head_poisoned: AtomicBool::new(true),
256            critical_write_poisoned: AtomicBool::new(false),
257        }
258    }
259}
260
261impl ReceiptCommitWriterHealth {
262    /// Record accept-time liveness anchors. `first_accept_unix_ms` is set once,
263    /// for operator display. `backlog_started_unix_ms` is (re)stamped whenever an
264    /// enqueue takes `inflight` from 0 to 1 (`previous_inflight == 0`), marking
265    /// the start of the current unserviced backlog. A backlog that only grows
266    /// keeps its original start; the next backlog after the writer fully drains
267    /// resets it. The stall clock reads this so a writer that wedges before its
268    /// first commit is still caught, while a writer resuming after an idle period
269    /// is measured from the fresh work rather than a stale last commit.
270    fn note_accept(&self, previous_inflight: u64) {
271        let now = current_unix_ms();
272        let _ =
273            self.first_accept_unix_ms
274                .compare_exchange(0, now, Ordering::SeqCst, Ordering::SeqCst);
275        if previous_inflight == 0 {
276            self.backlog_started_unix_ms.store(now, Ordering::SeqCst);
277        }
278    }
279
280    /// Record that the commit actor has disconnected so the liveness classifier
281    /// reports the writer `Dead`. The classifier keys the dead verdict on the
282    /// "unavailable" marker in `last_error`, so an actor observed gone at enqueue
283    /// time must set it: otherwise the next liveness sample can still read
284    /// `Healthy` and admit a tool side effect whose receipt can never be
285    /// persisted.
286    fn note_writer_unavailable(&self) {
287        if let Ok(mut last_error) = self.last_error.lock() {
288            *last_error = Some("sqlite receipt commit actor is unavailable".to_string());
289        }
290    }
291
292    /// Count a command as occupying a channel slot. Called before every
293    /// `try_send`; a rejected send undoes it with `note_channel_send_rejected`,
294    /// and the actor calls `note_channel_dequeue` once when it pulls the
295    /// command. Incrementing before the send (not after) keeps the actor from
296    /// dequeuing and decrementing before this increment lands, which would leak
297    /// the count, mirroring the `inflight` accounting.
298    fn note_channel_send(&self) {
299        self.queue_depth.fetch_add(1, Ordering::SeqCst);
300    }
301
302    fn note_channel_send_rejected(&self) {
303        atomic_saturating_sub(&self.queue_depth, 1);
304    }
305
306    fn note_channel_dequeue(&self) {
307        atomic_saturating_sub(&self.queue_depth, 1);
308    }
309
310    fn note_timeout(&self, message: &str) {
311        if let Ok(mut last_error) = self.last_error.lock() {
312            if last_error
313                .as_deref()
314                .is_none_or(is_receipt_writer_timeout_marker)
315            {
316                *last_error = Some(message.to_string());
317            }
318        }
319    }
320
321    fn clear_timeout_error_if_drained(&self) {
322        if let Ok(mut last_error) = self.last_error.lock() {
323            if self.timed_out_inflight.load(Ordering::SeqCst) == 0
324                && last_error
325                    .as_deref()
326                    .is_some_and(is_receipt_writer_timeout_marker)
327            {
328                *last_error = None;
329            }
330        }
331    }
332
333    /// Publish whether the writer head is poisoned. Written only by the actor
334    /// thread at every head-state transition (seed, post-write resync, reseed)
335    /// and read by `writer_serving_closed` from other threads.
336    fn set_head_poisoned(&self, poisoned: bool) {
337        self.head_poisoned.store(poisoned, Ordering::SeqCst);
338    }
339
340    fn store_head_snapshot(&self, head: &VerifiedHead) {
341        self.head_checkpoint_seq
342            .store(head.checkpoint_seq(), Ordering::SeqCst);
343        self.head_checkpointed_entry_seq
344            .store(head.checkpointed_entry_seq(), Ordering::SeqCst);
345        self.head_claim_log_count
346            .store(head.claim_log_count, Ordering::SeqCst);
347        self.head_claim_log_max_seq
348            .store(head.claim_log_max_seq, Ordering::SeqCst);
349    }
350}
351
352const WRITER_COMMAND_PENDING: u8 = 0;
353const WRITER_COMMAND_TIMED_OUT: u8 = 1;
354const WRITER_COMMAND_COMPLETED: u8 = 2;
355
356struct WriterCommandCompletion {
357    state: Arc<AtomicU8>,
358    health: Arc<ReceiptCommitWriterHealth>,
359}
360
361struct WriterCommandTimeout {
362    state: Arc<AtomicU8>,
363    health: Arc<ReceiptCommitWriterHealth>,
364}
365
366fn writer_command_tracker(
367    health: &Arc<ReceiptCommitWriterHealth>,
368) -> (WriterCommandCompletion, WriterCommandTimeout) {
369    let state = Arc::new(AtomicU8::new(WRITER_COMMAND_PENDING));
370    (
371        WriterCommandCompletion {
372            state: Arc::clone(&state),
373            health: Arc::clone(health),
374        },
375        WriterCommandTimeout {
376            state,
377            health: Arc::clone(health),
378        },
379    )
380}
381
382impl WriterCommandCompletion {
383    fn complete(&mut self) {
384        if self.state.swap(WRITER_COMMAND_COMPLETED, Ordering::SeqCst) == WRITER_COMMAND_TIMED_OUT {
385            atomic_saturating_sub(&self.health.timed_out_inflight, 1);
386            self.health.clear_timeout_error_if_drained();
387        }
388    }
389}
390
391impl Drop for WriterCommandCompletion {
392    fn drop(&mut self) {
393        self.complete();
394    }
395}
396
397impl WriterCommandTimeout {
398    /// Register a caller-visible timeout only while the actor still owns this
399    /// command. Increment-before-CAS prevents actor completion from racing past
400    /// the outstanding count; a completion that won first undoes the increment.
401    fn note_timeout(&self, message: &str) {
402        self.health.timed_out_total.fetch_add(1, Ordering::SeqCst);
403        self.health
404            .timed_out_inflight
405            .fetch_add(1, Ordering::SeqCst);
406        if self
407            .state
408            .compare_exchange(
409                WRITER_COMMAND_PENDING,
410                WRITER_COMMAND_TIMED_OUT,
411                Ordering::SeqCst,
412                Ordering::SeqCst,
413            )
414            .is_err()
415        {
416            atomic_saturating_sub(&self.health.timed_out_inflight, 1);
417            self.health.clear_timeout_error_if_drained();
418            return;
419        }
420
421        self.health.note_timeout(message);
422        // Completion may have won immediately after the CAS and cleared the
423        // count before the descriptive marker was published.
424        if self.state.load(Ordering::SeqCst) == WRITER_COMMAND_COMPLETED {
425            self.health.clear_timeout_error_if_drained();
426        }
427    }
428}
429
430struct ReceiptCommitRequest {
431    receipt: ChioReceipt,
432    raw_json: String,
433    /// When true, `ensure_receipt_lineage_statement_for_receipt_id_tx` runs
434    /// inside the same batch transaction as the receipt insert (trait-append
435    /// paths). Canonical inherent paths pass `false`.
436    ensure_lineage: bool,
437    response: mpsc::SyncSender<Result<u64, ReceiptStoreError>>,
438}
439
440/// Deferred response sender for a `Write` job. The
441/// actor invokes it AFTER `resync_head_after_write` so a committed write whose
442/// head resync then fails returns the resync error instead of a stale `Ok`.
443/// Called with `Ok(())` when resync succeeded (or never ran) to send the job's
444/// own outcome, or `Err(resync_error)` to override a committed job's `Ok` with
445/// the resync failure.
446///
447/// Returns `true` when the job's FINAL outcome (after any resync override) was
448/// `Ok`, so the actor can reconcile `committed_total` / `failed_total` for
449/// writer-routed receipts. This responder is the
450/// only place that knows the resync-adjusted outcome, so it reports the signal
451/// out of band (the actual `Result` still travels to the caller's channel).
452type WriterResponder = Box<dyn FnOnce(Result<(), ReceiptStoreError>) -> bool + Send + 'static>;
453
454/// A single-writer job. Runs the caller's closure on the writer connection and
455/// returns a [`WriterResponder`] so the ACTOR controls when the caller's result
456/// is sent: the response is withheld until the post-write head resync outcome
457/// is known.
458type WriterClosure = Box<
459    dyn FnOnce(Result<&mut SqliteStoreConnection, ReceiptStoreError>) -> WriterResponder
460        + Send
461        + 'static,
462>;
463
464enum ReceiptCommitCommand {
465    Append(Box<ReceiptCommitRequest>),
466    AppendWithTimeout {
467        request: Box<ReceiptCommitRequest>,
468        completion: WriterCommandCompletion,
469    },
470    Flush(mpsc::SyncSender<Result<(), ReceiptStoreError>>),
471    /// Generic single-writer job. Runs on the writer connection after any
472    /// in-flight append batch has committed. The closure receives `Err` when
473    /// the actor cannot provide a healthy writer connection (fail-closed).
474    ///
475    /// `appends_receipts` is true for jobs that insert tool/child receipt rows
476    /// (child receipts, authorization-consuming appends), which populate
477    /// `claim_receipt_log_entries` via the projection triggers. On a
478    /// non-incremental (full-verification) store, the pre-write check runs the
479    /// full claim-log validation for these. Metadata-only Write jobs leave it
480    /// false and skip the O(N) scan.
481    Write {
482        job: WriterClosure,
483        appends_receipts: bool,
484        fail_closed_on_error: bool,
485        completion: WriterCommandCompletion,
486    },
487    /// Rerun the full verification on the writer connection and, on success,
488    /// adopt the fresh head (clears a poisoned head). Audit-repair path.
489    ReseedHead(mpsc::SyncSender<Result<(), ReceiptStoreError>>),
490    /// Install (or replace) the background checkpoint signer on the actor
491    /// thread. Delivered over the command channel: no shared state, no lock.
492    InstallSigner(BackgroundCheckpointSigner),
493    #[cfg(test)]
494    RestartSupervisor,
495    /// Run a checkpoint-aligned co-archive-and-delete on the writer connection.
496    /// Serialized with appends by the single writer; drains any in-flight
497    /// append batch first. Returns the number of tool-receipt rows archived.
498    Rotate {
499        config: Box<RetentionConfig>,
500        response: mpsc::SyncSender<Result<u64, ReceiptStoreError>>,
501    },
502    /// Recover a store whose claim-log rows survived a source-row delete:
503    /// remove the orphaned projection rows. Runs unconditionally regardless of
504    /// head state, like `ReseedHead`
505    /// (the entire point is to repair a poisoned head), and on success
506    /// reseeds the head so the same store instance is appendable again
507    /// without requiring a fresh open. Returns the number of rows removed.
508    RetentionRepair {
509        archive_path: String,
510        response: mpsc::SyncSender<Result<u64, ReceiptStoreError>>,
511    },
512}
513
514impl ReceiptCommitCommand {
515    fn into_append(
516        self,
517    ) -> Result<(Box<ReceiptCommitRequest>, Option<WriterCommandCompletion>), Self> {
518        match self {
519            Self::Append(request) => Ok((request, None)),
520            Self::AppendWithTimeout {
521                request,
522                completion,
523            } => Ok((request, Some(completion))),
524            other => Err(other),
525        }
526    }
527}
528
529impl ReceiptCommitActor {
530    fn start(pool: Pool<SqliteConnectionManager>, incremental_verification: bool) -> Self {
531        let (sender, receiver) = receipt_commit_channel();
532        let health = Arc::new(ReceiptCommitWriterHealth::default());
533        let actor_health = Arc::clone(&health);
534        let thread_id = Arc::new(OnceLock::new());
535        let actor_thread_id = Arc::clone(&thread_id);
536        let config = SupervisorConfig {
537            name: "chio-receipt-writer",
538            // Durable receipt persistence is on the money path: a degraded writer must
539            // fail evaluations closed rather than execute tools without a receipt.
540            tcb_critical: true,
541            // Any writer fault is immediately operator-visible on this surface.
542            trip_after: 1,
543            max_restarts: 5,
544            base_backoff: Duration::from_millis(100),
545            max_backoff: Duration::from_secs(30),
546        };
547        // The loop body borrows the receiver so it survives a restart with the same
548        // still-open channel; the caller's sender stays valid across restarts. The
549        // pool and health handle are cheap to clone (an Arc bump each) per attempt.
550        let mut checkpoint_signer = None;
551        let supervisor = SupervisedThread::spawn(config, move |_shutdown| {
552            let _ = actor_thread_id.set(thread::current().id());
553            receipt_commit_actor_loop(
554                pool.clone(),
555                &receiver,
556                Arc::clone(&actor_health),
557                incremental_verification,
558                &mut checkpoint_signer,
559            )
560        });
561        let supervisor_health = supervisor.health();
562        Self {
563            sender,
564            health,
565            worker: Arc::new(ReceiptCommitWorker {
566                join: Some(SupervisedReceiptWriter {
567                    supervisor: Some(supervisor),
568                    health: supervisor_health,
569                    thread_id,
570                }),
571            }),
572        }
573    }
574
575    /// Typed error for a commit writer that is no longer serving, carrying the
576    /// supervisor's restart count and last recorded reason so the condition is
577    /// inspectable rather than an opaque pool-error string.
578    fn writer_dead_error(&self) -> ReceiptStoreError {
579        self.worker.writer_dead_error()
580    }
581
582    /// True when durable persistence can no longer be trusted, so the kernel
583    /// pre-dispatch gate must fail closed. This is either the supervised writer
584    /// thread leaving the healthy state, or a poisoned verified head: the thread
585    /// is alive but every append is rejected with a Conflict until an operator
586    /// reseeds, which the thread flag alone cannot see.
587    fn writer_serving_closed(&self) -> bool {
588        self.worker
589            .health()
590            .is_none_or(HealthFlag::is_serving_closed)
591            || self.health.head_poisoned.load(Ordering::SeqCst)
592    }
593
594    /// The supervised writer's severity and cumulative restart count, for the
595    /// health report.
596    fn writer_health_summary(&self) -> (HealthLevel, u64) {
597        match self.worker.health() {
598            Some(health) => {
599                let snapshot = health.snapshot();
600                (snapshot.level, snapshot.restart_total)
601            }
602            None => (HealthLevel::Failed, 0),
603        }
604    }
605
606    fn append(
607        &self,
608        receipt: ChioReceipt,
609        raw_json: String,
610        ensure_lineage: bool,
611    ) -> Result<u64, ReceiptStoreError> {
612        let (response, result) = mpsc::sync_channel(1);
613        let command = ReceiptCommitCommand::Append(Box::new(ReceiptCommitRequest {
614            receipt,
615            raw_json,
616            ensure_lineage,
617            response,
618        }));
619        // Increment `inflight` BEFORE handing the command to the worker. If we
620        // wait until after `try_send`, the worker can dequeue, commit, and run
621        // `atomic_saturating_sub(&health.inflight, n)` (see
622        // `commit_receipt_batch`) before this thread observes the send result.
623        // That race saturates `inflight` to 0 and leaks the increment, leaving
624        // `health.writer.inflight` permanently misreporting drained writes.
625        // The worker decrements unconditionally on dequeue, so the pre-send
626        // increment pairs correctly. Any failure of `try_send` undoes the
627        // speculative increment before returning.
628        let previous_inflight = self.health.inflight.fetch_add(1, Ordering::SeqCst);
629        self.health.note_channel_send();
630        match self.sender.try_send(command) {
631            Ok(()) => {
632                self.health.accepted_total.fetch_add(1, Ordering::SeqCst);
633                self.health.note_accept(previous_inflight);
634            }
635            Err(mpsc::TrySendError::Full(_)) => {
636                atomic_saturating_sub(&self.health.inflight, 1);
637                self.health.note_channel_send_rejected();
638                self.health.saturated_total.fetch_add(1, Ordering::SeqCst);
639                return Err(receipt_actor_saturated_error());
640            }
641            Err(mpsc::TrySendError::Disconnected(_)) => {
642                atomic_saturating_sub(&self.health.inflight, 1);
643                self.health.note_channel_send_rejected();
644                self.health.note_writer_unavailable();
645                return Err(self.writer_dead_error());
646            }
647        }
648        match result.recv() {
649            Ok(result) => result,
650            Err(_) => {
651                atomic_saturating_sub(&self.health.inflight, 1);
652                self.health.failed_total.fetch_add(1, Ordering::SeqCst);
653                self.health.note_writer_unavailable();
654                Err(self.writer_dead_error())
655            }
656        }
657    }
658
659    /// Bounded variant of `append`: identical up to the response wait, which is
660    /// capped at `timeout`. On expiry it does NOT decrement `inflight`. The
661    /// `try_send` succeeded, so the command is still queued or running on the
662    /// worker, which owns `inflight` and decrements it exactly once when it
663    /// drains the batch. Decrementing here too would double-count a slow-but-live
664    /// append and, under concurrency, could drive `inflight` to zero while work
665    /// is still queued, making writer health look drained before the actor
666    /// catches up. The timeout still fails this caller loudly and records this
667    /// specific command as timed out until the actor drains it. Terminal
668    /// committed/failed counters remain actor-owned and are updated exactly once.
669    fn append_with_timeout(
670        &self,
671        receipt: ChioReceipt,
672        raw_json: String,
673        ensure_lineage: bool,
674        timeout: Duration,
675    ) -> Result<u64, ReceiptStoreError> {
676        let (response, result) = mpsc::sync_channel(1);
677        let (completion, timeout_tracker) = writer_command_tracker(&self.health);
678        let command = ReceiptCommitCommand::AppendWithTimeout {
679            request: Box::new(ReceiptCommitRequest {
680                receipt,
681                raw_json,
682                ensure_lineage,
683                response,
684            }),
685            completion,
686        };
687        let previous_inflight = self.health.inflight.fetch_add(1, Ordering::SeqCst);
688        self.health.note_channel_send();
689        match self.sender.try_send(command) {
690            Ok(()) => {
691                self.health.accepted_total.fetch_add(1, Ordering::SeqCst);
692                self.health.note_accept(previous_inflight);
693            }
694            Err(mpsc::TrySendError::Full(_)) => {
695                atomic_saturating_sub(&self.health.inflight, 1);
696                self.health.note_channel_send_rejected();
697                self.health.saturated_total.fetch_add(1, Ordering::SeqCst);
698                return Err(receipt_actor_saturated_error());
699            }
700            Err(mpsc::TrySendError::Disconnected(_)) => {
701                atomic_saturating_sub(&self.health.inflight, 1);
702                self.health.note_channel_send_rejected();
703                self.health.note_writer_unavailable();
704                return Err(receipt_actor_unavailable_error());
705            }
706        }
707        match result.recv_timeout(timeout) {
708            Ok(result) => result,
709            Err(mpsc::RecvTimeoutError::Timeout) => {
710                timeout_tracker.note_timeout(RECEIPT_APPEND_TIMEOUT_MARKER);
711                Err(receipt_actor_append_timeout_error(timeout))
712            }
713            Err(mpsc::RecvTimeoutError::Disconnected) => {
714                atomic_saturating_sub(&self.health.inflight, 1);
715                self.health.failed_total.fetch_add(1, Ordering::SeqCst);
716                if let Ok(mut last_error) = self.health.last_error.lock() {
717                    *last_error = Some("sqlite receipt commit actor is unavailable".to_string());
718                }
719                Err(receipt_actor_unavailable_error())
720            }
721        }
722    }
723
724    fn flush(&self) -> Result<(), ReceiptStoreError> {
725        self.flush_with_receiver(|receiver| {
726            receiver.recv().map_err(|_| self.writer_dead_error())?
727        })
728    }
729
730    fn flush_with_timeout(&self, timeout: Duration) -> Result<(), ReceiptStoreError> {
731        self.flush_with_receiver(|receiver| match receiver.recv_timeout(timeout) {
732            Ok(result) => result,
733            Err(mpsc::RecvTimeoutError::Timeout) => Err(receipt_actor_flush_timeout_error(timeout)),
734            Err(mpsc::RecvTimeoutError::Disconnected) => Err(self.writer_dead_error()),
735        })
736    }
737
738    fn flush_with_receiver(
739        &self,
740        receive: impl FnOnce(
741            mpsc::Receiver<Result<(), ReceiptStoreError>>,
742        ) -> Result<(), ReceiptStoreError>,
743    ) -> Result<(), ReceiptStoreError> {
744        let (response, result) = mpsc::sync_channel(1);
745        self.health.note_channel_send();
746        match self.sender.try_send(ReceiptCommitCommand::Flush(response)) {
747            Ok(()) => {}
748            Err(mpsc::TrySendError::Full(_)) => {
749                self.health.note_channel_send_rejected();
750                self.health.saturated_total.fetch_add(1, Ordering::SeqCst);
751                return Err(receipt_actor_saturated_error());
752            }
753            Err(mpsc::TrySendError::Disconnected(_)) => {
754                self.health.note_channel_send_rejected();
755                self.health.note_writer_unavailable();
756                return Err(self.writer_dead_error());
757            }
758        }
759        receive(result)
760    }
761
762    #[cfg(test)]
763    fn reseed_head(&self) -> Result<(), ReceiptStoreError> {
764        let (response, result) = mpsc::sync_channel(1);
765        match self
766            .sender
767            .try_send(ReceiptCommitCommand::ReseedHead(response))
768        {
769            Ok(()) => {}
770            Err(mpsc::TrySendError::Full(_)) => return Err(receipt_actor_saturated_error()),
771            Err(mpsc::TrySendError::Disconnected(_)) => return Err(self.writer_dead_error()),
772        }
773        result.recv().map_err(|_| self.writer_dead_error())?
774    }
775
776    #[cfg(test)]
777    fn install_signer(&self, signer: BackgroundCheckpointSigner) -> Result<(), ReceiptStoreError> {
778        match self
779            .sender
780            .try_send(ReceiptCommitCommand::InstallSigner(signer))
781        {
782            Ok(()) => Ok(()),
783            Err(mpsc::TrySendError::Full(_)) => Err(receipt_actor_saturated_error()),
784            Err(mpsc::TrySendError::Disconnected(_)) => Err(self.writer_dead_error()),
785        }
786    }
787
788    /// Wall-clock (unix-ms) at which the current unserviced backlog began, or
789    /// `None` when no backlog has started. The liveness classifier anchors its
790    /// stall clock here so idle-then-fresh work is not mistaken for a wedge.
791    fn backlog_started_unix_ms(&self) -> Option<u64> {
792        match self.health.backlog_started_unix_ms.load(Ordering::SeqCst) {
793            0 => None,
794            value => Some(value),
795        }
796    }
797
798    fn writer_counters(&self) -> ReceiptWriterCounters {
799        let last_commit_unix_ms = match self.health.last_commit_unix_ms.load(Ordering::SeqCst) {
800            0 => None,
801            value => Some(value),
802        };
803        let first_accept_unix_ms = match self.health.first_accept_unix_ms.load(Ordering::SeqCst) {
804            0 => None,
805            value => Some(value),
806        };
807        let last_error = self
808            .health
809            .last_error
810            .lock()
811            .map(|error| error.clone())
812            .unwrap_or_else(|_| Some("receipt commit writer health lock poisoned".to_string()));
813        ReceiptWriterCounters {
814            accepted_total: self.health.accepted_total.load(Ordering::SeqCst),
815            committed_total: self.health.committed_total.load(Ordering::SeqCst),
816            failed_total: self.health.failed_total.load(Ordering::SeqCst),
817            saturated_total: self.health.saturated_total.load(Ordering::SeqCst),
818            inflight: self.health.inflight.load(Ordering::SeqCst),
819            timed_out_total: self.health.timed_out_total.load(Ordering::SeqCst),
820            timed_out_inflight: self.health.timed_out_inflight.load(Ordering::SeqCst),
821            queue_depth: self.health.queue_depth.load(Ordering::SeqCst),
822            last_commit_unix_ms,
823            first_accept_unix_ms,
824            last_error,
825        }
826    }
827}
828
829impl Drop for SupervisedReceiptWriter {
830    fn drop(&mut self) {
831        let Some(supervisor) = self.supervisor.take() else {
832            return;
833        };
834        if self.thread_id.get() == Some(&thread::current().id()) {
835            let _ = thread::Builder::new()
836                .name("chio-receipt-writer-reaper".to_string())
837                .spawn(move || {
838                    let _ = supervisor.join();
839                });
840        } else {
841            let _ = supervisor.join();
842        }
843    }
844}
845
846/// Cloneable handle for running arbitrary write transactions on the single
847/// writer connection. Closures MUST NOT call back into `SqliteReceiptStore`
848/// methods that enqueue writer commands (that would deadlock the actor on
849/// itself); they receive the writer connection directly instead.
850pub(crate) struct WriterHandle {
851    sender: mpsc::SyncSender<ReceiptCommitCommand>,
852    health: Arc<ReceiptCommitWriterHealth>,
853    worker: Arc<ReceiptCommitWorker>,
854    settlement_store_binding: Option<chio_settle::SettlementStoreBinding>,
855}
856
857impl WriterHandle {
858    pub(crate) const fn settlement_store_binding(
859        &self,
860    ) -> Option<chio_settle::SettlementStoreBinding> {
861        self.settlement_store_binding
862    }
863
864    /// Run one write job on the single writer connection and return its
865    /// typed result. Fail-closed on saturation or a dead writer. Use for
866    /// metadata-only writes (capability, liability, underwriting, IOU,
867    /// session anchors) that do not insert receipt rows.
868    pub(crate) fn run_write<T, F>(&self, job: F) -> Result<T, ReceiptStoreError>
869    where
870        F: FnOnce(&mut SqliteStoreConnection) -> Result<T, ReceiptStoreError> + Send + 'static,
871        T: Send + 'static,
872    {
873        self.run_write_kind(job, false, false)
874    }
875
876    /// Run one receipt-appending write job (child receipts,
877    /// authorization-consuming appends). These insert `claim_receipt_log_entries`
878    /// rows via the projection triggers, so the non-incremental fallback
879    /// pre-check runs the full claim-log validation (fail-closed).
880    pub(crate) fn run_write_receipt<T, F>(&self, job: F) -> Result<T, ReceiptStoreError>
881    where
882        F: FnOnce(&mut SqliteStoreConnection) -> Result<T, ReceiptStoreError> + Send + 'static,
883        T: Send + 'static,
884    {
885        self.run_write_kind(job, true, false)
886    }
887
888    fn run_critical_receipt_write<T, F>(&self, job: F) -> Result<T, ReceiptStoreError>
889    where
890        F: FnOnce(&mut SqliteStoreConnection) -> Result<T, ReceiptStoreError> + Send + 'static,
891        T: Send + 'static,
892    {
893        self.run_write_kind(job, true, true)
894    }
895
896    /// Bounded variant of [`run_write_receipt`]: identical up to the response
897    /// wait, which is capped at `timeout`. On expiry it fails this caller loudly
898    /// without decrementing `inflight` (the actor still owns the queued job and
899    /// decrements it exactly once when it drains). A genuinely wedged writer
900    /// therefore keeps `inflight` elevated, which is the signal the liveness
901    /// probe reads, and this caller does not pin the kernel-wide receipt write
902    /// lock waiting on it.
903    pub(crate) fn run_write_receipt_with_timeout<T, F>(
904        &self,
905        job: F,
906        timeout: Duration,
907    ) -> Result<T, ReceiptStoreError>
908    where
909        F: FnOnce(&mut SqliteStoreConnection) -> Result<T, ReceiptStoreError> + Send + 'static,
910        T: Send + 'static,
911    {
912        self.run_write_kind_with_timeout(job, true, false, timeout)
913    }
914
915    /// Bounded variant of a critical receipt write. Transaction failures retain
916    /// the critical poisoning semantics of [`run_critical_receipt_write`], while
917    /// the caller's response wait is capped at `timeout`.
918    fn run_critical_receipt_write_with_timeout<T, F>(
919        &self,
920        job: F,
921        timeout: Duration,
922    ) -> Result<T, ReceiptStoreError>
923    where
924        F: FnOnce(&mut SqliteStoreConnection) -> Result<T, ReceiptStoreError> + Send + 'static,
925        T: Send + 'static,
926    {
927        self.run_write_kind_with_timeout(job, true, true, timeout)
928    }
929
930    /// Bounded variant of [`run_write`]: a metadata-only write (capability
931    /// lineage, session anchors) whose response wait is capped at `timeout`.
932    /// Fail-closed and inflight-preserving on expiry exactly like
933    /// [`run_write_receipt_with_timeout`], so a hot-path metadata write cannot
934    /// hang the caller on a wedged writer.
935    pub(crate) fn run_write_with_timeout<T, F>(
936        &self,
937        job: F,
938        timeout: Duration,
939    ) -> Result<T, ReceiptStoreError>
940    where
941        F: FnOnce(&mut SqliteStoreConnection) -> Result<T, ReceiptStoreError> + Send + 'static,
942        T: Send + 'static,
943    {
944        self.run_write_kind_with_timeout(job, false, false, timeout)
945    }
946
947    fn run_write_kind_with_timeout<T, F>(
948        &self,
949        job: F,
950        appends_receipts: bool,
951        fail_closed_on_error: bool,
952        timeout: Duration,
953    ) -> Result<T, ReceiptStoreError>
954    where
955        F: FnOnce(&mut SqliteStoreConnection) -> Result<T, ReceiptStoreError> + Send + 'static,
956        T: Send + 'static,
957    {
958        let (result, timeout_tracker) =
959            self.enqueue_write_job(job, appends_receipts, fail_closed_on_error)?;
960        match result.recv_timeout(timeout) {
961            Ok(outcome) => outcome,
962            Err(mpsc::RecvTimeoutError::Timeout) => {
963                timeout_tracker.note_timeout(RECEIPT_WRITE_TIMEOUT_MARKER);
964                Err(receipt_actor_write_timeout_error(timeout))
965            }
966            Err(mpsc::RecvTimeoutError::Disconnected) => {
967                atomic_saturating_sub(&self.health.inflight, 1);
968                self.health.failed_total.fetch_add(1, Ordering::SeqCst);
969                // Record the writer death so the next liveness sample reports the
970                // writer Dead. The classifier keys the Dead verdict on the
971                // "unavailable" marker, so without setting it a disconnected
972                // writer with `inflight` compensated and `failed_total` matching
973                // `accepted_total` would sample as Healthy and admit a tool side
974                // effect before receipt persistence fails.
975                self.health.note_writer_unavailable();
976                Err(self.worker.writer_dead_error())
977            }
978        }
979    }
980
981    fn run_write_kind<T, F>(
982        &self,
983        job: F,
984        appends_receipts: bool,
985        fail_closed_on_error: bool,
986    ) -> Result<T, ReceiptStoreError>
987    where
988        F: FnOnce(&mut SqliteStoreConnection) -> Result<T, ReceiptStoreError> + Send + 'static,
989        T: Send + 'static,
990    {
991        let (result, _timeout_tracker) =
992            self.enqueue_write_job(job, appends_receipts, fail_closed_on_error)?;
993        match result.recv() {
994            Ok(outcome) => outcome,
995            Err(_) => {
996                // Accepted-then-lost: the actor took the command but exited
997                // before delivering a response (actor death; job panics are
998                // caught and answered above). The job-completion decrement (the
999                // `WriterInflightGuard` in the actor's `Write` arm) may never
1000                // have run - the command could have been
1001                // lost while still queued, before the arm was entered - so undo
1002                // the speculative pre-send increment and record the failure,
1003                // mirroring the append path's recv-Err handling, so
1004                // writer.inflight does not report a permanently-stuck write. If
1005                // the actor instead died mid-arm and the guard already fired,
1006                // `atomic_saturating_sub` keeps this compensating release from
1007                // underflowing.
1008                atomic_saturating_sub(&self.health.inflight, 1);
1009                self.health.failed_total.fetch_add(1, Ordering::SeqCst);
1010                // Record the writer death so the next liveness sample reports the
1011                // writer Dead rather than Healthy (see the bounded-write arm).
1012                self.health.note_writer_unavailable();
1013                Err(self.worker.writer_dead_error())
1014            }
1015        }
1016    }
1017
1018    /// Box a write job, speculatively account it as in-flight, and enqueue it on
1019    /// the commit actor. Returns the response receiver on a successful enqueue;
1020    /// the caller decides how long to wait. A `Full`/`Disconnected` send undoes
1021    /// the speculative `inflight` increment before returning (fail-closed).
1022    fn enqueue_write_job<T, F>(
1023        &self,
1024        job: F,
1025        appends_receipts: bool,
1026        fail_closed_on_error: bool,
1027    ) -> Result<
1028        (
1029            mpsc::Receiver<Result<T, ReceiptStoreError>>,
1030            WriterCommandTimeout,
1031        ),
1032        ReceiptStoreError,
1033    >
1034    where
1035        F: FnOnce(&mut SqliteStoreConnection) -> Result<T, ReceiptStoreError> + Send + 'static,
1036        T: Send + 'static,
1037    {
1038        let (response, result) = mpsc::sync_channel(1);
1039        let (completion, timeout_tracker) = writer_command_tracker(&self.health);
1040        let writer_health = Arc::clone(&self.health);
1041        let boxed: WriterClosure = Box::new(move |connection| {
1042            let outcome = match connection {
1043                // Panic isolation: `job` is one of the many rerouted write
1044                // families (lineage, liability,
1045                // underwriting, reconciliation, capability, federated, IOU,
1046                // checkpoint, reseed) now running on the single writer
1047                // thread. `AssertUnwindSafe` is sound here because the
1048                // writer actor re-acquires a fresh connection from the pool
1049                // for every command (see `handle_non_append_command`); a
1050                // caught panic fails only THIS job (fail-closed) and no
1051                // state from the panicking closure is reused afterward.
1052                Ok(connection) => {
1053                    std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| job(connection)))
1054                        .unwrap_or_else(|payload| Err(receipt_writer_job_panic_error(&payload)))
1055                }
1056                Err(error) => Err(error),
1057            };
1058            // Defer the send: the actor calls this
1059            // responder with the post-write head resync outcome. A resync
1060            // failure overrides a committed job's `Ok` with the resync error; a
1061            // job that already failed keeps its own error.
1062            let responder: WriterResponder =
1063                Box::new(move |resync: Result<(), ReceiptStoreError>| {
1064                    let final_outcome = match (outcome, resync) {
1065                        (job_outcome, Ok(())) => job_outcome,
1066                        (Err(job_error), Err(_)) => Err(job_error),
1067                        (Ok(_), Err(resync_error)) => Err(resync_error),
1068                    };
1069                    if fail_closed_on_error {
1070                        if let Err(error) = &final_outcome {
1071                            if let Ok(mut last_error) = writer_health.last_error.lock() {
1072                                *last_error = Some(error.to_string());
1073                            }
1074                            writer_health
1075                                .critical_write_poisoned
1076                                .store(true, Ordering::SeqCst);
1077                            writer_health.set_head_poisoned(true);
1078                        }
1079                    }
1080                    // Report the resync-adjusted outcome to the actor so it can
1081                    // reconcile committed/failed for this writer-routed job,
1082                    // then send the caller's result.
1083                    let committed = final_outcome.is_ok();
1084                    let _ = response.send(final_outcome);
1085                    committed
1086                });
1087            responder
1088        });
1089        // Pre-send increment: same race-avoidance invariant as
1090        // `ReceiptCommitActor::append` (see the comment at the `inflight`
1091        // increment in `append`). The actor decrements unconditionally on
1092        // dequeue; any send failure undoes the speculative increment.
1093        let previous_inflight = self.health.inflight.fetch_add(1, Ordering::SeqCst);
1094        self.health.note_channel_send();
1095        match self.sender.try_send(ReceiptCommitCommand::Write {
1096            job: boxed,
1097            appends_receipts,
1098            fail_closed_on_error,
1099            completion,
1100        }) {
1101            Ok(()) => {
1102                // Count writer-routed writes in health. A successful enqueue
1103                // mirrors the Append path's
1104                // `accepted_total` bump (see `append`): child receipts and
1105                // authorization-consuming receipts now go through
1106                // `run_write_receipt`, so without this a store dominated by
1107                // writer-routed receipts would advance the log while
1108                // `receipt_store_health().writer.accepted_total` stayed at zero.
1109                // O(1), fail-closed unchanged (a Full/Disconnected send still
1110                // returns before counting).
1111                self.health.accepted_total.fetch_add(1, Ordering::SeqCst);
1112                self.health.note_accept(previous_inflight);
1113            }
1114            Err(mpsc::TrySendError::Full(_)) => {
1115                atomic_saturating_sub(&self.health.inflight, 1);
1116                self.health.note_channel_send_rejected();
1117                self.health.saturated_total.fetch_add(1, Ordering::SeqCst);
1118                return Err(receipt_actor_saturated_error());
1119            }
1120            Err(mpsc::TrySendError::Disconnected(_)) => {
1121                atomic_saturating_sub(&self.health.inflight, 1);
1122                self.health.note_channel_send_rejected();
1123                self.health.note_writer_unavailable();
1124                return Err(self.worker.writer_dead_error());
1125            }
1126        }
1127        Ok((result, timeout_tracker))
1128    }
1129}
1130
1131fn receipt_commit_channel() -> (
1132    mpsc::SyncSender<ReceiptCommitCommand>,
1133    mpsc::Receiver<ReceiptCommitCommand>,
1134) {
1135    mpsc::sync_channel(RECEIPT_COMMIT_ACTOR_CHANNEL_CAPACITY)
1136}
1137
1138fn receipt_actor_unavailable_error() -> ReceiptStoreError {
1139    ReceiptStoreError::Pool("sqlite receipt commit actor is unavailable".to_string())
1140}
1141
1142fn receipt_actor_saturated_error() -> ReceiptStoreError {
1143    ReceiptStoreError::Pool("sqlite receipt commit queue saturated".to_string())
1144}
1145
1146fn receipt_actor_flush_timeout_error(timeout: Duration) -> ReceiptStoreError {
1147    ReceiptStoreError::Timeout {
1148        operation: "sqlite receipt commit flush".to_string(),
1149        timeout_ms: timeout.as_millis().min(u128::from(u64::MAX)) as u64,
1150    }
1151}
1152
1153fn receipt_actor_append_timeout_error(timeout: Duration) -> ReceiptStoreError {
1154    ReceiptStoreError::Timeout {
1155        operation: "sqlite receipt commit append".to_string(),
1156        timeout_ms: timeout.as_millis().min(u128::from(u64::MAX)) as u64,
1157    }
1158}
1159
1160fn receipt_actor_write_timeout_error(timeout: Duration) -> ReceiptStoreError {
1161    ReceiptStoreError::Timeout {
1162        operation: "sqlite receipt commit write".to_string(),
1163        timeout_ms: timeout.as_millis().min(u128::from(u64::MAX)) as u64,
1164    }
1165}
1166
1167/// Last verified position of the writer connection's view of the receipt
1168/// chain, owned exclusively by the commit-actor thread.
1169enum WriterHeadState {
1170    // Boxed: `VerifiedHead` embeds an `Option<KernelCheckpoint>`, which makes
1171    // this variant far larger than `Poisoned(String)` (clippy::large_enum_variant).
1172    Verified(Box<VerifiedHead>),
1173    /// Seeding or resync failed: every write is rejected with Conflict until
1174    /// `chio receipt audit --repair` reseeds (fail-closed).
1175    Poisoned(String),
1176}
1177
1178fn poisoned_head_error(message: &str) -> ReceiptStoreError {
1179    ReceiptStoreError::Conflict(format!(
1180        "receipt store verified head is unavailable ({message}); run `chio receipt audit --repair`"
1181    ))
1182}
1183
1184fn poison_head_from_writer_error(
1185    head_state: &mut WriterHeadState,
1186    health: &ReceiptCommitWriterHealth,
1187) {
1188    health.set_head_poisoned(true);
1189    *head_state = WriterHeadState::Poisoned(critical_writer_error_message(health));
1190}
1191
1192fn critical_writer_error_message(health: &ReceiptCommitWriterHealth) -> String {
1193    health
1194        .last_error
1195        .lock()
1196        .map(|error| error.clone())
1197        .unwrap_or_else(|poisoned| poisoned.into_inner().clone())
1198        .unwrap_or_else(|| "critical receipt writer job failed".to_string())
1199}
1200
1201fn receipt_commit_actor_loop(
1202    pool: Pool<SqliteConnectionManager>,
1203    receiver: &mpsc::Receiver<ReceiptCommitCommand>,
1204    health: Arc<ReceiptCommitWriterHealth>,
1205    incremental_verification: bool,
1206    checkpoint_signer: &mut Option<BackgroundCheckpointSigner>,
1207) -> SupervisedOutcome {
1208    let mut head_state = match pool
1209        .get()
1210        .map_err(|error| ReceiptStoreError::Pool(error.to_string()))
1211        .and_then(|connection| {
1212            if incremental_verification {
1213                seed_verified_head(&connection)
1214            } else {
1215                seed_head_snapshot(&connection)
1216            }
1217        }) {
1218        Ok(head) if health.critical_write_poisoned.load(Ordering::SeqCst) => {
1219            health.store_head_snapshot(&head);
1220            health.set_head_poisoned(true);
1221            WriterHeadState::Poisoned(critical_writer_error_message(&health))
1222        }
1223        Ok(head) => {
1224            health.store_head_snapshot(&head);
1225            // Seeding is authoritative: clear any poison a prior thread run (a
1226            // panic-then-restart of the supervised writer) may have published.
1227            health.set_head_poisoned(false);
1228            WriterHeadState::Verified(Box::new(head))
1229        }
1230        Err(error) => {
1231            if let Ok(mut last_error) = health.last_error.lock() {
1232                *last_error = Some(error.to_string());
1233            }
1234            health.set_head_poisoned(true);
1235            WriterHeadState::Poisoned(error.to_string())
1236        }
1237    };
1238
1239    let mut pending_flush_error: Option<ReceiptStoreError> = None;
1240    while let Ok(command) = receiver.recv() {
1241        // The command has left the channel; free its slot for the saturation
1242        // gate. Every command exits the channel through this recv or the batch
1243        // drain below, so both decrement exactly once per command.
1244        health.note_channel_dequeue();
1245        match command.into_append() {
1246            Ok((request, completion)) => {
1247                let mut requests = vec![*request];
1248                let mut completions: Vec<_> = completion.into_iter().collect();
1249                let mut flushes = Vec::new();
1250                let mut deferred: Option<ReceiptCommitCommand> = None;
1251                while requests.len() < RECEIPT_GROUP_COMMIT_MAX_BATCH {
1252                    let next = receiver.recv_timeout(RECEIPT_GROUP_COMMIT_FLUSH_DELAY);
1253                    if next.is_ok() {
1254                        health.note_channel_dequeue();
1255                    }
1256                    match next {
1257                        Ok(command) => match command.into_append() {
1258                            Ok((request, completion)) => {
1259                                requests.push(*request);
1260                                completions.extend(completion);
1261                            }
1262                            Err(ReceiptCommitCommand::Flush(response)) => {
1263                                flushes.push(response);
1264                                break;
1265                            }
1266                            Err(other) => {
1267                                // Non-append commands (Write, InstallSigner,
1268                                // ReseedHead) execute strictly after the batch
1269                                // they interrupted commits.
1270                                deferred = Some(other);
1271                                break;
1272                            }
1273                        },
1274                        Err(mpsc::RecvTimeoutError::Timeout) => break,
1275                        Err(mpsc::RecvTimeoutError::Disconnected) => break,
1276                    }
1277                }
1278                // Panic isolation: `commit_receipt_batch` runs on the single
1279                // writer thread. A
1280                // panic anywhere inside it (the append transaction, the
1281                // lineage fold) must fail THIS batch, not kill the thread.
1282                // Clone the response channels before handing `requests` /
1283                // `flushes` to the panicking call: if it unwinds, those
1284                // values are dropped mid-function and the only way left to
1285                // answer every caller is through these pre-cloned senders.
1286                let request_responses: Vec<_> = requests
1287                    .iter()
1288                    .map(|request| request.response.clone())
1289                    .collect();
1290                // The co-drained Flush waiters are NOT passed into
1291                // `commit_receipt_batch`; they are released below, AFTER the
1292                // checkpoint build, so a flush is a genuine checkpoint
1293                // barrier. Keeping them in the loop lets the
1294                // success and panic paths fan them out at one point, and
1295                // because they are not moved into the panicking call they
1296                // survive an unwind untouched.
1297                pending_flush_error =
1298                    match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
1299                        commit_receipt_batch_with_completions(
1300                            &pool,
1301                            &mut head_state,
1302                            incremental_verification,
1303                            requests,
1304                            &health,
1305                            completions,
1306                        )
1307                    })) {
1308                        Ok(flush_error) => flush_error,
1309                        Err(payload) => {
1310                            // A panic inside the append or lineage commit is at
1311                            // least as serious as the store-wide append faults
1312                            // `commit_receipt_batch` already poisons on: the
1313                            // batch's durable position can no longer be trusted.
1314                            // Poison the head so the pre-dispatch gate fails closed
1315                            // rather than admitting more tools whose receipts may
1316                            // not persist, until an operator reseeds.
1317                            let panic_error = receipt_writer_job_panic_error(&payload);
1318                            health.set_head_poisoned(true);
1319                            head_state = WriterHeadState::Poisoned(panic_error.to_string());
1320                            Some(fan_out_batch_panic_error(
1321                                &health,
1322                                request_responses,
1323                                panic_error,
1324                            ))
1325                        }
1326                    };
1327                // Checkpoint construction runs AFTER commit_receipt_batch has
1328                // already sent every APPEND durability response, so ADR-0013
1329                // append latency is not extended by checkpoint building; but it
1330                // runs BEFORE the co-drained Flush waiters are released, so a
1331                // flush cannot return until the owed checkpoints for the drained
1332                // appends are built (the flush-as-checkpoint barrier). A
1333                // build failure is recorded via `last_error` and does not poison
1334                // the head, and is surfaced to the co-drained Flush waiters of
1335                // THIS batch via `flush_barrier_error`. It is deliberately NOT
1336                // written back into `pending_flush_error`: that keeps the build
1337                // error scoped to this batch's barrier and preserves the
1338                // established contract of a later STANDALONE flush (which
1339                // reflects append durability; background-build health is already
1340                // surfaced through `last_error`/`receipt_store_health`).
1341                let mut flush_barrier_error = pending_flush_error
1342                    .as_ref()
1343                    .map(receipt_store_error_snapshot);
1344                if pending_flush_error.is_none() {
1345                    if let WriterHeadState::Verified(head) = &mut head_state {
1346                        if let Some(error) = build_due_checkpoints_and_record(
1347                            &pool,
1348                            head,
1349                            checkpoint_signer,
1350                            &health,
1351                        ) {
1352                            flush_barrier_error = Some(error);
1353                        }
1354                    }
1355                }
1356                // Release the co-drained Flush waiters now that owed checkpoints
1357                // are built (the checkpoint barrier). An append error or a
1358                // checkpoint-build failure reaches them as an Err; otherwise Ok.
1359                for response in flushes {
1360                    let result = match &flush_barrier_error {
1361                        Some(error) => Err(receipt_store_error_snapshot(error)),
1362                        None => Ok(()),
1363                    };
1364                    let _ = response.send(result);
1365                }
1366                if let Some(command) = deferred {
1367                    if let Some(outcome) = handle_non_append_command(
1368                        &pool,
1369                        &mut head_state,
1370                        incremental_verification,
1371                        &health,
1372                        checkpoint_signer,
1373                        &mut pending_flush_error,
1374                        command,
1375                    ) {
1376                        return outcome;
1377                    }
1378                }
1379            }
1380            Err(ReceiptCommitCommand::Flush(response)) => {
1381                let result = match &pending_flush_error {
1382                    Some(error) => Err(receipt_store_error_snapshot(error)),
1383                    None => Ok(()),
1384                };
1385                let _ = response.send(result);
1386            }
1387            Err(other) => {
1388                if let Some(outcome) = handle_non_append_command(
1389                    &pool,
1390                    &mut head_state,
1391                    incremental_verification,
1392                    &health,
1393                    checkpoint_signer,
1394                    &mut pending_flush_error,
1395                    other,
1396                ) {
1397                    return outcome;
1398                }
1399            }
1400        }
1401    }
1402    SupervisedOutcome::Shutdown
1403}
1404
1405fn handle_non_append_command(
1406    pool: &Pool<SqliteConnectionManager>,
1407    head_state: &mut WriterHeadState,
1408    incremental_verification: bool,
1409    health: &ReceiptCommitWriterHealth,
1410    checkpoint_signer: &mut Option<BackgroundCheckpointSigner>,
1411    pending_flush_error: &mut Option<ReceiptStoreError>,
1412    command: ReceiptCommitCommand,
1413) -> Option<SupervisedOutcome> {
1414    match command {
1415        ReceiptCommitCommand::Write {
1416            job,
1417            appends_receipts,
1418            fail_closed_on_error,
1419            mut completion,
1420        } => {
1421            // Hold the writer `inflight` count for the DURATION of this Write
1422            // job rather than releasing it immediately on dequeue, so a health
1423            // poll during a slow or stuck liability/checkpoint write reports
1424            // `inflight > 0`. The pre-send increment in
1425            // `WriterHandle::run_write_kind` is adopted by this RAII guard.
1426            //
1427            // The guard is released (`drop`) IMMEDIATELY BEFORE each
1428            // `respond(...)` on every exit path, so a caller that observes its
1429            // own response never sees itself still counted inflight. This
1430            // mirrors the Append path, which decrements in `commit_receipt_batch`
1431            // BEFORE fanning out its responses. The decrement stays deferred
1432            // until each respond, so inflight remains up through the job body and
1433            // the head resync (the response itself is deferred until then). The
1434            // guard's Drop still backstops any exit that panics before a respond
1435            // runs; `atomic_saturating_sub` keeps a rare overlap with the
1436            // caller's recv-Err compensation (actor-thread death) from
1437            // underflowing.
1438            let inflight_guard = WriterInflightGuard::new(&health.inflight);
1439            let mut connection = match pool.get() {
1440                Ok(connection) => connection,
1441                Err(error) => {
1442                    // No write ran (no connection), so there is no resync to
1443                    // gate on: send the pool error now (`Ok(())` = nothing to
1444                    // override). Count the failed outcome.
1445                    let respond = job(Err(ReceiptStoreError::Pool(error.to_string())));
1446                    // Decrement before the response reaches the caller.
1447                    drop(inflight_guard);
1448                    let committed = respond(Ok(()));
1449                    record_write_job_outcome(health, committed);
1450                    completion.complete();
1451                    if fail_closed_on_error && !committed {
1452                        poison_head_from_writer_error(head_state, health);
1453                    }
1454                    return None;
1455                }
1456            };
1457            match head_state {
1458                WriterHeadState::Poisoned(message) => {
1459                    let respond = job(Err(poisoned_head_error(message)));
1460                    // Decrement before the response reaches the caller.
1461                    drop(inflight_guard);
1462                    let committed = respond(Ok(()));
1463                    record_write_job_outcome(health, committed);
1464                    completion.complete();
1465                }
1466                WriterHeadState::Verified(head) => {
1467                    // Pre-check (fail-closed): same predecessor check the
1468                    // append path runs, so writer-routed appends (child
1469                    // receipts, consuming auth) are equally protected. On the
1470                    // non-incremental (full-verification) fallback, a
1471                    // receipt-appending job also runs the full claim-log
1472                    // validation, so uncheckpointed projection drift is caught
1473                    // before the
1474                    // write commits. Metadata-only Write jobs skip the O(N)
1475                    // scan.
1476                    let pre_check = if incremental_verification {
1477                        // Verify the checkpoint head, THEN validate the adopted
1478                        // claim-log delta before the job commits: a
1479                        // receipt-appending writer job must reject a
1480                        // stale/invalid baseline BEFORE its durable insert, the
1481                        // same way the append path does, not durably write and
1482                        // only poison the head in the post-write resync.
1483                        match verify_head_against_latest_checkpoint(&connection, head) {
1484                            Ok(()) => validate_writer_adopted_claim_log_baseline(
1485                                &connection,
1486                                head,
1487                                appends_receipts,
1488                            ),
1489                            Err(error) => Err(error),
1490                        }
1491                    } else {
1492                        verify_latest_checkpoint_integrity(&connection).and_then(|()| {
1493                            if appends_receipts {
1494                                validate_claim_receipt_log_entries(&connection)
1495                            } else {
1496                                Ok(())
1497                            }
1498                        })
1499                    };
1500                    if let Err(error) = pre_check {
1501                        let respond = job(Err(error));
1502                        // Decrement before the response reaches the caller.
1503                        drop(inflight_guard);
1504                        let committed = respond(Ok(()));
1505                        record_write_job_outcome(health, committed);
1506                        completion.complete();
1507                        if fail_closed_on_error && !committed {
1508                            poison_head_from_writer_error(head_state, health);
1509                        }
1510                        return None;
1511                    }
1512                    // Capture the head's checkpoint position BEFORE the job
1513                    // runs: a writer-routed recovery
1514                    // (`create_next_receipt_checkpoint`) that creates/adopts the
1515                    // missing checkpoint advances this during the resync below.
1516                    let pre_checkpoint_seq = head.checkpoint_seq();
1517                    // Run the job but DEFER its response: the caller must not
1518                    // observe `Ok` until
1519                    // `resync_head_after_write` confirms the head. A committed
1520                    // write whose resync then fails receives the resync error,
1521                    // not a stale `Ok`.
1522                    let respond = job(Ok(&mut connection));
1523                    // Post-resync: absorb whatever the closure committed
1524                    // (claim-log rows via projection triggers, checkpoint
1525                    // rows via the manual path) so the next append's
1526                    // cross-check cannot false-Conflict.
1527                    match resync_head_after_write(&connection, head) {
1528                        Ok(()) => {
1529                            // Reconcile committed/failed for this writer-routed
1530                            // job using the responder's resync-adjusted outcome
1531                            // signal. Decrement before the response reaches the
1532                            // caller; the post-response catch-up build below
1533                            // reads no inflight state.
1534                            drop(inflight_guard);
1535                            let committed = respond(Ok(()));
1536                            record_write_job_outcome(health, committed);
1537                            completion.complete();
1538                            if fail_closed_on_error && !committed {
1539                                poison_head_from_writer_error(head_state, health);
1540                                return None;
1541                            }
1542                            health.store_head_snapshot(head);
1543                            // Clear a stale checkpoint error after a manual
1544                            // recovery: a writer-routed
1545                            // op such as `create_next_receipt_checkpoint` can
1546                            // build/adopt the missing checkpoint inside the job,
1547                            // advancing the head's checkpoint seq during the resync
1548                            // above. `build_due_checkpoints_and_record` below then
1549                            // finds nothing due (`Ok(false)`) and would leave a
1550                            // prior background-build `last_error` in place, so
1551                            // `receipt_store_health` keeps reporting the store
1552                            // unhealthy after the repair. Clear it here when the
1553                            // checkpoint chain actually advanced (clear only on
1554                            // an actual advance, never on an idle refresh); a
1555                            // real later build failure re-sets it below.
1556                            if head.checkpoint_seq() > pre_checkpoint_seq {
1557                                if let Ok(mut last_error) = health.last_error.lock() {
1558                                    *last_error = None;
1559                                }
1560                            }
1561                            // Writer-routed appends (child receipts, consuming
1562                            // auth) can cross the threshold too; no
1563                            // pending_flush_error guard here since a Write job is
1564                            // not part of a batch. The writer pool holds exactly
1565                            // one connection (DEFAULT_WRITER_POOL_MAX_SIZE = 1):
1566                            // drop this one before build_due_checkpoints_and_record
1567                            // acquires its own, or `pool.get()` would block on
1568                            // itself.
1569                            drop(connection);
1570                            // Gate the catch-up build on a full-verified head,
1571                            // mirroring the InstallSigner defer. On a
1572                            // non-incremental (suspect)
1573                            // store `seed_head_snapshot` leaves the head
1574                            // UNVALIDATED; only a receipt-appending Write reran the
1575                            // full claim-log validation in the pre-check above, so
1576                            // a metadata-only `run_write` did NOT. Building here
1577                            // would checkpoint unaudited claim-log rows before the
1578                            // deferred full validation ever runs (fail-closed
1579                            // violation). Build only when the head is genuinely
1580                            // verified: incremental mode (seed_verified_head +
1581                            // per-append verify) OR a receipt-appending job that
1582                            // just ran the full validation.
1583                            if incremental_verification || appends_receipts {
1584                                build_due_checkpoints_and_record(
1585                                    pool,
1586                                    head,
1587                                    checkpoint_signer,
1588                                    health,
1589                                );
1590                            }
1591                        }
1592                        Err(error) => {
1593                            if let Ok(mut last_error) = health.last_error.lock() {
1594                                *last_error = Some(error.to_string());
1595                            }
1596                            let poison_message = error.to_string();
1597                            // Surface the resync failure to the caller: a write
1598                            // that returned `Ok` from its closure must NOT report
1599                            // success when the head is now poisoned. Count the
1600                            // failed outcome. Decrement before the response
1601                            // reaches the caller.
1602                            drop(inflight_guard);
1603                            let committed = respond(Err(error));
1604                            record_write_job_outcome(health, committed);
1605                            completion.complete();
1606                            health.set_head_poisoned(true);
1607                            *head_state = WriterHeadState::Poisoned(poison_message);
1608                        }
1609                    }
1610                }
1611            }
1612        }
1613        ReceiptCommitCommand::Rotate { config, response } => {
1614            // Unconditional decrement pairs with the pre-send increment in
1615            // `SqliteReceiptStore::dispatch_rotate` (mirrors the Write arm's
1616            // dequeue decrement above). It runs before every early return
1617            // below, so no dequeue path (poisoned head, pool-acquire error,
1618            // the panic-guarded rotation, success, or error) can leak the
1619            // in-flight rotation writer.
1620            atomic_saturating_sub(&health.inflight, 1);
1621            // Fail-closed: rotation deletes evidence, so it must never run on a
1622            // store whose chain integrity is unverified. Refuse on a poisoned
1623            // head (mirrors the Write arm) and point at the repair path.
1624            if let WriterHeadState::Poisoned(message) = head_state {
1625                let _ = response.send(Err(poisoned_head_error(message)));
1626                return None;
1627            }
1628            let mut connection = match pool.get() {
1629                Ok(connection) => connection,
1630                Err(error) => {
1631                    let _ = response.send(Err(ReceiptStoreError::Pool(error.to_string())));
1632                    return None;
1633                }
1634            };
1635            // Fail-closed: rotation deletes evidence, so it must audit the FULL
1636            // persisted checkpoint chain against the live claim log before pruning,
1637            // in BOTH verification modes. A non-incremental store seeds its head
1638            // via `seed_head_snapshot`, which defers the checkpoint-chain audit to
1639            // the next append, so its Verified state is not proof of integrity. An
1640            // incremental store maintains a per-append verified head, but that head
1641            // only attests NEW appends: it never notices a retroactive deletion of
1642            // a checkpoint-covered source row AND its claim-log projection row after
1643            // the store was opened. That drift leaves the source and projection sets
1644            // matching (so the projection audit below passes) while the covering
1645            // checkpoint's claim-log range falls short of its signed tree_size, and
1646            // rotating over it would co-archive only the survivors, delete the rest,
1647            // and stamp a watermark the archive cannot back. The full chain audit
1648            // rejects exactly that. Rotation is off the append hot path, so the O(N)
1649            // rebuild is affordable here even in incremental mode.
1650            let verified_latest_checkpoint = match verify_checkpoint_chain_integrity(&connection) {
1651                Ok(latest) => latest,
1652                Err(error) => {
1653                    let _ = response.send(Err(error));
1654                    return None;
1655                }
1656            };
1657            // The claim-log projection audit runs before EVERY rotation,
1658            // regardless of verification mode. A store in the drift shape (source
1659            // receipts deleted but their claim-log rows left behind, the shape
1660            // `retention_repair` recovers from) is NOT caught by the per-append
1661            // verified head an incremental store maintains: that head verifies new
1662            // appends, never a retroactive source-row deletion. Rotating over such
1663            // a store would co-archive orphaned claim-log rows without their
1664            // receipts (`verify_co_archival_complete` only compares surviving
1665            // source rows) and then delete the live claim log, destroying the
1666            // evidence repair needs to recover. Refuse fail-closed instead.
1667            if let Err(error) = validate_claim_receipt_log_entries(&connection) {
1668                let _ = response.send(Err(error));
1669                return None;
1670            }
1671            // In incremental mode the rotation trusts the per-append verified head
1672            // rather than an O(N) rebuild on the append hot path, but that head can
1673            // lag `kernel_checkpoints`: a second store instance or an operator
1674            // import may have appended checkpoint rows this handle has not yet
1675            // adopted, so `head.checkpointed_entry_seq()` stays at the boundary the
1676            // head was seeded at until a later append catches it up. Capping the
1677            // archival watermark at that stale boundary would make a quiet store
1678            // archive nothing on every retention interval despite holding aged,
1679            // checkpointed receipts. The full chain audit just above validated
1680            // every persisted checkpoint, so cap instead at the freshest VERIFIED
1681            // boundary: the latest persisted checkpoint's batch_end_seq (pinned
1682            // from that audit, so a checkpoint appended after it cannot widen the
1683            // cap). Non-incremental mode runs the same audit in its rotation path
1684            // and computes the watermark from every persisted checkpoint, so it
1685            // needs no cap.
1686            let verified_ceiling = if incremental_verification {
1687                Some(
1688                    verified_latest_checkpoint
1689                        .as_ref()
1690                        .map_or(0, |checkpoint| checkpoint.body.batch_end_seq),
1691                )
1692            } else {
1693                None
1694            };
1695            // Panic isolation: the writer actor re-acquires a fresh connection
1696            // for every command, so a caught panic fails only THIS rotation
1697            // (fail-closed) and no state from the panicking closure is reused
1698            // afterward.
1699            let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
1700                evidence_retention::rotate_on_writer_connection(
1701                    &mut connection,
1702                    &config,
1703                    verified_ceiling,
1704                )
1705            }))
1706            .unwrap_or_else(|payload| Err(receipt_writer_job_panic_error(&payload)));
1707            // After a successful bottom-of-log delete the cached head's
1708            // latest_checkpoint and claim_log_max_seq are unchanged (rotation
1709            // never deletes checkpoints and never touches the max entry_seq),
1710            // but claim_log_count shrank. Refresh it so diagnostics stay
1711            // accurate; correctness does not depend on this (no hot path
1712            // asserts count equality).
1713            if outcome.is_ok() {
1714                if let WriterHeadState::Verified(head) = head_state {
1715                    if let Ok((count, max_seq)) = claim_log_delta_count_and_max_seq(&connection, 0)
1716                    {
1717                        head.claim_log_count = count;
1718                        head.claim_log_max_seq = max_seq;
1719                    }
1720                    health.store_head_snapshot(head);
1721                }
1722            }
1723            let _ = response.send(outcome);
1724        }
1725        ReceiptCommitCommand::InstallSigner(signer) => {
1726            *checkpoint_signer = Some(signer);
1727            // Install-time catch-up. The store can
1728            // open on a DB that already has >= max_batch uncheckpointed
1729            // claim-log entries (a crash between the durable append response and
1730            // the background build, or enabling checkpointing on an existing
1731            // store). Without building here, the owed checkpoint waits for some
1732            // future Append/Write, so a quiet restarted store stays
1733            // uncheckpointed indefinitely despite checkpointing being enabled.
1734            // Run the existing bounded builder now so any already-owed
1735            // checkpoints (head.claim_log_max_seq - checkpointed_entry_seq >=
1736            // max_batch) are built at install time (O(b) per checkpoint, loops
1737            // until caught up; NOT a full verify). Fail-closed:
1738            // build_due_checkpoints_and_record records last_error and never
1739            // panics the actor.
1740            //
1741            // Deferred-seed gate: only build at
1742            // install when the head has actually been VALIDATED. With
1743            // `incremental_verification = false` the actor seeds via
1744            // `seed_head_snapshot`, which INTENTIONALLY skips the full claim-log
1745            // + checkpoint-chain audit (deferred to the next append/verify), so
1746            // the seeded head is `Verified` but UNVALIDATED. Building catch-up
1747            // checkpoints over that range would checkpoint unaudited data (a
1748            // fail-closed violation), so defer it in that mode: the next
1749            // receipt-appending append/Write runs the deferred full validation
1750            // and THEN builds the owed checkpoints. In the normal incremental
1751            // mode the seeded head is genuinely verified, so the owed
1752            // checkpoints still build here.
1753            if incremental_verification {
1754                if let WriterHeadState::Verified(head) = head_state {
1755                    build_due_checkpoints_and_record(pool, head, checkpoint_signer, health);
1756                }
1757            }
1758        }
1759        ReceiptCommitCommand::ReseedHead(response) => {
1760            let outcome = if health.critical_write_poisoned.load(Ordering::SeqCst) {
1761                Err(ReceiptStoreError::Conflict(format!(
1762                    "{}; repair the critical receipt projection and reopen the receipt store",
1763                    critical_writer_error_message(health)
1764                )))
1765            } else {
1766                pool.get()
1767                    .map_err(|error| ReceiptStoreError::Pool(error.to_string()))
1768                    .and_then(|connection| {
1769                        support::audit_receipt_cost_projection(&connection)?;
1770                        // Reseed always runs the FULL verification. This is the
1771                        // `chio receipt audit --repair`
1772                        // recovery path: it clears a poisoned head and must establish
1773                        // a genuinely CLEAN, fully-verified head, so it runs
1774                        // `seed_verified_head` (full claim-log validation +
1775                        // checkpoint-chain audit) regardless of the hot-path
1776                        // `incremental_verification` mode. Using the cheap
1777                        // `seed_head_snapshot` here would let `--repair` clear
1778                        // `last_error` and mark the head `Verified` while the on-disk
1779                        // log is still corrupt (repair theater). This is the recovery
1780                        // path, not per-append, so it is a recovery-path cost. NOTE
1781                        // the deliberate difference from the InstallSigner catch-up:
1782                        // that path DEFERS in
1783                        // `incremental_verification = false` because `seed_head_snapshot`
1784                        // leaves the head UNVALIDATED; reseed full-verifies, so it does
1785                        // not defer.
1786                        seed_verified_head(&connection)
1787                    })
1788            };
1789            let result = match outcome {
1790                Ok(head) => {
1791                    health.store_head_snapshot(&head);
1792                    if let Ok(mut last_error) = health.last_error.lock() {
1793                        *last_error = None;
1794                    }
1795                    // Clear the actor loop's stale flush error: a prior append
1796                    // poisoned the head and set
1797                    // `pending_flush_error`, but this reseed has just revalidated
1798                    // the DB and replaced the head. Without clearing it, a
1799                    // subsequent STANDALONE `flush_receipt_writes()` (no queued
1800                    // writes) would keep returning the stale append error even
1801                    // though the store recovered. Fail-closed is unaffected: a
1802                    // real later batch failure re-sets `pending_flush_error`.
1803                    *pending_flush_error = None;
1804                    health.set_head_poisoned(false);
1805                    *head_state = WriterHeadState::Verified(Box::new(head));
1806                    // Build owed checkpoints after a successful reseed. If the
1807                    // background signer was installed
1808                    // while the head was poisoned, its install-time catch-up
1809                    // was skipped, so a quiet store with >= max_batch
1810                    // uncheckpointed claim-log entries would stay uncheckpointed
1811                    // until some future write. Run the SAME bounded builder now.
1812                    // Unlike the InstallSigner catch-up (which gates on
1813                    // `incremental_verification` because its deferred seed is
1814                    // unvalidated), this is unconditional: the reseed just
1815                    // full-verified the head, so building over that range never
1816                    // checkpoints unaudited data. Bounded (O(b) per owed
1817                    // checkpoint), a recovery-path build (not per-append).
1818                    // No-op when no signer is present. Fail-closed:
1819                    // `build_due_checkpoints_and_record` records `last_error` on a
1820                    // build failure and never re-poisons the freshly verified head.
1821                    if let WriterHeadState::Verified(head) = head_state {
1822                        build_due_checkpoints_and_record(pool, head, checkpoint_signer, health);
1823                    }
1824                    Ok(())
1825                }
1826                Err(error) => {
1827                    if let Ok(mut last_error) = health.last_error.lock() {
1828                        *last_error = Some(error.to_string());
1829                    }
1830                    health.set_head_poisoned(true);
1831                    *head_state = WriterHeadState::Poisoned(error.to_string());
1832                    Err(error)
1833                }
1834            };
1835            let _ = response.send(result);
1836        }
1837        ReceiptCommitCommand::RetentionRepair {
1838            archive_path,
1839            response,
1840        } => {
1841            // Unconditional decrement pairs with the pre-send increment in
1842            // `SqliteReceiptStore::retention_repair` (mirrors the Rotate arm's
1843            // dequeue decrement above).
1844            atomic_saturating_sub(&health.inflight, 1);
1845            // Runs regardless of `head_state` (like ReseedHead): the whole
1846            // point of this command is to repair a store whose head is
1847            // already Poisoned by the drift the repair removes, so gating it
1848            // on `WriterHeadState::Verified` (the Write arm's guard) would
1849            // make it unusable on exactly the store it exists to fix.
1850            let outcome = pool
1851                .get()
1852                .map_err(|error| ReceiptStoreError::Pool(error.to_string()))
1853                .and_then(|mut connection| {
1854                    evidence_retention::retention_repair_on_writer(&mut connection, &archive_path)
1855                });
1856            if outcome.is_ok() {
1857                // Reseed the head so this same store instance is appendable
1858                // immediately, mirroring ReseedHead: the repair just removed
1859                // the drift that poisoned it (or was a no-op on an already
1860                // healthy store). A reseed failure here does not change the
1861                // repair's own outcome -- the archive rows are already
1862                // committed -- but it does update head_state/health so a
1863                // subsequent health check or write surfaces the real cause
1864                // instead of a stale poisoned message.
1865                let reseed = pool
1866                    .get()
1867                    .map_err(|error| ReceiptStoreError::Pool(error.to_string()))
1868                    .and_then(|connection| {
1869                        if incremental_verification {
1870                            seed_verified_head(&connection)
1871                        } else {
1872                            seed_head_snapshot(&connection)
1873                        }
1874                    });
1875                match reseed {
1876                    Ok(head) => {
1877                        health.store_head_snapshot(&head);
1878                        if let Ok(mut last_error) = health.last_error.lock() {
1879                            *last_error = None;
1880                        }
1881                        // Clear the actor loop's stale flush error, mirroring the
1882                        // ReseedHead recovery path: if an earlier append poisoned
1883                        // the head and set `pending_flush_error`, the repair has
1884                        // now reseeded a revalidated head, so a subsequent
1885                        // STANDALONE `flush_receipt_writes()` must not keep
1886                        // returning the pre-repair append error. Fail-closed is
1887                        // unaffected: a real later batch failure re-sets it.
1888                        *pending_flush_error = None;
1889                        *head_state = WriterHeadState::Verified(Box::new(head));
1890                    }
1891                    Err(error) => {
1892                        if let Ok(mut last_error) = health.last_error.lock() {
1893                            *last_error = Some(error.to_string());
1894                        }
1895                        *head_state = WriterHeadState::Poisoned(error.to_string());
1896                    }
1897                }
1898            }
1899            let _ = response.send(outcome);
1900        }
1901        // Append/Flush are handled by the main loop; reaching here is
1902        // impossible by construction but must stay fail-safe.
1903        ReceiptCommitCommand::Append(request) => {
1904            let _ = request
1905                .response
1906                .send(Err(receipt_actor_unavailable_error()));
1907        }
1908        ReceiptCommitCommand::AppendWithTimeout {
1909            request,
1910            mut completion,
1911        } => {
1912            completion.complete();
1913            let _ = request
1914                .response
1915                .send(Err(receipt_actor_unavailable_error()));
1916        }
1917        ReceiptCommitCommand::Flush(response) => {
1918            let _ = response.send(Err(receipt_actor_unavailable_error()));
1919        }
1920        #[cfg(test)]
1921        ReceiptCommitCommand::RestartSupervisor => return Some(SupervisedOutcome::Restart),
1922    }
1923    None
1924}
1925
1926/// Build every checkpoint the head owes and, on success, refresh the health
1927/// head snapshot; on failure, record the error without poisoning the head or
1928/// failing the append/write that triggered it (checkpoint construction never
1929/// blocks an already-durable commit). Returns the recorded
1930/// error (if any) so a flush-as-checkpoint-barrier caller can surface it to its
1931/// co-drained flush waiters; the durable append/write path ignores the return
1932/// and stays fail-closed via `last_error`.
1933fn build_due_checkpoints_and_record(
1934    pool: &Pool<SqliteConnectionManager>,
1935    head: &mut VerifiedHead,
1936    checkpoint_signer: &Option<BackgroundCheckpointSigner>,
1937    health: &ReceiptCommitWriterHealth,
1938) -> Option<ReceiptStoreError> {
1939    let signer = checkpoint_signer.as_ref()?;
1940    // Panic isolation: a panic mid-build
1941    // (Merkle build, Ed25519 sign, serde) must not kill the writer thread.
1942    // A committed or peer-adopted checkpoint can advance the verified head
1943    // before a later panic drops its frontier. Record `last_error` and rebuild.
1944    let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
1945        build_due_checkpoints(pool, head, signer)
1946    }))
1947    .unwrap_or_else(|payload| Err(receipt_writer_job_panic_error(&payload)));
1948    match result {
1949        Ok(advanced) => {
1950            health.store_head_snapshot(head);
1951            // Clear a stale background error only after this attempt advances
1952            // the verified head, whether through our commit or peer-winner
1953            // adoption. An idle refresh must not mask a current error.
1954            if advanced {
1955                if let Ok(mut last_error) = health.last_error.lock() {
1956                    *last_error = None;
1957                }
1958            }
1959            None
1960        }
1961        Err(error) => {
1962            if let Ok(mut last_error) = health.last_error.lock() {
1963                *last_error = Some(error.to_string());
1964            }
1965            Some(error)
1966        }
1967    }
1968}
1969
1970fn build_due_checkpoints(
1971    pool: &Pool<SqliteConnectionManager>,
1972    head: &mut VerifiedHead,
1973    signer: &BackgroundCheckpointSigner,
1974) -> Result<bool, ReceiptStoreError> {
1975    if signer.max_batch == 0 {
1976        return Ok(false); // ADR-0008: batch_size 0 disables checkpointing
1977    }
1978    let mut connection = pool
1979        .get()
1980        .map_err(|error| ReceiptStoreError::Pool(error.to_string()))?;
1981    let checkpoint_seq_before_refresh = head.checkpoint_seq();
1982    // Shared-file freshness: on a shared receipt DB
1983    // another writer can commit a checkpoint AFTER this actor's append
1984    // pre-check but BEFORE its batch tx. `append_receipt_batch` then adopts that
1985    // writer's claim-log rows via the baseline delta yet leaves
1986    // `head.latest_checkpoint` stale, so building from the stale position would
1987    // try to rebuild an already-committed checkpoint and fail with "already
1988    // exists with different content" (the clock-skew case the idempotent-
1989    // identical guard does not cover). Refresh the head against the latest
1990    // persisted checkpoint first so that checkpoint is ADOPTED, not rebuilt.
1991    // This is an O(1) latest-row read + digest adopt (plus bounded catch-up),
1992    // NOT a full chain verify, so the incremental hot path stays flat per
1993    // append.
1994    verify_head_against_latest_checkpoint(&connection, head)?;
1995    let refreshed = head.checkpoint_seq() > checkpoint_seq_before_refresh;
1996    maybe_build_checkpoint(&mut connection, head, signer).map(|advanced| refreshed || advanced)
1997}
1998
1999/// Build every checkpoint the head owes: count-based ADR-0008 trigger, range
2000/// derived from the cached head (NOT next_checkpoint_range_for_connection,
2001/// which runs a full chain verify). Cost per checkpoint is O(b) over the batch
2002/// plus O(log n) over the cached chain frontier; the frontier is rebuilt from
2003/// the database only on a cache miss (first issuance after seed or resync).
2004/// Returns true when this builder commits or boundedly adopts a checkpoint.
2005fn maybe_build_checkpoint(
2006    connection: &mut SqliteStoreConnection,
2007    head: &mut VerifiedHead,
2008    signer: &BackgroundCheckpointSigner,
2009) -> Result<bool, ReceiptStoreError> {
2010    if signer.max_batch == 0 {
2011        return Ok(false);
2012    }
2013    if head
2014        .claim_log_max_seq
2015        .saturating_sub(head.checkpointed_entry_seq())
2016        < signer.max_batch
2017    {
2018        return Ok(false);
2019    }
2020    // Chain leaves for every persisted checkpoint, extended in-loop as new
2021    // checkpoints commit; the cached head and the persisted chain must agree
2022    // on length before any of them are committed to a new chain_root.
2023    // O(n) exactly once per head, then extended in place. A cache miss runs the
2024    // full semantic chain audit before reusing its returned frontier. The
2025    // frontier still reproduces the predecessor's signed chain_root inside the
2026    // builder, so caching it does not weaken the check it replaces.
2027    let cached = head
2028        .chain_frontier
2029        .as_ref()
2030        .filter(|frontier| frontier.leaf_count() == head.checkpoint_seq())
2031        .cloned();
2032    let mut advanced = false;
2033    let mut chain_frontier = match cached {
2034        Some(frontier) => frontier,
2035        None => {
2036            let (frontier, cache_advanced) =
2037                build_checkpoint_after_frontier_cache_miss(connection, head, signer)?;
2038            advanced = cache_advanced;
2039            frontier
2040        }
2041    };
2042    if chain_frontier.leaf_count() != head.checkpoint_seq() {
2043        return Err(ReceiptStoreError::Conflict(format!(
2044            "persisted chain covers {} checkpoints but the head is at {}",
2045            chain_frontier.leaf_count(),
2046            head.checkpoint_seq()
2047        )));
2048    }
2049    while head
2050        .claim_log_max_seq
2051        .saturating_sub(head.checkpointed_entry_seq())
2052        >= signer.max_batch
2053    {
2054        let start_seq = head.checkpointed_entry_seq().saturating_add(1);
2055        let end_seq = start_seq.saturating_add(signer.max_batch - 1);
2056        ensure_claim_log_range_contiguous(connection, start_seq, end_seq, "checkpoint range")?;
2057        let receipt_bytes = load_claim_tree_canonical_bytes_range(connection, start_seq, end_seq)?
2058            .into_iter()
2059            .map(|(_, bytes)| bytes)
2060            .collect::<Vec<_>>();
2061        let checkpoint_seq = head
2062            .checkpoint_seq()
2063            .checked_add(1)
2064            .ok_or_else(|| ReceiptStoreError::Conflict("checkpoint_seq overflow".to_string()))?;
2065        // O(b) Merkle build over the batch, plus O(log n) over the chain
2066        // frontier; the predecessor digest comes from the cached head.
2067        let checkpoint = chio_kernel::checkpoint::build_checkpoint_with_chain_frontier(
2068            checkpoint_seq,
2069            start_seq,
2070            end_seq,
2071            &receipt_bytes,
2072            &signer.keypair,
2073            head.latest_checkpoint.as_ref(),
2074            &chain_frontier,
2075        )
2076        .map_err(checkpoint_error_to_receipt_store)?;
2077        #[cfg(test)]
2078        if test_hooks::panic_during_checkpoint_build(signer.max_batch) {
2079            panic!("injected test panic during background checkpoint build");
2080        }
2081        #[cfg(test)]
2082        if test_hooks::fail_checkpoint_build(signer.max_batch) {
2083            return Err(ReceiptStoreError::Conflict(
2084                "injected test checkpoint build failure".to_string(),
2085            ));
2086        }
2087        // The insert returns the checkpoint now persisted at this seq: either
2088        // the one we just built, or a concurrently committed winner (clock-skew
2089        // sibling) it validated and adopted. Catch the cached head up to THAT
2090        // checkpoint so a later verify_head_against_latest_checkpoint does not
2091        // see our discarded byte-different build diverge from the persisted row.
2092        let (adopted, adopted_frontier) = insert_background_checkpoint_guarded(
2093            connection,
2094            head.latest_checkpoint.as_ref(),
2095            &chain_frontier,
2096            &checkpoint,
2097        )?;
2098        chain_frontier = adopted_frontier;
2099        head.latest_checkpoint = Some(adopted);
2100        advanced = true;
2101    }
2102    head.chain_frontier = Some(chain_frontier);
2103    Ok(advanced)
2104}
2105
2106/// Head-resync rule: one indexed delta aggregate plus one
2107/// latest-checkpoint row read after every Write closure.
2108fn resync_head_after_write(
2109    connection: &Connection,
2110    head: &mut VerifiedHead,
2111) -> Result<(), ReceiptStoreError> {
2112    let pre_resync_max = head.claim_log_max_seq;
2113    let (delta_count, post_max) = claim_log_delta_count_and_max_seq(connection, pre_resync_max)?;
2114    // Validate the ADOPTED resync delta before advancing the head. A Write
2115    // closure can commit claim_receipt_log_entries rows
2116    // past this actor's head (another shared-DB writer, or a receipt-appending
2117    // Write job), and this resync absorbs them via COUNT/MAX. Without
2118    // validating them, an orphan/divergent row would be trusted and later
2119    // appends would skip it as already-verified, so a background checkpoint
2120    // could cover an unaudited entry. Re-validate JUST the
2121    // (pre_resync_max, post_max] delta against the source receipt tables
2122    // (O(delta)); the full-log validator is NOT called. Single-writer common
2123    // case: no other writer, empty delta, no-op. Fail-closed: an
2124    // orphan/divergent delta returns the error, which the Write arm turns into
2125    // a poisoned head.
2126    if delta_count > 0 {
2127        validate_adopted_claim_log_delta(connection, pre_resync_max, post_max)?;
2128    }
2129    head.claim_log_count = head.claim_log_count.saturating_add(delta_count);
2130    head.claim_log_max_seq = post_max;
2131    verify_head_against_latest_checkpoint(connection, head)
2132}
2133
2134#[cfg(test)]
2135fn commit_receipt_batch(
2136    pool: &Pool<SqliteConnectionManager>,
2137    head_state: &mut WriterHeadState,
2138    incremental_verification: bool,
2139    requests: Vec<ReceiptCommitRequest>,
2140    health: &ReceiptCommitWriterHealth,
2141) -> Option<ReceiptStoreError> {
2142    commit_receipt_batch_with_completions(
2143        pool,
2144        head_state,
2145        incremental_verification,
2146        requests,
2147        health,
2148        Vec::new(),
2149    )
2150}
2151
2152fn commit_receipt_batch_with_completions(
2153    pool: &Pool<SqliteConnectionManager>,
2154    head_state: &mut WriterHeadState,
2155    incremental_verification: bool,
2156    requests: Vec<ReceiptCommitRequest>,
2157    health: &ReceiptCommitWriterHealth,
2158    mut completions: Vec<WriterCommandCompletion>,
2159) -> Option<ReceiptStoreError> {
2160    let batch_outcome = match head_state {
2161        WriterHeadState::Verified(head) => {
2162            match append_receipt_batch(pool, head, incremental_verification, &requests) {
2163                Ok(results) => {
2164                    health.store_head_snapshot(head);
2165                    Ok(results)
2166                }
2167                Err(error) => Err(error),
2168            }
2169        }
2170        WriterHeadState::Poisoned(message) => Ok(receipt_batch_error_results(
2171            requests.len(),
2172            poisoned_head_error(message),
2173        )),
2174    };
2175    let results = match batch_outcome {
2176        Ok(results) => results,
2177        Err(error) => {
2178            // A store-wide append fault (a failed checkpoint or predecessor
2179            // verification, a transaction that will not open, a disk-full commit)
2180            // rejects every receipt in this batch and will reject every future
2181            // append until an operator reseeds. Poison the head so the
2182            // pre-dispatch gate fails closed rather than letting tools run against
2183            // a store that can no longer persist their receipts.
2184            let message = error.to_string();
2185            let results = receipt_batch_error_results(requests.len(), error);
2186            health.set_head_poisoned(true);
2187            *head_state = WriterHeadState::Poisoned(message);
2188            results
2189        }
2190    };
2191    let flush_error = results
2192        .iter()
2193        .find_map(|result| result.as_ref().err().map(receipt_store_error_snapshot));
2194    let committed = results.iter().filter(|result| result.is_ok()).count() as u64;
2195    let failed = results.iter().filter(|result| result.is_err()).count() as u64;
2196    if committed > 0 {
2197        health
2198            .committed_total
2199            .fetch_add(committed, Ordering::SeqCst);
2200        health
2201            .last_commit_unix_ms
2202            .store(current_unix_ms(), Ordering::SeqCst);
2203    }
2204    if failed > 0 {
2205        health.failed_total.fetch_add(failed, Ordering::SeqCst);
2206    }
2207    atomic_saturating_sub(&health.inflight, results.len() as u64);
2208    for completion in &mut completions {
2209        completion.complete();
2210    }
2211    if let Some(error) = &flush_error {
2212        if let Ok(mut last_error) = health.last_error.lock() {
2213            *last_error = Some(error.to_string());
2214        }
2215    } else {
2216        health.clear_timeout_error_if_drained();
2217    }
2218    // APPEND durability responses fan out here (ADR-0013): a durable append
2219    // response is never delayed by checkpoint construction. The co-drained
2220    // Flush waiters are released by the caller AFTER the checkpoint build, so a
2221    // flush is a genuine checkpoint barrier.
2222    for (request, result) in requests.into_iter().zip(results) {
2223        let _ = request.response.send(result);
2224    }
2225    flush_error
2226}
2227
2228fn current_unix_ms() -> u64 {
2229    SystemTime::now()
2230        .duration_since(UNIX_EPOCH)
2231        .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
2232        .unwrap_or(0)
2233}
2234
2235fn atomic_saturating_sub(value: &AtomicU64, amount: u64) {
2236    let mut current = value.load(Ordering::SeqCst);
2237    loop {
2238        let next = current.saturating_sub(amount);
2239        match value.compare_exchange(current, next, Ordering::SeqCst, Ordering::SeqCst) {
2240            Ok(_) => return,
2241            Err(observed) => current = observed,
2242        }
2243    }
2244}
2245
2246/// Classify commit-writer liveness from a counter snapshot. Kept pure so the
2247/// wedged / saturated / dead transitions are unit-testable without a live actor.
2248///
2249/// - `Dead`: the actor channel has disconnected (its last error reports the
2250///   writer unavailable). Nothing can drain, so admission must stop.
2251/// - `Wedged`: work is still queued or running (`inflight > 0`, or more appends
2252///   were accepted than have completed) and no progress has been made within
2253///   `stall_threshold_ms`. The stall clock is anchored to the more recent of the
2254///   last commit and the start of the current backlog, so a writer that wedges
2255///   before its first commit is caught, while a writer resuming after an idle
2256///   period (an old last commit but freshly enqueued work) is measured from that
2257///   fresh work rather than judged wedged the instant it accepts. A timed-out
2258///   command is tracked separately until that exact command drains, so a
2259///   timeout closes liveness immediately without corrupting terminal counters.
2260/// - `Saturated`: the commit channel is full right now (`queue_depth` has
2261///   reached the channel capacity), so the next send would be rejected. This
2262///   reads `queue_depth` rather than `inflight` so a drained but still-committing
2263///   batch, whose slots are already free, does not read as a full channel and
2264///   deny admission when the next append would in fact be accepted. Deny
2265///   admission rather than run a side effect whose receipt cannot be enqueued.
2266/// - `Healthy`: none of the above.
2267fn classify_writer_liveness(
2268    counters: &ReceiptWriterCounters,
2269    stall_threshold_ms: u64,
2270    channel_capacity: u64,
2271    backlog_started_unix_ms: Option<u64>,
2272    now_unix_ms: u64,
2273) -> chio_kernel::ReceiptWriterLiveness {
2274    use chio_kernel::ReceiptWriterLiveness as Liveness;
2275    if counters
2276        .last_error
2277        .as_deref()
2278        .is_some_and(|error| error.contains("unavailable"))
2279    {
2280        return Liveness::Dead;
2281    }
2282    if counters.timed_out_inflight > 0 {
2283        return Liveness::Wedged;
2284    }
2285    let backlogged = counters.inflight > 0
2286        || counters.accepted_total
2287            > counters
2288                .committed_total
2289                .saturating_add(counters.failed_total);
2290    // Anchor to the more recent of the last commit and the current backlog
2291    // start. Using the last commit alone marks a writer wedged after any idle
2292    // period (its last commit is naturally old); using the backlog start alone
2293    // would ignore progress a busy writer is still making.
2294    let progress_reference = counters
2295        .last_commit_unix_ms
2296        .into_iter()
2297        .chain(backlog_started_unix_ms)
2298        .max();
2299    let stalled = match progress_reference {
2300        Some(reference) => now_unix_ms.saturating_sub(reference) >= stall_threshold_ms,
2301        None => false,
2302    };
2303    if backlogged && stalled {
2304        return Liveness::Wedged;
2305    }
2306    if counters.queue_depth >= channel_capacity {
2307        return Liveness::Saturated;
2308    }
2309    Liveness::Healthy
2310}
2311
2312/// Fold the writer-liveness verdict into the store's top-level health boolean.
2313/// A wedged, saturated, or dead writer makes the store unhealthy even when the
2314/// checkpoint chain is intact and no error has been recorded yet: the
2315/// pre-dispatch gate is already denying tool calls against that writer, so a
2316/// health surface that stayed green would contradict it. `Healthy` and the
2317/// permissive `Unknown` (no async writer, or a read-only observer that cannot
2318/// see writer liveness) do not downgrade health.
2319fn receipt_store_healthy(
2320    checkpoint_healthy: bool,
2321    writer_last_error: Option<&str>,
2322    writer_liveness: chio_kernel::ReceiptWriterLiveness,
2323) -> bool {
2324    use chio_kernel::ReceiptWriterLiveness as Liveness;
2325    checkpoint_healthy
2326        && writer_last_error.is_none()
2327        && !matches!(
2328            writer_liveness,
2329            Liveness::Wedged | Liveness::Saturated | Liveness::Dead
2330        )
2331}
2332
2333#[cfg(test)]
2334mod writer_liveness_classifier_tests {
2335    use super::*;
2336    use chio_kernel::ReceiptWriterLiveness as Liveness;
2337
2338    const CAPACITY: u64 = RECEIPT_COMMIT_ACTOR_CHANNEL_CAPACITY as u64;
2339    const NOW: u64 = 1_000_000;
2340    const STALL_MS: u64 = 10_000;
2341
2342    #[test]
2343    fn timed_out_inflight_append_reports_wedged() {
2344        // A caller timeout is not a terminal failure. The actor still owns the
2345        // command, so accepted remains ahead of terminal outcomes until it drains.
2346        let counters = ReceiptWriterCounters {
2347            accepted_total: 1,
2348            inflight: 1,
2349            ..ReceiptWriterCounters::default()
2350        };
2351        assert_eq!(
2352            classify_writer_liveness(&counters, STALL_MS, CAPACITY, Some(NOW - 20_000), NOW),
2353            Liveness::Wedged
2354        );
2355    }
2356
2357    #[test]
2358    fn outstanding_timeout_reports_wedged_before_the_stall_threshold() {
2359        let counters = ReceiptWriterCounters {
2360            accepted_total: 1,
2361            inflight: 1,
2362            timed_out_total: 1,
2363            timed_out_inflight: 1,
2364            last_commit_unix_ms: None,
2365            last_error: Some("sqlite receipt commit append timed out".to_string()),
2366            ..ReceiptWriterCounters::default()
2367        };
2368        assert_eq!(
2369            classify_writer_liveness(&counters, STALL_MS, CAPACITY, Some(NOW - 6_000), NOW),
2370            Liveness::Wedged
2371        );
2372    }
2373
2374    #[test]
2375    fn never_committed_backlog_reports_wedged() {
2376        // Wedged before the first commit: `last_commit_unix_ms` is `None`, so the
2377        // stall clock must fall back to the current backlog start.
2378        let counters = ReceiptWriterCounters {
2379            accepted_total: 1,
2380            inflight: 1,
2381            last_commit_unix_ms: None,
2382            ..ReceiptWriterCounters::default()
2383        };
2384        assert_eq!(
2385            classify_writer_liveness(&counters, STALL_MS, CAPACITY, Some(NOW - 20_000), NOW),
2386            Liveness::Wedged
2387        );
2388    }
2389
2390    #[test]
2391    fn honors_configured_stall_threshold() {
2392        // Same backlog with a commit 600ms ago: wedged under a fail-fast 500ms
2393        // threshold, healthy under a lenient 10s threshold. Proves the threshold
2394        // is a parameter, not a hardcoded constant.
2395        let counters = ReceiptWriterCounters {
2396            accepted_total: 2,
2397            committed_total: 1,
2398            inflight: 1,
2399            last_commit_unix_ms: Some(NOW - 600),
2400            ..ReceiptWriterCounters::default()
2401        };
2402        assert_eq!(
2403            classify_writer_liveness(&counters, 500, CAPACITY, None, NOW),
2404            Liveness::Wedged
2405        );
2406        assert_eq!(
2407            classify_writer_liveness(&counters, 10_000, CAPACITY, None, NOW),
2408            Liveness::Healthy
2409        );
2410    }
2411
2412    #[test]
2413    fn full_commit_channel_reports_saturated() {
2414        // Channel full right now but still committing (recent commit): a new send
2415        // would be rejected, so admission must be denied even though the writer
2416        // is not wedged.
2417        let counters = ReceiptWriterCounters {
2418            accepted_total: CAPACITY + 5,
2419            committed_total: 4,
2420            inflight: CAPACITY,
2421            queue_depth: CAPACITY,
2422            last_commit_unix_ms: Some(NOW - 100),
2423            ..ReceiptWriterCounters::default()
2424        };
2425        assert_eq!(
2426            classify_writer_liveness(&counters, STALL_MS, CAPACITY, None, NOW),
2427            Liveness::Saturated
2428        );
2429        assert!(!Liveness::Saturated.healthy());
2430    }
2431
2432    #[test]
2433    fn a_drained_but_committing_batch_is_not_reported_saturated() {
2434        // The actor has drained a full batch out of the channel and is committing
2435        // it: `inflight` still counts that batch, but its channel slots are
2436        // already free, so the next send would succeed. Saturation reads
2437        // `queue_depth`, so this must classify Healthy rather than Saturated.
2438        // Reading `inflight` here wrongly denied admission under heavy but
2439        // healthy load.
2440        let counters = ReceiptWriterCounters {
2441            accepted_total: CAPACITY + RECEIPT_GROUP_COMMIT_MAX_BATCH as u64,
2442            committed_total: 0,
2443            inflight: CAPACITY,
2444            queue_depth: CAPACITY - RECEIPT_GROUP_COMMIT_MAX_BATCH as u64,
2445            last_commit_unix_ms: Some(NOW - 100),
2446            ..ReceiptWriterCounters::default()
2447        };
2448        assert_eq!(
2449            classify_writer_liveness(&counters, STALL_MS, CAPACITY, Some(NOW - 100), NOW),
2450            Liveness::Healthy
2451        );
2452    }
2453
2454    #[test]
2455    fn idle_writer_with_fresh_backlog_is_not_wedged() {
2456        // After a long idle period the last commit is naturally old, but a newly
2457        // enqueued write has only just started. The stall clock must anchor to the
2458        // fresh backlog start, not the stale last commit, or the writer is marked
2459        // wedged and admission denied the instant it accepts work after a quiet
2460        // period.
2461        let counters = ReceiptWriterCounters {
2462            accepted_total: 6,
2463            committed_total: 5,
2464            inflight: 1,
2465            last_commit_unix_ms: Some(NOW - 60_000),
2466            ..ReceiptWriterCounters::default()
2467        };
2468        assert_eq!(
2469            classify_writer_liveness(&counters, STALL_MS, CAPACITY, Some(NOW - 100), NOW),
2470            Liveness::Healthy,
2471            "fresh work after idle must not be judged wedged by the stale last commit"
2472        );
2473        // The same stale commit WITH a backlog that has itself gone unserviced
2474        // past the threshold is a genuine wedge.
2475        assert_eq!(
2476            classify_writer_liveness(&counters, STALL_MS, CAPACITY, Some(NOW - 20_000), NOW),
2477            Liveness::Wedged,
2478            "a backlog stalled past the threshold must still report wedged"
2479        );
2480    }
2481
2482    #[test]
2483    fn unavailable_writer_reports_dead() {
2484        let counters = ReceiptWriterCounters {
2485            last_error: Some("sqlite receipt commit actor is unavailable".to_string()),
2486            ..ReceiptWriterCounters::default()
2487        };
2488        assert_eq!(
2489            classify_writer_liveness(&counters, STALL_MS, CAPACITY, None, NOW),
2490            Liveness::Dead
2491        );
2492    }
2493
2494    #[test]
2495    fn drained_writer_reports_healthy() {
2496        let counters = ReceiptWriterCounters {
2497            accepted_total: 10,
2498            committed_total: 10,
2499            inflight: 0,
2500            last_commit_unix_ms: Some(NOW - 50),
2501            ..ReceiptWriterCounters::default()
2502        };
2503        assert_eq!(
2504            classify_writer_liveness(&counters, STALL_MS, CAPACITY, None, NOW),
2505            Liveness::Healthy
2506        );
2507    }
2508
2509    #[test]
2510    fn a_non_healthy_writer_makes_the_store_unhealthy() {
2511        // Checkpoint chain intact and no recorded error, but the writer is not
2512        // making progress: the pre-dispatch gate is denying tool calls, so the
2513        // top-level health boolean must not stay green.
2514        assert!(!receipt_store_healthy(true, None, Liveness::Wedged));
2515        assert!(!receipt_store_healthy(true, None, Liveness::Saturated));
2516        assert!(!receipt_store_healthy(true, None, Liveness::Dead));
2517    }
2518
2519    #[test]
2520    fn healthy_and_unknown_writers_do_not_downgrade_store_health() {
2521        assert!(receipt_store_healthy(true, None, Liveness::Healthy));
2522        // Unknown is the permissive verdict (no async writer, or a read-only
2523        // observer that cannot see writer liveness).
2524        assert!(receipt_store_healthy(true, None, Liveness::Unknown));
2525        // A recorded writer error or an unhealthy checkpoint chain still fails
2526        // closed regardless of a healthy liveness verdict.
2527        assert!(!receipt_store_healthy(false, None, Liveness::Healthy));
2528        assert!(!receipt_store_healthy(
2529            true,
2530            Some("checkpoint build failed"),
2531            Liveness::Healthy
2532        ));
2533    }
2534}
2535
2536/// Holds the writer `inflight` count for the DURATION of a writer-routed `Write`
2537/// job. The pre-send increment in `WriterHandle::run_write_kind` is ADOPTED by
2538/// this guard, so `receipt_store_health` reports `inflight > 0` while a slow or
2539/// stuck writer-routed op (pool acquire, pre-check, closure, resync) is actually
2540/// running. The `Write` arm releases it (`drop`) IMMEDIATELY BEFORE each
2541/// `respond(...)`, so a caller that observes its own response never sees itself
2542/// still counted inflight, mirroring the Append path, which decrements in
2543/// `commit_receipt_batch` BEFORE fanning out its results. Still Drop-based, so
2544/// any exit that panics before a respond runs releases exactly once; a release
2545/// overlap with the caller's recv-Err compensation under actor-thread death
2546/// saturates at zero via `atomic_saturating_sub` rather than underflowing.
2547struct WriterInflightGuard<'a> {
2548    inflight: &'a AtomicU64,
2549}
2550
2551impl<'a> WriterInflightGuard<'a> {
2552    fn new(inflight: &'a AtomicU64) -> Self {
2553        Self { inflight }
2554    }
2555}
2556
2557impl Drop for WriterInflightGuard<'_> {
2558    fn drop(&mut self) {
2559        atomic_saturating_sub(self.inflight, 1);
2560    }
2561}
2562
2563/// Reconcile a writer-routed `Write` job's health counters. Child receipts
2564/// and authorization-consuming appends run through
2565/// `WriterHandle::run_write_receipt`, and metadata-only writes through
2566/// `run_write`; both are `accepted_total`-counted at enqueue, but their
2567/// success/failure OUTCOME was never folded into `committed_total` /
2568/// `failed_total`, so accepted / committed / failed did not reconcile and a
2569/// store dominated by writer-routed receipts undercounted commits. The actor
2570/// calls this exactly once per `Write` with the responder's resync-adjusted
2571/// signal (O(1) per write). A committed outcome also refreshes
2572/// `last_commit_unix_ms`, mirroring the Append path (`commit_receipt_batch`).
2573fn record_write_job_outcome(health: &ReceiptCommitWriterHealth, committed: bool) {
2574    if committed {
2575        health.committed_total.fetch_add(1, Ordering::SeqCst);
2576        health
2577            .last_commit_unix_ms
2578            .store(current_unix_ms(), Ordering::SeqCst);
2579        // Clear only a timeout marker whose owning command has drained. A
2580        // successful earlier command cannot clear a later queued timeout, and a
2581        // genuine writer/head error is never treated as timeout state.
2582        health.clear_timeout_error_if_drained();
2583    } else {
2584        health.failed_total.fetch_add(1, Ordering::SeqCst);
2585    }
2586}
2587
2588/// Background checkpoint signer, installed once by the kernel after `open`
2589/// and before serving. `max_batch = 0` disables
2590/// checkpointing (ADR-0008 semantics).
2591#[derive(Clone)]
2592pub struct BackgroundCheckpointSigner {
2593    pub keypair: Arc<Keypair>,
2594    pub max_batch: u64,
2595}
2596
2597/// Last verified position of the receipt chain. Owned exclusively by the
2598/// commit-actor thread; never shared, never locked.
2599#[derive(Clone, Debug, Default)]
2600pub(crate) struct VerifiedHead {
2601    /// The newest checkpoint the actor has verified, already parsed and
2602    /// signature-checked once. `None` before the first checkpoint.
2603    latest_checkpoint: Option<KernelCheckpoint>,
2604    /// Frontier of the checkpoint-chain tree as of `latest_checkpoint`, kept
2605    /// so issuing a checkpoint costs O(log n) hashes instead of rehashing the
2606    /// whole chain. `None` means "not known here": catch-up or issuance rebuilds
2607    /// it from the persisted chain once and caches it again. Every path that
2608    /// moves `latest_checkpoint` without extending this must clear it.
2609    chain_frontier: Option<CheckpointChainFrontier>,
2610    /// Row count of `claim_receipt_log_entries` as last verified.
2611    claim_log_count: u64,
2612    /// MAX(entry_seq) of `claim_receipt_log_entries` as last verified.
2613    claim_log_max_seq: u64,
2614}
2615
2616impl VerifiedHead {
2617    pub(crate) fn checkpoint_seq(&self) -> u64 {
2618        self.latest_checkpoint
2619            .as_ref()
2620            .map_or(0, |checkpoint| checkpoint.body.checkpoint_seq)
2621    }
2622
2623    pub(crate) fn checkpointed_entry_seq(&self) -> u64 {
2624        self.latest_checkpoint
2625            .as_ref()
2626            .map_or(0, |checkpoint| checkpoint.body.batch_end_seq)
2627    }
2628}
2629
2630/// Writer-actor head snapshot exposed to `flush_report` and diagnostics.
2631/// Values are read from the health struct's atomics, written
2632/// only by the actor thread.
2633pub(crate) struct WriterHeadSnapshot {
2634    pub(crate) checkpoint_seq: u64,
2635    pub(crate) checkpointed_entry_seq: u64,
2636    // Read only by tests (`incremental_append_updates_the_head_and_stays_correct`,
2637    // `writer_routed_inserts_do_not_false_conflict_the_next_append`): they
2638    // cross-check the actor-maintained head against a full re-verification.
2639    // `flush_report` does not need the claim-log counters today.
2640    #[cfg_attr(not(test), allow(dead_code))]
2641    pub(crate) claim_log_count: u64,
2642    #[cfg_attr(not(test), allow(dead_code))]
2643    pub(crate) claim_log_max_seq: u64,
2644}
2645
2646/// Seed the verified head by running the existing FULL verification exactly
2647/// once (the startup path for the O(N) check; also the audit-repair path).
2648fn seed_verified_head(connection: &Connection) -> Result<VerifiedHead, ReceiptStoreError> {
2649    validate_claim_receipt_log_entries(connection)?;
2650    let (latest_checkpoint, chain_frontier) =
2651        verify_checkpoint_chain_integrity_with_frontier(connection)?;
2652    let (claim_log_count, claim_log_max_seq) = claim_log_delta_count_and_max_seq(connection, 0)?;
2653    Ok(VerifiedHead {
2654        latest_checkpoint,
2655        chain_frontier: Some(chain_frontier),
2656        claim_log_count,
2657        claim_log_max_seq,
2658    })
2659}
2660
2661/// Cheap head snapshot for `incremental_verification = false` stores: the
2662/// full per-append verification still runs on that path, so seeding only
2663/// parses the single latest checkpoint row (one signature check) plus two
2664/// aggregates. This keeps a suspect database openable for A/B verification.
2665fn seed_head_snapshot(connection: &Connection) -> Result<VerifiedHead, ReceiptStoreError> {
2666    let latest_checkpoint = load_latest_persisted_checkpoint_row(connection)?
2667        .map(parse_persisted_checkpoint_row)
2668        .transpose()?;
2669    let (claim_log_count, claim_log_max_seq) = claim_log_delta_count_and_max_seq(connection, 0)?;
2670    Ok(VerifiedHead {
2671        latest_checkpoint,
2672        chain_frontier: None,
2673        claim_log_count,
2674        claim_log_max_seq,
2675    })
2676}
2677
2678/// COUNT/MAX over `entry_seq > floor_entry_seq`: an indexed range scan over
2679/// the delta only (O(b)). An unscoped COUNT(*) would rescan the whole index
2680/// and reintroduce O(N). Returns `(delta_count, max_entry_seq)` where the max
2681/// falls back to `floor_entry_seq` for an empty delta.
2682fn claim_log_delta_count_and_max_seq(
2683    connection: &Connection,
2684    floor_entry_seq: u64,
2685) -> Result<(u64, u64), ReceiptStoreError> {
2686    let floor = sqlite_i64(floor_entry_seq, "claim log delta floor entry_seq")?;
2687    let (count, max_seq) = connection.query_row(
2688        "SELECT COUNT(*), COALESCE(MAX(entry_seq), ?1) FROM claim_receipt_log_entries WHERE entry_seq > ?1",
2689        params![floor],
2690        |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?)),
2691    )?;
2692    Ok((
2693        sqlite_u64(count, "claim log delta count")?,
2694        sqlite_u64(max_seq, "claim log delta max entry_seq")?,
2695    ))
2696}
2697
2698/// Fail-closed pre-job guard for a RECEIPT-APPENDING writer-routed job (child
2699/// receipts, authorization-consuming appends). The
2700/// incremental writer pre-check only re-verified the checkpoint HEAD; it did
2701/// NOT validate the `claim_receipt_log_entries` rows an out-of-band writer (a
2702/// second store instance, an operator repair) may have committed AHEAD of this
2703/// actor's head. Without this guard the job would DURABLY insert its receipt
2704/// and only afterwards, in `resync_head_after_write`, discover the bad/orphan
2705/// adopted row and poison the head - a fail-OPEN durable write. Validate the
2706/// ADOPTED delta (head.claim_log_max_seq, current_max] with the SAME bounded
2707/// `validate_adopted_claim_log_delta` the append path runs, BEFORE the job
2708/// commits, so a stale/invalid baseline denies the write with no durable
2709/// insert. Delta-bounded: single-writer no-stale-head case has an EMPTY delta
2710/// (pre_delta = 0) and is a no-op, and the full-log validator is NEVER called,
2711/// so the flat per-append cost holds. Metadata-only writes insert no
2712/// claim-log rows, so they skip this (appends_receipts = false).
2713fn validate_writer_adopted_claim_log_baseline(
2714    connection: &Connection,
2715    head: &VerifiedHead,
2716    appends_receipts: bool,
2717) -> Result<(), ReceiptStoreError> {
2718    if !appends_receipts {
2719        return Ok(());
2720    }
2721    let (pre_delta, baseline_max) =
2722        claim_log_delta_count_and_max_seq(connection, head.claim_log_max_seq)?;
2723    if pre_delta > 0 {
2724        validate_adopted_claim_log_delta(connection, head.claim_log_max_seq, baseline_max)?;
2725    }
2726    Ok(())
2727}
2728
2729/// O(1) predecessor check: the persisted latest checkpoint must still match
2730/// the verified head (one indexed row read + RFC 8785 canonical body digest
2731/// compare). When the persisted chain has moved FORWARD, verify only the new
2732/// checkpoints (bounded catch-up); every other divergence is a fail-closed
2733/// `Conflict` pointing at `chio receipt audit`.
2734fn verify_head_against_latest_checkpoint(
2735    connection: &Connection,
2736    head: &mut VerifiedHead,
2737) -> Result<(), ReceiptStoreError> {
2738    let persisted = load_latest_persisted_checkpoint_row(connection)?;
2739    let cached_seq = head.checkpoint_seq();
2740    match persisted {
2741        None if head.latest_checkpoint.is_none() => Ok(()),
2742        None => Err(ReceiptStoreError::Conflict(
2743            "latest checkpoint disappeared behind the verified head; run `chio receipt audit`"
2744                .to_string(),
2745        )),
2746        Some(row) if row.checkpoint_seq < cached_seq => Err(ReceiptStoreError::Conflict(format!(
2747            "checkpoint chain regressed from verified head {cached_seq} to {}; run `chio receipt audit`",
2748            row.checkpoint_seq
2749        ))),
2750        Some(row) if row.checkpoint_seq == cached_seq => {
2751            let Some(cached) = head.latest_checkpoint.as_ref() else {
2752                return Err(ReceiptStoreError::Conflict(
2753                    "checkpoint presence diverged from verified head; run `chio receipt audit`"
2754                        .to_string(),
2755                ));
2756            };
2757            // Body-only deserialize: parse_persisted_checkpoint_row would run
2758            // chio_kernel::checkpoint::validate_checkpoint and re-verify the
2759            // signature, putting one Ed25519 verify back on every append. The
2760            // cached head was signature-checked at seed time.
2761            let persisted_body: KernelCheckpointBody = serde_json::from_str(&row.statement_json)?;
2762            let persisted_digest = chio_kernel::checkpoint::checkpoint_body_sha256(&persisted_body)
2763                .map_err(checkpoint_error_to_receipt_store)?;
2764            let cached_digest = chio_kernel::checkpoint::checkpoint_body_sha256(&cached.body)
2765                .map_err(checkpoint_error_to_receipt_store)?;
2766            if persisted_digest != cached_digest {
2767                return Err(ReceiptStoreError::Conflict(
2768                    "latest checkpoint diverged from verified head; run `chio receipt audit`"
2769                        .to_string(),
2770                ));
2771            }
2772            // Full-column tamper catch: the body digest above covers ONLY what
2773            // statement_json serializes. The kernel_checkpoints row also stores
2774            // batch_start_seq/batch_end_seq/tree_size/merkle_root/issued_at/
2775            // kernel_key as their own columns; any one of them corrupted out of
2776            // band (immutability trigger bypassed) while statement_json is
2777            // untouched would pass the digest check yet leave a signed-body-bound
2778            // column diverged. `ensure_checkpoint_columns_match_body` reconciles
2779            // every such column against the (signature-verified) signed body it
2780            // is meant to mirror. This is O(1) int/string equality over the one
2781            // already-read row, NOT a per-append Ed25519 re-verify.
2782            ensure_checkpoint_columns_match_body(&row, &persisted_body)?;
2783            // The `signature` column is the signature OVER the body, not a body
2784            // field, so it is not covered above; compare it against the cached
2785            // head, which was signature-verified at seed/catch-up time (O(1)
2786            // string equality, no crypto).
2787            if row.signature_hex != cached.signature.to_hex() {
2788                return Err(ReceiptStoreError::Conflict(
2789                    "latest checkpoint signature column diverged from verified head; run `chio receipt audit`"
2790                        .to_string(),
2791                ));
2792            }
2793            // Recheck the latest checkpoint's transparency projection rows.
2794            // The body-digest / column / signature
2795            // checks above re-verify the `kernel_checkpoints` row on every
2796            // append, but the projection rows (`checkpoint_tree_heads`,
2797            // `checkpoint_predecessor_witnesses`,
2798            // `checkpoint_publication_metadata`) were validated only when this
2799            // checkpoint was first adopted (seed or catch-up). A projection row
2800            // tampered out of band (immutability guards momentarily absent, then
2801            // restored) while the checkpoint seq is UNCHANGED would otherwise be
2802            // trusted as verified until the next open/health/audit. Rechecking it
2803            // here closes that gap symmetrically with the per-append column
2804            // recheck: O(1) (three indexed single-row projection lookups plus an
2805            // O(1) derivation from the already-parsed checkpoint body, NO
2806            // batch/leaf scan and NO full-history walk), so the incremental
2807            // hot path stays flat per append. Fail-closed on any divergence.
2808            validate_checkpoint_projection_rows(connection, &row, cached)?;
2809            Ok(())
2810        }
2811        Some(row) => catch_up_verified_head_to(connection, head, row.checkpoint_seq),
2812    }
2813}
2814
2815/// Verify and adopt checkpoints `head.checkpoint_seq()+1 ..= latest_seq`.
2816/// O(new checkpoints): each row is parsed (one signature check), predecessor-
2817/// linked to the cached head, range-checked against the claim log, AND its
2818/// transparency projection rows validated before it
2819/// advances the head. Used when another writer instance (second kernel on the
2820/// same file, operator CLI) legitimately extended the chain. In the single-
2821/// writer hot path the head is never behind, so this loop body does not run
2822/// (zero added per-append cost); each caught-up checkpoint is O(b) for its own
2823/// batch, never a full-history walk.
2824fn catch_up_verified_head_to(
2825    connection: &Connection,
2826    head: &mut VerifiedHead,
2827    latest_seq: u64,
2828) -> Result<(), ReceiptStoreError> {
2829    let mut cursor = head.checkpoint_seq();
2830    // A checkpoint fully covered by a trusted archival watermark has had its
2831    // claim-log rows co-archived and deleted, so its Merkle range is served from
2832    // the archive exactly as the full chain walk exempts it. Without this the
2833    // incremental catch-up path would rebuild the deleted prefix from the live
2834    // claim log and fail: a stale writer that had not yet adopted a checkpoint
2835    // another handle archived could never catch up across the boundary, and its
2836    // next append would poison the head. Computed once for the caught-up span.
2837    let watermark = trusted_retention_watermark(connection)?;
2838    while cursor < latest_seq {
2839        let next_seq = cursor.saturating_add(1);
2840        let Some(row) = load_persisted_checkpoint_row(connection, next_seq)? else {
2841            return Err(ReceiptStoreError::Conflict(format!(
2842                "checkpoint chain gap at {next_seq} behind latest {latest_seq}; run `chio receipt audit`"
2843            )));
2844        };
2845        let checkpoint = parse_persisted_checkpoint_row(row.clone())?;
2846        match head.latest_checkpoint.as_ref() {
2847            Some(predecessor) => {
2848                chio_kernel::checkpoint::validate_checkpoint_predecessor(predecessor, &checkpoint)
2849                    .map_err(checkpoint_error_to_receipt_store)?;
2850            }
2851            None => validate_checkpoint_base(&checkpoint)?,
2852        }
2853        if checkpoint.body.batch_end_seq > watermark {
2854            validate_checkpoint_against_claim_log(connection, &checkpoint)?;
2855        }
2856        // Projection validation before adoption: the
2857        // catch-up path verified signature + predecessor + claim-log range but
2858        // not the transparency projection rows that full
2859        // `verify_checkpoint_chain_integrity` rejects. Adopting a checkpoint with
2860        // missing/divergent projection rows would advance `head.latest_checkpoint`
2861        // and let subsequent appends build on an audit-invalid chain. Validate ONLY
2862        // this adopted checkpoint's projection rows (O(b) for its batch, not full
2863        // history), fail closed on any divergence.
2864        validate_checkpoint_projection_rows(connection, &row, &checkpoint)?;
2865        // Check any signed chain root before adopting. Legacy v1 leaves still
2866        // extend the frontier so a later v2 root commits the complete history.
2867        head.chain_frontier = Some(advance_verified_checkpoint_chain_frontier(
2868            connection,
2869            head.chain_frontier.as_ref(),
2870            head.latest_checkpoint.as_ref(),
2871            &checkpoint,
2872        )?);
2873        head.latest_checkpoint = Some(checkpoint);
2874        cursor = next_seq;
2875    }
2876    Ok(())
2877}
2878
2879/// Insert one receipt (and, when requested, its lineage statement) within the
2880/// caller's transaction, returning the claim-log `entry_seq`. Split out of
2881/// `append_receipt_batch` so each record can run inside its own SAVEPOINT: a
2882/// per-receipt failure is returned as this record's `Err`
2883/// instead of aborting the whole coalesced batch. Receipt + lineage stay one
2884/// unit - a lineage failure returns `Err`, and the caller's savepoint rollback
2885/// undoes the receipt too, so no receipt-without-lineage state is possible.
2886fn append_single_receipt_record(
2887    tx: &rusqlite::Transaction<'_>,
2888    request: &ReceiptCommitRequest,
2889) -> Result<u64, ReceiptStoreError> {
2890    let seq = append_chio_receipt_tx(tx, &request.receipt, &request.raw_json)?;
2891    if request.ensure_lineage {
2892        #[cfg(test)]
2893        if test_hooks::fail_between_receipt_and_lineage() {
2894            return Err(ReceiptStoreError::Conflict(
2895                "injected failure between receipt insert and lineage insert".to_string(),
2896            ));
2897        }
2898        ensure_receipt_lineage_statement_for_receipt_id_tx(tx, &request.receipt.id)?;
2899    }
2900    Ok(seq)
2901}
2902
2903/// Append a coalesced group-commit batch.
2904///
2905/// `Err` is a STORE-WIDE fault that rejects the batch and poisons the head until
2906/// an operator reseeds. `Ok` carries one result per request; a malformed receipt
2907/// fails only its own savepoint and leaves the head intact.
2908fn append_receipt_batch(
2909    pool: &Pool<SqliteConnectionManager>,
2910    head: &mut VerifiedHead,
2911    incremental_verification: bool,
2912    requests: &[ReceiptCommitRequest],
2913) -> Result<Vec<Result<u64, ReceiptStoreError>>, ReceiptStoreError> {
2914    let mut connection = pool
2915        .get()
2916        .map_err(|error| ReceiptStoreError::Pool(error.to_string()))?;
2917    ensure_checkpoint_transparency_guards(&connection)?;
2918    if incremental_verification {
2919        // O(1) predecessor check (+ bounded catch-up), not a chain rebuild.
2920        verify_head_against_latest_checkpoint(&connection, head)?;
2921    } else {
2922        validate_claim_receipt_log_entries(&connection)?;
2923    }
2924    let tx = connection
2925        .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)
2926        .map_err(ReceiptStoreError::Sqlite)?;
2927    if !incremental_verification {
2928        verify_latest_checkpoint_integrity(&tx)?;
2929    }
2930    // Baseline inside the IMMEDIATE tx: rows another store instance committed
2931    // since our last look are adopted as pre-existing, so the cross-check
2932    // below measures exactly what THIS batch inserted.
2933    let (pre_delta, baseline_max) = claim_log_delta_count_and_max_seq(&tx, head.claim_log_max_seq)?;
2934    // Validate the ADOPTED baseline delta before trusting it. Rows another
2935    // store instance committed since our last look
2936    // (head.claim_log_max_seq + 1 ..= baseline_max) are absorbed as
2937    // pre-existing baseline. A full per-append validation would reject an
2938    // out-of-band mismatched/orphan claim_receipt_log_entries row
2939    // in that range. Re-validate JUST that bounded delta against the source
2940    // receipt tables (O(delta)); the full-log validator is NOT called. In the
2941    // single-writer hot path the head is never stale, so pre_delta is 0 and
2942    // this is a no-op (zero added cost).
2943    if pre_delta > 0 {
2944        validate_adopted_claim_log_delta(&tx, head.claim_log_max_seq, baseline_max)?;
2945    }
2946    let mut results = Vec::with_capacity(requests.len());
2947    for request in requests {
2948        #[cfg(test)]
2949        if test_hooks::panic_during_append_batch(&request.receipt.content_hash) {
2950            panic!("injected test panic during append batch");
2951        }
2952        // Per-record SAVEPOINT: a coalesced group-commit
2953        // batch mixes independent producers. A per-receipt failure (a conflicting
2954        // duplicate raw JSON, a lineage insert failure) must fail ONLY that
2955        // record, not roll back and error every unrelated valid append sharing
2956        // the same group-commit window. Wrap each record so a failure ROLLBACK TO
2957        // the savepoint undoes JUST this record's partial work - its receipt row,
2958        // its projection-trigger claim-log row, and its AUTOINCREMENT entry_seq,
2959        // which SQLite restores with the savepoint so surviving rows stay
2960        // contiguous - and the loop continues with the others. Two extra SQL
2961        // statements per record: O(1) per record, O(b) per batch, never a
2962        // full-history scan, so the flat per-append cost holds.
2963        tx.execute_batch("SAVEPOINT chio_append_record")
2964            .map_err(ReceiptStoreError::Sqlite)?;
2965        match append_single_receipt_record(&tx, request) {
2966            Ok(seq) => {
2967                tx.execute_batch("RELEASE chio_append_record")
2968                    .map_err(ReceiptStoreError::Sqlite)?;
2969                results.push(Ok(seq));
2970            }
2971            Err(error) => {
2972                // Fail THIS record closed and undo only its work, then keep going
2973                // for the others. A savepoint that will not unwind is a
2974                // transaction-integrity fault, so fail the whole batch closed in
2975                // that (unexpected) case.
2976                tx.execute_batch("ROLLBACK TO chio_append_record; RELEASE chio_append_record")
2977                    .map_err(ReceiptStoreError::Sqlite)?;
2978                results.push(Err(error));
2979            }
2980        }
2981    }
2982    // Idempotent duplicates return the existing entry_seq without adding a
2983    // projection row (append_chio_receipt_tx: ON CONFLICT(receipt_id) DO
2984    // NOTHING at receipt_store.rs:972, byte-identical duplicate branch at
2985    // :992-1011). Only entry_seqs beyond the baseline count as new rows, and
2986    // only DISTINCT ones: two byte-identical receipts landing in a single
2987    // group-commit batch (a concurrent duplicate append) both return the SAME
2988    // entry_seq from the idempotent branch while inserting exactly one
2989    // projection row. Deduplicating the new seqs keeps `inserted` equal to the
2990    // distinct row count so the cross-check below does not false-trigger the
2991    // projection-drift Conflict and roll back a valid idempotent batch.
2992    let inserted = results
2993        .iter()
2994        .filter_map(|result| result.as_ref().ok())
2995        .filter(|seq| **seq > baseline_max)
2996        .copied()
2997        .collect::<std::collections::BTreeSet<u64>>()
2998        .len() as u64;
2999    #[cfg(feature = "chaos-test-hooks")]
3000    chaos_test_hooks::pause_after_receipt_write_before_commit(inserted > 0)?;
3001    // O(b) projection cross-check over the delta only: the claim-log
3002    // projection triggers (bootstrap/open.rs:676 tool, :711 child) must have
3003    // advanced the projection by exactly the rows this batch inserted.
3004    let (delta_count, post_max) = claim_log_delta_count_and_max_seq(&tx, baseline_max)?;
3005    if delta_count != inserted || post_max < baseline_max {
3006        return Err(ReceiptStoreError::Conflict(
3007            "claim receipt log projection drift on append; run `chio receipt audit`".to_string(),
3008        ));
3009    }
3010    // Validate the NEWLY-projected rows before advancing the head. The
3011    // count/MAX cross-check above only proves the projection
3012    // advanced by the right NUMBER of rows; `append_chio_receipt_tx` verifies
3013    // only the projected `receipt_id`/`raw_json`, so a tampered projection
3014    // trigger could emit one row per insert whose `timestamp`, `tool_name`, or
3015    // attribution columns diverge from the source receipt and still pass here.
3016    // A full per-append validation would reject that drift on the next
3017    // append; without validating it now the head advances and future
3018    // appends treat the bad row as already verified. Re-validate JUST the
3019    // (baseline_max, post_max] delta this batch projected with the same
3020    // full-field validator (O(delta): the batch inserts a bounded number of
3021    // rows, so the flat per-append cost holds and the full-log validator is
3022    // NEVER called). Gated on a non-empty delta (an all-idempotent
3023    // batch projects nothing, so this is a no-op). Fail-closed: a divergent row
3024    // returns the Conflict before `tx.commit()`, so the head never advances.
3025    if delta_count > 0 {
3026        validate_adopted_claim_log_delta(&tx, baseline_max, post_max)?;
3027    }
3028    tx.commit().map_err(ReceiptStoreError::Sqlite)?;
3029    head.claim_log_count = head
3030        .claim_log_count
3031        .saturating_add(pre_delta)
3032        .saturating_add(delta_count);
3033    head.claim_log_max_seq = post_max.max(baseline_max);
3034    Ok(results)
3035}
3036
3037fn receipt_batch_error_results(
3038    count: usize,
3039    error: ReceiptStoreError,
3040) -> Vec<Result<u64, ReceiptStoreError>> {
3041    let snapshot = receipt_store_error_snapshot(&error);
3042    let mut original = Some(error);
3043    (0..count)
3044        .map(|_| {
3045            Err(original
3046                .take()
3047                .unwrap_or_else(|| receipt_store_error_snapshot(&snapshot)))
3048        })
3049        .collect()
3050}
3051
3052fn receipt_store_error_snapshot(error: &ReceiptStoreError) -> ReceiptStoreError {
3053    match error {
3054        ReceiptStoreError::Sqlite(error) => {
3055            ReceiptStoreError::Sqlite(rusqlite::Error::ToSqlConversionFailure(Box::new(
3056                std::io::Error::other(error.to_string()),
3057            )))
3058        }
3059        ReceiptStoreError::Pool(message) => ReceiptStoreError::Pool(message.clone()),
3060        ReceiptStoreError::Timeout {
3061            operation,
3062            timeout_ms,
3063        } => ReceiptStoreError::Timeout {
3064            operation: operation.clone(),
3065            timeout_ms: *timeout_ms,
3066        },
3067        ReceiptStoreError::Json(error) => ReceiptStoreError::Json(serde_json::Error::io(
3068            std::io::Error::other(error.to_string()),
3069        )),
3070        ReceiptStoreError::Io(error) => {
3071            ReceiptStoreError::Io(std::io::Error::new(error.kind(), error.to_string()))
3072        }
3073        ReceiptStoreError::CryptoDecode(message) => {
3074            ReceiptStoreError::CryptoDecode(message.clone())
3075        }
3076        ReceiptStoreError::Canonical(message) => ReceiptStoreError::Canonical(message.clone()),
3077        ReceiptStoreError::InvalidOutcome(message) => {
3078            ReceiptStoreError::InvalidOutcome(message.clone())
3079        }
3080        ReceiptStoreError::ReadBoundary(message) => {
3081            ReceiptStoreError::ReadBoundary(message.clone())
3082        }
3083        ReceiptStoreError::Conflict(message) => ReceiptStoreError::Conflict(message.clone()),
3084        ReceiptStoreError::NotFound(message) => ReceiptStoreError::NotFound(message.clone()),
3085        ReceiptStoreError::Unsupported(message) => ReceiptStoreError::Unsupported(message.clone()),
3086        ReceiptStoreError::Fenced => ReceiptStoreError::Fenced,
3087        ReceiptStoreError::OutcomeUnknown(message) => {
3088            ReceiptStoreError::OutcomeUnknown(message.clone())
3089        }
3090        ReceiptStoreError::RetentionArchiveIncomplete {
3091            table,
3092            live,
3093            archived,
3094        } => ReceiptStoreError::RetentionArchiveIncomplete {
3095            table,
3096            live: *live,
3097            archived: *archived,
3098        },
3099        ReceiptStoreError::RetentionWatermarkRegression { attempted, current } => {
3100            ReceiptStoreError::RetentionWatermarkRegression {
3101                attempted: *attempted,
3102                current: *current,
3103            }
3104        }
3105        ReceiptStoreError::ArchivedRangeProjection { watermark } => {
3106            ReceiptStoreError::ArchivedRangeProjection {
3107                watermark: *watermark,
3108            }
3109        }
3110        ReceiptStoreError::RetentionTenantScopeUnsupported => {
3111            ReceiptStoreError::RetentionTenantScopeUnsupported
3112        }
3113        ReceiptStoreError::WriterDead {
3114            restarts,
3115            last_error,
3116        } => ReceiptStoreError::WriterDead {
3117            restarts: *restarts,
3118            last_error: last_error.clone(),
3119        },
3120    }
3121}
3122
3123/// Convert a caught panic payload into a typed, fail-closed error. Panic
3124/// payloads are almost always `&'static str` (a `panic!("literal")`) or
3125/// `String` (a formatted `panic!("{}", ..)`); anything else degrades to a
3126/// generic message rather than unwrapping (house rule: no unwrap/expect in
3127/// non-test code).
3128fn receipt_writer_job_panic_error(payload: &(dyn std::any::Any + Send)) -> ReceiptStoreError {
3129    let message = payload
3130        .downcast_ref::<&str>()
3131        .map(|message| (*message).to_string())
3132        .or_else(|| payload.downcast_ref::<String>().cloned())
3133        .unwrap_or_else(|| "non-string panic payload".to_string());
3134    ReceiptStoreError::Canonical(format!("receipt writer job panicked: {message}"))
3135}
3136
3137/// Panic isolation: `commit_receipt_batch`
3138/// runs on the single writer thread, so a panic anywhere inside it (append
3139/// transaction, lineage fold) must not kill that thread. By the time this
3140/// runs, `requests` has already been moved into the panicking call and dropped
3141/// during unwind, so the pre-cloned request response senders are the only way
3142/// left to answer every appender in the batch. The co-drained Flush waiters are
3143/// NOT moved into the panicking call: they survive
3144/// the unwind in the actor loop, which fans out the returned error to them
3145/// after this. This mirrors `receipt_batch_error_results`'s uniform fan-out and
3146/// the health bookkeeping `commit_receipt_batch` would otherwise have performed
3147/// itself.
3148fn fan_out_batch_panic_error(
3149    health: &ReceiptCommitWriterHealth,
3150    request_responses: Vec<mpsc::SyncSender<Result<u64, ReceiptStoreError>>>,
3151    error: ReceiptStoreError,
3152) -> ReceiptStoreError {
3153    let batch_len = request_responses.len() as u64;
3154    health.failed_total.fetch_add(batch_len, Ordering::SeqCst);
3155    atomic_saturating_sub(&health.inflight, batch_len);
3156    if let Ok(mut last_error) = health.last_error.lock() {
3157        *last_error = Some(error.to_string());
3158    }
3159    for response in request_responses {
3160        let _ = response.send(Err(receipt_store_error_snapshot(&error)));
3161    }
3162    error
3163}
3164
3165#[cfg(test)]
3166pub(crate) mod test_hooks {
3167    use std::sync::atomic::{AtomicBool, Ordering};
3168
3169    /// When set, `append_receipt_batch` fails the batch between the receipt
3170    /// insert and the lineage ensure, proving the fold is one transaction.
3171    pub(crate) static FAIL_BETWEEN_RECEIPT_AND_LINEAGE: AtomicBool = AtomicBool::new(false);
3172
3173    pub(crate) fn fail_between_receipt_and_lineage() -> bool {
3174        FAIL_BETWEEN_RECEIPT_AND_LINEAGE.load(Ordering::SeqCst)
3175    }
3176
3177    /// When set, `maybe_build_checkpoint` panics after computing the
3178    /// checkpoint body but before opening its write transaction, proving the
3179    /// background-checkpoint catch_unwind wrap keeps the writer actor alive
3180    /// and leaves `head.latest_checkpoint` unadvanced. Tests run in parallel
3181    /// within this binary and this flag is process-global, so the panic is
3182    /// additionally gated on `PANIC_DURING_CHECKPOINT_BUILD_MARKER_MAX_BATCH`
3183    /// (a `max_batch` value no other test in this crate uses): a test whose
3184    /// signer does not use that exact batch size never panics, even if the
3185    /// flag happens to be `true` while it runs.
3186    pub(crate) static PANIC_DURING_CHECKPOINT_BUILD: AtomicBool = AtomicBool::new(false);
3187
3188    pub(crate) const PANIC_DURING_CHECKPOINT_BUILD_MARKER_MAX_BATCH: u64 = 5;
3189
3190    pub(crate) fn panic_during_checkpoint_build(max_batch: u64) -> bool {
3191        max_batch == PANIC_DURING_CHECKPOINT_BUILD_MARKER_MAX_BATCH
3192            && PANIC_DURING_CHECKPOINT_BUILD.load(Ordering::SeqCst)
3193    }
3194
3195    /// When set, `maybe_build_checkpoint` returns a fail-closed `Err` (a
3196    /// NON-panic checkpoint-build failure) for a signer using
3197    /// `FAIL_CHECKPOINT_BUILD_MARKER_MAX_BATCH`, proving a build failure is
3198    /// surfaced to a co-drained flush waiter (the flush-as-checkpoint
3199    /// barrier). It uses a DISTINCT marker from
3200    /// `PANIC_DURING_CHECKPOINT_BUILD` so the two process-global flags cannot
3201    /// interfere across the crate's parallel tests.
3202    pub(crate) static FAIL_CHECKPOINT_BUILD: AtomicBool = AtomicBool::new(false);
3203
3204    pub(crate) const FAIL_CHECKPOINT_BUILD_MARKER_MAX_BATCH: u64 = 7;
3205
3206    pub(crate) fn fail_checkpoint_build(max_batch: u64) -> bool {
3207        max_batch == FAIL_CHECKPOINT_BUILD_MARKER_MAX_BATCH
3208            && FAIL_CHECKPOINT_BUILD.load(Ordering::SeqCst)
3209    }
3210
3211    /// When set, `append_receipt_batch` panics before inserting the next
3212    /// request in the batch, proving the append-batch catch_unwind wrap in
3213    /// `receipt_commit_actor_loop` keeps the writer actor alive and fans out
3214    /// a typed error to every request in the interrupted batch. Gated on a
3215    /// `content_hash` marker for the same cross-test isolation reason as
3216    /// `PANIC_DURING_CHECKPOINT_BUILD` above (this flag is process-global,
3217    /// and other tests append receipts concurrently in the same binary).
3218    /// `content_hash`, not `receipt.id`, is the marker: `ChioReceipt::sign`
3219    /// always overwrites `id` with a content-derived hash
3220    /// (`prepare_receipt_body_for_signing`), so a caller-chosen `id` string
3221    /// does not survive signing, but a caller-chosen `content_hash` does.
3222    pub(crate) static PANIC_DURING_APPEND_BATCH: AtomicBool = AtomicBool::new(false);
3223
3224    pub(crate) const PANIC_DURING_APPEND_BATCH_MARKER_RECEIPT_ID: &str =
3225        "rcpt-test-hook-panic-during-append-batch";
3226
3227    /// `sample_receipt_with_id(id)` sets `content_hash: format!("content-{id}")`;
3228    /// this must match that pattern for `PANIC_DURING_APPEND_BATCH_MARKER_RECEIPT_ID`.
3229    pub(crate) const PANIC_DURING_APPEND_BATCH_MARKER_CONTENT_HASH: &str =
3230        "content-rcpt-test-hook-panic-during-append-batch";
3231
3232    pub(crate) fn panic_during_append_batch(content_hash: &str) -> bool {
3233        content_hash == PANIC_DURING_APPEND_BATCH_MARKER_CONTENT_HASH
3234            && PANIC_DURING_APPEND_BATCH.load(Ordering::SeqCst)
3235    }
3236}
3237
3238#[path = "receipt_store/bootstrap.rs"]
3239mod bootstrap;
3240mod chaos_test_hooks;
3241#[path = "receipt_store/evidence_retention.rs"]
3242mod evidence_retention;
3243#[path = "receipt_store/liability_claims.rs"]
3244mod liability_claims;
3245#[path = "receipt_store/liability_market.rs"]
3246mod liability_market;
3247#[path = "receipt_store/reports.rs"]
3248mod reports;
3249#[path = "receipt_store/support.rs"]
3250pub(crate) mod support;
3251#[cfg(test)]
3252#[path = "receipt_store/tests.rs"]
3253mod tests;
3254#[path = "receipt_store/underwriting_credit.rs"]
3255mod underwriting_credit;
3256
3257use support::*;
3258pub(crate) use support::{decode_verified_child_receipt, decode_verified_chio_receipt, sqlite_u64};
3259
3260impl SqliteReceiptStore {
3261    /// Reader-pool connection. READS ONLY: every write transaction must go
3262    /// through `writer_handle().run_write` (single-writer discipline). The
3263    /// reader pool is asserted read-only by
3264    /// `reader_pool_never_begins_a_write_transaction` in tests.
3265    pub(crate) fn connection(&self) -> Result<SqliteStoreConnection, ReceiptStoreError> {
3266        self.pool
3267            .get()
3268            .map_err(|error| ReceiptStoreError::Pool(error.to_string()))
3269    }
3270
3271    #[cfg(test)]
3272    pub(crate) fn reader_connection_for_test(
3273        &self,
3274    ) -> Result<SqliteStoreConnection, ReceiptStoreError> {
3275        self.connection()
3276    }
3277
3278    pub(crate) fn writer_handle(&self) -> WriterHandle {
3279        WriterHandle {
3280            sender: self.receipt_commit_actor.sender.clone(),
3281            health: Arc::clone(&self.receipt_commit_actor.health),
3282            worker: Arc::clone(&self.receipt_commit_actor.worker),
3283            settlement_store_binding: self.settlement_store_binding,
3284        }
3285    }
3286
3287    /// Highest tool-receipt replication seq, or 0 on an empty store. Single
3288    /// indexed MAX read; does not materialize the store.
3289    pub fn max_tool_receipt_seq(&self) -> Result<u64, ReceiptStoreError> {
3290        let connection = self.connection()?;
3291        let seq: i64 = connection.query_row(
3292            "SELECT COALESCE(MAX(seq), 0) FROM chio_tool_receipts",
3293            [],
3294            |row| row.get(0),
3295        )?;
3296        Ok(seq.max(0) as u64)
3297    }
3298
3299    /// Highest child-receipt replication seq, or 0 on an empty store.
3300    pub fn max_child_receipt_seq(&self) -> Result<u64, ReceiptStoreError> {
3301        let connection = self.connection()?;
3302        let seq: i64 = connection.query_row(
3303            "SELECT COALESCE(MAX(seq), 0) FROM chio_child_receipts",
3304            [],
3305            |row| row.get(0),
3306        )?;
3307        Ok(seq.max(0) as u64)
3308    }
3309
3310    /// Multi-tenant receipt isolation: toggle strict-isolation
3311    /// mode on tenant-scoped queries.
3312    ///
3313    /// When `strict = true`, a `tenant_filter = Some(id)` query returns
3314    /// ONLY rows whose `tenant_id = id`. Pre-multitenant receipts with
3315    /// `tenant_id IS NULL` are excluded.
3316    ///
3317    /// When `strict = false`, the same query also includes rows where
3318    /// `tenant_id IS NULL` -- the pre-multitenant "public" fallback
3319    /// set -- so pre-multitenant (NULL-tagged) receipts remain visible during
3320    /// an explicit compatibility window.
3321    ///
3322    /// A `tenant_filter = None` admin / compat query always returns
3323    /// every row regardless of this setting.
3324    pub fn with_strict_tenant_isolation(&self, strict: bool) {
3325        self.strict_tenant_isolation
3326            .store(strict, std::sync::atomic::Ordering::SeqCst);
3327    }
3328
3329    /// Read the current strict-tenant-isolation setting.
3330    #[must_use]
3331    pub fn strict_tenant_isolation_enabled(&self) -> bool {
3332        self.strict_tenant_isolation
3333            .load(std::sync::atomic::Ordering::SeqCst)
3334    }
3335
3336    /// Read-only after open (staged-rollout flag).
3337    #[must_use]
3338    pub fn incremental_verification_enabled(&self) -> bool {
3339        self.incremental_verification
3340    }
3341
3342    pub(crate) fn writer_head_snapshot(&self) -> WriterHeadSnapshot {
3343        let health = &self.receipt_commit_actor.health;
3344        WriterHeadSnapshot {
3345            checkpoint_seq: health.head_checkpoint_seq.load(Ordering::SeqCst),
3346            checkpointed_entry_seq: health.head_checkpointed_entry_seq.load(Ordering::SeqCst),
3347            claim_log_count: health.head_claim_log_count.load(Ordering::SeqCst),
3348            claim_log_max_seq: health.head_claim_log_max_seq.load(Ordering::SeqCst),
3349        }
3350    }
3351
3352    pub fn append_chio_receipt_canonical(
3353        &self,
3354        canonical: Arc<CanonicalBytes>,
3355    ) -> Result<(), ReceiptStoreError> {
3356        self.append_chio_receipt_canonical_returning_seq(canonical)
3357            .map(|_| ())
3358    }
3359
3360    pub fn append_chio_receipt_canonical_bytes(
3361        &self,
3362        canonical: Arc<CanonicalBytes>,
3363    ) -> Result<(), ReceiptStoreError> {
3364        self.append_chio_receipt_canonical(canonical)
3365    }
3366
3367    pub fn append_chio_receipt_canonical_returning_seq(
3368        &self,
3369        canonical: Arc<CanonicalBytes>,
3370    ) -> Result<u64, ReceiptStoreError> {
3371        let receipt = decode_canonical_chio_receipt(canonical.as_ref())?;
3372        let raw_json = canonical_receipt_json(canonical.as_ref())?;
3373        self.append_verified_chio_receipt_record(&receipt, raw_json, false)
3374    }
3375
3376    pub fn append_chio_receipt_canonical_bytes_returning_seq(
3377        &self,
3378        canonical: Arc<CanonicalBytes>,
3379    ) -> Result<u64, ReceiptStoreError> {
3380        self.append_chio_receipt_canonical_returning_seq(canonical)
3381    }
3382
3383    fn append_verified_chio_receipt_record(
3384        &self,
3385        receipt: &ChioReceipt,
3386        raw_json: &str,
3387        ensure_lineage: bool,
3388    ) -> Result<u64, ReceiptStoreError> {
3389        ensure_chio_receipt_verified(receipt)?;
3390        sqlite_i64(receipt.timestamp, "receipt timestamp")?;
3391        self.receipt_commit_actor
3392            .append(receipt.clone(), raw_json.to_string(), ensure_lineage)
3393    }
3394
3395    fn append_verified_chio_receipt_record_with_timeout(
3396        &self,
3397        receipt: &ChioReceipt,
3398        raw_json: &str,
3399        ensure_lineage: bool,
3400        timeout: Duration,
3401    ) -> Result<u64, ReceiptStoreError> {
3402        ensure_chio_receipt_verified(receipt)?;
3403        sqlite_i64(receipt.timestamp, "receipt timestamp")?;
3404        self.receipt_commit_actor.append_with_timeout(
3405            receipt.clone(),
3406            raw_json.to_string(),
3407            ensure_lineage,
3408            timeout,
3409        )
3410    }
3411
3412    /// Best-effort writer liveness derived from the commit-actor counters,
3413    /// assessed against the operator-configured `stall_threshold`. See
3414    /// [`classify_writer_liveness`] for the transition rules.
3415    pub fn writer_liveness(&self, stall_threshold: Duration) -> chio_kernel::ReceiptWriterLiveness {
3416        classify_writer_liveness(
3417            &self.receipt_commit_actor.writer_counters(),
3418            u64::try_from(stall_threshold.as_millis()).unwrap_or(u64::MAX),
3419            RECEIPT_COMMIT_ACTOR_CHANNEL_CAPACITY as u64,
3420            self.receipt_commit_actor.backlog_started_unix_ms(),
3421            current_unix_ms(),
3422        )
3423    }
3424
3425    pub fn append_chio_receipt_consuming_authorization(
3426        &self,
3427        receipt: &ChioReceipt,
3428        consumption: &AuthorizationReceiptConsumption,
3429    ) -> Result<(), ReceiptStoreError> {
3430        ensure_chio_receipt_verified(receipt)?;
3431        if receipt.id != consumption.consumer_receipt_id {
3432            return Err(ReceiptStoreError::Conflict(
3433                "authorization consumption consumer receipt id does not match appended receipt"
3434                    .to_string(),
3435            ));
3436        }
3437        if receipt.tenant_id.as_deref() != consumption.tenant_id.as_deref() {
3438            return Err(ReceiptStoreError::Conflict(
3439                "authorization consumption tenant id does not match appended receipt".to_string(),
3440            ));
3441        }
3442        sqlite_i64(receipt.timestamp, "receipt timestamp")?;
3443        let raw_json = canonical_json_bytes(receipt)
3444            .map_err(|error| ReceiptStoreError::Canonical(error.to_string()))?;
3445        let raw_json = std::str::from_utf8(raw_json.as_slice()).map_err(|error| {
3446            ReceiptStoreError::Canonical(format!("canonical receipt bytes are not UTF-8: {error}"))
3447        })?;
3448        let raw_json = raw_json.to_string();
3449        let receipt = receipt.clone();
3450        let consumption = consumption.clone();
3451        self.writer_handle().run_write_receipt(move |connection| {
3452            ensure_checkpoint_transparency_guards(connection)?;
3453            let tx =
3454                connection.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?;
3455            consume_authorization_receipt_tx(&tx, &consumption)?;
3456            append_chio_receipt_tx(&tx, &receipt, &raw_json)?;
3457            ensure_receipt_lineage_statement_for_receipt_id_tx(&tx, &receipt.id)?;
3458            tx.commit()?;
3459            Ok(())
3460        })
3461    }
3462
3463    pub fn flush_receipt_writes(&self) -> Result<ReceiptFlushReport, ReceiptStoreError> {
3464        self.receipt_commit_actor.flush()?;
3465        let wal_checkpoint = Some(self.wal_checkpoint_passive()?);
3466        self.flush_report(wal_checkpoint)
3467    }
3468
3469    /// Recover a store whose claim-log projection rows survived a source-row
3470    /// delete. Fail-closed: only removes claim-log `extra` rows (absent from
3471    /// both source tables) that are (a) present in the named archive and (b) at
3472    /// or below the smallest checkpoint batch_end_seq that covers them, so the
3473    /// uncheckpointed suffix is never touched. Returns the number of rows
3474    /// removed.
3475    ///
3476    /// Dispatched as its own writer-actor command (`RetentionRepair`), not
3477    /// `writer_handle().run_write`: a `Write` job is rejected outright while
3478    /// the head is `Poisoned` (see `handle_non_append_command`'s `Write`
3479    /// arm), which is exactly the state a bricked store's writer actor is in
3480    /// on open, so `run_write` can never reach the store this method exists
3481    /// to repair. `RetentionRepair` runs unconditionally on the single writer
3482    /// connection (still fully serialized with every other writer command,
3483    /// same single-writer discipline as `run_write`), like `ReseedHead` and
3484    /// `Rotate`, and reseeds the head on success so the same store instance
3485    /// is appendable again without requiring a fresh open.
3486    pub fn retention_repair(&self, archive_path: &str) -> Result<u64, ReceiptStoreError> {
3487        let (response, result) = mpsc::sync_channel(1);
3488        let health = &self.receipt_commit_actor.health;
3489        // In-flight writer, same accounting discipline as a rotation
3490        // (`dispatch_rotate`): increment before handing the command to the
3491        // actor so a concurrent `receipt_store_health` cannot observe a
3492        // dequeued-but-uncounted repair. The `RetentionRepair` arm
3493        // decrements unconditionally on dequeue; any send/recv failure here
3494        // undoes the speculative increment so a rejected repair never leaks
3495        // inflight.
3496        health.inflight.fetch_add(1, Ordering::SeqCst);
3497        if let Err(error) =
3498            self.receipt_commit_actor
3499                .sender
3500                .try_send(ReceiptCommitCommand::RetentionRepair {
3501                    archive_path: archive_path.to_string(),
3502                    response,
3503                })
3504        {
3505            atomic_saturating_sub(&health.inflight, 1);
3506            return Err(match error {
3507                mpsc::TrySendError::Full(_) => receipt_actor_saturated_error(),
3508                mpsc::TrySendError::Disconnected(_) => receipt_actor_unavailable_error(),
3509            });
3510        }
3511        match result.recv() {
3512            Ok(outcome) => outcome,
3513            Err(_) => {
3514                atomic_saturating_sub(&health.inflight, 1);
3515                Err(receipt_actor_unavailable_error())
3516            }
3517        }
3518    }
3519
3520    pub fn audit_receipt_cost_projection(&self) -> Result<(), ReceiptStoreError> {
3521        let connection = self.connection()?;
3522        support::audit_receipt_cost_projection(&connection)
3523    }
3524
3525    /// Rerun the one-time full verification on the writer connection and
3526    /// adopt the resulting head. This is the `chio receipt audit --repair`
3527    /// entry point; it is also safe to call on a healthy store.
3528    pub fn reseed_verified_head(&self) -> Result<(), ReceiptStoreError> {
3529        let (response, result) = mpsc::sync_channel(1);
3530        self.receipt_commit_actor.health.note_channel_send();
3531        match self
3532            .receipt_commit_actor
3533            .sender
3534            .try_send(ReceiptCommitCommand::ReseedHead(response))
3535        {
3536            Ok(()) => {}
3537            Err(mpsc::TrySendError::Full(_)) => {
3538                self.receipt_commit_actor
3539                    .health
3540                    .note_channel_send_rejected();
3541                return Err(receipt_actor_saturated_error());
3542            }
3543            Err(mpsc::TrySendError::Disconnected(_)) => {
3544                self.receipt_commit_actor
3545                    .health
3546                    .note_channel_send_rejected();
3547                // A disconnected admin send is the first observation that the
3548                // writer is gone. Record the unavailable marker so the next
3549                // liveness sample reports the writer Dead immediately, rather
3550                // than staying Healthy until a later append also disconnects.
3551                self.receipt_commit_actor.health.note_writer_unavailable();
3552                return Err(receipt_actor_unavailable_error());
3553            }
3554        }
3555        result
3556            .recv()
3557            .map_err(|_| receipt_actor_unavailable_error())?
3558    }
3559
3560    /// Install the background checkpoint signer. Idempotent per store (a
3561    /// second call replaces the signer). Until called, the store appends
3562    /// without producing checkpoints.
3563    pub fn enable_background_checkpoints(
3564        &self,
3565        signer: BackgroundCheckpointSigner,
3566    ) -> Result<(), ReceiptStoreError> {
3567        self.receipt_commit_actor.health.note_channel_send();
3568        match self
3569            .receipt_commit_actor
3570            .sender
3571            .try_send(ReceiptCommitCommand::InstallSigner(signer))
3572        {
3573            Ok(()) => Ok(()),
3574            Err(mpsc::TrySendError::Full(_)) => {
3575                self.receipt_commit_actor
3576                    .health
3577                    .note_channel_send_rejected();
3578                Err(receipt_actor_saturated_error())
3579            }
3580            Err(mpsc::TrySendError::Disconnected(_)) => {
3581                self.receipt_commit_actor
3582                    .health
3583                    .note_channel_send_rejected();
3584                // See `reseed_verified_head`: a disconnected admin send must
3585                // flip liveness to Dead now, not on a later append.
3586                self.receipt_commit_actor.health.note_writer_unavailable();
3587                Err(receipt_actor_unavailable_error())
3588            }
3589        }
3590    }
3591
3592    pub fn flush_receipt_writes_with_timeout(
3593        &self,
3594        timeout: Duration,
3595    ) -> Result<ReceiptFlushReport, ReceiptStoreError> {
3596        self.receipt_commit_actor.flush_with_timeout(timeout)?;
3597        let wal_checkpoint = Some(self.wal_checkpoint_passive()?);
3598        self.flush_report(wal_checkpoint)
3599    }
3600
3601    /// Record the outcome of a background retention rotation into health.
3602    /// `None` clears a prior failure after a successful rotation; `Some(message)`
3603    /// records a rotation error or panic so a persistently failing background
3604    /// maintenance task surfaces as unhealthy in `receipt_store_health` instead
3605    /// of the failure living only in the worker's logs. Called by the kernel
3606    /// maintenance worker on the store handle it holds; the health snapshot is
3607    /// shared across all handles of this store, so the failure is visible to any
3608    /// other handle sampling health.
3609    pub fn record_retention_rotation_outcome(&self, failure: Option<&str>) {
3610        if let Ok(mut retention_error) = self.receipt_commit_actor.health.retention_error.lock() {
3611            *retention_error = failure.map(ToString::to_string);
3612        }
3613    }
3614
3615    pub fn receipt_store_health(&self) -> Result<ReceiptStoreHealthReport, ReceiptStoreError> {
3616        self.validate_claim_receipt_log_projection_current()?;
3617        let status = self.receipt_checkpoint_status(Some(1))?;
3618        // A checkpoint-chain error already produced an unhealthy status: the
3619        // checkpointed boundary drops to 0, so the committed floor (which folds
3620        // in a retention watermark whose live prefix rows were archived and
3621        // deleted) reads as a whole-log backlog. Re-probing that range would
3622        // decode rows retention intentionally removed and turn the prepared
3623        // unhealthy report (carrying the checkpoint_error operators need) into a
3624        // hard error. Only decode-probe the uncheckpointed suffix when the
3625        // checkpoint status itself is clean.
3626        if status.checkpoint_error.is_none()
3627            && status.latest_committed_entry_seq > status.latest_checkpointed_entry_seq
3628        {
3629            let connection = self.connection()?;
3630            let start_seq = status.latest_checkpointed_entry_seq + 1;
3631            load_claim_tree_canonical_bytes_range(
3632                &connection,
3633                start_seq,
3634                status.latest_committed_entry_seq,
3635            )?;
3636        }
3637        let writer_counters = self.receipt_commit_actor.writer_counters();
3638        let (writer_level, writer_restart_total) =
3639            self.receipt_commit_actor.writer_health_summary();
3640        // The authoritative pre-dispatch gate reads the watchdog verdict against
3641        // the operator-configured stall threshold; this report has no config in
3642        // scope, so it both labels and folds health against the shipped default.
3643        let writer_liveness = self.writer_liveness(std::time::Duration::from_millis(
3644            chio_kernel::DEFAULT_RECEIPT_WRITER_STALL_MS,
3645        ));
3646        let retention_error = self
3647            .receipt_commit_actor
3648            .health
3649            .retention_error
3650            .lock()
3651            .ok()
3652            .and_then(|guard| guard.clone());
3653        // A writer that cannot be trusted to persist can never read green. This is
3654        // the same predicate the pre-dispatch gate denies on, so readiness and the
3655        // gate never disagree: it covers a wedged, saturated, or dead writer by
3656        // liveness, a dead or degraded supervised thread by level, and a poisoned
3657        // verified head via `writer_serving_closed`, where the thread is still
3658        // Healthy and no batch recorded a `last_error` yet every append is already
3659        // rejected. A persistently failing background retention rotation also
3660        // forces unhealthy so a silently unenforced retention policy is not masked.
3661        let healthy = receipt_store_healthy(
3662            status.healthy,
3663            writer_counters.last_error.as_deref(),
3664            writer_liveness,
3665        ) && !self.receipt_commit_actor.writer_serving_closed()
3666            && matches!(writer_level, HealthLevel::Healthy)
3667            && retention_error.is_none();
3668        let (uncheckpointed_start_seq, uncheckpointed_end_seq) = uncheckpointed_range(
3669            status.latest_checkpointed_entry_seq,
3670            status.latest_committed_entry_seq,
3671        );
3672        Ok(ReceiptStoreHealthReport {
3673            healthy,
3674            writer: writer_counters,
3675            writer_liveness: writer_liveness.as_label().to_string(),
3676            latest_committed_entry_seq: status.latest_committed_entry_seq,
3677            latest_checkpoint_seq: status.latest_checkpoint_seq,
3678            latest_checkpointed_entry_seq: status.latest_checkpointed_entry_seq,
3679            uncheckpointed_start_seq,
3680            uncheckpointed_end_seq,
3681            checkpoint_error: status.checkpoint_error,
3682            db_size_bytes: self.db_size_bytes().ok(),
3683            retention_watermark_entry_seq: status.retention_watermark_entry_seq,
3684            retention_error,
3685            writer_level,
3686            writer_restart_total,
3687        })
3688    }
3689
3690    /// Whether the commit writer can no longer be trusted to persist receipts, so
3691    /// the kernel pre-dispatch gate must deny before a tool executes. True when the
3692    /// supervised writer thread has left the healthy state, and also when the
3693    /// writer's verified head is poisoned: the thread is alive but every append is
3694    /// rejected until an operator reseeds.
3695    pub fn writer_serving_closed(&self) -> bool {
3696        self.receipt_commit_actor.writer_serving_closed()
3697    }
3698
3699    /// Sample receipt-store health from a READ-ONLY connection.
3700    ///
3701    /// The SIEM serve-mode watchdog observes a receipt DB the kernel owns; it
3702    /// must not create it, switch it to WAL, or spin a writer pool on it,
3703    /// matching the read-only receipt-polling contract. `open` does all three, so
3704    /// it cannot be used on a read-only mount and would create an empty DB on a
3705    /// mistyped path. This opens a single READ_ONLY connection instead: a missing
3706    /// file reports `NotFound` rather than being created, and a read-only mount
3707    /// is sampled without any write attempt.
3708    ///
3709    /// A read-only observer cannot see the owning writer's in-memory counters, so
3710    /// `writer` is defaulted; the checkpoint-progress fields the watchdog gauges
3711    /// consume (committed/checkpointed seqs and the uncheckpointed range) are
3712    /// computed from the read connection with the same helpers as
3713    /// `receipt_store_health`.
3714    pub fn receipt_store_health_read_only(
3715        path: &Path,
3716    ) -> Result<ReceiptStoreHealthReport, ReceiptStoreError> {
3717        let connection = Connection::open_with_flags(
3718            path,
3719            rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX,
3720        )
3721        .map_err(|error| {
3722            if error.sqlite_error_code() == Some(rusqlite::ErrorCode::CannotOpen) {
3723                ReceiptStoreError::NotFound(format!(
3724                    "receipt database {} does not exist",
3725                    path.display()
3726                ))
3727            } else {
3728                ReceiptStoreError::Sqlite(error)
3729            }
3730        })?;
3731        let live_committed_entry_seq = latest_claim_log_entry_seq(&connection)?;
3732        let retention_watermark_entry_seq = support::retention_watermark(&connection)?;
3733        // A full rotation deletes every live claim-log row, so the live
3734        // MAX(entry_seq) drops to 0 while the latest checkpoint still sits at the
3735        // archived watermark. Committed progress must fold in the archived prefix,
3736        // otherwise this read-only watchdog reports a healthy, fully-archived
3737        // store as behind its checkpoints (committed 0 < checkpointed W). Floor
3738        // the committed seq at the watermark.
3739        let latest_committed_entry_seq =
3740            live_committed_entry_seq.max(retention_watermark_entry_seq.unwrap_or(0));
3741        // Catch a checkpoint-chain-integrity failure into a report with the
3742        // checkpoint_error set rather than propagating Err. The watchdog samples
3743        // this on a fixed interval; if corruption made this return Err, the
3744        // sampler would log-and-skip with NO gauge update, so a corrupt store
3745        // would look silent instead of alarming. Mirror the
3746        // fail-open shape of `receipt_checkpoint_status` so the watchdog still
3747        // emits a large-backlog gauge (checkpointed defaults to 0 -> the
3748        // uncheckpointed range spans the whole committed log) with the
3749        // checkpoint_error attached.
3750        match verify_checkpoint_chain_integrity(&connection) {
3751            Ok(latest) => {
3752                let latest_checkpoint_seq = latest
3753                    .as_ref()
3754                    .map(|checkpoint| checkpoint.body.checkpoint_seq);
3755                let latest_checkpointed_entry_seq = latest
3756                    .as_ref()
3757                    .map_or(0, |checkpoint| checkpoint.body.batch_end_seq);
3758                let (uncheckpointed_start_seq, uncheckpointed_end_seq) =
3759                    uncheckpointed_range(latest_checkpointed_entry_seq, latest_committed_entry_seq);
3760                Ok(ReceiptStoreHealthReport {
3761                    healthy: latest_committed_entry_seq >= latest_checkpointed_entry_seq,
3762                    writer: ReceiptWriterCounters::default(),
3763                    // A read-only observer cannot see the owning writer's
3764                    // in-memory liveness, so it is reported as unknown.
3765                    writer_liveness: chio_kernel::ReceiptWriterLiveness::Unknown
3766                        .as_label()
3767                        .to_string(),
3768                    latest_committed_entry_seq,
3769                    latest_checkpoint_seq,
3770                    latest_checkpointed_entry_seq,
3771                    uncheckpointed_start_seq,
3772                    uncheckpointed_end_seq,
3773                    checkpoint_error: None,
3774                    db_size_bytes: None,
3775                    retention_watermark_entry_seq,
3776                    // A read-only observer cannot see the owning writer's
3777                    // in-memory background-retention state, so it is defaulted
3778                    // like the writer counters above.
3779                    retention_error: None,
3780                    // A read-only observer cannot see the owning writer's supervisor.
3781                    writer_level: HealthLevel::default(),
3782                    writer_restart_total: 0,
3783                })
3784            }
3785            Err(error) => {
3786                let (uncheckpointed_start_seq, uncheckpointed_end_seq) =
3787                    uncheckpointed_range(0, latest_committed_entry_seq);
3788                Ok(ReceiptStoreHealthReport {
3789                    healthy: false,
3790                    writer: ReceiptWriterCounters::default(),
3791                    writer_liveness: chio_kernel::ReceiptWriterLiveness::Unknown
3792                        .as_label()
3793                        .to_string(),
3794                    latest_committed_entry_seq,
3795                    latest_checkpoint_seq: None,
3796                    latest_checkpointed_entry_seq: 0,
3797                    uncheckpointed_start_seq,
3798                    uncheckpointed_end_seq,
3799                    checkpoint_error: Some(error.to_string()),
3800                    db_size_bytes: None,
3801                    retention_watermark_entry_seq,
3802                    retention_error: None,
3803                    writer_level: HealthLevel::default(),
3804                    writer_restart_total: 0,
3805                })
3806            }
3807        }
3808    }
3809
3810    pub fn latest_committed_entry_seq(&self) -> Result<u64, ReceiptStoreError> {
3811        let connection = self.connection()?;
3812        // After a full-prefix rotation the live claim-log table is empty, so
3813        // MAX(entry_seq) drops to 0 while the latest checkpoint and the
3814        // retention watermark still sit at the archived boundary W. Committed
3815        // progress must fold in the archived prefix; floor the live committed
3816        // seq at the watermark so a direct trait caller does not see committed
3817        // regress to 0 behind its checkpoints. Mirrors receipt_checkpoint_status,
3818        // receipt_store_health_read_only, and flush_report.
3819        let live = latest_claim_log_entry_seq(&connection)?;
3820        let watermark = support::retention_watermark(&connection)?.unwrap_or(0);
3821        Ok(live.max(watermark))
3822    }
3823
3824    pub fn latest_checkpointed_entry_seq(&self) -> Result<u64, ReceiptStoreError> {
3825        let connection = self.connection()?;
3826        latest_checkpointed_entry_seq(&connection)
3827    }
3828
3829    pub fn next_checkpoint_range(
3830        &self,
3831        max_batch: u64,
3832    ) -> Result<Option<ReceiptCheckpointRange>, ReceiptStoreError> {
3833        let connection = self.connection()?;
3834        next_checkpoint_range_for_connection(&connection, max_batch)
3835    }
3836
3837    pub fn receipt_checkpoint_status(
3838        &self,
3839        max_batch: Option<u64>,
3840    ) -> Result<ReceiptCheckpointStatusReport, ReceiptStoreError> {
3841        self.validate_claim_receipt_log_projection_current()?;
3842        let connection = self.connection()?;
3843        // Read once and reuse across every branch below: the watermark is
3844        // reported even on an error/unhealthy status so retention visibility
3845        // does not depend on checkpoint health.
3846        let retention_watermark_entry_seq = support::retention_watermark(&connection)?;
3847        // After a full-prefix rotation the live claim-log table is empty, so
3848        // MAX(entry_seq) drops to 0 while the latest checkpoint and the
3849        // retention watermark still sit at the archived boundary W. Committed
3850        // progress must fold in the archived prefix; floor the live committed
3851        // seq at the watermark so a fully-archived store does not report
3852        // committed regressing to 0 behind its checkpoints. Mirrors
3853        // receipt_store_health_read_only.
3854        let latest_committed_entry_seq = latest_claim_log_entry_seq(&connection)?
3855            .max(retention_watermark_entry_seq.unwrap_or(0));
3856        match verify_checkpoint_chain_integrity(&connection) {
3857            Ok(latest) => {
3858                let latest_checkpoint_seq = latest
3859                    .as_ref()
3860                    .map(|checkpoint| checkpoint.body.checkpoint_seq);
3861                let latest_checkpointed_entry_seq = latest
3862                    .as_ref()
3863                    .map_or(0, |checkpoint| checkpoint.body.batch_end_seq);
3864                if latest_committed_entry_seq > latest_checkpointed_entry_seq {
3865                    let start_seq = latest_checkpointed_entry_seq + 1;
3866                    if let Err(error) = ensure_claim_log_range_contiguous(
3867                        &connection,
3868                        start_seq,
3869                        latest_committed_entry_seq,
3870                        "uncheckpointed range",
3871                    ) {
3872                        return Ok(ReceiptCheckpointStatusReport {
3873                            healthy: false,
3874                            latest_committed_entry_seq,
3875                            latest_checkpoint_seq,
3876                            latest_checkpointed_entry_seq,
3877                            next_range: None,
3878                            checkpoint_error: Some(error.to_string()),
3879                            retention_watermark_entry_seq,
3880                        });
3881                    }
3882                }
3883                let next_range = match max_batch {
3884                    Some(max_batch) => {
3885                        next_checkpoint_range_for_connection(&connection, max_batch)?
3886                    }
3887                    None => None,
3888                };
3889                Ok(ReceiptCheckpointStatusReport {
3890                    healthy: true,
3891                    latest_committed_entry_seq,
3892                    latest_checkpoint_seq,
3893                    latest_checkpointed_entry_seq,
3894                    next_range,
3895                    checkpoint_error: None,
3896                    retention_watermark_entry_seq,
3897                })
3898            }
3899            Err(error) => Ok(ReceiptCheckpointStatusReport {
3900                healthy: false,
3901                latest_committed_entry_seq,
3902                latest_checkpoint_seq: None,
3903                latest_checkpointed_entry_seq: 0,
3904                next_range: None,
3905                checkpoint_error: Some(error.to_string()),
3906                retention_watermark_entry_seq,
3907            }),
3908        }
3909    }
3910
3911    pub fn create_next_receipt_checkpoint(
3912        &self,
3913        max_batch: u64,
3914        keypair: &Keypair,
3915    ) -> Result<ReceiptCheckpointCreateReport, ReceiptStoreError> {
3916        let keypair = keypair.clone();
3917        self.writer_handle().run_write(move |connection| {
3918            validate_claim_receipt_log_entries(connection)?;
3919            create_next_receipt_checkpoint_atomic(connection, max_batch, &keypair)
3920        })
3921    }
3922
3923    fn flush_report(
3924        &self,
3925        wal_checkpoint: Option<ReceiptWalCheckpointReport>,
3926    ) -> Result<ReceiptFlushReport, ReceiptStoreError> {
3927        let head = self.writer_head_snapshot();
3928        let connection = self.connection()?;
3929        // After a full-prefix rotation the live claim-log table is empty, so
3930        // MAX(entry_seq) drops to 0 while the latest checkpoint and the retention
3931        // watermark still sit at the archived boundary W. Committed progress must
3932        // fold in the archived prefix; floor the live committed seq at the
3933        // watermark so a fully-archived store does not report committed
3934        // regressing to 0 behind its checkpoints and corrupt operators' flush
3935        // metrics. Mirrors receipt_checkpoint_status and
3936        // receipt_store_health_read_only.
3937        let latest_committed_entry_seq = latest_claim_log_entry_seq(&connection)?
3938            .max(support::retention_watermark(&connection)?.unwrap_or(0));
3939        // The writer head snapshot is only refreshed by this handle's own
3940        // appends/writes. When another store instance or the operator CLI
3941        // extends the checkpoint chain and this handle has had no intervening
3942        // local write, the head atomics are stale and would overstate the
3943        // uncheckpointed range. Read the persisted checkpoint head from the DB
3944        // (read-only reader-pool query, not a writer-head mutation) and take
3945        // the higher of the two so the report reflects the current chain.
3946        // Only trust the persisted latest checkpoint if its signed body
3947        // VERIFIES: `parse_persisted_checkpoint_row` checks
3948        // column/body agreement AND the signature, so a tampered or out-of-band
3949        // row with an inflated `batch_end_seq` cannot make the flush report a
3950        // false `checkpointed_entry_seq` and hide the uncheckpointed range. On a
3951        // verification failure fall back to ONLY the actor's verified head (via
3952        // the `.max` below). Reader-pool READ, no write; single latest-row
3953        // body verification, not a full chain verify.
3954        //
3955        // Chain-connectivity guard: a single-row parse
3956        // does NOT catch a latest checkpoint that individually verifies yet is
3957        // DISCONNECTED from the chain (skipped `checkpoint_seq` or wrong
3958        // predecessor), which a full `verify_checkpoint_chain_integrity`
3959        // catches. Additionally require
3960        // the latest checkpoint to link to its immediate predecessor before
3961        // trusting its `batch_end_seq`; a disconnected latest is dropped (fall
3962        // back to the actor's verified head). This is a bounded O(1) predecessor
3963        // read on the operator/health surface, NOT a full O(N) chain walk on the
3964        // per-append hot path.
3965        //
3966        // Claim-log content guard: a separate process
3967        // advancing `kernel_checkpoints` on a shared DB can persist a latest row
3968        // that parses (columns match its signed body) AND links to its predecessor
3969        // yet whose `merkle_root`/`tree_size`/`batch_end_seq` describe a batch this
3970        // database's `claim_receipt_log_entries` never actually contained (an
3971        // imported/foreign checkpoint). A full `verify_checkpoint_chain_integrity`
3972        // rebuilds the checkpoint Merkle range from the local claim log; without
3973        // that content check here an
3974        // inflated `batch_end_seq` would make this report advertise a false
3975        // `checkpointed_entry_seq` and hide the uncheckpointed range. Rebuild the
3976        // latest checkpoint's Merkle range from the LOCAL claim log and drop it on
3977        // mismatch (fall back to the actor's verified head). Bounded O(b) over the
3978        // single latest checkpoint's own batch on the operator/health surface, NOT
3979        // a full O(N) chain walk on the per-append hot path.
3980        //
3981        // Watermark exemption: a checkpoint fully covered by a TRUSTED archival
3982        // watermark has had its claim-log rows co-archived and deleted, so a live
3983        // Merkle rebuild would fail for a perfectly valid checkpoint. Mirror the
3984        // full chain verification (`verify_checkpoint_chain_integrity`): skip only
3985        // the live rebuild for a watermark-covered checkpoint and trust the archive
3986        // to serve that deep verification; its signature, column agreement, and
3987        // chain connectivity above still run. Without this, a fully-archived latest
3988        // checkpoint is discarded and flush reports a stale `checkpointed_entry_seq`
3989        // and a spurious uncheckpointed range until a later write catches the head
3990        // up. `trusted_retention_watermark` is fail-closed (0 unless the boundary,
3991        // the absent live prefix, and the backing archive all check out), so a
3992        // forged watermark cannot suppress the rebuild for an unarchived range.
3993        let trusted_watermark = trusted_retention_watermark(&connection)?;
3994        let verified_persisted = load_latest_persisted_checkpoint_row(&connection)?
3995            .and_then(|row| parse_persisted_checkpoint_row(row).ok())
3996            .filter(|checkpoint| {
3997                latest_checkpoint_is_chain_connected(&connection, checkpoint).is_ok()
3998                    && (checkpoint.body.batch_end_seq <= trusted_watermark
3999                        || validate_checkpoint_against_claim_log(&connection, checkpoint).is_ok())
4000            });
4001        let persisted_checkpoint_seq = verified_persisted
4002            .as_ref()
4003            .map_or(0, |checkpoint| checkpoint.body.checkpoint_seq);
4004        let persisted_checkpointed_entry_seq = verified_persisted
4005            .as_ref()
4006            .map_or(0, |checkpoint| checkpoint.body.batch_end_seq);
4007        let checkpoint_seq = head.checkpoint_seq.max(persisted_checkpoint_seq);
4008        let latest_checkpointed_entry_seq = head
4009            .checkpointed_entry_seq
4010            .max(persisted_checkpointed_entry_seq);
4011        let latest_checkpoint_seq = (checkpoint_seq > 0).then_some(checkpoint_seq);
4012        let (uncheckpointed_start_seq, uncheckpointed_end_seq) =
4013            uncheckpointed_range(latest_checkpointed_entry_seq, latest_committed_entry_seq);
4014        Ok(ReceiptFlushReport {
4015            writer: self.receipt_commit_actor.writer_counters(),
4016            latest_committed_entry_seq,
4017            latest_checkpoint_seq,
4018            latest_checkpointed_entry_seq,
4019            uncheckpointed_start_seq,
4020            uncheckpointed_end_seq,
4021            wal_checkpoint,
4022            db_size_bytes: self.db_size_bytes().ok(),
4023        })
4024    }
4025
4026    fn validate_claim_receipt_log_projection_current(&self) -> Result<(), ReceiptStoreError> {
4027        let connection = self.connection()?;
4028        validate_claim_receipt_log_entries(&connection)
4029    }
4030
4031    fn wal_checkpoint_passive(&self) -> Result<ReceiptWalCheckpointReport, ReceiptStoreError> {
4032        let connection = self.connection()?;
4033        let (busy, log_frames, checkpointed_frames) =
4034            connection.query_row("PRAGMA wal_checkpoint(PASSIVE)", [], |row| {
4035                Ok((
4036                    row.get::<_, i64>(0)?,
4037                    row.get::<_, i64>(1)?,
4038                    row.get::<_, i64>(2)?,
4039                ))
4040            })?;
4041        Ok(ReceiptWalCheckpointReport {
4042            busy: sqlite_u64(busy, "wal checkpoint busy")?,
4043            log_frames: wal_checkpoint_frame_count(log_frames, "wal checkpoint log frames")?,
4044            checkpointed_frames: wal_checkpoint_frame_count(
4045                checkpointed_frames,
4046                "wal checkpointed frames",
4047            )?,
4048        })
4049    }
4050}
4051
4052/// `PRAGMA wal_checkpoint` reports -1 for the log/checkpointed frame columns
4053/// when there is nothing to checkpoint (an already-empty WAL). Under
4054/// concurrent `flush_receipt_writes()` callers this is routine: one caller's
4055/// PASSIVE checkpoint truncates the WAL, and a second caller racing right
4056/// behind it observes the now-empty WAL and gets -1/-1 from SQLite even
4057/// though `busy` is 0 (success). That is success-with-nothing-to-do, not an
4058/// error, so it is normalized to 0 rather than rejected by `sqlite_u64`.
4059fn wal_checkpoint_frame_count(value: i64, field: &str) -> Result<u64, ReceiptStoreError> {
4060    if value == -1 {
4061        return Ok(0);
4062    }
4063    sqlite_u64(value, field)
4064}
4065
4066fn uncheckpointed_range(checkpointed: u64, committed: u64) -> (Option<u64>, Option<u64>) {
4067    if committed > checkpointed {
4068        (Some(checkpointed + 1), Some(committed))
4069    } else {
4070        (None, None)
4071    }
4072}
4073
4074fn latest_claim_log_entry_seq(connection: &Connection) -> Result<u64, ReceiptStoreError> {
4075    connection
4076        .query_row(
4077            "SELECT COALESCE(MAX(entry_seq), 0) FROM claim_receipt_log_entries",
4078            [],
4079            |row| row.get::<_, i64>(0),
4080        )
4081        .map_err(ReceiptStoreError::from)
4082        .and_then(|value| sqlite_u64(value, "latest claim receipt log entry_seq"))
4083}
4084
4085fn latest_checkpointed_entry_seq(connection: &Connection) -> Result<u64, ReceiptStoreError> {
4086    verify_checkpoint_chain_integrity(connection)
4087        .map(|latest| latest.map_or(0, |checkpoint| checkpoint.body.batch_end_seq))
4088}
4089
4090fn next_checkpoint_range_for_connection(
4091    connection: &Connection,
4092    max_batch: u64,
4093) -> Result<Option<ReceiptCheckpointRange>, ReceiptStoreError> {
4094    if max_batch == 0 {
4095        return Err(ReceiptStoreError::Conflict(
4096            "checkpoint max_batch must be greater than zero".to_string(),
4097        ));
4098    }
4099    let latest_committed = latest_claim_log_entry_seq(connection)?;
4100    let latest_checkpointed = latest_checkpointed_entry_seq(connection)?;
4101    if latest_committed <= latest_checkpointed {
4102        return Ok(None);
4103    }
4104    let start_seq = latest_checkpointed + 1;
4105    let end_seq = latest_committed.min(start_seq.saturating_add(max_batch - 1));
4106    ensure_claim_log_range_contiguous(connection, start_seq, end_seq, "checkpoint range")?;
4107    Ok(Some(ReceiptCheckpointRange { start_seq, end_seq }))
4108}
4109
4110fn ensure_claim_log_range_contiguous(
4111    connection: &Connection,
4112    start_seq: u64,
4113    end_seq: u64,
4114    context: &str,
4115) -> Result<(), ReceiptStoreError> {
4116    if end_seq < start_seq {
4117        return Err(ReceiptStoreError::Conflict(format!(
4118            "claim receipt log {context} end {end_seq} is before start {start_seq}"
4119        )));
4120    }
4121    let (count, min_seq, max_seq) = connection.query_row(
4122        r#"
4123        SELECT COUNT(*), MIN(entry_seq), MAX(entry_seq)
4124        FROM claim_receipt_log_entries
4125        WHERE entry_seq >= ?1 AND entry_seq <= ?2
4126        "#,
4127        params![
4128            sqlite_i64(start_seq, "claim log range start_seq")?,
4129            sqlite_i64(end_seq, "claim log range end_seq")?,
4130        ],
4131        |row| {
4132            Ok((
4133                row.get::<_, i64>(0)?,
4134                row.get::<_, Option<i64>>(1)?,
4135                row.get::<_, Option<i64>>(2)?,
4136            ))
4137        },
4138    )?;
4139    let expected = end_seq - start_seq + 1;
4140    let count = sqlite_u64(count, "claim receipt log range count")?;
4141    let min_seq = min_seq
4142        .map(|value| sqlite_u64(value, "claim receipt log range min_seq"))
4143        .transpose()?;
4144    let max_seq = max_seq
4145        .map(|value| sqlite_u64(value, "claim receipt log range max_seq"))
4146        .transpose()?;
4147    if count != expected || min_seq != Some(start_seq) || max_seq != Some(end_seq) {
4148        return Err(ReceiptStoreError::Conflict(format!(
4149            "claim receipt log has a gap in {context} {start_seq}..={end_seq}"
4150        )));
4151    }
4152    Ok(())
4153}
4154
4155fn claim_log_entry_seq_for_source_tx(
4156    tx: &rusqlite::Transaction<'_>,
4157    receipt_kind: &str,
4158    source_seq: u64,
4159) -> Result<u64, ReceiptStoreError> {
4160    let source_seq_i64 = sqlite_i64(source_seq, "claim receipt source_seq")?;
4161    let (entry_seq, log_receipt_id, log_raw_json, source_receipt_id, source_raw_json) =
4162        match receipt_kind {
4163            "tool_receipt" => tx.query_row(
4164                r#"
4165                SELECT l.entry_seq, l.receipt_id, l.raw_json, r.receipt_id, r.raw_json
4166                FROM claim_receipt_log_entries l
4167                JOIN chio_tool_receipts r ON r.seq = l.source_seq
4168                WHERE l.receipt_kind = ?1 AND l.source_seq = ?2
4169                "#,
4170                params![receipt_kind, source_seq_i64],
4171                |row| {
4172                    Ok((
4173                        row.get::<_, i64>(0)?,
4174                        row.get::<_, String>(1)?,
4175                        row.get::<_, String>(2)?,
4176                        row.get::<_, String>(3)?,
4177                        row.get::<_, String>(4)?,
4178                    ))
4179                },
4180            ),
4181            "child_receipt" => tx.query_row(
4182                r#"
4183                SELECT l.entry_seq, l.receipt_id, l.raw_json, r.receipt_id, r.raw_json
4184                FROM claim_receipt_log_entries l
4185                JOIN chio_child_receipts r ON r.seq = l.source_seq
4186                WHERE l.receipt_kind = ?1 AND l.source_seq = ?2
4187                "#,
4188                params![receipt_kind, source_seq_i64],
4189                |row| {
4190                    Ok((
4191                        row.get::<_, i64>(0)?,
4192                        row.get::<_, String>(1)?,
4193                        row.get::<_, String>(2)?,
4194                        row.get::<_, String>(3)?,
4195                        row.get::<_, String>(4)?,
4196                    ))
4197                },
4198            ),
4199            other => {
4200                return Err(ReceiptStoreError::Conflict(format!(
4201                    "unsupported claim receipt log kind `{other}`"
4202                )));
4203            }
4204        }
4205        .optional()?
4206        .ok_or_else(|| {
4207            ReceiptStoreError::Conflict(format!(
4208                "claim receipt log entry missing for {receipt_kind} source seq {source_seq}"
4209            ))
4210        })?;
4211    if log_receipt_id != source_receipt_id || log_raw_json != source_raw_json {
4212        return Err(ReceiptStoreError::Conflict(format!(
4213            "claim receipt log entry for {receipt_kind} source seq {source_seq} diverges from source row"
4214        )));
4215    }
4216    sqlite_positive_u64(entry_seq, "claim receipt log entry_seq")
4217}
4218
4219fn append_chio_receipt_tx(
4220    tx: &rusqlite::Transaction<'_>,
4221    receipt: &ChioReceipt,
4222    raw_json: &str,
4223) -> Result<u64, ReceiptStoreError> {
4224    append_chio_receipt_tx_with_insert_status(tx, receipt, raw_json).map(|(seq, _)| seq)
4225}
4226
4227fn append_chio_receipt_tx_with_insert_status(
4228    tx: &rusqlite::Transaction<'_>,
4229    receipt: &ChioReceipt,
4230    raw_json: &str,
4231) -> Result<(u64, bool), ReceiptStoreError> {
4232    let (cost_currency, cost_charged_be) = receipt_cost_projection(receipt)?;
4233    let attribution = extract_receipt_attribution(receipt);
4234    let mut subject_key = attribution.subject_key;
4235    let mut issuer_key = attribution.issuer_key;
4236    if subject_key.is_none() || issuer_key.is_none() {
4237        if let Some((lineage_subject_key, lineage_issuer_key)) = tx
4238            .query_row(
4239                "SELECT subject_key, issuer_key FROM capability_lineage WHERE capability_id = ?1",
4240                params![receipt.capability_id.as_str()],
4241                |row| {
4242                    Ok((
4243                        row.get::<_, Option<String>>(0)?,
4244                        row.get::<_, Option<String>>(1)?,
4245                    ))
4246                },
4247            )
4248            .optional()?
4249        {
4250            if subject_key.is_none() {
4251                subject_key = lineage_subject_key;
4252            }
4253            if issuer_key.is_none() {
4254                issuer_key = lineage_issuer_key;
4255            }
4256        }
4257    }
4258    let source_seq = tx
4259        .query_row(
4260            r#"
4261        INSERT INTO chio_tool_receipts (receipt_id, timestamp, capability_id, subject_key, issuer_key, grant_index, tool_server, tool_name, decision_kind, policy_hash, content_hash, tenant_id, raw_json, cost_currency, cost_charged_be) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(receipt_id) DO NOTHING RETURNING seq
4262        "#,
4263            params![
4264                receipt.id.as_str(),
4265                sqlite_i64(receipt.timestamp, "receipt timestamp")?,
4266                receipt.capability_id.as_str(),
4267                subject_key,
4268                issuer_key,
4269                attribution.grant_index.map(i64::from),
4270                receipt.tool_server.as_str(),
4271                receipt.tool_name.as_str(),
4272                receipt_decision_kind(receipt),
4273                receipt.policy_hash.as_str(),
4274                receipt.content_hash.as_str(),
4275                receipt.tenant_id.as_deref(),
4276                raw_json,
4277                cost_currency.as_deref(),
4278                cost_charged_be.as_deref(),
4279            ],
4280            |row| row.get::<_, i64>(0),
4281        )
4282        .optional()?;
4283    let Some(source_seq) = source_seq else {
4284        let (existing_source_seq, existing_raw_json, existing_currency, existing_key) = tx.query_row(
4285            "SELECT seq, raw_json, cost_currency, cost_charged_be FROM chio_tool_receipts WHERE receipt_id = ?1",
4286            params![receipt.id.as_str()],
4287            |row| {
4288                Ok((
4289                    row.get::<_, i64>(0)?,
4290                    row.get::<_, String>(1)?,
4291                    row.get::<_, Option<String>>(2)?,
4292                    row.get::<_, Option<Vec<u8>>>(3)?,
4293                ))
4294            },
4295        )?;
4296        let existing_source_seq =
4297            sqlite_positive_u64(existing_source_seq, "tool receipt source_seq")?;
4298        if existing_raw_json != raw_json {
4299            return Err(ReceiptStoreError::Conflict(format!(
4300                "tool receipt `{}` already exists with different content",
4301                receipt.id
4302            )));
4303        }
4304        if existing_currency != cost_currency || existing_key != cost_charged_be {
4305            return Err(ReceiptStoreError::Conflict(format!(
4306                "tool receipt `{}` already exists with different cost projection",
4307                receipt.id
4308            )));
4309        }
4310        decode_verified_chio_receipt(
4311            &existing_raw_json,
4312            "persisted duplicate tool receipt",
4313            Some(existing_source_seq),
4314        )?;
4315        return claim_log_entry_seq_for_source_tx(tx, "tool_receipt", existing_source_seq)
4316            .map(|seq| (seq, false));
4317    };
4318    let source_seq = sqlite_positive_u64(source_seq, "tool receipt source_seq")?;
4319    claim_log_entry_seq_for_source_tx(tx, "tool_receipt", source_seq).map(|seq| (seq, true))
4320}
4321
4322fn consume_authorization_receipt_tx(
4323    tx: &rusqlite::Transaction<'_>,
4324    consumption: &AuthorizationReceiptConsumption,
4325) -> Result<(), ReceiptStoreError> {
4326    if consumption.authorization_receipt_id.trim().is_empty()
4327        || consumption.consumer_receipt_id.trim().is_empty()
4328        || consumption.request_id.trim().is_empty()
4329        || consumption.session_id.trim().is_empty()
4330        || consumption.tool_call_id.trim().is_empty()
4331        || consumption.parameter_hash.trim().is_empty()
4332    {
4333        return Err(ReceiptStoreError::Conflict(
4334            "authorization receipt consumption requires non-empty binding fields".to_string(),
4335        ));
4336    }
4337    // Tenant id may be `None` for non-enterprise / single-tenant deployments,
4338    // but if it is `Some(_)` it must not be an empty / whitespace-only string.
4339    if matches!(&consumption.tenant_id, Some(tenant) if tenant.trim().is_empty()) {
4340        return Err(ReceiptStoreError::Conflict(
4341            "authorization receipt consumption tenant id must not be empty when present"
4342                .to_string(),
4343        ));
4344    }
4345    let consumed_at = sqlite_i64(
4346        consumption.consumed_at_unix_ms,
4347        "authorization receipt consumed_at_unix_ms",
4348    )?;
4349    let authorization_tenant = tx
4350        .query_row(
4351            "SELECT tenant_id FROM chio_tool_receipts WHERE receipt_id = ?1",
4352            params![consumption.authorization_receipt_id.as_str()],
4353            |row| row.get::<_, Option<String>>(0),
4354        )
4355        .optional()?
4356        .ok_or_else(|| {
4357            ReceiptStoreError::NotFound(format!(
4358                "authorization receipt {} was not found",
4359                consumption.authorization_receipt_id
4360            ))
4361        })?;
4362    if authorization_tenant.as_deref() != consumption.tenant_id.as_deref() {
4363        return Err(ReceiptStoreError::Conflict(
4364            "authorization receipt tenant id does not match consumption tenant".to_string(),
4365        ));
4366    }
4367    match tx.execute(
4368        r#"
4369        INSERT INTO chio_authorization_receipt_consumptions (
4370            authorization_receipt_id,
4371            consumer_receipt_id,
4372            request_id,
4373            session_id,
4374            tool_call_id,
4375            tenant_id,
4376            parameter_hash,
4377            consumed_at_unix_ms
4378        ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)
4379        "#,
4380        params![
4381            consumption.authorization_receipt_id.as_str(),
4382            consumption.consumer_receipt_id.as_str(),
4383            consumption.request_id.as_str(),
4384            consumption.session_id.as_str(),
4385            consumption.tool_call_id.as_str(),
4386            consumption.tenant_id.as_deref(),
4387            consumption.parameter_hash.as_str(),
4388            consumed_at,
4389        ],
4390    ) {
4391        Ok(_) => Ok(()),
4392        Err(error)
4393            if matches!(
4394                error.sqlite_error_code(),
4395                Some(rusqlite::ErrorCode::ConstraintViolation)
4396            ) =>
4397        {
4398            Err(ReceiptStoreError::Conflict(
4399                "authorization receipt already consumed".to_string(),
4400            ))
4401        }
4402        Err(error) => Err(ReceiptStoreError::Sqlite(error)),
4403    }
4404}
4405
4406fn decode_canonical_chio_receipt(
4407    canonical: &CanonicalBytes,
4408) -> Result<ChioReceipt, ReceiptStoreError> {
4409    let receipt: ChioReceipt =
4410        serde_json::from_slice(canonical.as_bytes()).map_err(ReceiptStoreError::from)?;
4411    let expected = canonical_json_bytes(&receipt)
4412        .map_err(|error| ReceiptStoreError::Canonical(error.to_string()))?;
4413    if expected.as_slice() != canonical.as_bytes() {
4414        return Err(ReceiptStoreError::Canonical(
4415            "canonical receipt bytes do not match ChioReceipt serialization".to_string(),
4416        ));
4417    }
4418    Ok(receipt)
4419}
4420
4421fn canonical_receipt_json(canonical: &CanonicalBytes) -> Result<&str, ReceiptStoreError> {
4422    std::str::from_utf8(canonical.as_bytes()).map_err(|error| {
4423        ReceiptStoreError::Canonical(format!("canonical receipt bytes are not UTF-8: {error}"))
4424    })
4425}
4426#[cfg(test)]
4427mod receipt_commit_actor_tests {
4428    use super::*;
4429
4430    #[test]
4431    fn writer_health_starts_with_a_poisoned_head_until_seeding_clears_it() {
4432        // The commit writer seeds its verified head asynchronously on the actor
4433        // thread. Until that seed succeeds, durable persistence is unproven, so a
4434        // freshly constructed health mirror must report a poisoned head. Starting
4435        // open would let a corrupt or still-attaching store pass
4436        // `writer_serving_closed` and execute a tool before its first append can
4437        // reject, which is exactly the fail-open window the pre-dispatch gate
4438        // exists to prevent.
4439        let health = ReceiptCommitWriterHealth::default();
4440        assert!(
4441            health.head_poisoned.load(Ordering::SeqCst),
4442            "writer health must start head-poisoned (serving closed) until a seeded head clears it"
4443        );
4444    }
4445
4446    fn idle_worker() -> Arc<ReceiptCommitWorker> {
4447        Arc::new(ReceiptCommitWorker { join: None })
4448    }
4449
4450    fn actor_test_receipt() -> Result<ChioReceipt, ReceiptStoreError> {
4451        let keypair = chio_core::crypto::Keypair::generate();
4452        ChioReceipt::sign(
4453            chio_core::receipt::body::ChioReceiptBody {
4454                id: "rcpt-actor-test".to_string(),
4455                timestamp: 1,
4456                capability_id: "cap-actor".to_string(),
4457                tool_server: "shell".to_string(),
4458                tool_name: "bash".to_string(),
4459                action: chio_core::receipt::decision::ToolCallAction::from_parameters(
4460                    serde_json::json!({}),
4461                )
4462                .map_err(|error| ReceiptStoreError::Canonical(error.to_string()))?,
4463                decision: Some(Decision::Allow),
4464                receipt_kind: Default::default(),
4465                boundary_class: Default::default(),
4466                observation_outcome: None,
4467                tool_origin: Default::default(),
4468                redaction_mode: Default::default(),
4469                actor_chain: Vec::new(),
4470                content_hash: "content".to_string(),
4471                policy_hash: "policy".to_string(),
4472                evidence: Vec::new(),
4473                metadata: None,
4474                trust_level: chio_core::receipt::kinds::TrustLevel::default(),
4475                tenant_id: None,
4476                kernel_key: keypair.public_key(),
4477                bbs_projection_version: None,
4478            },
4479            &keypair,
4480        )
4481        .map_err(|error| ReceiptStoreError::CryptoDecode(error.to_string()))
4482    }
4483
4484    #[test]
4485    fn receipt_commit_actor_channel_has_fixed_capacity() -> Result<(), Box<dyn std::error::Error>> {
4486        let (sender, _receiver) = receipt_commit_channel();
4487        for _ in 0..RECEIPT_COMMIT_ACTOR_CHANNEL_CAPACITY {
4488            let (response, _result) = mpsc::sync_channel(1);
4489            sender.try_send(ReceiptCommitCommand::Flush(response))?;
4490        }
4491
4492        let (response, _result) = mpsc::sync_channel(1);
4493        match sender.try_send(ReceiptCommitCommand::Flush(response)) {
4494            Err(mpsc::TrySendError::Full(_)) => Ok(()),
4495            Err(mpsc::TrySendError::Disconnected(_)) => {
4496                Err("commit actor channel disconnected unexpectedly".into())
4497            }
4498            Ok(()) => Err("commit actor channel accepted beyond fixed capacity".into()),
4499        }
4500    }
4501
4502    #[test]
4503    fn receipt_commit_actor_append_fails_closed_when_queue_is_full(
4504    ) -> Result<(), Box<dyn std::error::Error>> {
4505        let (sender, _receiver) = receipt_commit_channel();
4506        let health = Arc::new(ReceiptCommitWriterHealth::default());
4507        for _ in 0..RECEIPT_COMMIT_ACTOR_CHANNEL_CAPACITY {
4508            let (response, _result) = mpsc::sync_channel(1);
4509            sender.try_send(ReceiptCommitCommand::Flush(response))?;
4510        }
4511        let actor = ReceiptCommitActor {
4512            sender,
4513            health,
4514            worker: idle_worker(),
4515        };
4516
4517        let error = actor.append(actor_test_receipt()?, "{}".to_string(), false);
4518
4519        assert!(error
4520            .err()
4521            .ok_or("expected queue saturation error")?
4522            .to_string()
4523            .contains("sqlite receipt commit queue saturated"));
4524        Ok(())
4525    }
4526
4527    #[test]
4528    fn receipt_commit_actor_flush_honors_timeout() -> Result<(), Box<dyn std::error::Error>> {
4529        let (sender, _receiver) = receipt_commit_channel();
4530        let health = Arc::new(ReceiptCommitWriterHealth::default());
4531        let actor = ReceiptCommitActor {
4532            sender,
4533            health,
4534            worker: idle_worker(),
4535        };
4536
4537        let error = actor.flush_with_timeout(Duration::from_millis(1));
4538
4539        match error.err().ok_or("expected flush timeout error")? {
4540            ReceiptStoreError::Timeout {
4541                operation,
4542                timeout_ms,
4543            } => {
4544                assert_eq!(operation, "sqlite receipt commit flush");
4545                assert_eq!(timeout_ms, 1);
4546            }
4547            other => {
4548                return Err(
4549                    std::io::Error::other(format!("expected timeout error, got {other}")).into(),
4550                );
4551            }
4552        }
4553        Ok(())
4554    }
4555
4556    #[test]
4557    fn append_with_timeout_maps_to_timeout_and_keeps_inflight_elevated(
4558    ) -> Result<(), Box<dyn std::error::Error>> {
4559        // A commit actor whose worker never drains: try_send queues the command,
4560        // but no reply ever arrives, so the bounded wait elapses.
4561        let (sender, _receiver) = receipt_commit_channel();
4562        let health = Arc::new(ReceiptCommitWriterHealth::default());
4563        let actor = ReceiptCommitActor {
4564            sender,
4565            health,
4566            worker: idle_worker(),
4567        };
4568        let inflight_before = actor.health.inflight.load(Ordering::SeqCst);
4569
4570        let start = std::time::Instant::now();
4571        let error = actor.append_with_timeout(
4572            actor_test_receipt()?,
4573            "{}".to_string(),
4574            false,
4575            Duration::from_millis(250),
4576        );
4577        assert!(start.elapsed() < Duration::from_secs(2));
4578
4579        match error.err().ok_or("expected append timeout error")? {
4580            ReceiptStoreError::Timeout { operation, .. } => {
4581                assert_eq!(operation, "sqlite receipt commit append");
4582            }
4583            other => {
4584                return Err(
4585                    std::io::Error::other(format!("expected timeout error, got {other}")).into(),
4586                );
4587            }
4588        }
4589        // The timeout side must not decrement inflight; ownership stays with the
4590        // actor, so a genuinely wedged writer keeps inflight elevated.
4591        assert_eq!(
4592            actor.health.inflight.load(Ordering::SeqCst),
4593            inflight_before + 1
4594        );
4595        assert_eq!(actor.health.failed_total.load(Ordering::SeqCst), 0);
4596        assert_eq!(actor.health.timed_out_inflight.load(Ordering::SeqCst), 1);
4597        assert_eq!(actor.health.timed_out_total.load(Ordering::SeqCst), 1);
4598        Ok(())
4599    }
4600
4601    #[test]
4602    fn enqueue_on_a_disconnected_actor_records_writer_dead(
4603    ) -> Result<(), Box<dyn std::error::Error>> {
4604        // The commit actor has exited, so its receiver is gone and `try_send`
4605        // fails Disconnected before any response channel exists. That enqueue
4606        // path must record the writer death, or the next liveness sample keeps
4607        // reporting the writer Healthy and admits a tool side effect whose
4608        // receipt can never be persisted.
4609        let (sender, receiver) = receipt_commit_channel();
4610        drop(receiver);
4611        let health = Arc::new(ReceiptCommitWriterHealth::default());
4612        let actor = ReceiptCommitActor {
4613            sender,
4614            health,
4615            worker: idle_worker(),
4616        };
4617
4618        let error = actor.append_with_timeout(
4619            actor_test_receipt()?,
4620            "{}".to_string(),
4621            false,
4622            Duration::from_millis(250),
4623        );
4624        assert!(error
4625            .err()
4626            .ok_or("expected writer-unavailable error")?
4627            .to_string()
4628            .contains("unavailable"));
4629
4630        let counters = actor.writer_counters();
4631        assert!(
4632            counters
4633                .last_error
4634                .as_deref()
4635                .is_some_and(|error| error.contains("unavailable")),
4636            "the disconnected enqueue must record the writer death"
4637        );
4638        assert_eq!(
4639            classify_writer_liveness(
4640                &counters,
4641                10_000,
4642                RECEIPT_COMMIT_ACTOR_CHANNEL_CAPACITY as u64,
4643                None,
4644                1_000_000,
4645            ),
4646            chio_kernel::ReceiptWriterLiveness::Dead,
4647            "a disconnected writer must classify as Dead so admission stops"
4648        );
4649        Ok(())
4650    }
4651
4652    #[test]
4653    fn note_accept_restamps_backlog_start_only_on_a_fresh_backlog() {
4654        let health = ReceiptCommitWriterHealth::default();
4655
4656        // 0 -> 1 begins a backlog and stamps a real start time.
4657        health.note_accept(0);
4658        assert_ne!(
4659            health.backlog_started_unix_ms.load(Ordering::SeqCst),
4660            0,
4661            "the first enqueue of a backlog must stamp its start"
4662        );
4663
4664        // 1 -> 2 grows an ongoing backlog and must NOT move its start.
4665        health.backlog_started_unix_ms.store(1, Ordering::SeqCst);
4666        health.note_accept(1);
4667        assert_eq!(
4668            health.backlog_started_unix_ms.load(Ordering::SeqCst),
4669            1,
4670            "a growing backlog must keep its original start"
4671        );
4672
4673        // 0 -> 1 after the writer drained begins a NEW backlog and restamps.
4674        health.backlog_started_unix_ms.store(1, Ordering::SeqCst);
4675        health.note_accept(0);
4676        assert_ne!(
4677            health.backlog_started_unix_ms.load(Ordering::SeqCst),
4678            1,
4679            "a fresh backlog after draining must restamp the start"
4680        );
4681    }
4682
4683    #[test]
4684    fn timeout_tracker_attributes_timeout_until_its_actor_token_completes() {
4685        let health = Arc::new(ReceiptCommitWriterHealth::default());
4686        let (mut completion, timeout) = writer_command_tracker(&health);
4687
4688        timeout.note_timeout("sqlite receipt commit write timed out");
4689        assert_eq!(health.timed_out_inflight.load(Ordering::SeqCst), 1);
4690        assert_eq!(health.timed_out_total.load(Ordering::SeqCst), 1);
4691        assert!(health.last_error.lock().is_ok_and(|error| error
4692            .as_deref()
4693            .is_some_and(|message| message.contains("timed out"))));
4694
4695        completion.complete();
4696        assert_eq!(health.timed_out_inflight.load(Ordering::SeqCst), 0);
4697        assert_eq!(health.timed_out_total.load(Ordering::SeqCst), 1);
4698        assert!(health.last_error.lock().is_ok_and(|error| error.is_none()));
4699    }
4700
4701    #[test]
4702    fn timeout_tracker_undoes_registration_when_actor_completed_first() {
4703        let health = Arc::new(ReceiptCommitWriterHealth::default());
4704        let (mut completion, timeout) = writer_command_tracker(&health);
4705
4706        completion.complete();
4707        timeout.note_timeout("sqlite receipt commit write timed out");
4708
4709        assert_eq!(health.timed_out_inflight.load(Ordering::SeqCst), 0);
4710        assert_eq!(health.timed_out_total.load(Ordering::SeqCst), 1);
4711        assert!(health.last_error.lock().is_ok_and(|error| error.is_none()));
4712    }
4713
4714    #[test]
4715    fn timeout_tracker_preserves_a_genuine_error_containing_timeout_words() {
4716        let health = Arc::new(ReceiptCommitWriterHealth::default());
4717        if let Ok(mut last_error) = health.last_error.lock() {
4718            *last_error = Some("database lock timed out".to_string());
4719        }
4720        let (mut completion, timeout) = writer_command_tracker(&health);
4721
4722        timeout.note_timeout(RECEIPT_WRITE_TIMEOUT_MARKER);
4723        completion.complete();
4724
4725        assert_eq!(health.timed_out_total.load(Ordering::SeqCst), 1);
4726        assert_eq!(health.timed_out_inflight.load(Ordering::SeqCst), 0);
4727        assert!(health
4728            .last_error
4729            .lock()
4730            .is_ok_and(|error| { error.as_deref() == Some("database lock timed out") }));
4731    }
4732
4733    #[test]
4734    fn unrelated_commit_preserves_an_outstanding_timeout_marker() {
4735        let health = ReceiptCommitWriterHealth::default();
4736        health.timed_out_inflight.store(1, Ordering::SeqCst);
4737        if let Ok(mut last_error) = health.last_error.lock() {
4738            *last_error = Some("sqlite receipt commit write timed out".to_string());
4739        }
4740        // A completed command that was ahead of the timed-out command cannot
4741        // clear its marker while that specific command remains queued/running.
4742        record_write_job_outcome(&health, true);
4743        let preserved = match health.last_error.lock() {
4744            Ok(guard) => guard
4745                .as_deref()
4746                .is_some_and(|error| error.contains("timed out")),
4747            Err(_) => false,
4748        };
4749        assert!(
4750            preserved,
4751            "an unrelated earlier commit must not clear a queued command's timeout marker"
4752        );
4753        assert_eq!(health.committed_total.load(Ordering::SeqCst), 1);
4754
4755        atomic_saturating_sub(&health.timed_out_inflight, 1);
4756        health.clear_timeout_error_if_drained();
4757        assert!(health.last_error.lock().is_ok_and(|error| error.is_none()));
4758    }
4759
4760    #[test]
4761    fn committed_write_preserves_a_genuine_writer_error() {
4762        let health = ReceiptCommitWriterHealth::default();
4763        // A poisoned-head / checkpoint fault is not a stall marker and must
4764        // survive a later commit so the store keeps reporting the real fault.
4765        if let Ok(mut last_error) = health.last_error.lock() {
4766            *last_error = Some("receipt store verified head is unavailable".to_string());
4767        }
4768        record_write_job_outcome(&health, true);
4769        let preserved = match health.last_error.lock() {
4770            Ok(guard) => guard.as_deref() == Some("receipt store verified head is unavailable"),
4771            Err(_) => false,
4772        };
4773        assert!(
4774            preserved,
4775            "a committed write must not clear an unrelated writer error"
4776        );
4777    }
4778
4779    #[test]
4780    fn run_write_receipt_with_timeout_fails_closed_when_writer_never_drains(
4781    ) -> Result<(), Box<dyn std::error::Error>> {
4782        // Child receipts persist through `run_write_receipt`. Its bounded variant
4783        // must fail closed on a wedged writer instead of blocking the caller (and
4784        // the kernel-wide receipt write lock it holds) forever.
4785        let (sender, _receiver) = receipt_commit_channel();
4786        let health = Arc::new(ReceiptCommitWriterHealth::default());
4787        let handle = WriterHandle {
4788            sender,
4789            health: Arc::clone(&health),
4790            worker: idle_worker(),
4791            settlement_store_binding: None,
4792        };
4793        let inflight_before = health.inflight.load(Ordering::SeqCst);
4794
4795        let start = std::time::Instant::now();
4796        let error =
4797            handle.run_write_receipt_with_timeout(|_connection| Ok(()), Duration::from_millis(250));
4798        assert!(start.elapsed() < Duration::from_secs(2));
4799
4800        match error.err().ok_or("expected write timeout error")? {
4801            ReceiptStoreError::Timeout { operation, .. } => {
4802                assert_eq!(operation, "sqlite receipt commit write");
4803            }
4804            other => {
4805                return Err(
4806                    std::io::Error::other(format!("expected timeout error, got {other}")).into(),
4807                );
4808            }
4809        }
4810        // Ownership of the queued job stays with the actor, so the timeout side
4811        // must leave inflight elevated (the honest wedged-writer signal).
4812        assert_eq!(health.inflight.load(Ordering::SeqCst), inflight_before + 1);
4813        assert_eq!(health.failed_total.load(Ordering::SeqCst), 0);
4814        assert_eq!(health.timed_out_inflight.load(Ordering::SeqCst), 1);
4815        assert_eq!(health.timed_out_total.load(Ordering::SeqCst), 1);
4816        Ok(())
4817    }
4818
4819    #[test]
4820    fn earlier_success_does_not_clear_a_later_queued_timeout(
4821    ) -> Result<(), Box<dyn std::error::Error>> {
4822        let path = std::env::temp_dir().join(format!(
4823            "chio-write-timeout-ordering-{}-{}.sqlite3",
4824            std::process::id(),
4825            SystemTime::now()
4826                .duration_since(UNIX_EPOCH)
4827                .map(|duration| duration.as_nanos())
4828                .unwrap_or(0)
4829        ));
4830        let store = SqliteReceiptStore::open(&path)?;
4831        let baseline = store.receipt_commit_actor.writer_counters();
4832        let writer_a = store.writer_handle();
4833        let writer_b = store.writer_handle();
4834        let (a_started_tx, a_started_rx) = mpsc::sync_channel(1);
4835        let (a_release_tx, a_release_rx) = mpsc::sync_channel(1);
4836        let a = std::thread::spawn(move || {
4837            writer_a.run_write(move |_connection| {
4838                let _ = a_started_tx.send(());
4839                let _ = a_release_rx.recv();
4840                Ok(())
4841            })
4842        });
4843        a_started_rx.recv()?;
4844
4845        let (b_started_tx, b_started_rx) = mpsc::sync_channel(1);
4846        let (b_release_tx, b_release_rx) = mpsc::sync_channel(1);
4847        let b_error = writer_b
4848            .run_write_with_timeout(
4849                move |_connection| {
4850                    let _ = b_started_tx.send(());
4851                    let _ = b_release_rx.recv();
4852                    Ok(())
4853                },
4854                Duration::from_millis(25),
4855            )
4856            .err()
4857            .ok_or("B must time out while queued behind A")?;
4858        assert!(matches!(b_error, ReceiptStoreError::Timeout { .. }));
4859        assert_eq!(
4860            store
4861                .receipt_commit_actor
4862                .health
4863                .timed_out_inflight
4864                .load(Ordering::SeqCst),
4865            1
4866        );
4867        assert_eq!(
4868            store.writer_liveness(Duration::from_secs(60)),
4869            chio_kernel::ReceiptWriterLiveness::Wedged
4870        );
4871
4872        a_release_tx.send(())?;
4873        a.join().map_err(|_| "A writer thread panicked")??;
4874        b_started_rx.recv()?;
4875        let after_a = store.receipt_commit_actor.writer_counters();
4876        assert_eq!(after_a.accepted_total, baseline.accepted_total + 2);
4877        assert_eq!(after_a.committed_total, baseline.committed_total + 1);
4878        assert_eq!(after_a.failed_total, baseline.failed_total);
4879        assert_eq!(after_a.timed_out_total, baseline.timed_out_total + 1);
4880        assert_eq!(after_a.timed_out_inflight, 1);
4881        assert_eq!(
4882            store
4883                .receipt_commit_actor
4884                .health
4885                .timed_out_inflight
4886                .load(Ordering::SeqCst),
4887            1,
4888            "A's success must not clear B's outstanding timeout"
4889        );
4890        assert!(after_a
4891            .last_error
4892            .as_deref()
4893            .is_some_and(|error| error.contains("timed out")));
4894        assert_eq!(
4895            store.writer_liveness(Duration::from_secs(60)),
4896            chio_kernel::ReceiptWriterLiveness::Wedged
4897        );
4898
4899        b_release_tx.send(())?;
4900        assert!(wait_until(|| {
4901            store
4902                .receipt_commit_actor
4903                .health
4904                .timed_out_inflight
4905                .load(Ordering::SeqCst)
4906                == 0
4907                && store
4908                    .receipt_commit_actor
4909                    .health
4910                    .committed_total
4911                    .load(Ordering::SeqCst)
4912                    == baseline.committed_total + 2
4913        }));
4914        let drained = store.receipt_commit_actor.writer_counters();
4915        assert_eq!(drained.failed_total, baseline.failed_total);
4916        assert_eq!(drained.timed_out_total, baseline.timed_out_total + 1);
4917        assert_eq!(drained.timed_out_inflight, 0);
4918        assert_eq!(
4919            drained.accepted_total,
4920            drained.committed_total + drained.failed_total
4921        );
4922        assert!(drained.last_error.is_none());
4923        assert_eq!(
4924            store.writer_liveness(Duration::from_secs(60)),
4925            chio_kernel::ReceiptWriterLiveness::Healthy
4926        );
4927
4928        drop(store);
4929        let _ = fs::remove_file(path);
4930        Ok(())
4931    }
4932
4933    #[test]
4934    fn critical_write_timeout_keeps_inflight_and_late_failure_poisoning(
4935    ) -> Result<(), Box<dyn std::error::Error>> {
4936        let (sender, receiver) = receipt_commit_channel();
4937        let health = Arc::new(ReceiptCommitWriterHealth::default());
4938        let handle = WriterHandle {
4939            sender,
4940            health: Arc::clone(&health),
4941            worker: idle_worker(),
4942            settlement_store_binding: None,
4943        };
4944        let inflight_before = health.inflight.load(Ordering::SeqCst);
4945
4946        let error = handle
4947            .run_critical_receipt_write_with_timeout(
4948                |_connection| Ok(()),
4949                Duration::from_millis(10),
4950            )
4951            .err()
4952            .ok_or("expected critical write timeout")?;
4953        assert!(matches!(error, ReceiptStoreError::Timeout { .. }));
4954        assert_eq!(health.inflight.load(Ordering::SeqCst), inflight_before + 1);
4955        assert_eq!(health.failed_total.load(Ordering::SeqCst), 0);
4956        assert_eq!(health.timed_out_inflight.load(Ordering::SeqCst), 1);
4957        assert_eq!(health.timed_out_total.load(Ordering::SeqCst), 1);
4958        assert!(!health.critical_write_poisoned.load(Ordering::SeqCst));
4959
4960        match receiver.recv()? {
4961            ReceiptCommitCommand::Write {
4962                job,
4963                appends_receipts,
4964                fail_closed_on_error,
4965                mut completion,
4966            } => {
4967                assert!(appends_receipts);
4968                assert!(fail_closed_on_error);
4969                health.note_channel_dequeue();
4970                let respond = job(Err(ReceiptStoreError::Conflict(
4971                    "late critical write failure".to_string(),
4972                )));
4973                atomic_saturating_sub(&health.inflight, 1);
4974                let committed = respond(Ok(()));
4975                assert!(!committed);
4976                record_write_job_outcome(&health, committed);
4977                completion.complete();
4978            }
4979            _ => return Err("expected queued critical Write command".into()),
4980        }
4981        assert!(health.critical_write_poisoned.load(Ordering::SeqCst));
4982        assert!(health.head_poisoned.load(Ordering::SeqCst));
4983        assert_eq!(health.timed_out_inflight.load(Ordering::SeqCst), 0);
4984        assert_eq!(health.timed_out_total.load(Ordering::SeqCst), 1);
4985        assert_eq!(health.accepted_total.load(Ordering::SeqCst), 1);
4986        assert_eq!(health.committed_total.load(Ordering::SeqCst), 0);
4987        assert_eq!(health.failed_total.load(Ordering::SeqCst), 1);
4988        Ok(())
4989    }
4990
4991    #[test]
4992    fn run_write_with_timeout_fails_closed_when_writer_never_drains(
4993    ) -> Result<(), Box<dyn std::error::Error>> {
4994        // The hot-path capability snapshot persists through `run_write_with_timeout`.
4995        // Its bounded metadata variant must fail closed on a wedged writer instead
4996        // of blocking the caller forever.
4997        let (sender, _receiver) = receipt_commit_channel();
4998        let health = Arc::new(ReceiptCommitWriterHealth::default());
4999        let handle = WriterHandle {
5000            sender,
5001            health: Arc::clone(&health),
5002            worker: idle_worker(),
5003            settlement_store_binding: None,
5004        };
5005        let inflight_before = health.inflight.load(Ordering::SeqCst);
5006
5007        let start = std::time::Instant::now();
5008        let error = handle.run_write_with_timeout(|_connection| Ok(()), Duration::from_millis(250));
5009        assert!(start.elapsed() < Duration::from_secs(2));
5010
5011        match error.err().ok_or("expected write timeout error")? {
5012            ReceiptStoreError::Timeout { operation, .. } => {
5013                assert_eq!(operation, "sqlite receipt commit write");
5014            }
5015            other => {
5016                return Err(
5017                    std::io::Error::other(format!("expected timeout error, got {other}")).into(),
5018                );
5019            }
5020        }
5021        // Ownership of the queued job stays with the actor, so the timeout side
5022        // must leave inflight elevated (the honest wedged-writer signal).
5023        assert_eq!(health.inflight.load(Ordering::SeqCst), inflight_before + 1);
5024        assert_eq!(health.failed_total.load(Ordering::SeqCst), 0);
5025        assert_eq!(health.timed_out_inflight.load(Ordering::SeqCst), 1);
5026        assert_eq!(health.timed_out_total.load(Ordering::SeqCst), 1);
5027        Ok(())
5028    }
5029
5030    #[test]
5031    fn disconnected_bounded_write_records_writer_death_for_liveness(
5032    ) -> Result<(), Box<dyn std::error::Error>> {
5033        // The commit actor accepts a bounded child-receipt write, then dies
5034        // without responding, disconnecting the caller's response channel. The
5035        // write must record the writer death so the next pre-dispatch liveness
5036        // sample reports the writer Dead and denies admission, instead of
5037        // sampling Healthy once inflight is compensated and failed_total matches
5038        // accepted_total.
5039        let (sender, receiver) = receipt_commit_channel();
5040        let health = Arc::new(ReceiptCommitWriterHealth::default());
5041        let handle = WriterHandle {
5042            sender,
5043            health: Arc::clone(&health),
5044            worker: idle_worker(),
5045            settlement_store_binding: None,
5046        };
5047        // Actor thread: take the one queued command and drop it (die mid-flight),
5048        // which drops the deferred responder and disconnects the caller.
5049        let actor = std::thread::spawn(move || {
5050            if let Ok(command) = receiver.recv() {
5051                drop(command);
5052            }
5053            drop(receiver);
5054        });
5055
5056        let error =
5057            handle.run_write_receipt_with_timeout(|_connection| Ok(()), Duration::from_secs(5));
5058        actor.join().map_err(|_| "actor thread panicked")?;
5059        assert!(error.is_err(), "a disconnected writer must fail closed");
5060
5061        let counters = ReceiptCommitActor {
5062            sender: receipt_commit_channel().0,
5063            health: Arc::clone(&health),
5064            worker: idle_worker(),
5065        }
5066        .writer_counters();
5067        assert!(
5068            counters
5069                .last_error
5070                .as_deref()
5071                .is_some_and(|reason| reason.contains("unavailable")),
5072            "writer death must be recorded for the liveness probe"
5073        );
5074        assert_eq!(
5075            classify_writer_liveness(
5076                &counters,
5077                10_000,
5078                RECEIPT_COMMIT_ACTOR_CHANNEL_CAPACITY as u64,
5079                None,
5080                current_unix_ms(),
5081            ),
5082            chio_kernel::ReceiptWriterLiveness::Dead
5083        );
5084        Ok(())
5085    }
5086
5087    #[test]
5088    fn disconnected_reseed_flips_writer_liveness_dead_immediately(
5089    ) -> Result<(), Box<dyn std::error::Error>> {
5090        // A disconnected admin send (reseed after the commit actor has already
5091        // exited) is the first observation that the writer is gone. It must flip
5092        // liveness to Dead now, so the pre-dispatch gate denies admission before
5093        // a later append reconfirms the death.
5094        let path = std::env::temp_dir().join(format!(
5095            "chio-reseed-dead-{}-{}.sqlite3",
5096            std::process::id(),
5097            SystemTime::now()
5098                .duration_since(UNIX_EPOCH)
5099                .map(|d| d.as_nanos())
5100                .unwrap_or(0)
5101        ));
5102        let mut store = SqliteReceiptStore::open(&path)?;
5103
5104        // Replace the live commit actor with one whose receiver is dropped, so
5105        // the next admin send observes a dead actor. Overwriting the field drops
5106        // the original sender, letting the original actor thread exit cleanly.
5107        let (sender, receiver) = receipt_commit_channel();
5108        drop(receiver);
5109        store.receipt_commit_actor = ReceiptCommitActor {
5110            sender,
5111            health: Arc::new(ReceiptCommitWriterHealth::default()),
5112            worker: idle_worker(),
5113        };
5114
5115        let error = match store.reseed_verified_head() {
5116            Ok(()) => return Err("reseed against a dead actor must fail closed".into()),
5117            Err(error) => error,
5118        };
5119        assert!(error.to_string().contains("unavailable"));
5120
5121        assert_eq!(
5122            store.writer_liveness(Duration::from_secs(60)),
5123            chio_kernel::ReceiptWriterLiveness::Dead,
5124            "a disconnected reseed must flip writer liveness to Dead immediately"
5125        );
5126
5127        let _ = std::fs::remove_file(&path);
5128        Ok(())
5129    }
5130
5131    #[test]
5132    fn run_write_executes_jobs_serially_on_the_writer_thread(
5133    ) -> Result<(), Box<dyn std::error::Error>> {
5134        let path = std::env::temp_dir().join(format!(
5135            "chio-run-write-{}-{}.sqlite3",
5136            std::process::id(),
5137            SystemTime::now()
5138                .duration_since(UNIX_EPOCH)
5139                .map(|d| d.as_nanos())
5140                .unwrap_or(0)
5141        ));
5142        let store = SqliteReceiptStore::open(&path)?;
5143        let writer = store.writer_handle();
5144
5145        let first_thread = writer.run_write(|_connection| Ok(std::thread::current().id()))?;
5146        let second_thread = writer.run_write(|_connection| Ok(std::thread::current().id()))?;
5147
5148        assert_eq!(
5149            first_thread, second_thread,
5150            "all write jobs must run on the single writer thread"
5151        );
5152        assert_ne!(
5153            first_thread,
5154            std::thread::current().id(),
5155            "write jobs must not run on the caller thread"
5156        );
5157
5158        // The closure really gets a usable writer connection.
5159        let journal_mode = writer.run_write(|connection| {
5160            connection
5161                .query_row("PRAGMA journal_mode", [], |row| row.get::<_, String>(0))
5162                .map_err(ReceiptStoreError::from)
5163        })?;
5164        assert!(journal_mode.eq_ignore_ascii_case("wal"));
5165
5166        // Inflight accounting drains back to zero after the jobs complete.
5167        assert_eq!(
5168            store
5169                .receipt_commit_actor
5170                .health
5171                .inflight
5172                .load(Ordering::SeqCst),
5173            0
5174        );
5175
5176        let _ = fs::remove_file(path);
5177        Ok(())
5178    }
5179
5180    #[test]
5181    fn run_write_fails_closed_when_queue_is_full() -> Result<(), Box<dyn std::error::Error>> {
5182        let (sender, _receiver) = receipt_commit_channel();
5183        let health = Arc::new(ReceiptCommitWriterHealth::default());
5184        for _ in 0..RECEIPT_COMMIT_ACTOR_CHANNEL_CAPACITY {
5185            let (response, _result) = mpsc::sync_channel(1);
5186            sender.try_send(ReceiptCommitCommand::Flush(response))?;
5187        }
5188        let handle = WriterHandle {
5189            sender,
5190            health: Arc::clone(&health),
5191            worker: idle_worker(),
5192            settlement_store_binding: None,
5193        };
5194
5195        let error = handle.run_write(|_connection| Ok(()));
5196
5197        assert!(error
5198            .err()
5199            .ok_or("expected queue saturation error")?
5200            .to_string()
5201            .contains("sqlite receipt commit queue saturated"));
5202        assert_eq!(
5203            health.inflight.load(Ordering::SeqCst),
5204            0,
5205            "speculative inflight increment must be undone on saturation"
5206        );
5207        assert_eq!(health.saturated_total.load(Ordering::SeqCst), 1);
5208        Ok(())
5209    }
5210
5211    /// A writer-routed `Write` job (liability write, manual checkpoint creation)
5212    /// must keep `writer_inflight` nonzero for the DURATION of the job, not just
5213    /// at enqueue, so a health poll during a slow or stuck Write does not report
5214    /// `inflight: 0` and hide active writer work. The `WriterInflightGuard`
5215    /// holds the count until the job completes, mirroring the Append path.
5216    #[test]
5217    fn write_job_holds_inflight_for_its_duration() -> Result<(), Box<dyn std::error::Error>> {
5218        let path = std::env::temp_dir().join(format!(
5219            "chio-write-inflight-{}-{}.sqlite3",
5220            std::process::id(),
5221            SystemTime::now()
5222                .duration_since(UNIX_EPOCH)
5223                .map(|d| d.as_nanos())
5224                .unwrap_or(0)
5225        ));
5226        let store = SqliteReceiptStore::open(&path)?;
5227        let writer = store.writer_handle();
5228
5229        // Drain any open-time writer activity to a known baseline before running
5230        // the coordinated job.
5231        let drained_baseline = wait_until(|| {
5232            store
5233                .receipt_commit_actor
5234                .health
5235                .inflight
5236                .load(Ordering::SeqCst)
5237                == 0
5238        });
5239        assert!(drained_baseline, "writer failed to drain to baseline");
5240
5241        // Coordinate a Write job that blocks inside its closure until released.
5242        let (started_tx, started_rx) = mpsc::sync_channel::<()>(1);
5243        let (release_tx, release_rx) = mpsc::sync_channel::<()>(1);
5244        let worker = std::thread::spawn(move || {
5245            writer.run_write(move |_connection| {
5246                // Signal that the job is now executing on the writer thread, then
5247                // block until the test releases it.
5248                let _ = started_tx.send(());
5249                let _ = release_rx.recv();
5250                Ok(())
5251            })
5252        });
5253
5254        // The job is running: inflight must be nonzero for the DURATION of the
5255        // Write, not merely at enqueue.
5256        started_rx.recv().map_err(|_| "write job never started")?;
5257        assert_eq!(
5258            store
5259                .receipt_commit_actor
5260                .health
5261                .inflight
5262                .load(Ordering::SeqCst),
5263            1,
5264            "a running Write job must report inflight > 0"
5265        );
5266
5267        // Release the job and confirm inflight drains back to baseline. The
5268        // `WriterInflightGuard` decrements just BEFORE the caller's response is
5269        // delivered, so this is already at baseline once the worker join
5270        // returns; poll defensively regardless.
5271        release_tx.send(())?;
5272        worker
5273            .join()
5274            .map_err(|_| "write worker thread panicked")??;
5275        let drained = wait_until(|| {
5276            store
5277                .receipt_commit_actor
5278                .health
5279                .inflight
5280                .load(Ordering::SeqCst)
5281                == 0
5282        });
5283        assert!(
5284            drained,
5285            "inflight must return to baseline after the Write completes"
5286        );
5287
5288        let _ = fs::remove_file(path);
5289        Ok(())
5290    }
5291
5292    /// The `WriterInflightGuard` decrement must be SYNCHRONOUS with
5293    /// caller-return: the guard drops IMMEDIATELY BEFORE each `respond(...)`,
5294    /// matching the Append path's decrement-then-fan-out ordering
5295    /// (`commit_receipt_batch`), so caller-return implies the decrement already
5296    /// happened. If the guard instead dropped at the END of the Write arm (after
5297    /// `respond(...)` unblocked `run_write`), a caller could return while
5298    /// `inflight` was still counted, the exact window that would make
5299    /// `run_write_executes_jobs_serially_on_the_writer_thread` intermittently
5300    /// observe `inflight == 1`. This asserts the guarantee DIRECTLY and
5301    /// deterministically (no `wait_until`): right after `run_write` returns,
5302    /// `inflight` reads 0 on every one of many iterations.
5303    #[test]
5304    fn write_decrements_inflight_before_returning_to_caller(
5305    ) -> Result<(), Box<dyn std::error::Error>> {
5306        let path = std::env::temp_dir().join(format!(
5307            "chio-write-inflight-order-{}-{}.sqlite3",
5308            std::process::id(),
5309            SystemTime::now()
5310                .duration_since(UNIX_EPOCH)
5311                .map(|d| d.as_nanos())
5312                .unwrap_or(0)
5313        ));
5314        let store = SqliteReceiptStore::open(&path)?;
5315        let writer = store.writer_handle();
5316
5317        // Drain any open-time writer activity to a known baseline first.
5318        let drained_baseline = wait_until(|| {
5319            store
5320                .receipt_commit_actor
5321                .health
5322                .inflight
5323                .load(Ordering::SeqCst)
5324                == 0
5325        });
5326        assert!(drained_baseline, "writer failed to drain to baseline");
5327
5328        // Many iterations to expose the ordering race: if the guard dropped
5329        // AFTER the response reached the caller (while the writer thread still
5330        // had the head snapshot, error clear, connection drop and catch-up build
5331        // to run), this load could intermittently observe 1. Because the
5332        // decrement precedes the response, caller-return happens-before this
5333        // load and it must read 0 on EVERY iteration with no polling.
5334        for iteration in 0..512 {
5335            writer.run_write(|_connection| Ok(()))?;
5336            let observed = store
5337                .receipt_commit_actor
5338                .health
5339                .inflight
5340                .load(Ordering::SeqCst);
5341            assert_eq!(
5342                observed, 0,
5343                "caller returned from run_write with inflight still counted \
5344                 (iteration {iteration}); the decrement must precede the response"
5345            );
5346        }
5347
5348        let _ = fs::remove_file(path);
5349        Ok(())
5350    }
5351
5352    /// Poll `predicate` for up to ~1s (1ms steps), returning whether it held.
5353    fn wait_until(predicate: impl Fn() -> bool) -> bool {
5354        for _ in 0..1_000 {
5355            if predicate() {
5356                return true;
5357            }
5358            std::thread::sleep(std::time::Duration::from_millis(1));
5359        }
5360        predicate()
5361    }
5362}