Skip to main content

khive_storage/
error.rs

1//! Storage error types shared across all backend implementations.
2
3use std::borrow::Cow;
4use std::error::Error as StdError;
5use std::fmt;
6
7use thiserror::Error;
8
9use crate::blob::ContentRef;
10use crate::capability::StorageCapability;
11
12/// What is known about one request when its single-writer execution seam
13/// terminates or is retired.
14///
15/// The state is deliberately about the request, not about why the task
16/// stopped. Callers need this distinction to decide whether a write is known
17/// not to have started, known to have had its SQLite transaction rolled back,
18/// or may already have produced side effects.
19#[derive(Debug, Clone, Copy, PartialEq, Eq)]
20pub enum WriterTaskRequestState {
21    /// The request's operation closure was never invoked.
22    NotStarted,
23    /// The request ran inside `BEGIN IMMEDIATE`, and that transaction was
24    /// successfully rolled back on the connection that owned it. The request
25    /// may have returned an error, failed COMMIT, or panicked; none of its
26    /// wrapped SQLite writes committed.
27    TransactionRolledBack,
28    /// The request was accepted, but its exact outcome cannot be established;
29    /// it may already have produced side effects and must not be blindly
30    /// retried.
31    SideEffectsUnknown,
32}
33
34/// The SQLite boundary that returned a native write failure, independently of settlement.
35#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
36#[serde(rename_all = "snake_case")]
37pub enum SqliteWriteStage {
38    #[serde(rename = "sqlite_begin_busy")]
39    Begin,
40    #[serde(rename = "sqlite_statement_failure")]
41    Statement,
42    #[serde(rename = "sqlite_commit_failure")]
43    Commit,
44}
45
46impl SqliteWriteStage {
47    pub const fn as_str(self) -> &'static str {
48        match self {
49            Self::Begin => "sqlite_begin_busy",
50            Self::Statement => "sqlite_statement_failure",
51            Self::Commit => "sqlite_commit_failure",
52        }
53    }
54}
55
56/// Native evidence only; the enclosing error still owns retry and settlement policy.
57#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
58pub struct SqliteWriteFailure {
59    pub stage: SqliteWriteStage,
60    pub primary_code: i32,
61    pub extended_code: i32,
62    /// Existing cleanup evidence; false does not prove rollback or any final outcome.
63    #[serde(default)]
64    pub settlement_unknown: bool,
65}
66
67impl fmt::Display for WriterTaskRequestState {
68    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
69        f.write_str(match self {
70            Self::NotStarted => "not_started",
71            Self::TransactionRolledBack => "transaction_rolled_back",
72            Self::SideEffectsUnknown => "side_effects_unknown",
73        })
74    }
75}
76
77/// Which pre-write disk-admission step could not establish safety.
78#[derive(Debug, Clone, Copy, PartialEq, Eq)]
79pub enum CapacityUnavailablePhase {
80    Identity,
81    Lock,
82    LockContended,
83    Probe,
84}
85
86impl CapacityUnavailablePhase {
87    pub const fn as_str(self) -> &'static str {
88        match self {
89            Self::Identity => "identity",
90            Self::Lock => "lock",
91            Self::LockContended => "lock_contended",
92            Self::Probe => "probe",
93        }
94    }
95}
96
97impl fmt::Display for CapacityUnavailablePhase {
98    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
99        f.write_str(self.as_str())
100    }
101}
102
103/// Unified error type for all storage operations.
104#[derive(Debug, Error)]
105pub enum StorageError {
106    #[error("{source}")]
107    SqliteWrite {
108        failure: SqliteWriteFailure,
109        #[source]
110        source: Box<StorageError>,
111    },
112    #[error("{capability:?} resource not found: {resource} ({key})")]
113    NotFound {
114        capability: StorageCapability,
115        resource: &'static str,
116        key: String,
117    },
118
119    #[error("{capability:?} resource already exists: {resource} ({key})")]
120    AlreadyExists {
121        capability: StorageCapability,
122        resource: &'static str,
123        key: String,
124    },
125
126    #[error("conflict in {capability:?} during {operation}: {message}")]
127    Conflict {
128        capability: StorageCapability,
129        operation: Cow<'static, str>,
130        message: String,
131    },
132
133    #[error("invalid input for {capability:?} during {operation}: {message}")]
134    InvalidInput {
135        capability: StorageCapability,
136        operation: Cow<'static, str>,
137        message: String,
138    },
139
140    /// A write would persist credential-shaped input. The match contains only
141    /// safe detector metadata, never the submitted credential.
142    #[error("write blocked: {matched}")]
143    SecretDetected {
144        capability: StorageCapability,
145        operation: Cow<'static, str>,
146        matched: Box<crate::secret_scan::SecretMatch>,
147    },
148
149    #[error("unsupported operation for {capability:?}: {operation} ({message})")]
150    Unsupported {
151        capability: StorageCapability,
152        operation: Cow<'static, str>,
153        message: String,
154    },
155
156    /// The authoritative object is larger than the caller's declared
157    /// whole-buffer limit. `observed_at_least` is either opened-object
158    /// metadata used for early refusal or the running byte count that first
159    /// crossed the limit; metadata is not treated as the actual-byte bound.
160    #[error(
161        "blob {content_ref} exceeds the {max_bytes}-byte read limit (observed at least {observed_at_least} bytes)"
162    )]
163    BlobTooLarge {
164        content_ref: ContentRef,
165        max_bytes: u64,
166        observed_at_least: u64,
167    },
168
169    /// The complete bounded body disagreed with metadata obtained from the
170    /// same opened object / GET response.
171    #[error(
172        "blob {content_ref} metadata reports {metadata_bytes} bytes but the complete body contains {actual_bytes} bytes"
173    )]
174    BlobSizeMismatch {
175        content_ref: ContentRef,
176        metadata_bytes: u64,
177        actual_bytes: u64,
178    },
179
180    /// The complete, size-consistent bounded body did not hash to the
181    /// requested content-addressed reference.
182    #[error("blob digest mismatch: expected {expected}, computed {actual}")]
183    BlobDigestMismatch {
184        expected: ContentRef,
185        actual: ContentRef,
186    },
187
188    #[error("pool failure during {operation}: {message}")]
189    Pool {
190        operation: Cow<'static, str>,
191        message: String,
192    },
193
194    #[error("timeout during {operation}")]
195    Timeout { operation: Cow<'static, str> },
196
197    /// A bounded wait for storage admission (a reader/writer handle slot or a
198    /// pooled reader checkout) elapsed before anything was acquired. The
199    /// operation never started, so retrying cannot duplicate a side effect —
200    /// distinct from [`StorageError::Timeout`], which makes no claim about
201    /// whether work was in flight when the deadline expired.
202    #[error("admission timeout during {operation} after {timeout_ms}ms{pool}", pool = match .pool_identity {
203        Some(identity) => format!(" (pool: {identity})"),
204        None => String::new(),
205    })]
206    AdmissionTimeout {
207        operation: Cow<'static, str>,
208        /// The configured admission deadline that elapsed, in milliseconds.
209        timeout_ms: u64,
210        /// Canonical database path display, or `:memory:`, when the refusing pool is known.
211        pool_identity: Option<String>,
212    },
213
214    #[error("sql transaction failure during {operation}: {message}")]
215    Transaction {
216        operation: Cow<'static, str>,
217        message: String,
218    },
219
220    /// A cached read-only handle's admitted transaction pinned a WAL
221    /// snapshot past the configured `read_tx_max_age` bound and was
222    /// proactively rolled back so the next call can open a fresh snapshot
223    /// (#1846). Distinct from the generic [`StorageError::Transaction`]
224    /// variant — which also covers failed-cleanup and write-side ambiguity
225    /// cases that are not uniformly safe to retry — so callers (and MCP
226    /// dispatch) can recognize this specific, always-safe-to-retry
227    /// condition by variant rather than by parsing rendered text.
228    #[error(
229        "cached read-only transaction exceeded the maximum read-transaction age \
230         ({max_age_secs}s) during {operation} and was rolled back; retry to open a fresh \
231         read snapshot"
232    )]
233    ReadTransactionAgeEvicted {
234        operation: Cow<'static, str>,
235        max_age_secs: u64,
236    },
237
238    /// A cached read-only handle's admitted transaction pinned a WAL
239    /// snapshot past `read_tx_max_age`, and the proactive rollback used to
240    /// end the eviction (#1846) did not restore autocommit, or the rollback
241    /// itself failed. The connection is discarded either way rather than
242    /// returned to the pool. The age check that triggered this still ran
243    /// before any read on the connection, so — exactly like
244    /// [`StorageError::ReadTransactionAgeEvicted`] — retrying the caller's
245    /// operation on a fresh connection is always safe; this variant exists
246    /// only to keep that guarantee distinguishable from a clean eviction in
247    /// the rendered message and to keep [`StorageError::Transaction`] (whose
248    /// other cases are not uniformly safe to retry) out of this path.
249    #[error(
250        "cached read-only transaction exceeded the maximum read-transaction age \
251         ({max_age_secs}s) during {operation} but could not be cleanly rolled back \
252         ({message}); the connection was discarded, retry to open a fresh read snapshot"
253    )]
254    ReadTransactionAgeEvictionCleanupFailed {
255        operation: Cow<'static, str>,
256        max_age_secs: u64,
257        message: String,
258    },
259
260    #[error("serialization failure in {capability:?}: {message}")]
261    Serialization {
262        capability: StorageCapability,
263        message: String,
264    },
265
266    #[error("index maintenance failure in {capability:?}: {message}")]
267    IndexMaintenance {
268        capability: StorageCapability,
269        message: String,
270    },
271
272    #[error("backend driver error in {capability:?} during {operation}: {source}")]
273    Driver {
274        capability: StorageCapability,
275        operation: Cow<'static, str>,
276        #[source]
277        source: Box<dyn StdError + Send + Sync>,
278    },
279
280    /// The bounded write-queue channel (ADR-067 Component A) did not free
281    /// capacity within the caller-supplied deadline. Returned only when a
282    /// caller wraps `WriterTaskHandle::send`'s `channel.send().await` in a
283    /// `tokio::time::timeout`; there is no immediate-error `try_send` path.
284    #[error("write queue full: timed out after {timeout_ms}ms waiting for writer task capacity")]
285    WriteQueueFull { timeout_ms: u64 },
286
287    /// SQLite refused the writer task's `BEGIN IMMEDIATE` with
288    /// `SQLITE_BUSY`/`SQLITE_LOCKED` until the configured busy timeout. The
289    /// queue accepted the request, but its operation closure was never
290    /// invoked, so retrying that one failed operation is safe.
291    #[error(
292        "writer task could not begin within {timeout_ms}ms because SQLite remained busy; request was not executed"
293    )]
294    WriterTaskBusy { timeout_ms: u64 },
295
296    /// One transaction-wrapped writer request returned an error, and the
297    /// writer then proved that request's SQLite transaction was rolled back.
298    /// Unlike [`StorageError::WriterTaskTerminated`], this does not retire the
299    /// writer seam. `source` preserves the operation or COMMIT error so its
300    /// capability and retry policy remain independently inspectable.
301    #[error("writer task request failed (request_state={request_state}): {source}")]
302    WriterTaskRequestFailed {
303        request_state: WriterTaskRequestState,
304        #[source]
305        source: Box<StorageError>,
306    },
307
308    /// A single-writer execution seam has terminated permanently. This is the
309    /// historical writer-task variant and display name; the fail-closed
310    /// pool-mutex fallback also uses it when transaction finalization becomes
311    /// terminal. The state identifies what is known about this request at
312    /// that boundary. Retrying on the same pool cannot recover the retired
313    /// writer seam.
314    #[error("writer task terminated (request_state={request_state})")]
315    WriterTaskTerminated {
316        request_state: WriterTaskRequestState,
317        /// Original SQLITE_FULL primary/extended codes, when the terminal cause supplied them.
318        /// Evidence only: it does not change finality, retryability or the error source chain.
319        sqlite_full_codes: Option<(i32, i32)>,
320    },
321
322    /// COMMIT was issued, but this request lost its reply without a known failure.
323    /// This does not establish that the writer task itself terminated.
324    #[error("write outcome unknown after COMMIT: {source}")]
325    WriteOutcomeUnknown {
326        #[source]
327        source: Box<dyn StdError + Send + Sync>,
328    },
329
330    /// Strict routing refused before execution; the original owner failure keeps its policy.
331    #[error("writer owner unavailable during {operation}: {source}")]
332    WriterOwnerUnavailable {
333        operation: Cow<'static, str>,
334        #[source]
335        source: Box<StorageError>,
336    },
337
338    /// A logical write attempted to bypass the owner under strict routing.
339    #[error("strict writer routing refused a direct write during {operation}")]
340    WriterRouteViolation { operation: Cow<'static, str> },
341
342    /// An internal storage failure not attributable to a specific storage
343    /// capability.
344    #[error("internal storage error: {0}")]
345    Internal(String),
346
347    /// `KHIVE_WRITE_QUEUE=1` is set but the calling thread has no Tokio
348    /// runtime context, so the writer task cannot be spawned (ADR-067
349    /// Component A). Returned instead of panicking.
350    /// See `crates/khive-storage/docs/api/error-taxonomy.md#writertasknoruntime`.
351    #[error(
352        "KHIVE_WRITE_QUEUE=1 but no Tokio runtime context is available to spawn the writer task"
353    )]
354    WriterTaskNoRuntime,
355
356    /// A filesystem-backed capability refused a write because its free-space
357    /// floor would be violated. Blob writes account for the known byte count;
358    /// SQLite admission cannot know the next transaction's size.
359    #[error(
360        "refusing write on {capability:?} at {volume}: {available_bytes} bytes available, \
361         the {floor_bytes}-byte free-space floor plus {required_headroom_bytes} bytes of \
362         operation headroom would be violated"
363    )]
364    CapacityFloor {
365        capability: StorageCapability,
366        volume: String,
367        available_bytes: u64,
368        floor_bytes: u64,
369        required_headroom_bytes: u64,
370    },
371
372    /// A new logical write could not establish disk-admission safety.
373    #[error("capacity admission unavailable for {capability:?} in {phase} phase: {message}")]
374    CapacityUnavailable {
375        capability: StorageCapability,
376        phase: CapacityUnavailablePhase,
377        message: String,
378    },
379}
380
381impl StorageError {
382    /// Return native write evidence through a request-finality envelope.
383    pub fn sqlite_write_failure(&self) -> Option<SqliteWriteFailure> {
384        match self {
385            Self::SqliteWrite { failure, .. } => Some(*failure),
386            Self::WriterTaskRequestFailed { source, .. }
387            | Self::WriterOwnerUnavailable { source, .. } => source.sqlite_write_failure(),
388            _ => None,
389        }
390    }
391
392    /// Borrow the original policy error without discarding any request-finality envelope.
393    pub fn without_sqlite_write_stage(&self) -> &Self {
394        match self {
395            Self::SqliteWrite { source, .. } => source.without_sqlite_write_stage(),
396            _ => self,
397        }
398    }
399
400    /// The outer write boundary's finality; an inner cause cannot override it.
401    pub fn writer_request_state(&self) -> Option<WriterTaskRequestState> {
402        match self.without_sqlite_write_stage() {
403            Self::WriterTaskRequestFailed { request_state, .. }
404            | Self::WriterTaskTerminated { request_state, .. } => Some(*request_state),
405            Self::WriteOutcomeUnknown { .. } => Some(WriterTaskRequestState::SideEffectsUnknown),
406            Self::WriterOwnerUnavailable { .. } | Self::WriterRouteViolation { .. } => {
407                Some(WriterTaskRequestState::NotStarted)
408            }
409            _ => None,
410        }
411    }
412
413    /// Distinguish issued-COMMIT reply loss from other unknown request outcomes.
414    pub fn is_write_outcome_unknown(&self) -> bool {
415        matches!(
416            self.without_sqlite_write_stage(),
417            Self::WriteOutcomeUnknown { .. }
418        )
419    }
420
421    /// Attach native evidence once; the original error still owns Display and retry policy.
422    pub fn with_sqlite_write_failure(self, failure: SqliteWriteFailure) -> Self {
423        if self.sqlite_write_failure().is_some() {
424            self
425        } else {
426            Self::SqliteWrite {
427                failure,
428                source: Box::new(self),
429            }
430        }
431    }
432    /// Construct a terminal writer outcome without native SQLite evidence.
433    ///
434    /// The request state and existing retry policy are unchanged.
435    pub fn writer_task_terminated(request_state: WriterTaskRequestState) -> Self {
436        Self::WriterTaskTerminated {
437            request_state,
438            sqlite_full_codes: None,
439        }
440    }
441
442    /// Construct a `Driver` error wrapping a backend-specific error source.
443    pub fn driver(
444        capability: StorageCapability,
445        operation: impl Into<Cow<'static, str>>,
446        source: impl StdError + Send + Sync + 'static,
447    ) -> Self {
448        Self::Driver {
449            capability,
450            operation: operation.into(),
451            source: Box::new(source),
452        }
453    }
454
455    /// Return the storage capability surface that produced this error, if any.
456    pub fn capability(&self) -> Option<StorageCapability> {
457        match self {
458            Self::NotFound { capability, .. }
459            | Self::AlreadyExists { capability, .. }
460            | Self::Conflict { capability, .. }
461            | Self::InvalidInput { capability, .. }
462            | Self::SecretDetected { capability, .. }
463            | Self::Unsupported { capability, .. }
464            | Self::Serialization { capability, .. }
465            | Self::IndexMaintenance { capability, .. }
466            | Self::Driver { capability, .. }
467            | Self::CapacityFloor { capability, .. }
468            | Self::CapacityUnavailable { capability, .. } => Some(*capability),
469            Self::BlobTooLarge { .. }
470            | Self::BlobSizeMismatch { .. }
471            | Self::BlobDigestMismatch { .. } => Some(StorageCapability::Blob),
472            Self::WriterTaskRequestFailed { source, .. }
473            | Self::WriterOwnerUnavailable { source, .. }
474            | Self::SqliteWrite { source, .. } => source.capability(),
475            Self::Pool { .. }
476            | Self::Timeout { .. }
477            | Self::AdmissionTimeout { .. }
478            | Self::Transaction { .. }
479            | Self::ReadTransactionAgeEvicted { .. }
480            | Self::ReadTransactionAgeEvictionCleanupFailed { .. }
481            | Self::WriteQueueFull { .. }
482            | Self::WriterTaskBusy { .. }
483            | Self::WriterTaskTerminated { .. }
484            | Self::WriteOutcomeUnknown { .. }
485            | Self::WriterRouteViolation { .. }
486            | Self::Internal(..)
487            | Self::WriterTaskNoRuntime => None,
488        }
489    }
490
491    /// Whether this error is transient and the operation may succeed on retry.
492    pub fn is_retryable(&self) -> bool {
493        if let Self::WriterTaskRequestFailed { source, .. }
494        | Self::WriterOwnerUnavailable { source, .. }
495        | Self::SqliteWrite { source, .. } = self
496        {
497            return source.is_retryable();
498        }
499        matches!(
500            self,
501            Self::Pool { .. }
502                | Self::Timeout { .. }
503                | Self::AdmissionTimeout { .. }
504                | Self::Transaction { .. }
505                | Self::ReadTransactionAgeEvicted { .. }
506                | Self::ReadTransactionAgeEvictionCleanupFailed { .. }
507                | Self::WriteQueueFull { .. }
508                | Self::WriterTaskBusy { .. }
509                | Self::CapacityUnavailable {
510                    phase: CapacityUnavailablePhase::LockContended,
511                    ..
512                }
513        )
514    }
515
516    /// Whether this error is an FTS5 query-parser rejection of the MATCH
517    /// expression itself, as opposed to a connection/pool/driver-level
518    /// failure of the text-search backend.
519    ///
520    /// True only for `Driver` errors from the `Text` capability at the
521    /// `fts_search` operation whose message names one of SQLite's FTS5
522    /// parser failure modes (syntax error, stack overflow, unsupported
523    /// column/phrase/NEAR query); all other errors return `false`.
524    ///
525    /// Callers that fail-open the FTS leg of a hybrid search (degrading to
526    /// vector-only results on a bad query string) MUST gate on this
527    /// predicate rather than on `StorageError` broadly — treating every
528    /// `Err` as degradable turns a real backend outage into a silently-empty
529    /// "successful" search (issue #389).
530    /// See `crates/khive-storage/docs/api/error-taxonomy.md#is_fts5_syntax_error`.
531    pub fn is_fts5_syntax_error(&self) -> bool {
532        if let Self::WriterTaskRequestFailed { source, .. } | Self::SqliteWrite { source, .. } =
533            self
534        {
535            return source.is_fts5_syntax_error();
536        }
537        let Self::Driver {
538            capability,
539            operation,
540            source,
541        } = self
542        else {
543            return false;
544        };
545        if *capability != StorageCapability::Text || operation.as_ref() != "fts_search" {
546            return false;
547        }
548        let msg = source.to_string();
549        msg.contains("fts5: syntax error")
550            || msg.contains("fts5: parser stack overflow")
551            || msg.contains("fts5: column queries are not supported")
552            || msg.contains("fts5: phrase queries are not supported (detail")
553            || msg.contains("fts5: NEAR queries are not supported (detail")
554    }
555
556    /// Whether this error is a UNIQUE constraint violation from a raw SQL
557    /// `execute` (e.g. an `INSERT` racing an existing row under a natural
558    /// key). True only for `Driver` errors from the `Sql` capability whose
559    /// `operation` is one of `execute`, `pool_writer.execute`, or
560    /// `tx.execute`, and whose message contains `UNIQUE constraint failed`.
561    /// Batch/script operations are intentionally excluded.
562    ///
563    /// Callers that treat exact-key duplicates as a tolerated no-op
564    /// (ADR-081 §4 serve-ledger idempotency) MUST gate on this predicate
565    /// rather than swallowing every `Driver` error at `execute` — that would
566    /// also hide genuine write failures (disk full, corruption).
567    /// See `crates/khive-storage/docs/api/error-taxonomy.md#is_unique_constraint_violation`.
568    pub fn is_unique_constraint_violation(&self) -> bool {
569        if let Self::WriterTaskRequestFailed { source, .. } | Self::SqliteWrite { source, .. } =
570            self
571        {
572            return source.is_unique_constraint_violation();
573        }
574        let Self::Driver {
575            capability,
576            operation,
577            source,
578        } = self
579        else {
580            return false;
581        };
582        if *capability != StorageCapability::Sql {
583            return false;
584        }
585        if !matches!(
586            operation.as_ref(),
587            "execute" | "pool_writer.execute" | "tx.execute"
588        ) {
589            return false;
590        }
591        source.to_string().contains("UNIQUE constraint failed")
592    }
593}
594
595#[cfg(test)]
596#[path = "error_tests.rs"]
597mod tests;