Skip to main content

khive_runtime/
error.rs

1//! Runtime error types.
2
3use std::fmt;
4use std::time::Duration;
5
6use thiserror::Error;
7use uuid::Uuid;
8
9/// Convenience alias for `Result<T, RuntimeError>`.
10pub type RuntimeResult<T> = Result<T, RuntimeError>;
11
12/// Stable ADR-135 F6 stage and wire code for a finite-wait pooled writer
13/// checkout that expires before SQLite executes.
14pub const WRITER_POOL_CHECKOUT_TIMEOUT_STAGE: &str = "writer_pool_checkout_timeout";
15
16/// Stable wire code/stage for a bounded write-queue enqueue that never
17/// accepted the request within its configured deadline (#1382, #1643).
18pub const WRITER_QUEUE_SATURATED_STAGE: &str = "writer_queue_saturated";
19
20/// Stable wire code/stage for SQLite write-lock contention after the writer
21/// queue accepted a request but before its operation closure ran.
22pub const WRITER_TASK_BEGIN_BUSY_STAGE: &str = "writer_task_begin_busy";
23
24/// Stable wire code/stage for a bounded storage-admission wait (a
25/// reader/writer handle slot or a pooled reader checkout) that elapsed
26/// before anything was acquired. The operation never started, so the
27/// failure is safe to retry.
28pub const STORAGE_ADMISSION_TIMEOUT_STAGE: &str = "storage_admission_timeout";
29
30/// Stable wire code/stage for a cached read-only transaction proactively
31/// rolled back after pinning a WAL snapshot past the configured
32/// `read_tx_max_age` bound (#1846). Always safe to retry: the eviction only
33/// ever fires on a read-only handle, so no side effect can exist for a
34/// retry to duplicate. Shared by both
35/// [`khive_storage::StorageError::ReadTransactionAgeEvicted`] (clean
36/// rollback) and
37/// [`khive_storage::StorageError::ReadTransactionAgeEvictionCleanupFailed`]
38/// (rollback denied or failed outright) — the rendered `message` field
39/// distinguishes the two; retry behavior is identical for both.
40pub const READ_TX_AGE_EVICTED_STAGE: &str = "read_tx_age_evicted";
41
42/// Stable ADR-131:251 `scope` discriminator carried on a
43/// [`WRITER_QUEUE_SATURATED_STAGE`] failure — distinguishes write-queue
44/// admission saturation from other `unavailable` failure kinds that share
45/// the same `retryable: true` shape but are not bounded by the admission
46/// deadline (e.g. [`WRITER_POOL_CHECKOUT_TIMEOUT_STAGE`], which has no
47/// ADR-131-defined scope and carries `None`).
48pub const WRITER_ADMISSION_SCOPE: &str = "writer_admission";
49
50/// Structured context for a pre-execution admission failure: a finite-wait
51/// pooled writer checkout timeout, a bounded write-queue enqueue timeout, or
52/// a bounded storage-admission wait (reader/writer handle slot or pooled
53/// reader checkout). All happen before SQLite executes the request, so all
54/// are safe to classify as retryable — the request was never accepted, let
55/// alone started.
56#[derive(Debug, Clone, PartialEq, Eq)]
57pub struct AdmissionFailureContext {
58    /// Stable wire stage/code: one of [`WRITER_POOL_CHECKOUT_TIMEOUT_STAGE`],
59    /// [`WRITER_QUEUE_SATURATED_STAGE`], or
60    /// [`STORAGE_ADMISSION_TIMEOUT_STAGE`].
61    pub stage: &'static str,
62    /// The configured deadline that elapsed.
63    pub timeout: Duration,
64    /// Storage capability the request was scoped to, when known. The write
65    /// queue is process-local and unscoped by capability, so this is `None`
66    /// for [`WRITER_QUEUE_SATURATED_STAGE`].
67    pub capability: Option<khive_storage::StorageCapability>,
68    /// Storage operation name, when known.
69    pub operation: Option<String>,
70    /// ADR-131:251 `scope` discriminator. `Some(`[`WRITER_ADMISSION_SCOPE`]`)`
71    /// for [`WRITER_QUEUE_SATURATED_STAGE`]; `None` for
72    /// [`WRITER_POOL_CHECKOUT_TIMEOUT_STAGE`], which ADR-131 does not define a
73    /// scope for.
74    pub scope: Option<&'static str>,
75    /// ADR-131:251 `retry_after_ms` hint — set equal to the admission
76    /// deadline actually applied to the rejected operation, so a retrying
77    /// caller waits at least one full admission window before retrying.
78    /// `Some(timeout_ms)` for [`WRITER_QUEUE_SATURATED_STAGE`]; `None`
79    /// otherwise.
80    pub retry_after_ms: Option<u64>,
81}
82
83/// Structured context for a failure that is proven safe to retry.
84///
85/// This includes the two pre-admission failures above and writer-task BEGIN
86/// contention after queue acceptance but before operation execution. Callers
87/// must inspect `scope`: only `writer_admission` means the queue never
88/// accepted the operation.
89#[derive(Debug, Clone, PartialEq, Eq)]
90pub struct RetryableFailureContext {
91    /// Stable wire stage/code.
92    pub stage: &'static str,
93    /// Configured wait that ended before execution.
94    pub timeout: Duration,
95    /// Storage capability when the failure is capability-scoped.
96    pub capability: Option<khive_storage::StorageCapability>,
97    /// Storage operation when known.
98    pub operation: Option<String>,
99    /// Admission scope only when the queue never accepted the operation.
100    pub scope: Option<&'static str>,
101    /// Server backoff hint when a governing contract defines one.
102    pub retry_after_ms: Option<u64>,
103}
104
105impl From<AdmissionFailureContext> for RetryableFailureContext {
106    fn from(context: AdmissionFailureContext) -> Self {
107        Self {
108            stage: context.stage,
109            timeout: context.timeout,
110            capability: context.capability,
111            operation: context.operation,
112            scope: context.scope,
113            retry_after_ms: context.retry_after_ms,
114        }
115    }
116}
117
118/// Typed disposition for a channel message whose `comm.ingest` write failed.
119///
120/// Every bucket carries the stable [`RuntimeError`] variant name that selected
121/// it. Consumers can therefore make retry/quarantine decisions and report a
122/// reason without inspecting the error's rendered text.
123#[derive(Debug, Clone, Copy, PartialEq, Eq)]
124pub enum ChannelIngestFailureClass {
125    /// A pre-execution admission failure that is safe to retry indefinitely.
126    Retryable { reason: &'static str },
127    /// A deterministic policy refusal that should quarantine immediately.
128    Permanent { reason: &'static str },
129    /// An error with no explicit retry or permanent policy classification.
130    Unknown { reason: &'static str },
131}
132
133impl ChannelIngestFailureClass {
134    /// Stable lowercase bucket name persisted on quarantine notifications.
135    pub const fn name(self) -> &'static str {
136        match self {
137            Self::Retryable { .. } => "retryable",
138            Self::Permanent { .. } => "permanent",
139            Self::Unknown { .. } => "unknown",
140        }
141    }
142
143    /// Stable typed error-variant name; never derived from `Display` output.
144    pub const fn reason(self) -> &'static str {
145        match self {
146            Self::Retryable { reason } | Self::Permanent { reason } | Self::Unknown { reason } => {
147                reason
148            }
149        }
150    }
151}
152
153/// Structured context recovered from either a direct SQLite runtime error or
154/// a typed SQLite source preserved inside a storage-driver wrapper.
155#[derive(Debug, Clone, PartialEq, Eq)]
156pub struct WriterPoolCheckoutTimeoutContext {
157    /// Finite checkout deadline that elapsed before the pooled mutex was
158    /// acquired.
159    pub timeout: Duration,
160    /// Storage capability that wrapped the SQLite source, if one did.
161    pub capability: Option<khive_storage::StorageCapability>,
162    /// Storage operation that wrapped the SQLite source, if one did.
163    pub operation: Option<String>,
164}
165
166/// A guarded edge write (`link`/`link_many`) was refused because one or both
167/// endpoints no longer existed at write time. Names the exact endpoint(s)
168/// missing instead of a generic "source or target" message, and, for a batch
169/// write, which entry in the batch failed first.
170#[derive(Debug, Clone, PartialEq, Eq)]
171pub struct GuardedWriteFailure {
172    /// Index of the failing entry within a `link_many` batch. `None` for the
173    /// singleton `link` path, which has no batch to index into.
174    pub entry_index: Option<usize>,
175    /// The source endpoint id, present only when it is the (or one of the)
176    /// missing endpoint(s).
177    pub missing_source: Option<Uuid>,
178    /// The target endpoint id, present only when it is the (or one of the)
179    /// missing endpoint(s).
180    pub missing_target: Option<Uuid>,
181}
182
183impl fmt::Display for GuardedWriteFailure {
184    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
185        let mut missing = Vec::new();
186        if let Some(source) = self.missing_source {
187            missing.push(format!("source {source}"));
188        }
189        if let Some(target) = self.missing_target {
190            missing.push(format!("target {target}"));
191        }
192        let missing = if missing.is_empty() {
193            "endpoint(s)".to_string()
194        } else {
195            missing.join(" and ")
196        };
197        match self.entry_index {
198            Some(index) => write!(
199                f,
200                "batch entry {index}: {missing} no longer exist at write time"
201            ),
202            None => write!(f, "{missing} no longer exist at write time"),
203        }
204    }
205}
206
207impl std::error::Error for GuardedWriteFailure {}
208
209/// A single missing pack dependency.
210#[derive(Debug, Clone, PartialEq, Eq)]
211pub struct MissingPackDependency {
212    pub from: String,
213    pub requires: String,
214}
215
216impl fmt::Display for MissingPackDependency {
217    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
218        write!(
219            f,
220            "pack '{}' requires '{}', but '{}' is not in the loaded pack set",
221            self.from, self.requires, self.requires
222        )
223    }
224}
225
226impl std::error::Error for MissingPackDependency {}
227
228/// Multiple missing pack dependencies collected into one error.
229#[derive(Debug, Clone, PartialEq, Eq)]
230pub struct MissingPackDependencies {
231    pub missing: Vec<MissingPackDependency>,
232}
233
234impl fmt::Display for MissingPackDependencies {
235    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
236        let parts: Vec<String> = self.missing.iter().map(ToString::to_string).collect();
237        write!(f, "{}", parts.join("; "))
238    }
239}
240
241impl std::error::Error for MissingPackDependencies {}
242
243/// Circular pack dependency detected during topological sort.
244#[derive(Debug, Clone, PartialEq, Eq)]
245pub struct CircularPackDependency {
246    pub cycle: Vec<String>,
247}
248
249impl fmt::Display for CircularPackDependency {
250    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
251        write!(
252            f,
253            "circular dependency detected among packs: {}",
254            self.cycle.join(" -> ")
255        )
256    }
257}
258
259impl std::error::Error for CircularPackDependency {}
260
261/// All errors produced by the khive-runtime layer.
262///
263/// Variants cover storage, query, validation, namespace isolation, and permission failures.
264/// Callers should match on `InvalidInput` for bad arguments, `NotFound` for missing records,
265/// and `NamespaceMismatch` (reported as not-found) for cross-namespace access attempts.
266#[derive(Debug, Error)]
267pub enum RuntimeError {
268    #[error("storage: {0}")]
269    Storage(#[from] khive_storage::StorageError),
270
271    #[error("sqlite: {0}")]
272    Sqlite(#[from] khive_db::SqliteError),
273
274    #[error("query: {0}")]
275    Query(#[from] khive_query::QueryError),
276
277    #[error("not found: {0}")]
278    NotFound(String),
279
280    #[error("invalid input: {0}")]
281    InvalidInput(String),
282
283    #[error("invalid input: {0}")]
284    UnknownVerb(String),
285
286    #[error("unconfigured: {0} is not set")]
287    Unconfigured(String),
288
289    #[error("unknown embedding model: {0}")]
290    UnknownModel(String),
291
292    #[error("embedding: {0}")]
293    Embedding(#[from] lattice_embed::EmbedError),
294
295    #[error("ambiguous: {0}")]
296    Ambiguous(String),
297
298    #[error("fusion: {0}")]
299    Fusion(#[from] khive_fusion::FuseError),
300
301    /// `FusionStrategy::Custom { name, .. }` named a strategy no pack has
302    /// registered via `KhiveRuntime::register_fusion_strategy` (ADR-012).
303    /// Fails closed — never falls back to RRF or any other default.
304    #[error("unknown fusion strategy: {0}")]
305    UnknownFusionStrategy(String),
306
307    #[error("internal: {0}")]
308    Internal(String),
309
310    /// An `EventStore` was configured via `with_event_store` but does not
311    /// implement ADR-133's `preflight_event`/`append_events_idempotent`
312    /// pair (`EventStore::supports_idempotent_audit_batch` reports
313    /// `false`). Raised at `build()` time rather than left to fail every
314    /// audited dispatch silently.
315    #[error("audit batch incompatible event store: {0}")]
316    IncompatibleEventStore(String),
317
318    #[error("guarded edge write refused: {0}")]
319    GuardedWriteFailed(GuardedWriteFailure),
320
321    #[error("missing pack dependency: {0}")]
322    MissingPackDependency(MissingPackDependency),
323
324    #[error("missing pack dependencies: {0}")]
325    MissingPackDependencies(MissingPackDependencies),
326
327    #[error("{0}")]
328    CircularPackDependency(CircularPackDependency),
329
330    #[error("pack '{name}' registered twice (indices {first_idx} and {second_idx})")]
331    PackRedeclared {
332        name: String,
333        first_idx: usize,
334        second_idx: usize,
335    },
336
337    /// Two packs declared the same `Visibility::Verb` handler name.
338    /// `Visibility::Subhandler` entries are pack-prefixed and do not
339    /// participate in cross-pack collision checks.
340    #[error(
341        "verb collision: verb {verb:?} declared by both pack {first_pack:?} and pack \
342         {second_pack:?}; rename one handler or use Visibility::Subhandler for internal verbs"
343    )]
344    VerbCollision {
345        verb: String,
346        first_pack: String,
347        second_pack: String,
348    },
349
350    /// A handler advertised a parameter that request parsing owns at the envelope level.
351    #[error(
352        "pack {pack:?} handler {verb:?} declares request-envelope parameter {param:?}; rename the verb argument"
353    )]
354    ReservedEnvelopeParam {
355        pack: String,
356        verb: String,
357        param: String,
358    },
359
360    /// Gate denied this verb invocation.
361    ///
362    /// Returned by `VerbRegistry::dispatch` when the configured `Gate` returns
363    /// `GateDecision::Deny`. The pack is never invoked. The `reason` field
364    /// carries the deny message produced by the gate implementation.
365    #[error("permission denied for verb {verb:?}: {reason}")]
366    PermissionDenied { verb: String, reason: String },
367
368    /// The configured gate could not produce an authorization decision.
369    ///
370    /// This is distinct from [`Self::PermissionDenied`]: the pack or
371    /// intercepted operation is never invoked, but the gate did not answer
372    /// with an explicit denial.
373    #[error("gate unavailable for verb {verb:?}: {reason}")]
374    GateUnavailable { verb: String, reason: String },
375
376    /// A structured [`khive_types::KhiveError`] converted into the runtime
377    /// layer. The full structured error is preserved so callers can inspect
378    /// `kind`, `code`, `details`, and `retry_hint` without information loss.
379    #[error("{0}")]
380    Khive(khive_types::KhiveError),
381
382    /// Record exists but belongs to a different namespace than the provided token.
383    ///
384    /// Externally reported as "not found in this namespace" to avoid leaking
385    /// cross-namespace existence information (timing-oracle mitigation).
386    #[error("not found in this namespace")]
387    NamespaceMismatch { id: uuid::Uuid },
388
389    /// A short-prefix lookup matched more than one record.
390    ///
391    /// `prefix` is the 8+ hex-char prefix supplied by the caller.
392    /// `matches` holds the full UUIDs of all matching records (at most 2 are
393    /// reported to bound the scan — callers must supply the full UUID to disambiguate).
394    #[error("ambiguous prefix {prefix:?}: matches {}", format_uuid_list(matches))]
395    AmbiguousPrefix {
396        prefix: String,
397        matches: Vec<uuid::Uuid>,
398    },
399
400    /// Cross-backend `merge_entity` is unsupported in v1.
401    ///
402    /// Both entities must reside on the same backend. To merge entities on different
403    /// backends, manually export `from_id`, delete it, and re-import on `into_id`'s backend.
404    #[error(
405        "cross-backend merge is not supported: \
406         into_id {into_id} is on backend '{into_backend}', \
407         from_id {from_id} is on backend '{from_backend}'. \
408         Both entities must be on the same backend to merge."
409    )]
410    CrossBackendMergeUnsupported {
411        into_id: uuid::Uuid,
412        from_id: uuid::Uuid,
413        into_backend: String,
414        from_backend: String,
415    },
416
417    // ── Remote Resolution and Content-Hash Verification ──────────────────────
418    /// A `kg://` ref names a remote not declared in `schema.yaml`.
419    #[error("unknown remote: {name:?}")]
420    UnknownRemote { name: String },
421
422    /// A remote cache entry is absent and `--fetch` was not requested.
423    #[error("remote cache missing for remote={remote:?} namespace={namespace:?}")]
424    RemoteCacheMissing { remote: String, namespace: String },
425
426    /// A short ID matches multiple entities in the same namespace or remote cache.
427    #[error("ambiguous id {id:?}: matched {count} records")]
428    AmbiguousId { id: String, count: usize },
429
430    /// A write operation targeted a remote namespace, which is read-only.
431    #[error("cross-namespace write denied: cannot write to remote namespace {namespace:?}")]
432    CrossNamespaceWrite { namespace: String },
433
434    /// A remote fetch failed (network error, authentication failure, etc.).
435    #[error("remote fetch error for remote={remote:?}: {message}")]
436    RemoteFetchError { remote: String, message: String },
437
438    /// A caller-supplied write budget was exceeded during a Compound apply.
439    ///
440    /// `max_new_entries` is the limit passed by the caller. `attempted_new_entries`
441    /// is `consumed + 1`, i.e. the create that would have exceeded the cap.
442    /// `None` budget never produces this error (unlimited path).
443    #[error(
444        "write budget exceeded: max_new_entries={max_new_entries}, \
445         attempted_new_entries={attempted_new_entries}"
446    )]
447    WriteBudgetExceeded {
448        max_new_entries: u64,
449        attempted_new_entries: u64,
450    },
451
452    /// Write blocked: content matches a secret pattern.
453    ///
454    /// The `SecretMatch` carries the detector name and a masked excerpt
455    /// (`first6...Nchars`). The full candidate is never stored in the error.
456    /// Store a pointer (env-var name, keychain item) rather than the raw value.
457    #[error("write blocked: {0}")]
458    SecretDetected(crate::secret_gate::SecretMatch),
459
460    /// A bounded per-operation deadline elapsed before the operation
461    /// completed (#889). The operation may still be running in the
462    /// background (this is a client-observable timeout, not a cancellation
463    /// signal to the underlying work) — callers should treat this as "no
464    /// answer within budget", not "the operation failed or was rolled back".
465    ///
466    /// Distinct from `#836`'s narrower `ann_ready_timeout_ms`, which bounds
467    /// only a single cold-miss ANN-build wait inside the recall vector leg
468    /// and degrades to an in-band FTS-only result. This variant bounds the
469    /// *entire* operation end-to-end and is surfaced as a typed error so a
470    /// caller under sustained contention gets a fast, clear answer instead
471    /// of hanging until an upstream client-side ceiling (observed at 300s in
472    /// production, #889) fires instead.
473    #[error("{operation} exceeded its {budget_ms}ms deadline (elapsed {elapsed_ms}ms)")]
474    DeadlineExceeded {
475        operation: String,
476        budget_ms: u64,
477        elapsed_ms: u64,
478    },
479}
480
481impl RuntimeError {
482    /// Classify a failed inbound channel write without inspecting rendered
483    /// error text.
484    ///
485    /// Existing typed safe-retry failures remain retryable. Secret detection is
486    /// the first deterministic policy refusal and is permanent. `None` from
487    /// [`Self::retryable_failure_context`] is deliberately not interpreted as
488    /// permanent: every other variant starts in the bounded `Unknown` bucket.
489    pub fn channel_ingest_failure_class(&self) -> ChannelIngestFailureClass {
490        let reason = self.variant_name();
491        if self.retryable_failure_context().is_some() {
492            ChannelIngestFailureClass::Retryable { reason }
493        } else if matches!(self, Self::SecretDetected(_)) {
494            ChannelIngestFailureClass::Permanent { reason }
495        } else {
496            ChannelIngestFailureClass::Unknown { reason }
497        }
498    }
499
500    /// Stable top-level variant name used by typed policy classifiers.
501    const fn variant_name(&self) -> &'static str {
502        match self {
503            Self::Storage(_) => "Storage",
504            Self::Sqlite(_) => "Sqlite",
505            Self::Query(_) => "Query",
506            Self::NotFound(_) => "NotFound",
507            Self::InvalidInput(_) => "InvalidInput",
508            Self::UnknownVerb(_) => "UnknownVerb",
509            Self::Unconfigured(_) => "Unconfigured",
510            Self::UnknownModel(_) => "UnknownModel",
511            Self::Embedding(_) => "Embedding",
512            Self::Ambiguous(_) => "Ambiguous",
513            Self::Fusion(_) => "Fusion",
514            Self::UnknownFusionStrategy(_) => "UnknownFusionStrategy",
515            Self::Internal(_) => "Internal",
516            Self::GuardedWriteFailed(_) => "GuardedWriteFailed",
517            Self::MissingPackDependency(_) => "MissingPackDependency",
518            Self::MissingPackDependencies(_) => "MissingPackDependencies",
519            Self::CircularPackDependency(_) => "CircularPackDependency",
520            Self::PackRedeclared { .. } => "PackRedeclared",
521            Self::VerbCollision { .. } => "VerbCollision",
522            Self::ReservedEnvelopeParam { .. } => "ReservedEnvelopeParam",
523            Self::PermissionDenied { .. } => "PermissionDenied",
524            Self::GateUnavailable { .. } => "GateUnavailable",
525            Self::Khive(_) => "Khive",
526            Self::NamespaceMismatch { .. } => "NamespaceMismatch",
527            Self::AmbiguousPrefix { .. } => "AmbiguousPrefix",
528            Self::CrossBackendMergeUnsupported { .. } => "CrossBackendMergeUnsupported",
529            Self::UnknownRemote { .. } => "UnknownRemote",
530            Self::RemoteCacheMissing { .. } => "RemoteCacheMissing",
531            Self::AmbiguousId { .. } => "AmbiguousId",
532            Self::CrossNamespaceWrite { .. } => "CrossNamespaceWrite",
533            Self::RemoteFetchError { .. } => "RemoteFetchError",
534            Self::WriteBudgetExceeded { .. } => "WriteBudgetExceeded",
535            Self::SecretDetected(_) => "SecretDetected",
536            Self::DeadlineExceeded { .. } => "DeadlineExceeded",
537            Self::IncompatibleEventStore(_) => "IncompatibleEventStore",
538        }
539    }
540
541    /// Recover a finite-wait pool-checkout timeout without inspecting rendered
542    /// error text.
543    ///
544    /// Store implementations retain [`khive_db::SqliteError`] as the typed
545    /// source of `StorageError::Driver`; this method carries that structure
546    /// through the runtime wrapper for the MCP wire serializer. A direct
547    /// `RuntimeError::Sqlite` follows the same classification path.
548    pub fn writer_pool_checkout_timeout_context(&self) -> Option<WriterPoolCheckoutTimeoutContext> {
549        let (sqlite_error, capability, operation) = match self {
550            Self::Sqlite(error) => (error, None, None),
551            Self::Storage(khive_storage::StorageError::Driver {
552                capability,
553                operation,
554                source,
555            }) => (
556                source.downcast_ref::<khive_db::SqliteError>()?,
557                Some(*capability),
558                Some(operation.to_string()),
559            ),
560            _ => return None,
561        };
562
563        let khive_db::SqliteError::WriterPoolCheckoutTimeout { timeout } = sqlite_error else {
564            return None;
565        };
566        Some(WriterPoolCheckoutTimeoutContext {
567            timeout: *timeout,
568            capability,
569            operation,
570        })
571    }
572
573    /// Recover either pre-execution write-admission failure this process can
574    /// produce, by typed variant rather than rendered message text (#1643).
575    /// Both are safe to classify as retryable: the request was never
576    /// accepted, so no partial side effect can exist to roll back.
577    pub fn admission_failure_context(&self) -> Option<AdmissionFailureContext> {
578        if let Some(context) = self.writer_pool_checkout_timeout_context() {
579            return Some(AdmissionFailureContext {
580                stage: WRITER_POOL_CHECKOUT_TIMEOUT_STAGE,
581                timeout: context.timeout,
582                capability: context.capability,
583                operation: context.operation,
584                scope: None,
585                retry_after_ms: None,
586            });
587        }
588        if let Self::Storage(khive_storage::StorageError::WriteQueueFull { timeout_ms }) = self {
589            return Some(AdmissionFailureContext {
590                stage: WRITER_QUEUE_SATURATED_STAGE,
591                timeout: Duration::from_millis(*timeout_ms),
592                capability: None,
593                operation: None,
594                scope: Some(WRITER_ADMISSION_SCOPE),
595                retry_after_ms: Some(*timeout_ms),
596            });
597        }
598        if let Self::Storage(khive_storage::StorageError::AdmissionTimeout {
599            operation,
600            timeout_ms,
601        }) = self
602        {
603            return Some(AdmissionFailureContext {
604                stage: STORAGE_ADMISSION_TIMEOUT_STAGE,
605                timeout: Duration::from_millis(*timeout_ms),
606                capability: None,
607                operation: Some(operation.to_string()),
608                scope: None,
609                retry_after_ms: None,
610            });
611        }
612        None
613    }
614
615    /// Recover every typed failure for which this process can prove that
616    /// retrying the one failed operation cannot duplicate a side effect.
617    pub fn retryable_failure_context(&self) -> Option<RetryableFailureContext> {
618        if let Some(context) = self.admission_failure_context() {
619            return Some(context.into());
620        }
621        if let Self::Storage(khive_storage::StorageError::WriterTaskBusy { timeout_ms }) = self {
622            return Some(RetryableFailureContext {
623                stage: WRITER_TASK_BEGIN_BUSY_STAGE,
624                timeout: Duration::from_millis(*timeout_ms),
625                capability: None,
626                operation: Some("writer_task_begin".to_string()),
627                scope: None,
628                retry_after_ms: None,
629            });
630        }
631        if let Self::Storage(khive_storage::StorageError::ReadTransactionAgeEvicted {
632            operation,
633            max_age_secs,
634        }) = self
635        {
636            return Some(RetryableFailureContext {
637                stage: READ_TX_AGE_EVICTED_STAGE,
638                timeout: Duration::from_secs(*max_age_secs),
639                capability: Some(khive_storage::StorageCapability::Sql),
640                operation: Some(operation.to_string()),
641                scope: None,
642                retry_after_ms: None,
643            });
644        }
645        if let Self::Storage(
646            khive_storage::StorageError::ReadTransactionAgeEvictionCleanupFailed {
647                operation,
648                max_age_secs,
649                ..
650            },
651        ) = self
652        {
653            return Some(RetryableFailureContext {
654                stage: READ_TX_AGE_EVICTED_STAGE,
655                timeout: Duration::from_secs(*max_age_secs),
656                capability: Some(khive_storage::StorageCapability::Sql),
657                operation: Some(operation.to_string()),
658                scope: None,
659                retry_after_ms: None,
660            });
661        }
662        None
663    }
664}
665
666/// Resolve an FTS text-leg search result, failing loud on parser syntax
667/// errors instead of silently degrading to vector-only fusion.
668///
669/// A genuine backend outage (pool exhaustion, connection failure, etc.) is
670/// NOT a bad query and is returned as-is via the fallthrough `Err(e)` arm;
671/// `is_fts5_syntax_error` is the narrow gate that tells the two apart.
672pub fn fts_text_leg_or_err<T>(
673    result: Result<Vec<T>, RuntimeError>,
674    context: &'static str,
675    query: &str,
676) -> RuntimeResult<Vec<T>> {
677    match result {
678        Ok(hits) => Ok(hits),
679        Err(RuntimeError::Storage(se)) if se.is_fts5_syntax_error() => {
680            tracing::warn!(
681                error = %se,
682                query = %query,
683                context,
684                "FTS text leg failed on a parser syntax error; failing loud (#569)"
685            );
686            Err(RuntimeError::InvalidInput(format!(
687                "{context}: FTS query could not be parsed: {se}"
688            )))
689        }
690        Err(e) => Err(e),
691    }
692}
693
694fn format_uuid_list(uuids: &[uuid::Uuid]) -> String {
695    uuids
696        .iter()
697        .map(uuid::Uuid::to_string)
698        .collect::<Vec<_>>()
699        .join(", ")
700}
701
702/// Maps the dependency-light `khive-types` entity-type resolution error onto
703/// `RuntimeError::InvalidInput` at the pack boundary: `khive-types` cannot
704/// depend on `khive-runtime`, so it cannot produce `RuntimeError` directly.
705impl From<khive_types::EntityTypeError> for RuntimeError {
706    fn from(e: khive_types::EntityTypeError) -> Self {
707        Self::InvalidInput(e.to_string())
708    }
709}
710
711impl From<khive_types::KhiveError> for RuntimeError {
712    fn from(e: khive_types::KhiveError) -> Self {
713        Self::Khive(e)
714    }
715}
716
717#[cfg(test)]
718mod channel_ingest_failure_class_tests {
719    use super::{ChannelIngestFailureClass, RuntimeError};
720    use crate::secret_gate::SecretMatch;
721    use std::time::Duration;
722
723    #[test]
724    fn secret_detected_is_permanent_by_typed_variant_not_display_text() {
725        let first = RuntimeError::SecretDetected(SecretMatch {
726            detector: "fixture",
727            masked: "first-rendering".to_string(),
728        });
729        let second = RuntimeError::SecretDetected(SecretMatch {
730            detector: "fixture",
731            masked: "completely-different-rendering".to_string(),
732        });
733
734        assert_ne!(first.to_string(), second.to_string());
735        assert_eq!(
736            first.channel_ingest_failure_class(),
737            ChannelIngestFailureClass::Permanent {
738                reason: "SecretDetected"
739            }
740        );
741        assert_eq!(
742            second.channel_ingest_failure_class(),
743            ChannelIngestFailureClass::Permanent {
744                reason: "SecretDetected"
745            },
746            "rendered error details must not participate in ingest classification"
747        );
748    }
749
750    #[test]
751    fn admission_failures_are_retryable_and_unclassified_errors_are_unknown() {
752        let retryable =
753            RuntimeError::Storage(khive_storage::StorageError::WriteQueueFull { timeout_ms: 25 });
754        assert_eq!(
755            retryable.channel_ingest_failure_class(),
756            ChannelIngestFailureClass::Retryable { reason: "Storage" }
757        );
758
759        let begin_busy =
760            RuntimeError::Storage(khive_storage::StorageError::WriterTaskBusy { timeout_ms: 175 });
761        let context = begin_busy
762            .retryable_failure_context()
763            .expect("contended BEGIN must remain typed and retryable");
764        assert_eq!(context.stage, "writer_task_begin_busy");
765        assert_eq!(context.timeout, Duration::from_millis(175));
766        assert_eq!(context.operation.as_deref(), Some("writer_task_begin"));
767        assert_eq!(context.scope, None);
768        assert_eq!(context.retry_after_ms, None);
769        assert_eq!(
770            begin_busy.channel_ingest_failure_class(),
771            ChannelIngestFailureClass::Retryable { reason: "Storage" }
772        );
773
774        let unknown = RuntimeError::InvalidInput(
775            "write blocked: SecretDetected text must not affect classification".to_string(),
776        );
777        assert_eq!(
778            unknown.channel_ingest_failure_class(),
779            ChannelIngestFailureClass::Unknown {
780                reason: "InvalidInput"
781            },
782            "a rendered message resembling SecretDetected must remain Unknown unless its typed variant is SecretDetected"
783        );
784    }
785
786    #[test]
787    fn storage_admission_timeout_is_a_retryable_admission_failure() {
788        let admission = RuntimeError::Storage(khive_storage::StorageError::AdmissionTimeout {
789            operation: "sql_bridge.writer_handle".into(),
790            timeout_ms: 30_000,
791        });
792        let context = admission
793            .admission_failure_context()
794            .expect("a storage admission timeout happens before the operation starts");
795        assert_eq!(context.stage, super::STORAGE_ADMISSION_TIMEOUT_STAGE);
796        assert_eq!(context.timeout, Duration::from_millis(30_000));
797        assert_eq!(
798            context.operation.as_deref(),
799            Some("sql_bridge.writer_handle")
800        );
801        assert_eq!(context.scope, None);
802        assert_eq!(context.retry_after_ms, None);
803        assert_eq!(
804            admission.channel_ingest_failure_class(),
805            ChannelIngestFailureClass::Retryable { reason: "Storage" }
806        );
807
808        // The general mid-flight timeout stays unclassified: it proves
809        // nothing about whether work was in flight when the deadline expired.
810        let mid_flight = RuntimeError::Storage(khive_storage::StorageError::Timeout {
811            operation: "sql_bridge.reader_open".into(),
812        });
813        assert!(mid_flight.admission_failure_context().is_none());
814        assert!(mid_flight.retryable_failure_context().is_none());
815    }
816
817    /// Binds both age-eviction outcomes a cached-reader rollback can produce
818    /// (clean rollback vs. failed/denied cleanup) to the same retryable wire
819    /// stage: a regression that reverted either branch back to generic
820    /// `StorageError::Transaction` would drop this match arm entirely and
821    /// fail here, since `Transaction` does have a `retryable_failure_context`
822    /// mapping distinct from `super::READ_TX_AGE_EVICTED_STAGE`.
823    #[test]
824    fn both_read_tx_age_eviction_outcomes_map_to_the_same_retryable_stage() {
825        let clean = RuntimeError::Storage(khive_storage::StorageError::ReadTransactionAgeEvicted {
826            operation: "query_all".into(),
827            max_age_secs: 120,
828        });
829        let clean_context = clean
830            .retryable_failure_context()
831            .expect("a clean age eviction must be typed-retryable");
832        assert_eq!(clean_context.stage, super::READ_TX_AGE_EVICTED_STAGE);
833        assert_eq!(clean_context.timeout, Duration::from_secs(120));
834        assert_eq!(clean_context.operation.as_deref(), Some("query_all"));
835        assert_eq!(
836            clean_context.capability,
837            Some(khive_storage::StorageCapability::Sql)
838        );
839
840        let cleanup_failed = RuntimeError::Storage(
841            khive_storage::StorageError::ReadTransactionAgeEvictionCleanupFailed {
842                operation: "query_all".into(),
843                max_age_secs: 120,
844                message: "rollback failed: disk I/O error".into(),
845            },
846        );
847        let cleanup_failed_context = cleanup_failed
848            .retryable_failure_context()
849            .expect("a failed cleanup rollback must remain typed-retryable");
850        assert_eq!(
851            cleanup_failed_context.stage,
852            super::READ_TX_AGE_EVICTED_STAGE
853        );
854        assert_eq!(cleanup_failed_context.timeout, Duration::from_secs(120));
855        assert_eq!(
856            cleanup_failed_context.operation.as_deref(),
857            Some("query_all")
858        );
859        assert_eq!(
860            cleanup_failed_context.capability,
861            Some(khive_storage::StorageCapability::Sql)
862        );
863
864        assert_ne!(
865            clean.to_string(),
866            cleanup_failed.to_string(),
867            "the rendered message must distinguish a clean eviction from a failed cleanup even \
868             though both map to the same wire stage"
869        );
870        assert!(cleanup_failed.to_string().contains("rollback failed"));
871    }
872}