1use std::fmt;
4use std::time::Duration;
5
6use thiserror::Error;
7use uuid::Uuid;
8
9pub type RuntimeResult<T> = Result<T, RuntimeError>;
11
12pub const WRITER_POOL_CHECKOUT_TIMEOUT_STAGE: &str = "writer_pool_checkout_timeout";
15
16pub const WRITER_QUEUE_SATURATED_STAGE: &str = "writer_queue_saturated";
19
20pub const WRITER_TASK_BEGIN_BUSY_STAGE: &str = "writer_task_begin_busy";
23
24pub const STORAGE_ADMISSION_TIMEOUT_STAGE: &str = "storage_admission_timeout";
29
30pub const READ_TX_AGE_EVICTED_STAGE: &str = "read_tx_age_evicted";
41
42pub const WRITER_ADMISSION_SCOPE: &str = "writer_admission";
49
50#[derive(Debug, Clone, PartialEq, Eq)]
57pub struct AdmissionFailureContext {
58 pub stage: &'static str,
62 pub timeout: Duration,
64 pub capability: Option<khive_storage::StorageCapability>,
68 pub operation: Option<String>,
70 pub scope: Option<&'static str>,
75 pub retry_after_ms: Option<u64>,
81}
82
83#[derive(Debug, Clone, PartialEq, Eq)]
90pub struct RetryableFailureContext {
91 pub stage: &'static str,
93 pub timeout: Duration,
95 pub capability: Option<khive_storage::StorageCapability>,
97 pub operation: Option<String>,
99 pub scope: Option<&'static str>,
101 pub retry_after_ms: Option<u64>,
103}
104
105impl From<AdmissionFailureContext> for RetryableFailureContext {
106 fn from(context: AdmissionFailureContext) -> Self {
107 Self {
108 stage: context.stage,
109 timeout: context.timeout,
110 capability: context.capability,
111 operation: context.operation,
112 scope: context.scope,
113 retry_after_ms: context.retry_after_ms,
114 }
115 }
116}
117
118#[derive(Debug, Clone, Copy, PartialEq, Eq)]
124pub enum ChannelIngestFailureClass {
125 Retryable { reason: &'static str },
127 Permanent { reason: &'static str },
129 Unknown { reason: &'static str },
131}
132
133impl ChannelIngestFailureClass {
134 pub const fn name(self) -> &'static str {
136 match self {
137 Self::Retryable { .. } => "retryable",
138 Self::Permanent { .. } => "permanent",
139 Self::Unknown { .. } => "unknown",
140 }
141 }
142
143 pub const fn reason(self) -> &'static str {
145 match self {
146 Self::Retryable { reason } | Self::Permanent { reason } | Self::Unknown { reason } => {
147 reason
148 }
149 }
150 }
151}
152
153#[derive(Debug, Clone, PartialEq, Eq)]
156pub struct WriterPoolCheckoutTimeoutContext {
157 pub timeout: Duration,
160 pub capability: Option<khive_storage::StorageCapability>,
162 pub operation: Option<String>,
164}
165
166#[derive(Debug, Clone, PartialEq, Eq)]
171pub struct GuardedWriteFailure {
172 pub entry_index: Option<usize>,
175 pub missing_source: Option<Uuid>,
178 pub missing_target: Option<Uuid>,
181}
182
183impl fmt::Display for GuardedWriteFailure {
184 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
185 let mut missing = Vec::new();
186 if let Some(source) = self.missing_source {
187 missing.push(format!("source {source}"));
188 }
189 if let Some(target) = self.missing_target {
190 missing.push(format!("target {target}"));
191 }
192 let missing = if missing.is_empty() {
193 "endpoint(s)".to_string()
194 } else {
195 missing.join(" and ")
196 };
197 match self.entry_index {
198 Some(index) => write!(
199 f,
200 "batch entry {index}: {missing} no longer exist at write time"
201 ),
202 None => write!(f, "{missing} no longer exist at write time"),
203 }
204 }
205}
206
207impl std::error::Error for GuardedWriteFailure {}
208
209#[derive(Debug, Clone, PartialEq, Eq)]
211pub struct MissingPackDependency {
212 pub from: String,
213 pub requires: String,
214}
215
216impl fmt::Display for MissingPackDependency {
217 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
218 write!(
219 f,
220 "pack '{}' requires '{}', but '{}' is not in the loaded pack set",
221 self.from, self.requires, self.requires
222 )
223 }
224}
225
226impl std::error::Error for MissingPackDependency {}
227
228#[derive(Debug, Clone, PartialEq, Eq)]
230pub struct MissingPackDependencies {
231 pub missing: Vec<MissingPackDependency>,
232}
233
234impl fmt::Display for MissingPackDependencies {
235 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
236 let parts: Vec<String> = self.missing.iter().map(ToString::to_string).collect();
237 write!(f, "{}", parts.join("; "))
238 }
239}
240
241impl std::error::Error for MissingPackDependencies {}
242
243#[derive(Debug, Clone, PartialEq, Eq)]
245pub struct CircularPackDependency {
246 pub cycle: Vec<String>,
247}
248
249impl fmt::Display for CircularPackDependency {
250 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
251 write!(
252 f,
253 "circular dependency detected among packs: {}",
254 self.cycle.join(" -> ")
255 )
256 }
257}
258
259impl std::error::Error for CircularPackDependency {}
260
261#[derive(Debug, Error)]
267pub enum RuntimeError {
268 #[error("storage: {0}")]
269 Storage(#[from] khive_storage::StorageError),
270
271 #[error("sqlite: {0}")]
272 Sqlite(#[from] khive_db::SqliteError),
273
274 #[error("query: {0}")]
275 Query(#[from] khive_query::QueryError),
276
277 #[error("not found: {0}")]
278 NotFound(String),
279
280 #[error("invalid input: {0}")]
281 InvalidInput(String),
282
283 #[error("invalid input: {0}")]
284 UnknownVerb(String),
285
286 #[error("unconfigured: {0} is not set")]
287 Unconfigured(String),
288
289 #[error("unknown embedding model: {0}")]
290 UnknownModel(String),
291
292 #[error("embedding: {0}")]
293 Embedding(#[from] lattice_embed::EmbedError),
294
295 #[error("ambiguous: {0}")]
296 Ambiguous(String),
297
298 #[error("fusion: {0}")]
299 Fusion(#[from] khive_fusion::FuseError),
300
301 #[error("unknown fusion strategy: {0}")]
305 UnknownFusionStrategy(String),
306
307 #[error("internal: {0}")]
308 Internal(String),
309
310 #[error("audit batch incompatible event store: {0}")]
316 IncompatibleEventStore(String),
317
318 #[error("guarded edge write refused: {0}")]
319 GuardedWriteFailed(GuardedWriteFailure),
320
321 #[error("missing pack dependency: {0}")]
322 MissingPackDependency(MissingPackDependency),
323
324 #[error("missing pack dependencies: {0}")]
325 MissingPackDependencies(MissingPackDependencies),
326
327 #[error("{0}")]
328 CircularPackDependency(CircularPackDependency),
329
330 #[error("pack '{name}' registered twice (indices {first_idx} and {second_idx})")]
331 PackRedeclared {
332 name: String,
333 first_idx: usize,
334 second_idx: usize,
335 },
336
337 #[error(
341 "verb collision: verb {verb:?} declared by both pack {first_pack:?} and pack \
342 {second_pack:?}; rename one handler or use Visibility::Subhandler for internal verbs"
343 )]
344 VerbCollision {
345 verb: String,
346 first_pack: String,
347 second_pack: String,
348 },
349
350 #[error(
352 "pack {pack:?} handler {verb:?} declares request-envelope parameter {param:?}; rename the verb argument"
353 )]
354 ReservedEnvelopeParam {
355 pack: String,
356 verb: String,
357 param: String,
358 },
359
360 #[error("permission denied for verb {verb:?}: {reason}")]
366 PermissionDenied { verb: String, reason: String },
367
368 #[error("gate unavailable for verb {verb:?}: {reason}")]
374 GateUnavailable { verb: String, reason: String },
375
376 #[error("{0}")]
380 Khive(khive_types::KhiveError),
381
382 #[error("not found in this namespace")]
387 NamespaceMismatch { id: uuid::Uuid },
388
389 #[error("ambiguous prefix {prefix:?}: matches {}", format_uuid_list(matches))]
395 AmbiguousPrefix {
396 prefix: String,
397 matches: Vec<uuid::Uuid>,
398 },
399
400 #[error(
405 "cross-backend merge is not supported: \
406 into_id {into_id} is on backend '{into_backend}', \
407 from_id {from_id} is on backend '{from_backend}'. \
408 Both entities must be on the same backend to merge."
409 )]
410 CrossBackendMergeUnsupported {
411 into_id: uuid::Uuid,
412 from_id: uuid::Uuid,
413 into_backend: String,
414 from_backend: String,
415 },
416
417 #[error("unknown remote: {name:?}")]
420 UnknownRemote { name: String },
421
422 #[error("remote cache missing for remote={remote:?} namespace={namespace:?}")]
424 RemoteCacheMissing { remote: String, namespace: String },
425
426 #[error("ambiguous id {id:?}: matched {count} records")]
428 AmbiguousId { id: String, count: usize },
429
430 #[error("cross-namespace write denied: cannot write to remote namespace {namespace:?}")]
432 CrossNamespaceWrite { namespace: String },
433
434 #[error("remote fetch error for remote={remote:?}: {message}")]
436 RemoteFetchError { remote: String, message: String },
437
438 #[error(
444 "write budget exceeded: max_new_entries={max_new_entries}, \
445 attempted_new_entries={attempted_new_entries}"
446 )]
447 WriteBudgetExceeded {
448 max_new_entries: u64,
449 attempted_new_entries: u64,
450 },
451
452 #[error("write blocked: {0}")]
458 SecretDetected(crate::secret_gate::SecretMatch),
459
460 #[error("{operation} exceeded its {budget_ms}ms deadline (elapsed {elapsed_ms}ms)")]
474 DeadlineExceeded {
475 operation: String,
476 budget_ms: u64,
477 elapsed_ms: u64,
478 },
479}
480
481impl RuntimeError {
482 pub fn channel_ingest_failure_class(&self) -> ChannelIngestFailureClass {
490 let reason = self.variant_name();
491 if self.retryable_failure_context().is_some() {
492 ChannelIngestFailureClass::Retryable { reason }
493 } else if matches!(self, Self::SecretDetected(_)) {
494 ChannelIngestFailureClass::Permanent { reason }
495 } else {
496 ChannelIngestFailureClass::Unknown { reason }
497 }
498 }
499
500 const fn variant_name(&self) -> &'static str {
502 match self {
503 Self::Storage(_) => "Storage",
504 Self::Sqlite(_) => "Sqlite",
505 Self::Query(_) => "Query",
506 Self::NotFound(_) => "NotFound",
507 Self::InvalidInput(_) => "InvalidInput",
508 Self::UnknownVerb(_) => "UnknownVerb",
509 Self::Unconfigured(_) => "Unconfigured",
510 Self::UnknownModel(_) => "UnknownModel",
511 Self::Embedding(_) => "Embedding",
512 Self::Ambiguous(_) => "Ambiguous",
513 Self::Fusion(_) => "Fusion",
514 Self::UnknownFusionStrategy(_) => "UnknownFusionStrategy",
515 Self::Internal(_) => "Internal",
516 Self::GuardedWriteFailed(_) => "GuardedWriteFailed",
517 Self::MissingPackDependency(_) => "MissingPackDependency",
518 Self::MissingPackDependencies(_) => "MissingPackDependencies",
519 Self::CircularPackDependency(_) => "CircularPackDependency",
520 Self::PackRedeclared { .. } => "PackRedeclared",
521 Self::VerbCollision { .. } => "VerbCollision",
522 Self::ReservedEnvelopeParam { .. } => "ReservedEnvelopeParam",
523 Self::PermissionDenied { .. } => "PermissionDenied",
524 Self::GateUnavailable { .. } => "GateUnavailable",
525 Self::Khive(_) => "Khive",
526 Self::NamespaceMismatch { .. } => "NamespaceMismatch",
527 Self::AmbiguousPrefix { .. } => "AmbiguousPrefix",
528 Self::CrossBackendMergeUnsupported { .. } => "CrossBackendMergeUnsupported",
529 Self::UnknownRemote { .. } => "UnknownRemote",
530 Self::RemoteCacheMissing { .. } => "RemoteCacheMissing",
531 Self::AmbiguousId { .. } => "AmbiguousId",
532 Self::CrossNamespaceWrite { .. } => "CrossNamespaceWrite",
533 Self::RemoteFetchError { .. } => "RemoteFetchError",
534 Self::WriteBudgetExceeded { .. } => "WriteBudgetExceeded",
535 Self::SecretDetected(_) => "SecretDetected",
536 Self::DeadlineExceeded { .. } => "DeadlineExceeded",
537 Self::IncompatibleEventStore(_) => "IncompatibleEventStore",
538 }
539 }
540
541 pub fn writer_pool_checkout_timeout_context(&self) -> Option<WriterPoolCheckoutTimeoutContext> {
549 let (sqlite_error, capability, operation) = match self {
550 Self::Sqlite(error) => (error, None, None),
551 Self::Storage(khive_storage::StorageError::Driver {
552 capability,
553 operation,
554 source,
555 }) => (
556 source.downcast_ref::<khive_db::SqliteError>()?,
557 Some(*capability),
558 Some(operation.to_string()),
559 ),
560 _ => return None,
561 };
562
563 let khive_db::SqliteError::WriterPoolCheckoutTimeout { timeout } = sqlite_error else {
564 return None;
565 };
566 Some(WriterPoolCheckoutTimeoutContext {
567 timeout: *timeout,
568 capability,
569 operation,
570 })
571 }
572
573 pub fn admission_failure_context(&self) -> Option<AdmissionFailureContext> {
578 if let Some(context) = self.writer_pool_checkout_timeout_context() {
579 return Some(AdmissionFailureContext {
580 stage: WRITER_POOL_CHECKOUT_TIMEOUT_STAGE,
581 timeout: context.timeout,
582 capability: context.capability,
583 operation: context.operation,
584 scope: None,
585 retry_after_ms: None,
586 });
587 }
588 if let Self::Storage(khive_storage::StorageError::WriteQueueFull { timeout_ms }) = self {
589 return Some(AdmissionFailureContext {
590 stage: WRITER_QUEUE_SATURATED_STAGE,
591 timeout: Duration::from_millis(*timeout_ms),
592 capability: None,
593 operation: None,
594 scope: Some(WRITER_ADMISSION_SCOPE),
595 retry_after_ms: Some(*timeout_ms),
596 });
597 }
598 if let Self::Storage(khive_storage::StorageError::AdmissionTimeout {
599 operation,
600 timeout_ms,
601 }) = self
602 {
603 return Some(AdmissionFailureContext {
604 stage: STORAGE_ADMISSION_TIMEOUT_STAGE,
605 timeout: Duration::from_millis(*timeout_ms),
606 capability: None,
607 operation: Some(operation.to_string()),
608 scope: None,
609 retry_after_ms: None,
610 });
611 }
612 None
613 }
614
615 pub fn retryable_failure_context(&self) -> Option<RetryableFailureContext> {
618 if let Some(context) = self.admission_failure_context() {
619 return Some(context.into());
620 }
621 if let Self::Storage(khive_storage::StorageError::WriterTaskBusy { timeout_ms }) = self {
622 return Some(RetryableFailureContext {
623 stage: WRITER_TASK_BEGIN_BUSY_STAGE,
624 timeout: Duration::from_millis(*timeout_ms),
625 capability: None,
626 operation: Some("writer_task_begin".to_string()),
627 scope: None,
628 retry_after_ms: None,
629 });
630 }
631 if let Self::Storage(khive_storage::StorageError::ReadTransactionAgeEvicted {
632 operation,
633 max_age_secs,
634 }) = self
635 {
636 return Some(RetryableFailureContext {
637 stage: READ_TX_AGE_EVICTED_STAGE,
638 timeout: Duration::from_secs(*max_age_secs),
639 capability: Some(khive_storage::StorageCapability::Sql),
640 operation: Some(operation.to_string()),
641 scope: None,
642 retry_after_ms: None,
643 });
644 }
645 if let Self::Storage(
646 khive_storage::StorageError::ReadTransactionAgeEvictionCleanupFailed {
647 operation,
648 max_age_secs,
649 ..
650 },
651 ) = self
652 {
653 return Some(RetryableFailureContext {
654 stage: READ_TX_AGE_EVICTED_STAGE,
655 timeout: Duration::from_secs(*max_age_secs),
656 capability: Some(khive_storage::StorageCapability::Sql),
657 operation: Some(operation.to_string()),
658 scope: None,
659 retry_after_ms: None,
660 });
661 }
662 None
663 }
664}
665
666pub fn fts_text_leg_or_err<T>(
673 result: Result<Vec<T>, RuntimeError>,
674 context: &'static str,
675 query: &str,
676) -> RuntimeResult<Vec<T>> {
677 match result {
678 Ok(hits) => Ok(hits),
679 Err(RuntimeError::Storage(se)) if se.is_fts5_syntax_error() => {
680 tracing::warn!(
681 error = %se,
682 query = %query,
683 context,
684 "FTS text leg failed on a parser syntax error; failing loud (#569)"
685 );
686 Err(RuntimeError::InvalidInput(format!(
687 "{context}: FTS query could not be parsed: {se}"
688 )))
689 }
690 Err(e) => Err(e),
691 }
692}
693
694fn format_uuid_list(uuids: &[uuid::Uuid]) -> String {
695 uuids
696 .iter()
697 .map(uuid::Uuid::to_string)
698 .collect::<Vec<_>>()
699 .join(", ")
700}
701
702impl From<khive_types::EntityTypeError> for RuntimeError {
706 fn from(e: khive_types::EntityTypeError) -> Self {
707 Self::InvalidInput(e.to_string())
708 }
709}
710
711impl From<khive_types::KhiveError> for RuntimeError {
712 fn from(e: khive_types::KhiveError) -> Self {
713 Self::Khive(e)
714 }
715}
716
717#[cfg(test)]
718mod channel_ingest_failure_class_tests {
719 use super::{ChannelIngestFailureClass, RuntimeError};
720 use crate::secret_gate::SecretMatch;
721 use std::time::Duration;
722
723 #[test]
724 fn secret_detected_is_permanent_by_typed_variant_not_display_text() {
725 let first = RuntimeError::SecretDetected(SecretMatch {
726 detector: "fixture",
727 masked: "first-rendering".to_string(),
728 });
729 let second = RuntimeError::SecretDetected(SecretMatch {
730 detector: "fixture",
731 masked: "completely-different-rendering".to_string(),
732 });
733
734 assert_ne!(first.to_string(), second.to_string());
735 assert_eq!(
736 first.channel_ingest_failure_class(),
737 ChannelIngestFailureClass::Permanent {
738 reason: "SecretDetected"
739 }
740 );
741 assert_eq!(
742 second.channel_ingest_failure_class(),
743 ChannelIngestFailureClass::Permanent {
744 reason: "SecretDetected"
745 },
746 "rendered error details must not participate in ingest classification"
747 );
748 }
749
750 #[test]
751 fn admission_failures_are_retryable_and_unclassified_errors_are_unknown() {
752 let retryable =
753 RuntimeError::Storage(khive_storage::StorageError::WriteQueueFull { timeout_ms: 25 });
754 assert_eq!(
755 retryable.channel_ingest_failure_class(),
756 ChannelIngestFailureClass::Retryable { reason: "Storage" }
757 );
758
759 let begin_busy =
760 RuntimeError::Storage(khive_storage::StorageError::WriterTaskBusy { timeout_ms: 175 });
761 let context = begin_busy
762 .retryable_failure_context()
763 .expect("contended BEGIN must remain typed and retryable");
764 assert_eq!(context.stage, "writer_task_begin_busy");
765 assert_eq!(context.timeout, Duration::from_millis(175));
766 assert_eq!(context.operation.as_deref(), Some("writer_task_begin"));
767 assert_eq!(context.scope, None);
768 assert_eq!(context.retry_after_ms, None);
769 assert_eq!(
770 begin_busy.channel_ingest_failure_class(),
771 ChannelIngestFailureClass::Retryable { reason: "Storage" }
772 );
773
774 let unknown = RuntimeError::InvalidInput(
775 "write blocked: SecretDetected text must not affect classification".to_string(),
776 );
777 assert_eq!(
778 unknown.channel_ingest_failure_class(),
779 ChannelIngestFailureClass::Unknown {
780 reason: "InvalidInput"
781 },
782 "a rendered message resembling SecretDetected must remain Unknown unless its typed variant is SecretDetected"
783 );
784 }
785
786 #[test]
787 fn storage_admission_timeout_is_a_retryable_admission_failure() {
788 let admission = RuntimeError::Storage(khive_storage::StorageError::AdmissionTimeout {
789 operation: "sql_bridge.writer_handle".into(),
790 timeout_ms: 30_000,
791 });
792 let context = admission
793 .admission_failure_context()
794 .expect("a storage admission timeout happens before the operation starts");
795 assert_eq!(context.stage, super::STORAGE_ADMISSION_TIMEOUT_STAGE);
796 assert_eq!(context.timeout, Duration::from_millis(30_000));
797 assert_eq!(
798 context.operation.as_deref(),
799 Some("sql_bridge.writer_handle")
800 );
801 assert_eq!(context.scope, None);
802 assert_eq!(context.retry_after_ms, None);
803 assert_eq!(
804 admission.channel_ingest_failure_class(),
805 ChannelIngestFailureClass::Retryable { reason: "Storage" }
806 );
807
808 let mid_flight = RuntimeError::Storage(khive_storage::StorageError::Timeout {
811 operation: "sql_bridge.reader_open".into(),
812 });
813 assert!(mid_flight.admission_failure_context().is_none());
814 assert!(mid_flight.retryable_failure_context().is_none());
815 }
816
817 #[test]
824 fn both_read_tx_age_eviction_outcomes_map_to_the_same_retryable_stage() {
825 let clean = RuntimeError::Storage(khive_storage::StorageError::ReadTransactionAgeEvicted {
826 operation: "query_all".into(),
827 max_age_secs: 120,
828 });
829 let clean_context = clean
830 .retryable_failure_context()
831 .expect("a clean age eviction must be typed-retryable");
832 assert_eq!(clean_context.stage, super::READ_TX_AGE_EVICTED_STAGE);
833 assert_eq!(clean_context.timeout, Duration::from_secs(120));
834 assert_eq!(clean_context.operation.as_deref(), Some("query_all"));
835 assert_eq!(
836 clean_context.capability,
837 Some(khive_storage::StorageCapability::Sql)
838 );
839
840 let cleanup_failed = RuntimeError::Storage(
841 khive_storage::StorageError::ReadTransactionAgeEvictionCleanupFailed {
842 operation: "query_all".into(),
843 max_age_secs: 120,
844 message: "rollback failed: disk I/O error".into(),
845 },
846 );
847 let cleanup_failed_context = cleanup_failed
848 .retryable_failure_context()
849 .expect("a failed cleanup rollback must remain typed-retryable");
850 assert_eq!(
851 cleanup_failed_context.stage,
852 super::READ_TX_AGE_EVICTED_STAGE
853 );
854 assert_eq!(cleanup_failed_context.timeout, Duration::from_secs(120));
855 assert_eq!(
856 cleanup_failed_context.operation.as_deref(),
857 Some("query_all")
858 );
859 assert_eq!(
860 cleanup_failed_context.capability,
861 Some(khive_storage::StorageCapability::Sql)
862 );
863
864 assert_ne!(
865 clean.to_string(),
866 cleanup_failed.to_string(),
867 "the rendered message must distinguish a clean eviction from a failed cleanup even \
868 though both map to the same wire stage"
869 );
870 assert!(cleanup_failed.to_string().contains("rollback failed"));
871 }
872}