use std::fmt;
use std::time::Duration;
use thiserror::Error;
use uuid::Uuid;
pub type RuntimeResult<T> = Result<T, RuntimeError>;
pub const WRITER_POOL_CHECKOUT_TIMEOUT_STAGE: &str = "writer_pool_checkout_timeout";
pub const WRITER_QUEUE_SATURATED_STAGE: &str = "writer_queue_saturated";
pub const WRITER_TASK_BEGIN_BUSY_STAGE: &str = "writer_task_begin_busy";
pub const STORAGE_ADMISSION_TIMEOUT_STAGE: &str = "storage_admission_timeout";
pub const READ_TX_AGE_EVICTED_STAGE: &str = "read_tx_age_evicted";
pub const WRITER_ADMISSION_SCOPE: &str = "writer_admission";
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AdmissionFailureContext {
pub stage: &'static str,
pub timeout: Duration,
pub capability: Option<khive_storage::StorageCapability>,
pub operation: Option<String>,
pub scope: Option<&'static str>,
pub retry_after_ms: Option<u64>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RetryableFailureContext {
pub stage: &'static str,
pub timeout: Duration,
pub capability: Option<khive_storage::StorageCapability>,
pub operation: Option<String>,
pub scope: Option<&'static str>,
pub retry_after_ms: Option<u64>,
}
impl From<AdmissionFailureContext> for RetryableFailureContext {
fn from(context: AdmissionFailureContext) -> Self {
Self {
stage: context.stage,
timeout: context.timeout,
capability: context.capability,
operation: context.operation,
scope: context.scope,
retry_after_ms: context.retry_after_ms,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ChannelIngestFailureClass {
Retryable { reason: &'static str },
Permanent { reason: &'static str },
Unknown { reason: &'static str },
}
impl ChannelIngestFailureClass {
pub const fn name(self) -> &'static str {
match self {
Self::Retryable { .. } => "retryable",
Self::Permanent { .. } => "permanent",
Self::Unknown { .. } => "unknown",
}
}
pub const fn reason(self) -> &'static str {
match self {
Self::Retryable { reason } | Self::Permanent { reason } | Self::Unknown { reason } => {
reason
}
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct WriterPoolCheckoutTimeoutContext {
pub timeout: Duration,
pub capability: Option<khive_storage::StorageCapability>,
pub operation: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct GuardedWriteFailure {
pub entry_index: Option<usize>,
pub missing_source: Option<Uuid>,
pub missing_target: Option<Uuid>,
}
impl fmt::Display for GuardedWriteFailure {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let mut missing = Vec::new();
if let Some(source) = self.missing_source {
missing.push(format!("source {source}"));
}
if let Some(target) = self.missing_target {
missing.push(format!("target {target}"));
}
let missing = if missing.is_empty() {
"endpoint(s)".to_string()
} else {
missing.join(" and ")
};
match self.entry_index {
Some(index) => write!(
f,
"batch entry {index}: {missing} no longer exist at write time"
),
None => write!(f, "{missing} no longer exist at write time"),
}
}
}
impl std::error::Error for GuardedWriteFailure {}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MissingPackDependency {
pub from: String,
pub requires: String,
}
impl fmt::Display for MissingPackDependency {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"pack '{}' requires '{}', but '{}' is not in the loaded pack set",
self.from, self.requires, self.requires
)
}
}
impl std::error::Error for MissingPackDependency {}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MissingPackDependencies {
pub missing: Vec<MissingPackDependency>,
}
impl fmt::Display for MissingPackDependencies {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let parts: Vec<String> = self.missing.iter().map(ToString::to_string).collect();
write!(f, "{}", parts.join("; "))
}
}
impl std::error::Error for MissingPackDependencies {}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CircularPackDependency {
pub cycle: Vec<String>,
}
impl fmt::Display for CircularPackDependency {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"circular dependency detected among packs: {}",
self.cycle.join(" -> ")
)
}
}
impl std::error::Error for CircularPackDependency {}
#[derive(Debug, Error)]
pub enum RuntimeError {
#[error("storage: {0}")]
Storage(#[from] khive_storage::StorageError),
#[error("sqlite: {0}")]
Sqlite(#[from] khive_db::SqliteError),
#[error("query: {0}")]
Query(#[from] khive_query::QueryError),
#[error("not found: {0}")]
NotFound(String),
#[error("invalid input: {0}")]
InvalidInput(String),
#[error("invalid input: {0}")]
UnknownVerb(String),
#[error("unconfigured: {0} is not set")]
Unconfigured(String),
#[error("unknown embedding model: {0}")]
UnknownModel(String),
#[error("embedding: {0}")]
Embedding(#[from] lattice_embed::EmbedError),
#[error("ambiguous: {0}")]
Ambiguous(String),
#[error("fusion: {0}")]
Fusion(#[from] khive_fusion::FuseError),
#[error("unknown fusion strategy: {0}")]
UnknownFusionStrategy(String),
#[error("internal: {0}")]
Internal(String),
#[error("audit batch incompatible event store: {0}")]
IncompatibleEventStore(String),
#[error("guarded edge write refused: {0}")]
GuardedWriteFailed(GuardedWriteFailure),
#[error("missing pack dependency: {0}")]
MissingPackDependency(MissingPackDependency),
#[error("missing pack dependencies: {0}")]
MissingPackDependencies(MissingPackDependencies),
#[error("{0}")]
CircularPackDependency(CircularPackDependency),
#[error("pack '{name}' registered twice (indices {first_idx} and {second_idx})")]
PackRedeclared {
name: String,
first_idx: usize,
second_idx: usize,
},
#[error(
"verb collision: verb {verb:?} declared by both pack {first_pack:?} and pack \
{second_pack:?}; rename one handler or use Visibility::Subhandler for internal verbs"
)]
VerbCollision {
verb: String,
first_pack: String,
second_pack: String,
},
#[error(
"pack {pack:?} handler {verb:?} declares request-envelope parameter {param:?}; rename the verb argument"
)]
ReservedEnvelopeParam {
pack: String,
verb: String,
param: String,
},
#[error("permission denied for verb {verb:?}: {reason}")]
PermissionDenied { verb: String, reason: String },
#[error("gate unavailable for verb {verb:?}: {reason}")]
GateUnavailable { verb: String, reason: String },
#[error("{0}")]
Khive(khive_types::KhiveError),
#[error("not found in this namespace")]
NamespaceMismatch { id: uuid::Uuid },
#[error("ambiguous prefix {prefix:?}: matches {}", format_uuid_list(matches))]
AmbiguousPrefix {
prefix: String,
matches: Vec<uuid::Uuid>,
},
#[error(
"cross-backend merge is not supported: \
into_id {into_id} is on backend '{into_backend}', \
from_id {from_id} is on backend '{from_backend}'. \
Both entities must be on the same backend to merge."
)]
CrossBackendMergeUnsupported {
into_id: uuid::Uuid,
from_id: uuid::Uuid,
into_backend: String,
from_backend: String,
},
#[error("unknown remote: {name:?}")]
UnknownRemote { name: String },
#[error("remote cache missing for remote={remote:?} namespace={namespace:?}")]
RemoteCacheMissing { remote: String, namespace: String },
#[error("ambiguous id {id:?}: matched {count} records")]
AmbiguousId { id: String, count: usize },
#[error("cross-namespace write denied: cannot write to remote namespace {namespace:?}")]
CrossNamespaceWrite { namespace: String },
#[error("remote fetch error for remote={remote:?}: {message}")]
RemoteFetchError { remote: String, message: String },
#[error(
"write budget exceeded: max_new_entries={max_new_entries}, \
attempted_new_entries={attempted_new_entries}"
)]
WriteBudgetExceeded {
max_new_entries: u64,
attempted_new_entries: u64,
},
#[error("write blocked: {0}")]
SecretDetected(crate::secret_gate::SecretMatch),
#[error("{operation} exceeded its {budget_ms}ms deadline (elapsed {elapsed_ms}ms)")]
DeadlineExceeded {
operation: String,
budget_ms: u64,
elapsed_ms: u64,
},
}
impl RuntimeError {
pub fn channel_ingest_failure_class(&self) -> ChannelIngestFailureClass {
let reason = self.variant_name();
if self.retryable_failure_context().is_some() {
ChannelIngestFailureClass::Retryable { reason }
} else if matches!(self, Self::SecretDetected(_)) {
ChannelIngestFailureClass::Permanent { reason }
} else {
ChannelIngestFailureClass::Unknown { reason }
}
}
const fn variant_name(&self) -> &'static str {
match self {
Self::Storage(_) => "Storage",
Self::Sqlite(_) => "Sqlite",
Self::Query(_) => "Query",
Self::NotFound(_) => "NotFound",
Self::InvalidInput(_) => "InvalidInput",
Self::UnknownVerb(_) => "UnknownVerb",
Self::Unconfigured(_) => "Unconfigured",
Self::UnknownModel(_) => "UnknownModel",
Self::Embedding(_) => "Embedding",
Self::Ambiguous(_) => "Ambiguous",
Self::Fusion(_) => "Fusion",
Self::UnknownFusionStrategy(_) => "UnknownFusionStrategy",
Self::Internal(_) => "Internal",
Self::GuardedWriteFailed(_) => "GuardedWriteFailed",
Self::MissingPackDependency(_) => "MissingPackDependency",
Self::MissingPackDependencies(_) => "MissingPackDependencies",
Self::CircularPackDependency(_) => "CircularPackDependency",
Self::PackRedeclared { .. } => "PackRedeclared",
Self::VerbCollision { .. } => "VerbCollision",
Self::ReservedEnvelopeParam { .. } => "ReservedEnvelopeParam",
Self::PermissionDenied { .. } => "PermissionDenied",
Self::GateUnavailable { .. } => "GateUnavailable",
Self::Khive(_) => "Khive",
Self::NamespaceMismatch { .. } => "NamespaceMismatch",
Self::AmbiguousPrefix { .. } => "AmbiguousPrefix",
Self::CrossBackendMergeUnsupported { .. } => "CrossBackendMergeUnsupported",
Self::UnknownRemote { .. } => "UnknownRemote",
Self::RemoteCacheMissing { .. } => "RemoteCacheMissing",
Self::AmbiguousId { .. } => "AmbiguousId",
Self::CrossNamespaceWrite { .. } => "CrossNamespaceWrite",
Self::RemoteFetchError { .. } => "RemoteFetchError",
Self::WriteBudgetExceeded { .. } => "WriteBudgetExceeded",
Self::SecretDetected(_) => "SecretDetected",
Self::DeadlineExceeded { .. } => "DeadlineExceeded",
Self::IncompatibleEventStore(_) => "IncompatibleEventStore",
}
}
pub fn writer_pool_checkout_timeout_context(&self) -> Option<WriterPoolCheckoutTimeoutContext> {
let (sqlite_error, capability, operation) = match self {
Self::Sqlite(error) => (error, None, None),
Self::Storage(khive_storage::StorageError::Driver {
capability,
operation,
source,
}) => (
source.downcast_ref::<khive_db::SqliteError>()?,
Some(*capability),
Some(operation.to_string()),
),
_ => return None,
};
let khive_db::SqliteError::WriterPoolCheckoutTimeout { timeout } = sqlite_error else {
return None;
};
Some(WriterPoolCheckoutTimeoutContext {
timeout: *timeout,
capability,
operation,
})
}
pub fn admission_failure_context(&self) -> Option<AdmissionFailureContext> {
if let Some(context) = self.writer_pool_checkout_timeout_context() {
return Some(AdmissionFailureContext {
stage: WRITER_POOL_CHECKOUT_TIMEOUT_STAGE,
timeout: context.timeout,
capability: context.capability,
operation: context.operation,
scope: None,
retry_after_ms: None,
});
}
if let Self::Storage(khive_storage::StorageError::WriteQueueFull { timeout_ms }) = self {
return Some(AdmissionFailureContext {
stage: WRITER_QUEUE_SATURATED_STAGE,
timeout: Duration::from_millis(*timeout_ms),
capability: None,
operation: None,
scope: Some(WRITER_ADMISSION_SCOPE),
retry_after_ms: Some(*timeout_ms),
});
}
if let Self::Storage(khive_storage::StorageError::AdmissionTimeout {
operation,
timeout_ms,
}) = self
{
return Some(AdmissionFailureContext {
stage: STORAGE_ADMISSION_TIMEOUT_STAGE,
timeout: Duration::from_millis(*timeout_ms),
capability: None,
operation: Some(operation.to_string()),
scope: None,
retry_after_ms: None,
});
}
None
}
pub fn retryable_failure_context(&self) -> Option<RetryableFailureContext> {
if let Some(context) = self.admission_failure_context() {
return Some(context.into());
}
if let Self::Storage(khive_storage::StorageError::WriterTaskBusy { timeout_ms }) = self {
return Some(RetryableFailureContext {
stage: WRITER_TASK_BEGIN_BUSY_STAGE,
timeout: Duration::from_millis(*timeout_ms),
capability: None,
operation: Some("writer_task_begin".to_string()),
scope: None,
retry_after_ms: None,
});
}
if let Self::Storage(khive_storage::StorageError::ReadTransactionAgeEvicted {
operation,
max_age_secs,
}) = self
{
return Some(RetryableFailureContext {
stage: READ_TX_AGE_EVICTED_STAGE,
timeout: Duration::from_secs(*max_age_secs),
capability: Some(khive_storage::StorageCapability::Sql),
operation: Some(operation.to_string()),
scope: None,
retry_after_ms: None,
});
}
if let Self::Storage(
khive_storage::StorageError::ReadTransactionAgeEvictionCleanupFailed {
operation,
max_age_secs,
..
},
) = self
{
return Some(RetryableFailureContext {
stage: READ_TX_AGE_EVICTED_STAGE,
timeout: Duration::from_secs(*max_age_secs),
capability: Some(khive_storage::StorageCapability::Sql),
operation: Some(operation.to_string()),
scope: None,
retry_after_ms: None,
});
}
None
}
}
pub fn fts_text_leg_or_err<T>(
result: Result<Vec<T>, RuntimeError>,
context: &'static str,
query: &str,
) -> RuntimeResult<Vec<T>> {
match result {
Ok(hits) => Ok(hits),
Err(RuntimeError::Storage(se)) if se.is_fts5_syntax_error() => {
tracing::warn!(
error = %se,
query = %query,
context,
"FTS text leg failed on a parser syntax error; failing loud (#569)"
);
Err(RuntimeError::InvalidInput(format!(
"{context}: FTS query could not be parsed: {se}"
)))
}
Err(e) => Err(e),
}
}
fn format_uuid_list(uuids: &[uuid::Uuid]) -> String {
uuids
.iter()
.map(uuid::Uuid::to_string)
.collect::<Vec<_>>()
.join(", ")
}
impl From<khive_types::EntityTypeError> for RuntimeError {
fn from(e: khive_types::EntityTypeError) -> Self {
Self::InvalidInput(e.to_string())
}
}
impl From<khive_types::KhiveError> for RuntimeError {
fn from(e: khive_types::KhiveError) -> Self {
Self::Khive(e)
}
}
#[cfg(test)]
mod channel_ingest_failure_class_tests {
use super::{ChannelIngestFailureClass, RuntimeError};
use crate::secret_gate::SecretMatch;
use std::time::Duration;
#[test]
fn secret_detected_is_permanent_by_typed_variant_not_display_text() {
let first = RuntimeError::SecretDetected(SecretMatch {
detector: "fixture",
masked: "first-rendering".to_string(),
});
let second = RuntimeError::SecretDetected(SecretMatch {
detector: "fixture",
masked: "completely-different-rendering".to_string(),
});
assert_ne!(first.to_string(), second.to_string());
assert_eq!(
first.channel_ingest_failure_class(),
ChannelIngestFailureClass::Permanent {
reason: "SecretDetected"
}
);
assert_eq!(
second.channel_ingest_failure_class(),
ChannelIngestFailureClass::Permanent {
reason: "SecretDetected"
},
"rendered error details must not participate in ingest classification"
);
}
#[test]
fn admission_failures_are_retryable_and_unclassified_errors_are_unknown() {
let retryable =
RuntimeError::Storage(khive_storage::StorageError::WriteQueueFull { timeout_ms: 25 });
assert_eq!(
retryable.channel_ingest_failure_class(),
ChannelIngestFailureClass::Retryable { reason: "Storage" }
);
let begin_busy =
RuntimeError::Storage(khive_storage::StorageError::WriterTaskBusy { timeout_ms: 175 });
let context = begin_busy
.retryable_failure_context()
.expect("contended BEGIN must remain typed and retryable");
assert_eq!(context.stage, "writer_task_begin_busy");
assert_eq!(context.timeout, Duration::from_millis(175));
assert_eq!(context.operation.as_deref(), Some("writer_task_begin"));
assert_eq!(context.scope, None);
assert_eq!(context.retry_after_ms, None);
assert_eq!(
begin_busy.channel_ingest_failure_class(),
ChannelIngestFailureClass::Retryable { reason: "Storage" }
);
let unknown = RuntimeError::InvalidInput(
"write blocked: SecretDetected text must not affect classification".to_string(),
);
assert_eq!(
unknown.channel_ingest_failure_class(),
ChannelIngestFailureClass::Unknown {
reason: "InvalidInput"
},
"a rendered message resembling SecretDetected must remain Unknown unless its typed variant is SecretDetected"
);
}
#[test]
fn storage_admission_timeout_is_a_retryable_admission_failure() {
let admission = RuntimeError::Storage(khive_storage::StorageError::AdmissionTimeout {
operation: "sql_bridge.writer_handle".into(),
timeout_ms: 30_000,
});
let context = admission
.admission_failure_context()
.expect("a storage admission timeout happens before the operation starts");
assert_eq!(context.stage, super::STORAGE_ADMISSION_TIMEOUT_STAGE);
assert_eq!(context.timeout, Duration::from_millis(30_000));
assert_eq!(
context.operation.as_deref(),
Some("sql_bridge.writer_handle")
);
assert_eq!(context.scope, None);
assert_eq!(context.retry_after_ms, None);
assert_eq!(
admission.channel_ingest_failure_class(),
ChannelIngestFailureClass::Retryable { reason: "Storage" }
);
let mid_flight = RuntimeError::Storage(khive_storage::StorageError::Timeout {
operation: "sql_bridge.reader_open".into(),
});
assert!(mid_flight.admission_failure_context().is_none());
assert!(mid_flight.retryable_failure_context().is_none());
}
#[test]
fn both_read_tx_age_eviction_outcomes_map_to_the_same_retryable_stage() {
let clean = RuntimeError::Storage(khive_storage::StorageError::ReadTransactionAgeEvicted {
operation: "query_all".into(),
max_age_secs: 120,
});
let clean_context = clean
.retryable_failure_context()
.expect("a clean age eviction must be typed-retryable");
assert_eq!(clean_context.stage, super::READ_TX_AGE_EVICTED_STAGE);
assert_eq!(clean_context.timeout, Duration::from_secs(120));
assert_eq!(clean_context.operation.as_deref(), Some("query_all"));
assert_eq!(
clean_context.capability,
Some(khive_storage::StorageCapability::Sql)
);
let cleanup_failed = RuntimeError::Storage(
khive_storage::StorageError::ReadTransactionAgeEvictionCleanupFailed {
operation: "query_all".into(),
max_age_secs: 120,
message: "rollback failed: disk I/O error".into(),
},
);
let cleanup_failed_context = cleanup_failed
.retryable_failure_context()
.expect("a failed cleanup rollback must remain typed-retryable");
assert_eq!(
cleanup_failed_context.stage,
super::READ_TX_AGE_EVICTED_STAGE
);
assert_eq!(cleanup_failed_context.timeout, Duration::from_secs(120));
assert_eq!(
cleanup_failed_context.operation.as_deref(),
Some("query_all")
);
assert_eq!(
cleanup_failed_context.capability,
Some(khive_storage::StorageCapability::Sql)
);
assert_ne!(
clean.to_string(),
cleanup_failed.to_string(),
"the rendered message must distinguish a clean eviction from a failed cleanup even \
though both map to the same wire stage"
);
assert!(cleanup_failed.to_string().contains("rollback failed"));
}
}