1use khive_storage::{StorageCapability, StorageError};
8use serde_json::{json, Value};
9
10use crate::{DomainDisposition, RuntimeError};
11
12pub fn runtime_error_value(error: RuntimeError, disposition: DomainDisposition) -> Value {
19 let disposition = if disposition == DomainDisposition::Committed {
20 disposition
21 } else {
22 error.batch_write_disposition().unwrap_or(disposition)
23 };
24 runtime_error_value_inner(error, disposition)
25}
26
27pub enum FailureEntrySource {
29 Runtime {
31 error: RuntimeError,
32 boundary: DomainDisposition,
33 },
34 Value {
36 error: Value,
37 disposition: DomainDisposition,
38 },
39}
40
41pub struct FailureEntryParts {
43 error: Value,
44 disposition: DomainDisposition,
45}
46
47impl FailureEntryParts {
48 pub fn into_error(self) -> Value {
50 self.error
51 }
52
53 pub fn into_entry(self, tool: impl Into<String>) -> Value {
55 let mut entry = serde_json::Map::new();
56 entry.insert("ok".into(), Value::Bool(false));
57 entry.insert("tool".into(), Value::String(tool.into()));
58 entry.insert(
59 "domain_disposition".into(),
60 json!(self.disposition.as_str()),
61 );
62 entry.insert("error".into(), self.error);
63 Value::Object(entry)
64 }
65}
66
67pub fn failure_error_disposition(error: &Value) -> DomainDisposition {
71 match error.get("domain_disposition").and_then(Value::as_str) {
72 Some("committed") => DomainDisposition::Committed,
73 Some("not_committed") => DomainDisposition::NotCommitted,
74 _ => DomainDisposition::Unknown,
75 }
76}
77
78pub fn failure_entry_parts(source: FailureEntrySource) -> FailureEntryParts {
87 use crate::daemon::{drop_error_detail_iteratively, error_detail_value_within_limit};
88
89 let (error, boundary, disposition) = match source {
90 FailureEntrySource::Runtime { error, boundary } => {
91 let projected = runtime_error_value(error, boundary);
92 let disposition = failure_error_disposition(&projected);
93 (projected, boundary, disposition)
94 }
95 FailureEntrySource::Value { error, disposition } => (error, disposition, disposition),
96 };
97 let mut error = match error {
98 Value::Object(map) => map,
99 Value::String(message) => serde_json::Map::from_iter([
100 ("kind".into(), json!("runtime_error")),
101 ("message".into(), Value::String(message)),
102 ]),
103 other => {
104 drop_error_detail_iteratively(other);
105 serde_json::Map::from_iter([
106 ("kind".into(), json!("runtime_error")),
107 ("message".into(), json!("operation failed")),
108 ])
109 }
110 };
111 let committed = |value: DomainDisposition| value == DomainDisposition::Committed;
112 if let Some(result) = error.remove("domain_result") {
113 if !committed(boundary) {
114 drop_error_detail_iteratively(result);
115 } else if !error_detail_value_within_limit(&result) {
116 drop_error_detail_iteratively(result);
117 error.insert("code".into(), json!("result_too_deep"));
118 error.insert(
119 "message".into(),
120 json!("committed domain result omitted because it exceeds the nesting depth limit"),
121 );
122 } else if !committed(disposition) {
123 drop_error_detail_iteratively(result);
124 } else {
125 error.insert("domain_result".into(), result);
126 }
127 }
128 error.insert("domain_disposition".into(), json!(disposition.as_str()));
129 FailureEntryParts {
130 error: Value::Object(error),
131 disposition,
132 }
133}
134
135fn runtime_error_value_inner(error: RuntimeError, disposition: DomainDisposition) -> Value {
138 let sqlite_write_failure = match error.refusal_source() {
141 RuntimeError::Storage(error) => error.sqlite_write_failure(),
142 _ => None,
143 };
144 let outcome_has_precedence = error.writer_task_failure_context().is_some_and(|context| {
145 context.task_terminated
146 || context.request_state == khive_storage::WriterTaskRequestState::SideEffectsUnknown
147 });
148 let receipt_projection = crate::visibility_receipts::receipt_error_projection(&error);
149 let named_disposition = match &error {
150 _ if receipt_projection.is_some_and(|(_, replay)| replay) => Some("not_committed"),
151 error if error.is_stream_policy_refusal() => Some("not_committed"),
155 RuntimeError::RefusedWithReceipt(_) => Some("not_committed"),
158 RuntimeError::Khive(k) => match (k.kind(), k.details().and_then(|d| d.get("reason"))) {
159 (
160 khive_types::ErrorKind::Internal,
161 Some("post_commit_degraded" | "embedding_input_truncated"),
162 ) => Some("committed"),
163 (
165 khive_types::ErrorKind::Conflict,
166 Some("key_conflict" | "fence_conflict" | "idempotency_key_conflict"),
167 ) => Some("not_committed"),
168 (khive_types::ErrorKind::Unavailable, Some("key_holder_unresolved")) => Some("unknown"),
169 (khive_types::ErrorKind::Conflict, Some("seq_conflict")) => Some("not_committed"),
177 (khive_types::ErrorKind::Conflict, Some("unknown_op")) => Some("not_committed"),
178 (khive_types::ErrorKind::Conflict, Some("version_conflict" | "identity_conflict")) => {
179 Some("not_committed")
180 }
181 (khive_types::ErrorKind::Conflict, Some("expired" | "live_until_unreadable")) => {
182 Some("not_committed")
183 }
184 (khive_types::ErrorKind::NotFound, Some("stream_write_not_found")) => {
185 Some("not_committed")
186 }
187 (khive_types::ErrorKind::NotFound, Some("keyed_replay_holder_missing")) => {
190 Some("not_committed")
191 }
192 (
193 khive_types::ErrorKind::InvalidInput,
194 Some("member_unavailable" | "sequence_exhausted"),
195 ) => Some("not_committed"),
196 _ => None,
197 },
198 _ => None,
199 };
200 let denial_message =
203 matches!(error, RuntimeError::PermissionDenied { .. }).then(|| error.to_string());
204 let payload = match error {
205 RuntimeError::SearchRead { context } => {
206 let message = context.to_string();
207 let backend = context.backend().as_str().to_owned();
208 let substrate = context.substrate().as_str();
209 let cause_class = context.source().audit_error_class().as_str();
210 let retryable = context.source().retryable_failure_context().is_some();
211 let source = runtime_error_value_inner(context.into_source(), disposition);
214 let kind = source["kind"]
215 .as_str()
216 .expect("canonical error kind is a string");
217 assert!(
218 matches!(
219 kind,
220 "runtime_error"
221 | "obligation"
222 | "internal"
223 | "remote_fetch_error"
224 | "not_found"
225 | "storage"
226 | "unavailable"
227 | "invalid_input"
228 | "unauthorized"
229 | "conflict"
230 ),
231 "canonical error kind must have an explicit privacy classification"
232 );
233 let mut value = json!({
234 "kind": kind,
235 "code": "search_read_failed",
236 "message": message,
237 "cause_class": cause_class,
238 "domain_disposition": failure_error_disposition(&source).as_str(),
239 "search_context": {"backend": backend, "substrate": substrate},
240 });
241 if retryable {
242 value["retryable"] = json!(true);
243 }
244 return value;
245 }
246 RuntimeError::CanonicalUnsupportedOperation {
247 op_index,
248 operation,
249 } => {
250 return json!({
251 "kind": "runtime_error",
252 "code": "invalid_input",
253 "reason": "KnownUnimplemented",
254 "operation": operation,
255 "op_index": op_index,
256 "message": format!("canonical operation {op_index}: {operation} is KnownUnimplemented for canonical apply"),
257 "domain_disposition": "not_committed",
258 "retryable": false,
259 });
260 }
261 RuntimeError::CanonicalAdmission { op_index, source } => {
262 let mut value = runtime_error_value_inner(*source, DomainDisposition::NotCommitted);
266 value["op_index"] = json!(op_index);
267 value["domain_disposition"] = json!("not_committed");
268 value["retryable"] = json!(false);
269 return value;
270 }
271 RuntimeError::LintFix { source, detail } => {
272 let mut value = runtime_error_value_inner(*source, DomainDisposition::Unknown);
273 let object = value
274 .as_object_mut()
275 .expect("runtime error projection is an object");
276 let details = object.entry("details").or_insert_with(|| json!({}));
277 if !details.is_object() {
278 *details = json!({});
279 }
280 details["lint_fix"] = json!(detail);
281 value["domain_disposition"] = json!("unknown");
284 value["retryable"] = json!(false);
285 return value;
286 }
287
288 RuntimeError::WithResolution { context } => {
289 let crate::error::ResolutionFailureContext { source, resolution } = context;
290 let mut value = runtime_error_value_inner(*source, disposition);
291 let details = value
292 .as_object_mut()
293 .expect("runtime error projection is an object")
294 .entry("details")
295 .or_insert_with(|| json!({}));
296 if !details.is_object() {
297 *details = json!({});
298 }
299 details["resolution"] = json!(resolution);
300 return value;
301 }
302 RuntimeError::BatchWrite { context } => {
303 let crate::error::BatchWriteContext {
304 source, admission, ..
305 } = *context;
306 let mut value = runtime_error_value_inner(*source, disposition);
307 if let Some(report) = admission {
308 if report.validation_complete {
309 if let Some(first) = report.refused.first() {
310 if !value["details"].is_object() {
311 value["details"] = json!({});
312 }
313 value["details"]["index"] = json!(first.index.to_string());
314 value["details"]["field"] = json!(first.field);
315 value["details"]["reason"] = json!(first.reason);
316 }
317 } else {
318 value["unchecked_indices"] = json!(report.unchecked_indices);
319 }
320 value["refused"] = json!(report.refused);
321 value["validation_complete"] = json!(report.validation_complete);
322 }
323 value["domain_disposition"] = json!(disposition.as_str());
324 return value;
325 }
326 RuntimeError::RefusedWithEvents { context } => {
327 let crate::error::RefusalEventContext { source, recordings } = context;
328 let mut value = runtime_error_value_inner(*source, disposition);
331 if !recordings.is_empty() {
332 value["refusal_recorded"] = json!(recordings.iter().all(|recording| {
333 matches!(recording, crate::RefusalEventRecording::Recorded { .. })
334 }));
335 value["refusal_events"] = json!(recordings);
336 }
337 return value;
338 }
339 RuntimeError::PermissionDenied {
340 verb,
341 reason,
342 receipt,
343 } => json!({
344 "kind": "runtime_error",
345 "code": "permission_denied",
346 "message": denial_message.unwrap_or_default(),
347 "verb": verb,
348 "reason": reason,
349 "audit_event_id": receipt.audit_event_id.map(|id| id.to_string()),
350 "audit_outcome": receipt.audit_outcome.wire_code(),
351 }),
352 RuntimeError::SecretDetected(matched) => {
353 let message = RuntimeError::SecretDetected(matched.clone()).to_string();
354 json!({
355 "kind": "runtime_error",
356 "code": "secret_detected",
357 "detector": matched.detector,
358 "location": matched.location,
359 "message": message,
360 })
361 }
362 RuntimeError::RefusedWithReceipt(refusal) => {
363 let crate::error::ReceiptRefusal {
364 code,
365 message,
366 receipt_id,
367 reason,
368 detail,
369 } = *refusal;
370 let mut error = match detail {
373 Value::Object(members) => members,
374 _ => serde_json::Map::new(),
375 };
376 error.insert("kind".into(), json!("runtime_error"));
377 error.insert("code".into(), json!(code));
378 error.insert("message".into(), json!(message));
379 error.insert("receipt_id".into(), json!(receipt_id));
380 error.insert("reason".into(), json!(reason));
381 Value::Object(error)
382 }
383 RuntimeError::AuditObligation {
384 failure,
385 domain_result,
386 } => {
387 let mut error = serde_json::Map::from_iter([
388 ("kind".into(), json!("obligation")),
389 ("code".into(), json!(failure.wire_code())),
390 ("message".into(), json!(failure.to_string())),
391 ]);
392 error.insert("domain_result".into(), domain_result);
393 Value::Object(error)
394 }
395 RuntimeError::Khive(k) => {
396 let mut value = serde_json::to_value(&k)
397 .unwrap_or_else(|_| json!({"kind": "internal", "message": k.to_string()}));
398 if k.kind() == khive_types::ErrorKind::InvalidInput
399 && k.details().and_then(|d| d.get("reason")) == Some("external_id_unverifiable")
400 {
401 value["code"] = json!("external_id_unverifiable");
402 }
403 value
404 }
405 RuntimeError::RemoteFetchError { remote, message } => json!({
406 "kind": "remote_fetch_error",
407 "remote": remote,
408 "message": message,
409 }),
410 missing @ RuntimeError::NotFound(_) => json!({
411 "kind": "not_found",
412 "code": null,
413 "details": null,
414 "message": missing.to_string(),
415 }),
416 other @ (RuntimeError::Storage(_)
417 | RuntimeError::Query(_)
418 | RuntimeError::InvalidInput(_)
419 | RuntimeError::UnknownVerb(_)
420 | RuntimeError::Unconfigured(_)
421 | RuntimeError::UnknownModel(_)
422 | RuntimeError::Embedding(_)
423 | RuntimeError::Ambiguous(_)
424 | RuntimeError::Fusion(_)
425 | RuntimeError::UnknownFusionStrategy(_)
426 | RuntimeError::Internal(_)
427 | RuntimeError::IncompatibleEventStore(_)
428 | RuntimeError::GuardedWriteFailed(_)
429 | RuntimeError::MissingPackDependency(_)
430 | RuntimeError::MissingPackDependencies(_)
431 | RuntimeError::CircularPackDependency(_)
432 | RuntimeError::PackRedeclared { .. }
433 | RuntimeError::VerbCollision { .. }
434 | RuntimeError::ReservedEnvelopeParam { .. }
435 | RuntimeError::GateUnavailable { .. }
436 | RuntimeError::NamespaceMismatch { .. }
437 | RuntimeError::AmbiguousPrefix { .. }
438 | RuntimeError::CrossBackendMergeUnsupported { .. }
439 | RuntimeError::UnknownRemote { .. }
440 | RuntimeError::RemoteCacheMissing { .. }
441 | RuntimeError::AmbiguousId { .. }
442 | RuntimeError::CrossNamespaceWrite { .. }
443 | RuntimeError::WriteBudgetExceeded { .. }
444 | RuntimeError::DeadlineExceeded { .. }
445 | RuntimeError::AttachmentCutoverRequired { .. }) => {
446 if let Some(capacity) = sqlite_capacity_failure(&other) {
447 let mut value = capacity.into_value(other.to_string());
448 if let Some(context) = other.writer_task_failure_context() {
449 value["request_state"] = json!(context.request_state.to_string());
450 value["task_terminated"] = json!(context.task_terminated);
451 }
452 value
453 } else if let Some(context) = other.writer_task_failure_context() {
454 json!({"kind":"storage", "code":context.stage, "stage":context.stage,
455 "message":other.to_string(), "retryable":context.retryable,
456 "request_state":context.request_state.to_string(), "task_terminated":context.task_terminated})
457 } else if let Some(context) = other.retryable_failure_context() {
458 let timeout_ms = u64::try_from(context.timeout.as_millis()).unwrap_or(u64::MAX);
459 let mut value = json!({"kind":"unavailable", "code":context.stage, "stage":context.stage,
460 "message":other.to_string(), "retryable":true, "timeout_ms":timeout_ms,
461 "capability":context.capability.map(storage_capability_wire_name),
462 "operation":context.operation, "scope":context.scope, "retry_after_ms":context.retry_after_ms});
463 if let Some(pool_identity) = context.pool_identity {
464 value["pool_identity"] = json!(pool_identity);
465 }
466 value
467 } else {
468 json!({"kind":"runtime_error", "message":other.to_string()})
469 }
470 }
471 };
472 let mut value = payload;
473 if let Some(failure) = sqlite_write_failure {
474 if !outcome_has_precedence
476 && !failure.settlement_unknown
477 && failure.primary_code != rusqlite::ffi::SQLITE_FULL
478 {
479 value["code"] = json!(failure.stage.as_str());
480 value["stage"] = json!(failure.stage.as_str());
481 if value["kind"] == "runtime_error" {
482 value["kind"] = json!("storage");
483 }
484 }
485 value["sqlite_write_stage"] = json!(failure.stage.as_str());
486 value["sqlite_primary_code"] = json!(failure.primary_code);
487 value["sqlite_extended_code"] = json!(failure.extended_code);
488 }
489 if let Some((retryable, _)) = receipt_projection {
490 value["retryable"] = json!(retryable);
491 }
492 value["domain_disposition"] = json!(disposition.as_str());
493 if let Some(named) = named_disposition {
494 value["domain_disposition"] = json!(named);
495 }
496 value
497}
498
499enum SqliteCapacityFailure {
500 Refused {
501 volume: String,
502 available_bytes: u64,
503 reserve_bytes: u64,
504 required_headroom_bytes: u64,
505 },
506 Unavailable {
507 phase: khive_storage::CapacityUnavailablePhase,
508 },
509 NativeFull {
510 primary_code: i32,
511 extended_code: i32,
512 },
513}
514
515impl SqliteCapacityFailure {
516 fn into_value(self, message: String) -> Value {
517 match self {
518 Self::Refused {
519 volume,
520 available_bytes,
521 reserve_bytes,
522 required_headroom_bytes,
523 } => json!({
524 "kind": "storage",
525 "code": "sqlite_capacity_refused",
526 "stage": "sqlite_capacity_refused",
527 "message": message,
528 "retryable": false,
529 "capability": "sql",
530 "volume": volume,
531 "available_bytes": available_bytes,
532 "reserve_bytes": reserve_bytes,
533 "required_headroom_bytes": required_headroom_bytes,
534 }),
535 Self::Unavailable { phase } => json!({
536 "kind": "storage",
537 "code": "sqlite_capacity_unavailable",
538 "stage": "sqlite_capacity_unavailable",
539 "message": message,
540 "retryable": phase == khive_storage::CapacityUnavailablePhase::LockContended,
541 "capability": "sql",
542 "phase": phase.as_str(),
543 }),
544 Self::NativeFull {
545 primary_code,
546 extended_code,
547 } => json!({
548 "kind": "storage",
549 "code": "sqlite_disk_full",
550 "stage": "sqlite_disk_full",
551 "message": message,
552 "retryable": false,
553 "capability": "sql",
554 "sqlite_primary_code": primary_code,
555 "sqlite_extended_code": extended_code,
556 }),
557 }
558 }
559}
560
561fn sqlite_capacity_failure(error: &RuntimeError) -> Option<SqliteCapacityFailure> {
562 match error {
563 RuntimeError::Storage(storage) => sqlite_capacity_failure_from_storage(storage),
564 _ => None,
565 }
566}
567
568fn sqlite_capacity_failure_from_storage(error: &StorageError) -> Option<SqliteCapacityFailure> {
569 match error {
570 StorageError::CapacityFloor {
571 capability: StorageCapability::Sql,
572 volume,
573 available_bytes,
574 floor_bytes,
575 required_headroom_bytes,
576 } => Some(SqliteCapacityFailure::Refused {
577 volume: volume.clone(),
578 available_bytes: *available_bytes,
579 reserve_bytes: *floor_bytes,
580 required_headroom_bytes: *required_headroom_bytes,
581 }),
582 StorageError::CapacityUnavailable {
583 capability: StorageCapability::Sql,
584 phase,
585 ..
586 } => Some(SqliteCapacityFailure::Unavailable { phase: *phase }),
587 StorageError::WriterTaskTerminated {
588 sqlite_full_codes: Some((primary_code, extended_code)),
589 ..
590 } if *primary_code == rusqlite::ffi::SQLITE_FULL
591 && *extended_code >= 0
592 && (*extended_code & 0xff) == *primary_code =>
593 {
594 Some(SqliteCapacityFailure::NativeFull {
595 primary_code: *primary_code,
596 extended_code: *extended_code,
597 })
598 }
599 StorageError::SqliteWrite { failure, .. }
600 if failure.primary_code == rusqlite::ffi::SQLITE_FULL =>
601 {
602 Some(SqliteCapacityFailure::NativeFull {
603 primary_code: failure.primary_code,
604 extended_code: failure.extended_code,
605 })
606 }
607 StorageError::WriterTaskRequestFailed { source, .. }
608 | StorageError::SqliteWrite { source, .. } => sqlite_capacity_failure_from_storage(source),
609 StorageError::Driver { source, .. } => {
610 if let Some(sqlite) = source.downcast_ref::<khive_db::SqliteError>() {
611 sqlite_capacity_failure_from_sqlite(sqlite)
612 } else {
613 native_sqlite_full(source.as_ref())
614 }
615 }
616 _ => None,
617 }
618}
619
620fn sqlite_capacity_failure_from_sqlite(
621 error: &khive_db::SqliteError,
622) -> Option<SqliteCapacityFailure> {
623 match error {
624 khive_db::SqliteError::CapacityFloor {
625 volume,
626 available_bytes,
627 floor_bytes,
628 required_headroom_bytes,
629 } => Some(SqliteCapacityFailure::Refused {
630 volume: volume.clone(),
631 available_bytes: *available_bytes,
632 reserve_bytes: *floor_bytes,
633 required_headroom_bytes: *required_headroom_bytes,
634 }),
635 khive_db::SqliteError::CapacityUnavailable { phase, .. } => {
636 Some(SqliteCapacityFailure::Unavailable { phase: *phase })
637 }
638 khive_db::SqliteError::Rusqlite(sqlite)
639 | khive_db::SqliteError::Write { source: sqlite, .. } => native_sqlite_full(sqlite),
640 _ => None,
641 }
642}
643
644fn native_sqlite_full(error: &(dyn std::error::Error + 'static)) -> Option<SqliteCapacityFailure> {
645 let mut source = Some(error);
646 while let Some(current) = source {
647 if let Some(rusqlite::Error::SqliteFailure(code, _)) =
648 current.downcast_ref::<rusqlite::Error>()
649 {
650 if code.code == rusqlite::ErrorCode::DiskFull {
651 return Some(SqliteCapacityFailure::NativeFull {
652 primary_code: code.extended_code & 0xff,
653 extended_code: code.extended_code,
654 });
655 }
656 }
657 source = current.source();
658 }
659 None
660}
661
662fn storage_capability_wire_name(capability: StorageCapability) -> &'static str {
663 match capability {
664 StorageCapability::Sql => "sql",
665 StorageCapability::Notes => "notes",
666 StorageCapability::Entities => "entities",
667 StorageCapability::Graph => "graph",
668 StorageCapability::Events => "events",
669 StorageCapability::Vectors => "vectors",
670 StorageCapability::Sparse => "sparse",
671 StorageCapability::Text => "text",
672 StorageCapability::Blob => "blob",
673 StorageCapability::Attachments => "attachments",
674 }
675}
676
677#[cfg(test)]
678#[path = "error_projection_tests.rs"]
679mod tests;